Skip to content

Commit

Permalink
Handle failure better in service dataset creation.
Browse files Browse the repository at this point in the history
Wrapping withn try block calls in service's logic to have manager create
a dataset so that errors get turned into responses that make it back to
the client (rather than the connection just closing); also handling the
special case of invalid MinIO bucket names (i.e., dataset names) in a
customized, more informative way.
  • Loading branch information
robertbartel authored and hellkite500 committed Mar 15, 2024
1 parent b2987d7 commit b80d6d7
Showing 1 changed file with 32 additions and 10 deletions.
42 changes: 32 additions & 10 deletions python/services/dataservice/dmod/dataservice/service.py
Original file line number Diff line number Diff line change
Expand Up @@ -556,15 +556,37 @@ def _process_dataset_create(self, message: DatasetManagementMessage) -> DatasetM
return DatasetManagementResponse(action=message.management_action, success=False, message=msg,
reason="No Dataset Domain", dataset_name=message.dataset_name)

# Create the dataset
dataset_type = self._determine_dataset_type(message)
dataset = self._all_data_managers[dataset_type].create(name=message.dataset_name, category=message.data_category,
domain=message.data_domain, is_read_only=False)
# TODO: determine if there is an expectation to find data
# TODO: if so, attempt to find data, setting pending response based on result
return DatasetManagementResponse(action=message.management_action, success=True, reason="Dataset Created",
data_id=str(dataset.uuid), dataset_name=dataset.name,
is_awaiting=message.is_pending_data)

# Create the dataset
try:
dataset = self._all_data_managers[dataset_type].create(name=message.dataset_name,
category=message.data_category,
domain=message.data_domain, is_read_only=False)
# TODO: determine if there is an expectation to find data
# TODO: if so, attempt to find data, setting pending response based on result
return DatasetManagementResponse(action=message.management_action, success=True, reason="Dataset Created",
data_id=str(dataset.uuid), dataset_name=dataset.name,
is_awaiting=message.is_pending_data)
# Do something a little differently for this particular known special cases
except ValueError as e:
# TODO: (later) strictly speaking, this or something similar will probably applies to AWS S3 datasets later;
# see https://docs.aws.amazon.com/AmazonS3/latest/userguide/bucketnamingrules.html
if str(e) == f"invalid bucket name {message.dataset_name}" and dataset_type == DatasetType.OBJECT_STORE:
return DatasetManagementResponse(action=message.management_action, success=False,
reason=f"Unsupported Name For Backing Storage Type",
dataset_name=message.dataset_name, is_awaiting=message.is_pending_data,
message=f"Datasets of {dataset_type.name} type have additional name "
f"restrictions; names can consist only of lowercase letters, "
f"numbers, and hyphens (-)")
else:
return DatasetManagementResponse(action=message.management_action, success=False, message=str(e),
reason=f"Encountered {e.__class__.__name__}",
dataset_name=message.dataset_name, is_awaiting=message.is_pending_data)
except Exception as e:
return DatasetManagementResponse(action=message.management_action, success=False, message=str(e),
reason=f"Encountered {e.__class__.__name__}",
dataset_name=message.dataset_name, is_awaiting=message.is_pending_data)

def _process_dataset_delete(self, message: DatasetManagementMessage) -> DatasetManagementResponse:
"""
Expand Down Expand Up @@ -659,8 +681,8 @@ def _process_query(self, message: DatasetManagementMessage) -> DatasetManagement
dataset_name = message.dataset_name
list_of_files = self.get_known_datasets()[dataset_name].manager.list_files(dataset_name)
return DatasetManagementResponse(action=message.management_action, success=True, dataset_name=dataset_name,
reason='Obtained {} Items List',
data={DatasetManagementResponse._DATA_KEY_QUERY_RESULTS: list_of_files})
reason=f'Obtained {dataset_name} Items List',
data={"query_results": {QueryType.LIST_FILES.name: list_of_files}})
# TODO: (later) add support for messages with other query types also
else:
reason = 'Unsupported {} Query Type - {}'.format(DatasetQuery.__class__.__name__, query_type.name)
Expand Down

0 comments on commit b80d6d7

Please sign in to comment.