Skip to content

Commit

Permalink
fix feature none
Browse files Browse the repository at this point in the history
  • Loading branch information
huseinzol05 committed Oct 13, 2024
1 parent fe1c754 commit 07b82b1
Showing 1 changed file with 4 additions and 2 deletions.
6 changes: 4 additions & 2 deletions session/translation/end-to-end/run_t5_v2.py
Original file line number Diff line number Diff line change
Expand Up @@ -120,13 +120,15 @@ def __call__(self, features, return_tensors=None):
if return_tensors is None:
return_tensors = self.return_tensors

features = [f for f in features if f is not None]

label_name = 'labels'
labels = [feature[label_name] for feature in features if feature is not None]
labels = [feature[label_name] for feature in features]
# reconvert list[None] to None if necessary
# this might occur when we pass {..., "labels": None}
if labels is not None and all(label is None for label in labels):
labels = None
non_labels_features = [{k: v for k, v in feature.items() if k != label_name} for feature in features if feature is not None]
non_labels_features = [{k: v for k, v in feature.items() if k != label_name} for feature in features]

# run through tokenizer without labels to ensure no side effects
batch = pad_without_fast_tokenizer_warning(
Expand Down

0 comments on commit 07b82b1

Please sign in to comment.