Skip to content

Commit

Permalink
Fix not all exceptions being able to be pickled (#543)
Browse files Browse the repository at this point in the history
  • Loading branch information
gabrielmbmb authored Apr 16, 2024
1 parent 1ea4e95 commit 15d7f8c
Show file tree
Hide file tree
Showing 2 changed files with 8 additions and 3 deletions.
2 changes: 0 additions & 2 deletions src/distilabel/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,9 +13,7 @@
# limitations under the License.

from rich import traceback as rich_traceback
from tblib import pickling_support

__version__ = "1.0.0"

rich_traceback.install(show_locals=True)
pickling_support.install()
9 changes: 8 additions & 1 deletion src/distilabel/pipeline/local.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,8 @@
import traceback
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union, cast

import tblib

from distilabel.distiset import create_distiset
from distilabel.llms.mixins import CudaDevicePlacementMixin
from distilabel.pipeline.base import BasePipeline, _Batch, _BatchManager, _WriteBuffer
Expand Down Expand Up @@ -488,6 +490,9 @@ def _error_callback(self, e: BaseException) -> None:
with self.shared_info[_STEPS_LOADED_LOCK_KEY]:
self.shared_info[_STEPS_LOADED_KEY] = [_STEPS_LOADED_ERROR_CODE]
_SUBPROCESS_EXCEPTION = e.subprocess_exception
_SUBPROCESS_EXCEPTION.__traceback__ = tblib.Traceback.from_string(
e.formatted_traceback
).as_traceback()
return

# If the step is global, is not in the last trophic level and has no successors,
Expand Down Expand Up @@ -598,7 +603,9 @@ def __init__(
self.step = step
self.code = code
self.subprocess_exception = subprocess_exception
self.formatted_traceback = traceback.format_exc()
self.formatted_traceback = "".join(
traceback.format_exception(subprocess_exception)
)

@classmethod
def create_load_error(
Expand Down

0 comments on commit 15d7f8c

Please sign in to comment.