Merge branch 'main' into issue/225/ecsv

src/hipscat_import/pipeline_resume_plan.py

@@ -134,8 +134,21 @@ def wait_for_futures(self, futures, stage_name):
         ):
             if future.status == "error":
                 some_error = True
-                print(f"{stage_name} task {future.key} failed with message:")
-                print(future.exception())
+                exception = future.exception()
+                trace_strs = [
+                    f"{stage_name} task {future.key} failed with message:",
+                    f"  {type(exception).__name__}: {exception}",
+                    "  Traceback (most recent call last):",
+                ]
+                stack_trace = exception.__traceback__
+                while stack_trace is not None:
+                    filename = stack_trace.tb_frame.f_code.co_filename
+                    method_name = stack_trace.tb_frame.f_code.co_name
+                    line_number = stack_trace.tb_lineno
+                    trace_strs.append(f'    File "{filename}", line {line_number}, in {method_name}')
+                    stack_trace = stack_trace.tb_next
+                print("\n".join(trace_strs))
+
         if some_error:
             raise RuntimeError(f"Some {stage_name} stages failed. See logs for details.")
 

tests/hipscat_import/test_pipeline_resume_plan.py

@@ -97,8 +97,10 @@ def test_safe_to_resume(tmp_path):
 
 
 @pytest.mark.dask
-def test_wait_for_futures(tmp_path, dask_client):
-    """Test that we can wait around for futures to complete."""
+def test_wait_for_futures(tmp_path, dask_client, capsys):
+    """Test that we can wait around for futures to complete.
+
+    Additionally test that relevant parts of the traceback are printed to stdout."""
     plan = PipelineResumePlan(tmp_path=tmp_path, progress_bar=False, resume=False)
 
     def error_on_even(argument):
@@ -115,6 +117,10 @@ def error_on_even(argument):
     with pytest.raises(RuntimeError, match="Some test stages failed"):
         plan.wait_for_futures(futures, "test")
 
+        captured = capsys.readouterr()
+        assert "we are at odds with evens" in captured
+        assert "error_on_even" in captured
+
 
 def test_formatted_stage_name():
     """Test that we make pretty stage names for presenting in progress bars"""