Skip to content

Commit

Permalink
Remove the explicit task key. (#174)
Browse files Browse the repository at this point in the history
* Remove the explicit task key.

* Fix doc build error with dask client (#173) (#175)

* Give more failure info.

* Maybe the done files aren't written?

* Use same pip as local?

* Maybe tasks aren't scheduled because they're done.

* Undo red herring

* Restructure exception handling
  • Loading branch information
delucchi-cmu committed Nov 29, 2023
1 parent 4dae8de commit 7aea21f
Show file tree
Hide file tree
Showing 2 changed files with 1 addition and 5 deletions.
3 changes: 0 additions & 3 deletions src/hipscat_import/catalog/run_import.py
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,6 @@ def _map_pixels(args, client):
futures.append(
client.submit(
mr.map_to_pixels,
key=key,
input_file=file_path,
resume_path=args.resume_plan.tmp_path,
file_reader=reader_future,
Expand All @@ -53,7 +52,6 @@ def _split_pixels(args, alignment_future, client):
futures.append(
client.submit(
mr.split_pixels,
key=key,
input_file=file_path,
file_reader=reader_future,
highest_order=args.mapping_healpix_order,
Expand Down Expand Up @@ -85,7 +83,6 @@ def _reduce_pixels(args, destination_pixel_map, client):
futures.append(
client.submit(
mr.reduce_pixel_shards,
key=destination_pixel_key,
cache_shard_path=args.tmp_path,
resume_path=args.resume_plan.tmp_path,
reducing_key=destination_pixel_key,
Expand Down
3 changes: 1 addition & 2 deletions src/hipscat_import/soap/run_soap.py
Original file line number Diff line number Diff line change
Expand Up @@ -22,11 +22,10 @@ def run(args, client):
resume_plan = SoapPlan(args)
if not resume_plan.is_counting_done():
futures = []
for source_pixel, object_pixels, source_key in resume_plan.count_keys:
for source_pixel, object_pixels, _source_key in resume_plan.count_keys:
futures.append(
client.submit(
count_joins,
key=source_key,
soap_args=args,
source_pixel=source_pixel,
object_pixels=object_pixels,
Expand Down

0 comments on commit 7aea21f

Please sign in to comment.