fetchMetadata must use a List for deterministic batching

spotify · Dec 14, 2023 · a45419c · a45419c
1 parent d62b24f
commit a45419c
Showing 1 changed file with 2 additions and 2 deletions.
diff --git a/scio-smb/src/main/java/org/apache/beam/sdk/extensions/smb/BucketMetadataUtil.java b/scio-smb/src/main/java/org/apache/beam/sdk/extensions/smb/BucketMetadataUtil.java
@@ -69,7 +69,7 @@ int leastNumBuckets() {
     this.batchSize = batchSize;
   }
 
-  private <V> Map<ResourceId, BucketMetadata<?, ?, V>> fetchMetadata(Set<ResourceId> directories) {
+  private <V> Map<ResourceId, BucketMetadata<?, ?, V>> fetchMetadata(List<ResourceId> directories) {
     final int total = directories.size();
     final Map<ResourceId, BucketMetadata<?, ?, V>> metadata = new ConcurrentHashMap<>();
     int start = 0;
@@ -89,7 +89,7 @@ private <V> SourceMetadata<V> getSourceMetadata(
       BiFunction<BucketMetadata<?, ?, V>, BucketMetadata<?, ?, V>, Boolean>
           compatibilityCompareFn) {
     final Map<ResourceId, BucketMetadata<?, ?, V>> bucketMetadatas =
-        fetchMetadata(directories.keySet());
+        fetchMetadata(new ArrayList<>(directories.keySet()));
     Preconditions.checkState(!bucketMetadatas.isEmpty(), "Failed to find metadata");
 
     Map<ResourceId, SourceMetadataValue<V>> mapping = new HashMap<>();