[Enhancement] Support restore/rollback sync during conversion (1/2)

xtable-api/src/main/java/org/apache/xtable/model/InternalSnapshot.java

@@ -47,4 +47,6 @@ public class InternalSnapshot {
   List<PartitionFileGroup> partitionedDataFiles;
   // pending commits before latest commit on the table.
   @Builder.Default List<Instant> pendingCommits = Collections.emptyList();
+  // commit identifier in source table
+  @Builder.Default String sourceIdentifier = "";
 }

xtable-api/src/main/java/org/apache/xtable/model/TableChange.java

@@ -36,4 +36,7 @@ public class TableChange {
 
   /** The {@link InternalTable} at the commit time to which this table change belongs. */
   InternalTable tableAsOfChange;
+
+  // Commit identifier in source table
+  @Builder.Default String sourceIdentifier = "";
 }

xtable-api/src/main/java/org/apache/xtable/model/metadata/TableSyncMetadata.java

@@ -54,10 +54,20 @@ public class TableSyncMetadata {
   Instant lastInstantSynced;
   List<Instant> instantsToConsiderForNextSync;
   int version;
+  String sourceTableFormat;
+  String sourceIdentifier;
 
   public static TableSyncMetadata of(
-      Instant lastInstantSynced, List<Instant> instantsToConsiderForNextSync) {
-    return new TableSyncMetadata(lastInstantSynced, instantsToConsiderForNextSync, CURRENT_VERSION);
+      Instant lastInstantSynced,
+      List<Instant> instantsToConsiderForNextSync,
+      String sourceTableFormat,
+      String sourceIdentifier) {
+    return new TableSyncMetadata(
+        lastInstantSynced,
+        instantsToConsiderForNextSync,
+        CURRENT_VERSION,
+        sourceTableFormat,
+        sourceIdentifier);
   }
 
   public String toJson() {

xtable-api/src/main/java/org/apache/xtable/spi/extractor/ConversionSource.java

@@ -78,4 +78,13 @@ public interface ConversionSource<COMMIT> extends Closeable {
    *     false.
    */
   boolean isIncrementalSyncSafeFrom(Instant instant);
+
+  /**
+   * Extract the identifier of the provided commit, the identifier defined as 1. Snapshot ID in
+   * Iceberg 2. Version ID in Delta 3. timestamp in Hudi
+   *
+   * @param commit The provided commit
+   * @return the string version of commit identifier
+   */
+  String getCommitIdentifier(COMMIT commit);
 }

xtable-api/src/main/java/org/apache/xtable/spi/sync/ConversionTarget.java

@@ -90,4 +90,14 @@ public interface ConversionTarget {
 
   /** Initializes the client with provided configuration */
   void init(TargetTable targetTable, Configuration configuration);
+
+  /**
+   * Retrieves the commit identifier from the target table that corresponds to a given source table
+   * commit identifier
+   *
+   * @param sourceIdentifier the unique identifier of the source table commit
+   * @return an {@link Optional} containing the target commit identifier if a corresponding commit
+   *     exists, or an empty {@link Optional} if no match is found
+   */
+  Optional<String> getTargetCommitIdentifier(String sourceIdentifier);
 }

xtable-api/src/main/java/org/apache/xtable/spi/sync/TableFormatSync.java

@@ -73,7 +73,8 @@ public Map<String, SyncResult> syncSnapshot(
                 internalTable,
                 target -> target.syncFilesForSnapshot(snapshot.getPartitionedDataFiles()),
                 startTime,
-                snapshot.getPendingCommits()));
+                snapshot.getPendingCommits(),
+                snapshot.getSourceIdentifier()));
       } catch (Exception e) {
         log.error("Failed to sync snapshot", e);
         results.put(
@@ -121,7 +122,8 @@ public Map<String, List<SyncResult>> syncChanges(
                   change.getTableAsOfChange(),
                   target -> target.syncFilesForDiff(change.getFilesDiff()),
                   startTime,
-                  changes.getPendingCommits()));
+                  changes.getPendingCommits(),
+                  change.getSourceIdentifier()));
         } catch (Exception e) {
           log.error("Failed to sync table changes", e);
           resultsForFormat.add(buildResultForError(SyncMode.INCREMENTAL, startTime, e));
@@ -149,19 +151,26 @@ private SyncResult getSyncResult(
       InternalTable tableState,
       SyncFiles fileSyncMethod,
       Instant startTime,
-      List<Instant> pendingCommits) {
+      List<Instant> pendingCommits,
+      String sourceIdentifier) {
     // initialize the sync
     conversionTarget.beginSync(tableState);
+    // Persist the latest commit time in table properties for incremental syncs
+    // Syncing metadata must precede the following steps to ensure that the metadata is available
+    // before committing
+    TableSyncMetadata latestState =
+        TableSyncMetadata.of(
+            tableState.getLatestCommitTime(),
+            pendingCommits,
+            tableState.getTableFormat(),
+            sourceIdentifier);
+    conversionTarget.syncMetadata(latestState);
     // sync schema updates
     conversionTarget.syncSchema(tableState.getReadSchema());
     // sync partition updates
     conversionTarget.syncPartitionSpec(tableState.getPartitioningFields());
     // Update the files in the target table
     fileSyncMethod.sync(conversionTarget);
-    // Persist the latest commit time in table properties for incremental syncs.
-    TableSyncMetadata latestState =
-        TableSyncMetadata.of(tableState.getLatestCommitTime(), pendingCommits);
-    conversionTarget.syncMetadata(latestState);
     conversionTarget.completeSync();
 
     return SyncResult.builder()

xtable-api/src/test/java/org/apache/xtable/model/metadata/TestTableSyncMetadata.java

@@ -49,14 +49,17 @@ private static Stream<Arguments> provideMetadataAndJson() {
                 Instant.parse("2020-07-04T10:15:30.00Z"),
                 Arrays.asList(
                     Instant.parse("2020-08-21T11:15:30.00Z"),
-                    Instant.parse("2024-01-21T12:15:30.00Z"))),
-            "{\"lastInstantSynced\":\"2020-07-04T10:15:30Z\",\"instantsToConsiderForNextSync\":[\"2020-08-21T11:15:30Z\",\"2024-01-21T12:15:30Z\"],\"version\":0}"),
+                    Instant.parse("2024-01-21T12:15:30.00Z")),
+                "TEST",
+                "0"),
+            "{\"lastInstantSynced\":\"2020-07-04T10:15:30Z\",\"instantsToConsiderForNextSync\":[\"2020-08-21T11:15:30Z\",\"2024-01-21T12:15:30Z\"],\"version\":0,\"sourceTableFormat\":\"TEST\",\"sourceIdentifier\":\"0\"}"),
         Arguments.of(
-            TableSyncMetadata.of(Instant.parse("2020-07-04T10:15:30.00Z"), Collections.emptyList()),
-            "{\"lastInstantSynced\":\"2020-07-04T10:15:30Z\",\"instantsToConsiderForNextSync\":[],\"version\":0}"),
+            TableSyncMetadata.of(
+                Instant.parse("2020-07-04T10:15:30.00Z"), Collections.emptyList(), "TEST", "0"),
+            "{\"lastInstantSynced\":\"2020-07-04T10:15:30Z\",\"instantsToConsiderForNextSync\":[],\"version\":0,\"sourceTableFormat\":\"TEST\",\"sourceIdentifier\":\"0\"}"),
         Arguments.of(
-            TableSyncMetadata.of(Instant.parse("2020-07-04T10:15:30.00Z"), null),
-            "{\"lastInstantSynced\":\"2020-07-04T10:15:30Z\",\"version\":0}"));
+            TableSyncMetadata.of(Instant.parse("2020-07-04T10:15:30.00Z"), null, "TEST", "0"),
+            "{\"lastInstantSynced\":\"2020-07-04T10:15:30Z\",\"version\":0,\"sourceTableFormat\":\"TEST\",\"sourceIdentifier\":\"0\"}"));
   }
 
   @Test

xtable-api/src/test/java/org/apache/xtable/spi/sync/TestTableFormatSync.java

@@ -76,6 +76,7 @@ void syncSnapshotWithFailureForOneFormat() {
             .table(startingTableState)
             .partitionedDataFiles(fileGroups)
             .pendingCommits(pendingCommitInstants)
+            .sourceIdentifier("0")
             .build();
     when(mockConversionTarget1.getTableFormat()).thenReturn(TableFormat.ICEBERG);
     when(mockConversionTarget2.getTableFormat()).thenReturn(TableFormat.DELTA);
@@ -106,7 +107,10 @@ void syncSnapshotWithFailureForOneFormat() {
         failureResult.getStatus());
 
     verifyBaseConversionTargetCalls(
-        mockConversionTarget2, startingTableState, pendingCommitInstants);
+        mockConversionTarget2,
+        startingTableState,
+        pendingCommitInstants,
+        snapshot.getSourceIdentifier());
     verify(mockConversionTarget2).syncFilesForSnapshot(fileGroups);
     verify(mockConversionTarget2).completeSync();
     verify(mockConversionTarget1, never()).completeSync();
@@ -124,15 +128,27 @@ void syncChangesWithFailureForOneFormat() {
     InternalTable tableState1 = getTableState(1);
     DataFilesDiff dataFilesDiff1 = getFilesDiff(1);
     TableChange tableChange1 =
-        TableChange.builder().tableAsOfChange(tableState1).filesDiff(dataFilesDiff1).build();
+        TableChange.builder()
+            .tableAsOfChange(tableState1)
+            .filesDiff(dataFilesDiff1)
+            .sourceIdentifier("0")
+            .build();
     InternalTable tableState2 = getTableState(2);
     DataFilesDiff dataFilesDiff2 = getFilesDiff(2);
     TableChange tableChange2 =
-        TableChange.builder().tableAsOfChange(tableState2).filesDiff(dataFilesDiff2).build();
+        TableChange.builder()
+            .tableAsOfChange(tableState2)
+            .filesDiff(dataFilesDiff2)
+            .sourceIdentifier("1")
+            .build();
     InternalTable tableState3 = getTableState(3);
     DataFilesDiff dataFilesDiff3 = getFilesDiff(3);
     TableChange tableChange3 =
-        TableChange.builder().tableAsOfChange(tableState3).filesDiff(dataFilesDiff3).build();
+        TableChange.builder()
+            .tableAsOfChange(tableState3)
+            .filesDiff(dataFilesDiff3)
+            .sourceIdentifier("2")
+            .build();
 
     List<Instant> pendingCommitInstants = Collections.singletonList(Instant.now());
     when(mockConversionTarget1.getTableFormat()).thenReturn(TableFormat.ICEBERG);
@@ -151,10 +167,12 @@ void syncChangesWithFailureForOneFormat() {
     Map<ConversionTarget, TableSyncMetadata> conversionTargetWithMetadata = new HashMap<>();
     conversionTargetWithMetadata.put(
         mockConversionTarget1,
-        TableSyncMetadata.of(Instant.now().minus(1, ChronoUnit.HOURS), Collections.emptyList()));
+        TableSyncMetadata.of(
+            Instant.now().minus(1, ChronoUnit.HOURS), Collections.emptyList(), "TEST", "0"));
     conversionTargetWithMetadata.put(
         mockConversionTarget2,
-        TableSyncMetadata.of(Instant.now().minus(1, ChronoUnit.HOURS), Collections.emptyList()));
+        TableSyncMetadata.of(
+            Instant.now().minus(1, ChronoUnit.HOURS), Collections.emptyList(), "TEST", "1"));
 
     Map<String, List<SyncResult>> result =
         TableFormatSync.getInstance()
@@ -194,13 +212,29 @@ void syncChangesWithFailureForOneFormat() {
       assertSyncResultTimes(successResults.get(i), start);
     }
 
-    verifyBaseConversionTargetCalls(mockConversionTarget1, tableState1, pendingCommitInstants);
+    verifyBaseConversionTargetCalls(
+        mockConversionTarget1,
+        tableState1,
+        pendingCommitInstants,
+        tableChange1.getSourceIdentifier());
     verify(mockConversionTarget1).syncFilesForDiff(dataFilesDiff1);
-    verifyBaseConversionTargetCalls(mockConversionTarget2, tableState1, pendingCommitInstants);
+    verifyBaseConversionTargetCalls(
+        mockConversionTarget2,
+        tableState1,
+        pendingCommitInstants,
+        tableChange1.getSourceIdentifier());
     verify(mockConversionTarget2).syncFilesForDiff(dataFilesDiff1);
-    verifyBaseConversionTargetCalls(mockConversionTarget2, tableState2, pendingCommitInstants);
+    verifyBaseConversionTargetCalls(
+        mockConversionTarget2,
+        tableState2,
+        pendingCommitInstants,
+        tableChange2.getSourceIdentifier());
     verify(mockConversionTarget2).syncFilesForDiff(dataFilesDiff2);
-    verifyBaseConversionTargetCalls(mockConversionTarget2, tableState3, pendingCommitInstants);
+    verifyBaseConversionTargetCalls(
+        mockConversionTarget2,
+        tableState3,
+        pendingCommitInstants,
+        tableChange3.getSourceIdentifier());
     verify(mockConversionTarget2).syncFilesForDiff(dataFilesDiff3);
     verify(mockConversionTarget1, times(1)).completeSync();
     verify(mockConversionTarget2, times(3)).completeSync();
@@ -212,15 +246,27 @@ void syncChangesWithDifferentFormatsAndMetadata() {
     InternalTable tableState1 = getTableState(1);
     DataFilesDiff dataFilesDiff1 = getFilesDiff(1);
     TableChange tableChange1 =
-        TableChange.builder().tableAsOfChange(tableState1).filesDiff(dataFilesDiff1).build();
+        TableChange.builder()
+            .tableAsOfChange(tableState1)
+            .filesDiff(dataFilesDiff1)
+            .sourceIdentifier("0")
+            .build();
     InternalTable tableState2 = getTableState(2);
     DataFilesDiff dataFilesDiff2 = getFilesDiff(2);
     TableChange tableChange2 =
-        TableChange.builder().tableAsOfChange(tableState2).filesDiff(dataFilesDiff2).build();
+        TableChange.builder()
+            .tableAsOfChange(tableState2)
+            .filesDiff(dataFilesDiff2)
+            .sourceIdentifier("1")
+            .build();
     InternalTable tableState3 = getTableState(3);
     DataFilesDiff dataFilesDiff3 = getFilesDiff(3);
     TableChange tableChange3 =
-        TableChange.builder().tableAsOfChange(tableState3).filesDiff(dataFilesDiff3).build();
+        TableChange.builder()
+            .tableAsOfChange(tableState3)
+            .filesDiff(dataFilesDiff3)
+            .sourceIdentifier("2")
+            .build();
 
     List<Instant> pendingCommitInstants = Collections.singletonList(Instant.now());
     when(mockConversionTarget1.getTableFormat()).thenReturn(TableFormat.ICEBERG);
@@ -240,12 +286,17 @@ void syncChangesWithDifferentFormatsAndMetadata() {
         mockConversionTarget1,
         TableSyncMetadata.of(
             tableChange2.getTableAsOfChange().getLatestCommitTime(),
-            Collections.singletonList(tableChange1.getTableAsOfChange().getLatestCommitTime())));
+            Collections.singletonList(tableChange1.getTableAsOfChange().getLatestCommitTime()),
+            "TEST",
+            tableChange2.getSourceIdentifier()));
     // mockConversionTarget2 will have synced the first table change previously
     conversionTargetWithMetadata.put(
         mockConversionTarget2,
         TableSyncMetadata.of(
-            tableChange1.getTableAsOfChange().getLatestCommitTime(), Collections.emptyList()));
+            tableChange1.getTableAsOfChange().getLatestCommitTime(),
+            Collections.emptyList(),
+            "TEST",
+            tableChange1.getSourceIdentifier()));
 
     Map<String, List<SyncResult>> result =
         TableFormatSync.getInstance()
@@ -280,15 +331,31 @@ void syncChangesWithDifferentFormatsAndMetadata() {
     }
 
     // conversionTarget1 syncs table changes 1 and 3
-    verifyBaseConversionTargetCalls(mockConversionTarget1, tableState1, pendingCommitInstants);
+    verifyBaseConversionTargetCalls(
+        mockConversionTarget1,
+        tableState1,
+        pendingCommitInstants,
+        tableChange1.getSourceIdentifier());
     verify(mockConversionTarget1).syncFilesForDiff(dataFilesDiff1);
-    verifyBaseConversionTargetCalls(mockConversionTarget1, tableState3, pendingCommitInstants);
+    verifyBaseConversionTargetCalls(
+        mockConversionTarget1,
+        tableState3,
+        pendingCommitInstants,
+        tableChange3.getSourceIdentifier());
     verify(mockConversionTarget1).syncFilesForDiff(dataFilesDiff3);
     verify(mockConversionTarget1, times(2)).completeSync();
     // conversionTarget2 syncs table changes 2 and 3
-    verifyBaseConversionTargetCalls(mockConversionTarget2, tableState2, pendingCommitInstants);
+    verifyBaseConversionTargetCalls(
+        mockConversionTarget2,
+        tableState2,
+        pendingCommitInstants,
+        tableChange2.getSourceIdentifier());
     verify(mockConversionTarget2).syncFilesForDiff(dataFilesDiff2);
-    verifyBaseConversionTargetCalls(mockConversionTarget2, tableState3, pendingCommitInstants);
+    verifyBaseConversionTargetCalls(
+        mockConversionTarget2,
+        tableState3,
+        pendingCommitInstants,
+        tableChange3.getSourceIdentifier());
     verify(mockConversionTarget2).syncFilesForDiff(dataFilesDiff3);
     verify(mockConversionTarget2, times(2)).completeSync();
   }
@@ -299,7 +366,11 @@ void syncChangesOneFormatWithNoRequiredChanges() {
     InternalTable tableState1 = getTableState(1);
     DataFilesDiff dataFilesDiff1 = getFilesDiff(1);
     TableChange tableChange1 =
-        TableChange.builder().tableAsOfChange(tableState1).filesDiff(dataFilesDiff1).build();
+        TableChange.builder()
+            .tableAsOfChange(tableState1)
+            .filesDiff(dataFilesDiff1)
+            .sourceIdentifier("0")
+            .build();
 
     List<Instant> pendingCommitInstants = Collections.emptyList();
     when(mockConversionTarget1.getTableFormat()).thenReturn(TableFormat.ICEBERG);
@@ -315,11 +386,13 @@ void syncChangesOneFormatWithNoRequiredChanges() {
     Map<ConversionTarget, TableSyncMetadata> conversionTargetWithMetadata = new HashMap<>();
     // mockConversionTarget1 will have nothing to sync
     conversionTargetWithMetadata.put(
-        mockConversionTarget1, TableSyncMetadata.of(Instant.now(), Collections.emptyList()));
+        mockConversionTarget1,
+        TableSyncMetadata.of(Instant.now(), Collections.emptyList(), "TEST", "0"));
     // mockConversionTarget2 will have synced the first table change previously
     conversionTargetWithMetadata.put(
         mockConversionTarget2,
-        TableSyncMetadata.of(Instant.now().minus(1, ChronoUnit.HOURS), Collections.emptyList()));
+        TableSyncMetadata.of(
+            Instant.now().minus(1, ChronoUnit.HOURS), Collections.emptyList(), "TEST", "1"));
 
     Map<String, List<SyncResult>> result =
         TableFormatSync.getInstance()
@@ -338,7 +411,11 @@ void syncChangesOneFormatWithNoRequiredChanges() {
     verify(mockConversionTarget1, never()).syncFilesForDiff(any());
     verify(mockConversionTarget1, never()).completeSync();
 
-    verifyBaseConversionTargetCalls(mockConversionTarget2, tableState1, pendingCommitInstants);
+    verifyBaseConversionTargetCalls(
+        mockConversionTarget2,
+        tableState1,
+        pendingCommitInstants,
+        tableChange1.getSourceIdentifier());
     verify(mockConversionTarget2).syncFilesForDiff(dataFilesDiff1);
   }
 
@@ -379,12 +456,17 @@ private DataFilesDiff getFilesDiff(int id) {
   private void verifyBaseConversionTargetCalls(
       ConversionTarget mockConversionTarget,
       InternalTable startingTableState,
-      List<Instant> pendingCommitInstants) {
+      List<Instant> pendingCommitInstants,
+      String sourceIdentifier) {
     verify(mockConversionTarget).beginSync(startingTableState);
     verify(mockConversionTarget).syncSchema(startingTableState.getReadSchema());
     verify(mockConversionTarget).syncPartitionSpec(startingTableState.getPartitioningFields());
     verify(mockConversionTarget)
         .syncMetadata(
-            TableSyncMetadata.of(startingTableState.getLatestCommitTime(), pendingCommitInstants));
+            TableSyncMetadata.of(
+                startingTableState.getLatestCommitTime(),
+                pendingCommitInstants,
+                startingTableState.getTableFormat(),
+                sourceIdentifier));
   }
 }