fix: reading cdf from a checkpointed table

Signed-off-by: Stephen Carman <[email protected]>

crates/test/tests/data/checkpoint-cdf-table/_delta_log/00000000000000000003.json

@@ -0,0 +1,3 @@
+{"commitInfo":{"timestamp":1704559499570,"operation":"DELETE","operationParameters":{"predicate":"[\"(name#40 = Dennis)\"]"},"readVersion":2,"isolationLevel":"Serializable","isBlindAppend":false,"operationMetrics":{"numRemovedFiles":"1","numRemovedBytes":"917","numCopiedRows":"0","numDeletionVectorsAdded":"0","executionTimeMs":"3479","numDeletionVectorsUpdated":"0","numAddedFiles":"0","numDeletionVectorsRemoved":"0","numAddedChangeFiles":"1","numDeletedRows":"1","scanTimeMs":"3157","numAddedBytes":"0","rewriteTimeMs":"322"},"engineInfo":"Apache-Spark/3.5.0 Delta-Lake/3.0.0","txnId":"ef48960f-ceb5-4bc2-9b59-8c947083ae58"}}
+{"remove":{"path":"birthday=2023-12-29/part-00000-1ca113cd-a94c-46a8-9c5b-b99e676ddd06.c000.snappy.parquet","deletionTimestamp":1704559499540,"dataChange":true,"extendedFileMetadata":true,"partitionValues":{"birthday":"2023-12-29"},"size":917}}
+{"cdc":{"path":"_change_data/birthday=2023-12-29/cdc-00000-ed223ebe-3b27-44af-b2cf-91e882f4c500.c000.snappy.parquet","partitionValues":{"birthday":"2023-12-29"},"size":971,"dataChange":false}}

crates/test/tests/data/checkpoint-cdf-table/_delta_log/00000000000000000004.json

@@ -0,0 +1,5 @@
+{"commitInfo":{"timestamp":1736181198167,"operation":"WRITE","operationParameters":{"mode":"Append","partitionBy":"[]"},"readVersion":3,"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{"numFiles":"4","numOutputRows":"4","numOutputBytes":"2760"},"engineInfo":"Apache-Spark/3.5.1 Delta-Lake/3.2.1","txnId":"05de6624-a123-4c46-bf95-4dcc34b56aff"}}
+{"add":{"path":"birthday=2024-12-30/part-00000-735d4a7f-9956-46d5-8955-e9bc3599aa88.c000.snappy.parquet","partitionValues":{"birthday":"2024-12-30"},"size":701,"modificationTime":1736181198024,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":11,\"name\":\"Ossama\"},\"maxValues\":{\"id\":11,\"name\":\"Ossama\"},\"nullCount\":{\"id\":0,\"name\":0}}"}}
+{"add":{"path":"birthday=2023-12-29/part-00001-e041c37a-5bac-443c-a8c6-a3713894743d.c000.snappy.parquet","partitionValues":{"birthday":"2023-12-29"},"size":687,"modificationTime":1736181198024,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":12,\"name\":\"Nick\"},\"maxValues\":{\"id\":12,\"name\":\"Nick\"},\"nullCount\":{\"id\":0,\"name\":0}}"}}
+{"add":{"path":"birthday=2023-12-22/part-00002-fc3f3da0-9475-49db-a5be-f675a10bbe2c.c000.snappy.parquet","partitionValues":{"birthday":"2023-12-22"},"size":686,"modificationTime":1736181198024,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":13,\"name\":\"Ryan\"},\"maxValues\":{\"id\":13,\"name\":\"Ryan\"},\"nullCount\":{\"id\":0,\"name\":0}}"}}
+{"add":{"path":"birthday=2023-12-25/part-00003-4f6cd749-bd9f-4a4a-a594-66fc77d41c58.c000.snappy.parquet","partitionValues":{"birthday":"2023-12-25"},"size":686,"modificationTime":1736181198024,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":14,\"name\":\"Zach\"},\"maxValues\":{\"id\":14,\"name\":\"Zach\"},\"nullCount\":{\"id\":0,\"name\":0}}"}}

crates/test/tests/data/checkpoint-cdf-table/_delta_log/00000000000000000005.json

@@ -0,0 +1,13 @@
+{"commitInfo":{"timestamp":1736181499623,"operation":"UPDATE","operationParameters":{"predicate":"[\"(id#104 >= 11)\"]"},"readVersion":4,"isolationLevel":"Serializable","isBlindAppend":false,"operationMetrics":{"numRemovedFiles":"4","numRemovedBytes":"6939","numCopiedRows":"0","numDeletionVectorsAdded":"0","executionTimeMs":"6073","numDeletionVectorsUpdated":"0","scanTimeMs":"5118","numAddedFiles":"4","numUpdatedRows":"4","numDeletionVectorsRemoved":"0","numAddedChangeFiles":"4","numAddedBytes":"3628","rewriteTimeMs":"950"},"engineInfo":"Apache-Spark/3.5.1 Delta-Lake/3.2.1","txnId":"a53a1e14-a31b-43dc-837b-053f3c423cc4"}}
+{"add":{"path":"birthday=2024-12-30/part-00000-1f959cb4-ae21-4e3c-b9da-e1610fb63cae.c000.snappy.parquet","partitionValues":{"birthday":"2024-12-30"},"size":918,"modificationTime":1736181499498,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":12,\"name\":\"Ossama\"},\"maxValues\":{\"id\":12,\"name\":\"Ossama\"},\"nullCount\":{\"id\":0,\"name\":0}}"}}
+{"add":{"path":"birthday=2023-12-29/part-00001-21869311-b18b-4a90-800d-521fdeeb0917.c000.snappy.parquet","partitionValues":{"birthday":"2023-12-29"},"size":904,"modificationTime":1736181499498,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":13,\"name\":\"Nick\"},\"maxValues\":{\"id\":13,\"name\":\"Nick\"},\"nullCount\":{\"id\":0,\"name\":0}}"}}
+{"add":{"path":"birthday=2023-12-25/part-00002-90c97264-1f4e-4789-9879-8da4ac3a278c.c000.snappy.parquet","partitionValues":{"birthday":"2023-12-25"},"size":904,"modificationTime":1736181499498,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":15,\"name\":\"Zach\"},\"maxValues\":{\"id\":15,\"name\":\"Zach\"},\"nullCount\":{\"id\":0,\"name\":0}}"}}
+{"add":{"path":"birthday=2023-12-22/part-00003-50021c28-2b26-4382-9a0f-63f05671edef.c000.snappy.parquet","partitionValues":{"birthday":"2023-12-22"},"size":902,"modificationTime":1736181499498,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":14,\"name\":\"Ryan\"},\"maxValues\":{\"id\":14,\"name\":\"Ryan\"},\"nullCount\":{\"id\":0,\"name\":0}}"}}
+{"cdc":{"path":"_change_data/birthday=2024-12-30/cdc-00000-66f2943f-f545-4ad5-a29a-d41a6fc0964f.c000.snappy.parquet","partitionValues":{"birthday":"2024-12-30"},"size":1056,"dataChange":false}}
+{"cdc":{"path":"_change_data/birthday=2023-12-29/cdc-00001-8a2331ca-2aec-4763-9b72-0ef2ebf20c89.c000.snappy.parquet","partitionValues":{"birthday":"2023-12-29"},"size":1041,"dataChange":false}}
+{"cdc":{"path":"_change_data/birthday=2023-12-25/cdc-00002-1bf2daf8-1bef-483e-a298-60f36a9f14c7.c000.snappy.parquet","partitionValues":{"birthday":"2023-12-25"},"size":1041,"dataChange":false}}
+{"cdc":{"path":"_change_data/birthday=2023-12-22/cdc-00003-1d50571e-b1c0-46a6-8fc9-575036b63924.c000.snappy.parquet","partitionValues":{"birthday":"2023-12-22"},"size":1041,"dataChange":false}}
+{"remove":{"path":"birthday=2024-12-30/part-00000-735d4a7f-9956-46d5-8955-e9bc3599aa88.c000.snappy.parquet","deletionTimestamp":1736181499597,"dataChange":true,"extendedFileMetadata":true,"partitionValues":{"birthday":"2024-12-30"},"size":701}}
+{"remove":{"path":"birthday=2023-12-29/part-00001-e041c37a-5bac-443c-a8c6-a3713894743d.c000.snappy.parquet","deletionTimestamp":1736181499597,"dataChange":true,"extendedFileMetadata":true,"partitionValues":{"birthday":"2023-12-29"},"size":687}}
+{"remove":{"path":"birthday=2023-12-25/part-00003-4f6cd749-bd9f-4a4a-a594-66fc77d41c58.c000.snappy.parquet","deletionTimestamp":1736181499597,"dataChange":true,"extendedFileMetadata":true,"partitionValues":{"birthday":"2023-12-25"},"size":686}}
+{"remove":{"path":"birthday=2023-12-22/part-00002-fc3f3da0-9475-49db-a5be-f675a10bbe2c.c000.snappy.parquet","deletionTimestamp":1736181499597,"dataChange":true,"extendedFileMetadata":true,"partitionValues":{"birthday":"2023-12-22"},"size":686}}

crates/test/tests/data/checkpoint-cdf-table/_delta_log/_last_checkpoint

@@ -0,0 +1 @@
+{"version":3,"size":11,"sizeInBytes":18082,"numOfAddFiles":9,"checkpointSchema":{"type":"struct","fields":[{"name":"txn","type":{"type":"struct","fields":[{"name":"appId","type":"string","nullable":true,"metadata":{}},{"name":"version","type":"long","nullable":true,"metadata":{}},{"name":"lastUpdated","type":"long","nullable":true,"metadata":{}}]},"nullable":true,"metadata":{}},{"name":"add","type":{"type":"struct","fields":[{"name":"path","type":"string","nullable":true,"metadata":{}},{"name":"partitionValues","type":{"type":"map","keyType":"string","valueType":"string","valueContainsNull":true},"nullable":true,"metadata":{}},{"name":"size","type":"long","nullable":true,"metadata":{}},{"name":"modificationTime","type":"long","nullable":true,"metadata":{}},{"name":"dataChange","type":"boolean","nullable":true,"metadata":{}},{"name":"tags","type":{"type":"map","keyType":"string","valueType":"string","valueContainsNull":true},"nullable":true,"metadata":{}},{"name":"deletionVector","type":{"type":"struct","fields":[{"name":"storageType","type":"string","nullable":true,"metadata":{}},{"name":"pathOrInlineDv","type":"string","nullable":true,"metadata":{}},{"name":"offset","type":"integer","nullable":true,"metadata":{}},{"name":"sizeInBytes","type":"integer","nullable":true,"metadata":{}},{"name":"cardinality","type":"long","nullable":true,"metadata":{}},{"name":"maxRowIndex","type":"long","nullable":true,"metadata":{}}]},"nullable":true,"metadata":{}},{"name":"baseRowId","type":"long","nullable":true,"metadata":{}},{"name":"defaultRowCommitVersion","type":"long","nullable":true,"metadata":{}},{"name":"clusteringProvider","type":"string","nullable":true,"metadata":{}},{"name":"stats","type":"string","nullable":true,"metadata":{}},{"name":"partitionValues_parsed","type":{"type":"struct","fields":[{"name":"birthday","type":"date","nullable":true,"metadata":{}}]},"nullable":true,"metadata":{}}]},"nullable":true,"metadata":{}},{"name":"remove","type":{"type":"struct","fields":[{"name":"path","type":"string","nullable":true,"metadata":{}},{"name":"deletionTimestamp","type":"long","nullable":true,"metadata":{}},{"name":"dataChange","type":"boolean","nullable":true,"metadata":{}},{"name":"extendedFileMetadata","type":"boolean","nullable":true,"metadata":{}},{"name":"partitionValues","type":{"type":"map","keyType":"string","valueType":"string","valueContainsNull":true},"nullable":true,"metadata":{}},{"name":"size","type":"long","nullable":true,"metadata":{}},{"name":"deletionVector","type":{"type":"struct","fields":[{"name":"storageType","type":"string","nullable":true,"metadata":{}},{"name":"pathOrInlineDv","type":"string","nullable":true,"metadata":{}},{"name":"offset","type":"integer","nullable":true,"metadata":{}},{"name":"sizeInBytes","type":"integer","nullable":true,"metadata":{}},{"name":"cardinality","type":"long","nullable":true,"metadata":{}},{"name":"maxRowIndex","type":"long","nullable":true,"metadata":{}}]},"nullable":true,"metadata":{}},{"name":"baseRowId","type":"long","nullable":true,"metadata":{}},{"name":"defaultRowCommitVersion","type":"long","nullable":true,"metadata":{}}]},"nullable":true,"metadata":{}},{"name":"metaData","type":{"type":"struct","fields":[{"name":"id","type":"string","nullable":true,"metadata":{}},{"name":"name","type":"string","nullable":true,"metadata":{}},{"name":"description","type":"string","nullable":true,"metadata":{}},{"name":"format","type":{"type":"struct","fields":[{"name":"provider","type":"string","nullable":true,"metadata":{}},{"name":"options","type":{"type":"map","keyType":"string","valueType":"string","valueContainsNull":true},"nullable":true,"metadata":{}}]},"nullable":true,"metadata":{}},{"name":"schemaString","type":"string","nullable":true,"metadata":{}},{"name":"partitionColumns","type":{"type":"array","elementType":"string","containsNull":true},"nullable":true,"metadata":{}},{"name":"configuration","type":{"type":"map","keyType":"string","valueType":"string","valueContainsNull":true},"nullable":true,"metadata":{}},{"name":"createdTime","type":"long","nullable":true,"metadata":{}}]},"nullable":true,"metadata":{}},{"name":"protocol","type":{"type":"struct","fields":[{"name":"minReaderVersion","type":"integer","nullable":true,"metadata":{}},{"name":"minWriterVersion","type":"integer","nullable":true,"metadata":{}},{"name":"readerFeatures","type":{"type":"array","elementType":"string","containsNull":true},"nullable":true,"metadata":{}},{"name":"writerFeatures","type":{"type":"array","elementType":"string","containsNull":true},"nullable":true,"metadata":{}}]},"nullable":true,"metadata":{}},{"name":"domainMetadata","type":{"type":"struct","fields":[{"name":"domain","type":"string","nullable":true,"metadata":{}},{"name":"configuration","type":"string","nullable":true,"metadata":{}},{"name":"removed","type":"boolean","nullable":true,"metadata":{}}]},"nullable":true,"metadata":{}}]},"checksum":"d7e1e1a7cb6ef0cb2059567425b7a1c7"}