From 07b238de9f16235df3db65cbc2f2d79f8acd8967 Mon Sep 17 00:00:00 2001 From: Marc Handalian Date: Mon, 27 Jan 2025 15:11:04 -0800 Subject: [PATCH] initial commit streaming aggs with DF Signed-off-by: Marc Handalian --- .../arrow/spi/PartitionedStreamProducer.java | 16 + .../opensearch/arrow/spi/StreamManager.java | 3 + .../opensearch/arrow/spi/StreamProducer.java | 2 + libs/build.gradle | 1 + libs/datafusion/build.gradle | 163 + libs/datafusion/jni/Cargo.lock | 4947 +++++++++++++++++ libs/datafusion/jni/Cargo.toml | 33 + libs/datafusion/jni/src/lib.rs | 398 ++ libs/datafusion/jni/src/provider.rs | 216 + libs/datafusion/jni/src/provider/test/mod.rs | 261 + .../org.opensearch.datafusion/DataFrame.java | 73 + .../DataFrameStreamProducer.java | 132 + .../org.opensearch.datafusion/DataFusion.java | 83 + .../ObjectResultCallback.java | 16 + .../RecordBatchStream.java | 122 + .../SessionContext.java | 50 + .../package-info.java | 12 + .../arrow/flight/impl/BaseFlightProducer.java | 25 +- .../flight/impl/FlightStreamManager.java | 7 + server/build.gradle | 2 + .../action/search/FetchSearchPhase.java | 11 +- .../search/QueryPhaseResultConsumer.java | 21 +- .../action/search/SearchPhaseController.java | 262 +- .../action/search/StreamAsyncAction.java | 1 + .../arrow/custom/StreamManagerWrapper.java | 2 + .../bucket/BucketsAggregator.java | 4 + .../GlobalOrdinalsStringTermsAggregator.java | 24 +- .../support/StreamingAggregator.java | 105 +- 28 files changed, 6828 insertions(+), 164 deletions(-) create mode 100644 libs/arrow-spi/src/main/java/org/opensearch/arrow/spi/PartitionedStreamProducer.java create mode 100644 libs/datafusion/build.gradle create mode 100644 libs/datafusion/jni/Cargo.lock create mode 100644 libs/datafusion/jni/Cargo.toml create mode 100644 libs/datafusion/jni/src/lib.rs create mode 100644 libs/datafusion/jni/src/provider.rs create mode 100644 libs/datafusion/jni/src/provider/test/mod.rs create mode 100644 libs/datafusion/src/main/java/org.opensearch.datafusion/DataFrame.java create mode 100644 libs/datafusion/src/main/java/org.opensearch.datafusion/DataFrameStreamProducer.java create mode 100644 libs/datafusion/src/main/java/org.opensearch.datafusion/DataFusion.java create mode 100644 libs/datafusion/src/main/java/org.opensearch.datafusion/ObjectResultCallback.java create mode 100644 libs/datafusion/src/main/java/org.opensearch.datafusion/RecordBatchStream.java create mode 100644 libs/datafusion/src/main/java/org.opensearch.datafusion/SessionContext.java create mode 100644 libs/datafusion/src/main/java/org.opensearch.datafusion/package-info.java diff --git a/libs/arrow-spi/src/main/java/org/opensearch/arrow/spi/PartitionedStreamProducer.java b/libs/arrow-spi/src/main/java/org/opensearch/arrow/spi/PartitionedStreamProducer.java new file mode 100644 index 0000000000000..c7bfbd36e606b --- /dev/null +++ b/libs/arrow-spi/src/main/java/org/opensearch/arrow/spi/PartitionedStreamProducer.java @@ -0,0 +1,16 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.arrow.spi; + +import java.util.Set; + +public interface PartitionedStreamProducer extends StreamProducer { + Set partitions(); + void setRootTicket(StreamTicket ticket); +} diff --git a/libs/arrow-spi/src/main/java/org/opensearch/arrow/spi/StreamManager.java b/libs/arrow-spi/src/main/java/org/opensearch/arrow/spi/StreamManager.java index cdb83f032356a..668749c1269c5 100644 --- a/libs/arrow-spi/src/main/java/org/opensearch/arrow/spi/StreamManager.java +++ b/libs/arrow-spi/src/main/java/org/opensearch/arrow/spi/StreamManager.java @@ -11,6 +11,9 @@ import org.opensearch.common.annotation.ExperimentalApi; import org.opensearch.core.tasks.TaskId; +import java.util.Set; +import java.util.function.Function; + /** * Interface for managing Arrow data streams between producers and consumers. * StreamManager handles the registration of producers, stream access control via tickets, diff --git a/libs/arrow-spi/src/main/java/org/opensearch/arrow/spi/StreamProducer.java b/libs/arrow-spi/src/main/java/org/opensearch/arrow/spi/StreamProducer.java index c5cd6f16adfdd..9e1b9121ed808 100644 --- a/libs/arrow-spi/src/main/java/org/opensearch/arrow/spi/StreamProducer.java +++ b/libs/arrow-spi/src/main/java/org/opensearch/arrow/spi/StreamProducer.java @@ -14,6 +14,8 @@ import org.opensearch.core.tasks.TaskId; import java.io.Closeable; +import java.util.Collections; +import java.util.Set; /** * Represents a producer of Arrow streams. The producer first needs to define the job by implementing this interface and diff --git a/libs/build.gradle b/libs/build.gradle index 27ba3ad2102b8..45a30c0d70863 100644 --- a/libs/build.gradle +++ b/libs/build.gradle @@ -45,6 +45,7 @@ subprojects { Project depProject = project.project(dep.path) if (depProject != null && (false == depProject.path.equals(':libs:opensearch-core') && false == depProject.path.equals(':libs:opensearch-arrow-memory-shaded') && + false == depProject.path.equals(':libs:opensearch-arrow-spi') && false == depProject.path.equals(':libs:opensearch-common')) && depProject.path.startsWith(':libs')) { throw new InvalidUserDataException("projects in :libs " diff --git a/libs/datafusion/build.gradle b/libs/datafusion/build.gradle new file mode 100644 index 0000000000000..9a58792e099d0 --- /dev/null +++ b/libs/datafusion/build.gradle @@ -0,0 +1,163 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ +import org.opensearch.gradle.testclusters.OpenSearchCluster + +dependencies { + api project(':libs:opensearch-common') + implementation project(':libs:opensearch-arrow-spi') + + api 'org.apache.arrow:arrow-c-data:17.0.0' + base { + archivesName = 'opensearch-datafusion' + } + // logging + implementation "org.apache.logging.log4j:log4j-api:${versions.log4j}" + implementation "org.apache.logging.log4j:log4j-core:${versions.log4j}" + implementation "org.apache.logging.log4j:log4j-slf4j-impl:${versions.log4j}" + + // testing + testImplementation "com.carrotsearch.randomizedtesting:randomizedtesting-runner:${versions.randomizedrunner}" + testImplementation "junit:junit:${versions.junit}" + testImplementation "org.hamcrest:hamcrest:${versions.hamcrest}" + testImplementation(project(":test:framework")) { + exclude group: 'org.opensearch', module: 'opensearch-datafusion' + } +} + +//compileJava { +// dependsOn "cargoBuild" +//} + +tasks.named("assemble").configure { + dependsOn "cargoBuild" +} + +tasks.named('forbiddenApisMain').configure { + replaceSignatureFiles 'jdk-signatures' +} + +task cargoBuild(type:Exec) { + println("Running Cargo Build") + workingDir 'jni' + commandLine 'cargo', 'build', '--release' +} + +task copyNativeLib(type: Copy) { + from 'jni/target/release' + include '*.so', '*.dll', '*.dylib', '*.d' + into "$rootDir/native-lib" +} + +clean.doFirst { + delete "jni/target" + delete "$rootDir/native-lib" +} + +cargoBuild.finalizedBy copyNativeLib + +//build { +// dependsOn cargoBuild +//} + +clean.doFirst { + delete "jni/target" +} + +assemble.enabled = true + +//test { +// dependsOn cargoBuild +// systemProperty "java.library.path", "$rootDir/jni/target/release" +//} +// +//testClusters.runTask { +// dependsOn cargoBuild +// println("ROOT DIR"); +// print("$rootDir") +// systemProperty("java.library.path", "$rootDir/libs/datafusion/jni/target/release") +//} + + +// audit garbage +tasks.named('thirdPartyAudit').configure { + ignoreMissingClasses( + "com.conversantmedia.util.concurrent.SpinPolicy", + "com.fasterxml.jackson.dataformat.xml.JacksonXmlModule", + "com.fasterxml.jackson.dataformat.xml.XmlMapper", + "com.fasterxml.jackson.dataformat.xml.util.DefaultXmlPrettyPrinter", + "com.lmax.disruptor.EventFactory", + "com.lmax.disruptor.EventTranslator", + "com.lmax.disruptor.EventTranslatorTwoArg", + "com.lmax.disruptor.EventTranslatorVararg", + "com.lmax.disruptor.ExceptionHandler", + "com.lmax.disruptor.LifecycleAware", + "com.lmax.disruptor.RingBuffer", + "com.lmax.disruptor.Sequence", + "com.lmax.disruptor.SequenceBarrier", + "com.lmax.disruptor.SequenceReportingEventHandler", + "com.lmax.disruptor.TimeoutException", + "com.lmax.disruptor.WaitStrategy", + "com.lmax.disruptor.dsl.Disruptor", + "com.lmax.disruptor.dsl.ProducerType", + "javax.jms.Connection", + "javax.jms.ConnectionFactory", + "javax.jms.Destination", + "javax.jms.JMSException", + "javax.jms.MapMessage", + "javax.jms.Message", + "javax.jms.MessageConsumer", + "javax.jms.MessageProducer", + "javax.jms.Session", + "javax.mail.Authenticator", + "javax.mail.Message\$RecipientType", + "javax.mail.PasswordAuthentication", + "javax.mail.Session", + "javax.mail.Transport", + "javax.mail.internet.InternetAddress", + "javax.mail.internet.InternetHeaders", + "javax.mail.internet.MimeMessage", + "javax.mail.internet.MimeMultipart", + "javax.mail.internet.MimeUtility", + "org.apache.commons.compress.compressors.CompressorStreamFactory", + "org.apache.commons.compress.utils.IOUtils", + "org.apache.commons.csv.CSVFormat", + "org.apache.commons.csv.QuoteMode", + "org.apache.kafka.clients.producer.Producer", + "org.apache.kafka.clients.producer.RecordMetadata", + "org.apache.kafka.common.serialization.ByteArraySerializer", + "org.codehaus.stax2.XMLStreamWriter2", + "org.fusesource.jansi.Ansi", + "org.fusesource.jansi.AnsiRenderer\$Code", + "org.jctools.queues.MpscArrayQueue", + "org.osgi.framework.Bundle", + "org.osgi.framework.BundleActivator", + "org.osgi.framework.BundleContext", + "org.osgi.framework.BundleEvent", + "org.osgi.framework.BundleReference", + "org.osgi.framework.FrameworkUtil", + "org.osgi.framework.ServiceReference", + "org.osgi.framework.ServiceRegistration", + "org.osgi.framework.SynchronousBundleListener", + "org.osgi.framework.wiring.BundleWire", + "org.osgi.framework.wiring.BundleWiring", + "org.slf4j.ext.EventData", + "org.zeromq.SocketType", + "org.zeromq.ZContext", + "org.zeromq.ZMQ", + "org.zeromq.ZMQ\$Context", + "org.zeromq.ZMQ\$Socket", + "org.zeromq.ZMonitor", + "org.zeromq.ZMonitor\$Event", + "org.zeromq.ZMonitor\$ZEvent" + ) + + ignoreViolations( + "org.apache.logging.log4j.core.util.internal.UnsafeUtil", + "org.apache.logging.log4j.core.util.internal.UnsafeUtil\$1" + ) +} diff --git a/libs/datafusion/jni/Cargo.lock b/libs/datafusion/jni/Cargo.lock new file mode 100644 index 0000000000000..b0f63abdbe066 --- /dev/null +++ b/libs/datafusion/jni/Cargo.lock @@ -0,0 +1,4947 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "addr2line" +version = "0.24.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dfbe277e56a376000877090da837660b4427aad530e3028d44e0bffe4f89a1c1" +dependencies = [ + "gimli", +] + +[[package]] +name = "adler2" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "512761e0bb2578dd7380c6baaa0f4ce03e84f95e960231d1dec8bf4d7d6e2627" + +[[package]] +name = "ahash" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e89da841a80418a9b391ebaea17f5c112ffaaa96f621d2c285b5174da76b9011" +dependencies = [ + "cfg-if", + "const-random", + "getrandom", + "once_cell", + "version_check", + "zerocopy", +] + +[[package]] +name = "aho-corasick" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916" +dependencies = [ + "memchr", +] + +[[package]] +name = "alloc-no-stdlib" +version = "2.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc7bb162ec39d46ab1ca8c77bf72e890535becd1751bb45f64c597edb4c8c6b3" + +[[package]] +name = "alloc-stdlib" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94fb8275041c72129eb51b7d0322c29b8387a0386127718b096429201a5d6ece" +dependencies = [ + "alloc-no-stdlib", +] + +[[package]] +name = "allocator-api2" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" + +[[package]] +name = "android-tzdata" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e999941b234f3131b00bc13c22d06e8c5ff726d1b6318ac7eb276997bbb4fef0" + +[[package]] +name = "android_system_properties" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +dependencies = [ + "libc", +] + +[[package]] +name = "anstream" +version = "0.6.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8acc5369981196006228e28809f761875c0327210a891e941f4c683b3a99529b" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55cc3b69f167a1ef2e161439aa98aed94e6028e5f9a59be9a6ffb47aef1651f9" + +[[package]] +name = "anstyle-parse" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b2d16507662817a6a20a9ea92df6652ee4f94f914589377d69f3b21bc5798a9" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "79947af37f4177cfead1110013d678905c37501914fba0efea834c3fe9a8d60c" +dependencies = [ + "windows-sys 0.59.0", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca3534e77181a9cc07539ad51f2141fe32f6c3ffd4df76db8ad92346b003ae4e" +dependencies = [ + "anstyle", + "once_cell", + "windows-sys 0.59.0", +] + +[[package]] +name = "anyhow" +version = "1.0.95" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34ac096ce696dc2fcabef30516bb13c0a68a11d30131d3df6f04711467681b04" + +[[package]] +name = "approx" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cab112f0a86d568ea0e627cc1d6be74a1e9cd55214684db5561995f6dad897c6" +dependencies = [ + "num-traits", +] + +[[package]] +name = "arrayref" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76a2e8124351fda1ef8aaaa3bbd7ebbcb486bbcd4225aca0aa0d84bb2db8fecb" + +[[package]] +name = "arrayvec" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" + +[[package]] +name = "arrow" +version = "53.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c91839b07e474b3995035fd8ac33ee54f9c9ccbbb1ea33d9909c71bffdf1259d" +dependencies = [ + "arrow-arith", + "arrow-array", + "arrow-buffer", + "arrow-cast", + "arrow-csv", + "arrow-data", + "arrow-ipc", + "arrow-json", + "arrow-ord", + "arrow-row", + "arrow-schema", + "arrow-select", + "arrow-string", +] + +[[package]] +name = "arrow-arith" +version = "53.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "855c57c4efd26722b044dcd3e348252560e3e0333087fb9f6479dc0bf744054f" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "chrono", + "half", + "num", +] + +[[package]] +name = "arrow-array" +version = "53.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd03279cea46569acf9295f6224fbc370c5df184b4d2ecfe97ccb131d5615a7f" +dependencies = [ + "ahash", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "chrono", + "chrono-tz", + "half", + "hashbrown 0.15.2", + "num", +] + +[[package]] +name = "arrow-buffer" +version = "53.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e4a9b9b1d6d7117f6138e13bc4dd5daa7f94e671b70e8c9c4dc37b4f5ecfc16" +dependencies = [ + "bytes", + "half", + "num", +] + +[[package]] +name = "arrow-cast" +version = "53.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc70e39916e60c5b7af7a8e2719e3ae589326039e1e863675a008bee5ffe90fd" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "arrow-select", + "atoi", + "base64", + "chrono", + "comfy-table", + "half", + "lexical-core", + "num", + "ryu", +] + +[[package]] +name = "arrow-csv" +version = "53.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "789b2af43c1049b03a8d088ff6b2257cdcea1756cd76b174b1f2600356771b97" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-cast", + "arrow-data", + "arrow-schema", + "chrono", + "csv", + "csv-core", + "lazy_static", + "lexical-core", + "regex", +] + +[[package]] +name = "arrow-data" +version = "53.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4e75edf21ffd53744a9b8e3ed11101f610e7ceb1a29860432824f1834a1f623" +dependencies = [ + "arrow-buffer", + "arrow-schema", + "half", + "num", +] + +[[package]] +name = "arrow-flight" +version = "53.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c09b331887a526f203f2123444792aee924632bd08b9940435070901075832e" +dependencies = [ + "arrow-arith", + "arrow-array", + "arrow-buffer", + "arrow-cast", + "arrow-data", + "arrow-ipc", + "arrow-ord", + "arrow-row", + "arrow-schema", + "arrow-select", + "arrow-string", + "base64", + "bytes", + "futures", + "once_cell", + "paste", + "prost", + "prost-types", + "tokio", + "tonic", +] + +[[package]] +name = "arrow-ipc" +version = "53.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d186a909dece9160bf8312f5124d797884f608ef5435a36d9d608e0b2a9bcbf8" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-cast", + "arrow-data", + "arrow-schema", + "flatbuffers", + "lz4_flex", +] + +[[package]] +name = "arrow-json" +version = "53.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b66ff2fedc1222942d0bd2fd391cb14a85baa3857be95c9373179bd616753b85" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-cast", + "arrow-data", + "arrow-schema", + "chrono", + "half", + "indexmap 2.7.0", + "lexical-core", + "num", + "serde", + "serde_json", +] + +[[package]] +name = "arrow-ord" +version = "53.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ece7b5bc1180e6d82d1a60e1688c199829e8842e38497563c3ab6ea813e527fd" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "arrow-select", + "half", + "num", +] + +[[package]] +name = "arrow-row" +version = "53.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "745c114c8f0e8ce211c83389270de6fbe96a9088a7b32c2a041258a443fe83ff" +dependencies = [ + "ahash", + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "half", +] + +[[package]] +name = "arrow-schema" +version = "53.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b95513080e728e4cec37f1ff5af4f12c9688d47795d17cda80b6ec2cf74d4678" +dependencies = [ + "bitflags 2.8.0", + "serde", +] + +[[package]] +name = "arrow-select" +version = "53.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e415279094ea70323c032c6e739c48ad8d80e78a09bef7117b8718ad5bf3722" +dependencies = [ + "ahash", + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "num", +] + +[[package]] +name = "arrow-string" +version = "53.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11d956cae7002eb8d83a27dbd34daaea1cf5b75852f0b84deb4d93a276e92bbf" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "arrow-select", + "memchr", + "num", + "regex", + "regex-syntax 0.8.5", +] + +[[package]] +name = "async-compression" +version = "0.4.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df895a515f70646414f4b45c0b79082783b80552b373a68283012928df56f522" +dependencies = [ + "bzip2 0.4.4", + "flate2", + "futures-core", + "memchr", + "pin-project-lite", + "tokio", + "xz2", + "zstd", + "zstd-safe", +] + +[[package]] +name = "async-stream" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b5a71a6f37880a80d1d7f19efd781e4b5de42c88f0722cc13bcb6cc2cfe8476" +dependencies = [ + "async-stream-impl", + "futures-core", + "pin-project-lite", +] + +[[package]] +name = "async-stream-impl" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "async-trait" +version = "0.1.85" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f934833b4b7233644e5848f235df3f57ed8c80f1528a26c3dfa13d2147fa056" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "atoi" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f28d99ec8bfea296261ca1af174f24225171fea9664ba9003cbebee704810528" +dependencies = [ + "num-traits", +] + +[[package]] +name = "atomic-waker" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" + +[[package]] +name = "autocfg" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26" + +[[package]] +name = "axum" +version = "0.7.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edca88bc138befd0323b20752846e6587272d3b03b0343c8ea28a6f819e6e71f" +dependencies = [ + "async-trait", + "axum-core", + "bytes", + "futures-util", + "http", + "http-body", + "http-body-util", + "itoa", + "matchit", + "memchr", + "mime", + "percent-encoding", + "pin-project-lite", + "rustversion", + "serde", + "sync_wrapper", + "tower 0.5.2", + "tower-layer", + "tower-service", +] + +[[package]] +name = "axum-core" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09f2bd6146b97ae3359fa0cc6d6b376d9539582c7b4220f041a33ec24c226199" +dependencies = [ + "async-trait", + "bytes", + "futures-util", + "http", + "http-body", + "http-body-util", + "mime", + "pin-project-lite", + "rustversion", + "sync_wrapper", + "tower-layer", + "tower-service", +] + +[[package]] +name = "backtrace" +version = "0.3.74" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d82cb332cdfaed17ae235a638438ac4d4839913cc2af585c3c6746e8f8bee1a" +dependencies = [ + "addr2line", + "cfg-if", + "libc", + "miniz_oxide", + "object", + "rustc-demangle", + "windows-targets 0.52.6", +] + +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + +[[package]] +name = "bigdecimal" +version = "0.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f31f3af01c5c65a07985c804d3366560e6fa7883d640a122819b14ec327482c" +dependencies = [ + "autocfg", + "libm", + "num-bigint", + "num-integer", + "num-traits", +] + +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + +[[package]] +name = "bitflags" +version = "2.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f68f53c83ab957f72c32642f3868eec03eb974d1fb82e453128456482613d36" + +[[package]] +name = "blake2" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46502ad458c9a52b69d4d4d32775c788b7a1b85e8bc9d482d92250fc0e3f8efe" +dependencies = [ + "digest", +] + +[[package]] +name = "blake3" +version = "1.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8ee0c1824c4dea5b5f81736aff91bae041d2c07ee1192bec91054e10e3e601e" +dependencies = [ + "arrayref", + "arrayvec", + "cc", + "cfg-if", + "constant_time_eq", +] + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + +[[package]] +name = "bollard" +version = "0.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97ccca1260af6a459d75994ad5acc1651bcabcbdbc41467cc9786519ab854c30" +dependencies = [ + "base64", + "bollard-stubs", + "bytes", + "futures-core", + "futures-util", + "hex", + "http", + "http-body-util", + "hyper", + "hyper-named-pipe", + "hyper-util", + "hyperlocal", + "log", + "pin-project-lite", + "serde", + "serde_derive", + "serde_json", + "serde_repr", + "serde_urlencoded", + "thiserror 2.0.11", + "tokio", + "tokio-util", + "tower-service", + "url", + "winapi", +] + +[[package]] +name = "bollard-stubs" +version = "1.47.1-rc.27.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f179cfbddb6e77a5472703d4b30436bff32929c0aa8a9008ecf23d1d3cdd0da" +dependencies = [ + "serde", + "serde_repr", + "serde_with", +] + +[[package]] +name = "brotli" +version = "7.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc97b8f16f944bba54f0433f07e30be199b6dc2bd25937444bbad560bcea29bd" +dependencies = [ + "alloc-no-stdlib", + "alloc-stdlib", + "brotli-decompressor", +] + +[[package]] +name = "brotli-decompressor" +version = "4.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a45bd2e4095a8b518033b128020dd4a55aab1c0a381ba4404a472630f4bc362" +dependencies = [ + "alloc-no-stdlib", + "alloc-stdlib", +] + +[[package]] +name = "bumpalo" +version = "3.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "79296716171880943b8470b5f8d03aa55eb2e645a4874bdbb28adb49162e012c" + +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + +[[package]] +name = "bytes" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "325918d6fe32f23b19878fe4b34794ae41fc19ddbe53b10571a4874d44ffd39b" + +[[package]] +name = "bzip2" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bdb116a6ef3f6c3698828873ad02c3014b3c85cadb88496095628e3ef1e347f8" +dependencies = [ + "bzip2-sys", + "libc", +] + +[[package]] +name = "bzip2" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bafdbf26611df8c14810e268ddceda071c297570a5fb360ceddf617fe417ef58" +dependencies = [ + "bzip2-sys", + "libc", +] + +[[package]] +name = "bzip2-sys" +version = "0.1.11+1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "736a955f3fa7875102d57c82b8cac37ec45224a07fd32d58f9f7a186b6cd4cdc" +dependencies = [ + "cc", + "libc", + "pkg-config", +] + +[[package]] +name = "cc" +version = "1.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8293772165d9345bdaaa39b45b2109591e63fe5e6fbc23c6ff930a048aa310b" +dependencies = [ + "jobserver", + "libc", + "shlex", +] + +[[package]] +name = "cesu8" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d43a04d8753f35258c91f8ec639f792891f748a1edbd759cf1dcea3382ad83c" + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "chrono" +version = "0.4.39" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e36cc9d416881d2e24f9a963be5fb1cd90966419ac844274161d10488b3e825" +dependencies = [ + "android-tzdata", + "iana-time-zone", + "js-sys", + "num-traits", + "serde", + "wasm-bindgen", + "windows-targets 0.52.6", +] + +[[package]] +name = "chrono-tz" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd6dd8046d00723a59a2f8c5f295c515b9bb9a331ee4f8f3d4dd49e428acd3b6" +dependencies = [ + "chrono", + "chrono-tz-build", + "phf", +] + +[[package]] +name = "chrono-tz-build" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e94fea34d77a245229e7746bd2beb786cd2a896f306ff491fb8cecb3074b10a7" +dependencies = [ + "parse-zoneinfo", + "phf_codegen", +] + +[[package]] +name = "colorchoice" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b63caa9aa9397e2d9480a9b13673856c78d8ac123288526c37d7839f2a86990" + +[[package]] +name = "combine" +version = "4.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba5a308b75df32fe02788e748662718f03fde005016435c444eea572398219fd" +dependencies = [ + "bytes", + "memchr", +] + +[[package]] +name = "comfy-table" +version = "7.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24f165e7b643266ea80cb858aed492ad9280e3e05ce24d4a99d7d7b889b6a4d9" +dependencies = [ + "strum", + "strum_macros", + "unicode-width", +] + +[[package]] +name = "console" +version = "0.15.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea3c6ecd8059b57859df5c69830340ed3c41d30e3da0c1cbed90a96ac853041b" +dependencies = [ + "encode_unicode", + "libc", + "once_cell", + "windows-sys 0.59.0", +] + +[[package]] +name = "const-random" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87e00182fe74b066627d63b85fd550ac2998d4b0bd86bfed477a0ae4c7c71359" +dependencies = [ + "const-random-macro", +] + +[[package]] +name = "const-random-macro" +version = "0.1.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9d839f2a20b0aee515dc581a6172f2321f96cab76c1a38a4c584a194955390e" +dependencies = [ + "getrandom", + "once_cell", + "tiny-keccak", +] + +[[package]] +name = "constant_time_eq" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c74b8349d32d297c9134b8c88677813a227df8f779daa29bfc29c183fe3dca6" + +[[package]] +name = "core-foundation" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b55271e5c8c478ad3f38ad24ef34923091e0548492a266d19b3c0b4d82574c63" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" + +[[package]] +name = "cpufeatures" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16b80225097f2e5ae4e7179dd2266824648f3e2f49d9134d584b76389d31c4c3" +dependencies = [ + "libc", +] + +[[package]] +name = "crc32fast" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a97769d94ddab943e4510d138150169a2758b5ef3eb191a9ee688de3e23ef7b3" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" + +[[package]] +name = "crunchy" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a81dae078cea95a014a339291cec439d2f232ebe854a9d672b796c6afafa9b7" + +[[package]] +name = "crypto-common" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "csv" +version = "1.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "acdc4883a9c96732e4733212c01447ebd805833b7275a73ca3ee080fd77afdaf" +dependencies = [ + "csv-core", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "csv-core" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5efa2b3d7902f4b634a20cae3c9c4e6209dc4779feb6863329607560143efa70" +dependencies = [ + "memchr", +] + +[[package]] +name = "darling" +version = "0.20.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f63b86c8a8826a49b8c21f08a2d07338eec8d900540f8630dc76284be802989" +dependencies = [ + "darling_core", + "darling_macro", +] + +[[package]] +name = "darling_core" +version = "0.20.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95133861a8032aaea082871032f5815eb9e98cef03fa916ab4500513994df9e5" +dependencies = [ + "fnv", + "ident_case", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "darling_macro" +version = "0.20.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d336a2a514f6ccccaa3e09b02d41d35330c07ddf03a62165fcec10bb561c7806" +dependencies = [ + "darling_core", + "quote", + "syn", +] + +[[package]] +name = "dashmap" +version = "6.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5041cc499144891f3790297212f32a74fb938e5136a14943f338ef9e0ae276cf" +dependencies = [ + "cfg-if", + "crossbeam-utils", + "hashbrown 0.14.5", + "lock_api", + "once_cell", + "parking_lot_core", +] + +[[package]] +name = "data-encoding" +version = "2.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e60eed09d8c01d3cee5b7d30acb059b76614c918fa0f992e0dd6eeb10daad6f" + +[[package]] +name = "datafusion" +version = "44.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "014fc8c384ecacedaabb3bc8359c2a6c6e9d8f7bea65be3434eccacfc37f52d9" +dependencies = [ + "arrow", + "arrow-array", + "arrow-ipc", + "arrow-schema", + "async-compression", + "async-trait", + "bytes", + "bzip2 0.5.0", + "chrono", + "dashmap", + "datafusion-catalog", + "datafusion-common", + "datafusion-common-runtime", + "datafusion-execution", + "datafusion-expr", + "datafusion-functions", + "datafusion-functions-aggregate", + "datafusion-functions-nested", + "datafusion-functions-table", + "datafusion-functions-window", + "datafusion-optimizer", + "datafusion-physical-expr", + "datafusion-physical-expr-common", + "datafusion-physical-optimizer", + "datafusion-physical-plan", + "datafusion-sql", + "flate2", + "futures", + "glob", + "itertools 0.13.0", + "log", + "object_store", + "parking_lot", + "parquet", + "rand", + "regex", + "sqlparser", + "tempfile", + "tokio", + "tokio-util", + "url", + "uuid", + "xz2", + "zstd", +] + +[[package]] +name = "datafusion-catalog" +version = "44.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee60d33e210ef96070377ae667ece7caa0e959c8387496773d4a1a72f1a5012e" +dependencies = [ + "arrow-schema", + "async-trait", + "datafusion-common", + "datafusion-execution", + "datafusion-expr", + "datafusion-physical-plan", + "parking_lot", +] + +[[package]] +name = "datafusion-common" +version = "44.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b42b7d720fe21ed9cca2ebb635f3f13a12cfab786b41e0fba184fb2e620525b" +dependencies = [ + "ahash", + "arrow", + "arrow-array", + "arrow-buffer", + "arrow-schema", + "half", + "hashbrown 0.14.5", + "indexmap 2.7.0", + "libc", + "log", + "object_store", + "parquet", + "paste", + "recursive", + "sqlparser", + "tokio", + "web-time", +] + +[[package]] +name = "datafusion-common-runtime" +version = "44.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72fbf14d4079f7ce5306393084fe5057dddfdc2113577e0049310afa12e94281" +dependencies = [ + "log", + "tokio", +] + +[[package]] +name = "datafusion-doc" +version = "44.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c278dbd64860ed0bb5240fc1f4cb6aeea437153910aea69bcf7d5a8d6d0454f3" + +[[package]] +name = "datafusion-execution" +version = "44.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e22cb02af47e756468b3cbfee7a83e3d4f2278d452deb4b033ba933c75169486" +dependencies = [ + "arrow", + "dashmap", + "datafusion-common", + "datafusion-expr", + "futures", + "log", + "object_store", + "parking_lot", + "rand", + "tempfile", + "url", +] + +[[package]] +name = "datafusion-expr" +version = "44.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62298eadb1d15b525df1315e61a71519ffc563d41d5c3b2a30fda2d70f77b93c" +dependencies = [ + "arrow", + "chrono", + "datafusion-common", + "datafusion-doc", + "datafusion-expr-common", + "datafusion-functions-aggregate-common", + "datafusion-functions-window-common", + "datafusion-physical-expr-common", + "indexmap 2.7.0", + "paste", + "recursive", + "serde_json", + "sqlparser", +] + +[[package]] +name = "datafusion-expr-common" +version = "44.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dda7f73c5fc349251cd3dcb05773c5bf55d2505a698ef9d38dfc712161ea2f55" +dependencies = [ + "arrow", + "datafusion-common", + "itertools 0.13.0", +] + +[[package]] +name = "datafusion-functions" +version = "44.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd197f3b2975424d3a4898ea46651be855a46721a56727515dbd5c9e2fb597da" +dependencies = [ + "arrow", + "arrow-buffer", + "base64", + "blake2", + "blake3", + "chrono", + "datafusion-common", + "datafusion-doc", + "datafusion-execution", + "datafusion-expr", + "datafusion-expr-common", + "datafusion-macros", + "hashbrown 0.14.5", + "hex", + "itertools 0.13.0", + "log", + "md-5", + "rand", + "regex", + "sha2", + "unicode-segmentation", + "uuid", +] + +[[package]] +name = "datafusion-functions-aggregate" +version = "44.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aabbe48fba18f9981b134124381bee9e46f93518b8ad2f9721ee296cef5affb9" +dependencies = [ + "ahash", + "arrow", + "arrow-schema", + "datafusion-common", + "datafusion-doc", + "datafusion-execution", + "datafusion-expr", + "datafusion-functions-aggregate-common", + "datafusion-macros", + "datafusion-physical-expr", + "datafusion-physical-expr-common", + "half", + "log", + "paste", +] + +[[package]] +name = "datafusion-functions-aggregate-common" +version = "44.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7a3fefed9c8c11268d446d924baca8cabf52fe32f73fdaa20854bac6473590c" +dependencies = [ + "ahash", + "arrow", + "datafusion-common", + "datafusion-expr-common", + "datafusion-physical-expr-common", +] + +[[package]] +name = "datafusion-functions-nested" +version = "44.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6360f27464fab857bec698af39b2ae331dc07c8bf008fb4de387a19cdc6815a5" +dependencies = [ + "arrow", + "arrow-array", + "arrow-buffer", + "arrow-ord", + "arrow-schema", + "datafusion-common", + "datafusion-execution", + "datafusion-expr", + "datafusion-functions", + "datafusion-functions-aggregate", + "datafusion-physical-expr-common", + "itertools 0.13.0", + "log", + "paste", +] + +[[package]] +name = "datafusion-functions-table" +version = "44.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c35c070eb705c12795dab399c3809f4dfbc290678c624d3989490ca9b8449c1" +dependencies = [ + "arrow", + "async-trait", + "datafusion-catalog", + "datafusion-common", + "datafusion-expr", + "datafusion-physical-plan", + "parking_lot", + "paste", +] + +[[package]] +name = "datafusion-functions-window" +version = "44.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52229bca26b590b140900752226c829f15fc1a99840e1ca3ce1a9534690b82a8" +dependencies = [ + "datafusion-common", + "datafusion-doc", + "datafusion-expr", + "datafusion-functions-window-common", + "datafusion-macros", + "datafusion-physical-expr", + "datafusion-physical-expr-common", + "log", + "paste", +] + +[[package]] +name = "datafusion-functions-window-common" +version = "44.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "367befc303b64a668a10ae6988a064a9289e1999e71a7f8e526b6e14d6bdd9d6" +dependencies = [ + "datafusion-common", + "datafusion-physical-expr-common", +] + +[[package]] +name = "datafusion-macros" +version = "44.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f5de3c8f386ea991696553afe241a326ecbc3c98a12c562867e4be754d3a060c" +dependencies = [ + "quote", + "syn", +] + +[[package]] +name = "datafusion-optimizer" +version = "44.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53b520413906f755910422b016fb73884ae6e9e1b376de4f9584b6c0e031da75" +dependencies = [ + "arrow", + "chrono", + "datafusion-common", + "datafusion-expr", + "datafusion-physical-expr", + "indexmap 2.7.0", + "itertools 0.13.0", + "log", + "recursive", + "regex", + "regex-syntax 0.8.5", +] + +[[package]] +name = "datafusion-physical-expr" +version = "44.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "acd6ddc378f6ad19af95ccd6790dec8f8e1264bc4c70e99ddc1830c1a1c78ccd" +dependencies = [ + "ahash", + "arrow", + "arrow-array", + "arrow-buffer", + "arrow-schema", + "datafusion-common", + "datafusion-expr", + "datafusion-expr-common", + "datafusion-functions-aggregate-common", + "datafusion-physical-expr-common", + "half", + "hashbrown 0.14.5", + "indexmap 2.7.0", + "itertools 0.13.0", + "log", + "paste", + "petgraph", +] + +[[package]] +name = "datafusion-physical-expr-common" +version = "44.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06e6c05458eccd74b4c77ed6a1fe63d52434240711de7f6960034794dad1caf5" +dependencies = [ + "ahash", + "arrow", + "datafusion-common", + "datafusion-expr-common", + "hashbrown 0.14.5", + "itertools 0.13.0", +] + +[[package]] +name = "datafusion-physical-optimizer" +version = "44.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9dc3a82190f49c37d377f31317e07ab5d7588b837adadba8ac367baad5dc2351" +dependencies = [ + "arrow", + "datafusion-common", + "datafusion-execution", + "datafusion-expr-common", + "datafusion-physical-expr", + "datafusion-physical-plan", + "itertools 0.13.0", + "log", + "recursive", +] + +[[package]] +name = "datafusion-physical-plan" +version = "44.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a6608bc9844b4ddb5ed4e687d173e6c88700b1d0482f43894617d18a1fe75da" +dependencies = [ + "ahash", + "arrow", + "arrow-array", + "arrow-buffer", + "arrow-ord", + "arrow-schema", + "async-trait", + "chrono", + "datafusion-common", + "datafusion-common-runtime", + "datafusion-execution", + "datafusion-expr", + "datafusion-functions-window-common", + "datafusion-physical-expr", + "datafusion-physical-expr-common", + "futures", + "half", + "hashbrown 0.14.5", + "indexmap 2.7.0", + "itertools 0.13.0", + "log", + "parking_lot", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "datafusion-proto" +version = "44.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e23b0998195e495bfa7b37cdceb317129a6c40522219f6872d2e0c9ae9f4fcb" +dependencies = [ + "arrow", + "chrono", + "datafusion", + "datafusion-common", + "datafusion-expr", + "datafusion-proto-common", + "object_store", + "prost", +] + +[[package]] +name = "datafusion-proto-common" +version = "44.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfc59992a29eed2d2c1dd779deac99083b217774ebcf90ee121840607a4d866f" +dependencies = [ + "arrow", + "datafusion-common", + "prost", +] + +[[package]] +name = "datafusion-sql" +version = "44.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a884061c79b33d0c8e84a6f4f4be8bdc12c0f53f5af28ddf5d6d95ac0b15fdc" +dependencies = [ + "arrow", + "arrow-array", + "arrow-schema", + "bigdecimal", + "datafusion-common", + "datafusion-expr", + "indexmap 2.7.0", + "log", + "recursive", + "regex", + "sqlparser", +] + +[[package]] +name = "datafusion-table-providers" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8260cf26a6804e8eb8d87ff5a12cc3b02d2a5773324265c3b0fd2b91f5179a99" +dependencies = [ + "arrow-flight", + "async-trait", + "bigdecimal", + "byteorder", + "chrono", + "dashmap", + "datafusion", + "datafusion-proto", + "fallible-iterator", + "fundu", + "futures", + "geo-types", + "itertools 0.14.0", + "num-bigint", + "sea-query", + "secrecy", + "serde", + "serde_json", + "sha2", + "snafu", + "time", + "tokio", + "tonic", + "tracing", + "trust-dns-resolver", + "url", +] + +[[package]] +name = "datafusion_jni" +version = "0.1.0" +dependencies = [ + "anyhow", + "arrow", + "arrow-flight", + "async-trait", + "bollard", + "bytes", + "datafusion", + "datafusion-table-providers", + "futures", + "geozero", + "insta", + "jni", + "prost", + "rand", + "reqwest", + "rstest", + "serde_json", + "test-log", + "tokio", + "tokio-stream", + "tonic", + "tracing-subscriber", +] + +[[package]] +name = "deranged" +version = "0.3.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b42b6fa04a440b495c8b04d0e71b707c585f83cb9cb28cf8cd0d976c315e31b4" +dependencies = [ + "powerfmt", + "serde", +] + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "crypto-common", + "subtle", +] + +[[package]] +name = "displaydoc" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "either" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60b1af1c220855b6ceac025d3f6ecdd2b7c4894bfe9cd9bda4fbb4bc7c0d4cf0" + +[[package]] +name = "encode_unicode" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34aa73646ffb006b8f5147f3dc182bd4bcb190227ce861fc4a4844bf8e3cb2c0" + +[[package]] +name = "encoding_rs" +version = "0.8.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75030f3c4f45dafd7586dd6780965a8c7e8e285a5ecb86713e63a79c5b2766f3" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "enum-as-inner" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1e6a265c649f3f5979b601d26f1d05ada116434c87741c9493cb56218f76cbc" +dependencies = [ + "heck 0.5.0", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "env_filter" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "186e05a59d4c50738528153b83b0b0194d3a29507dfec16eccd4b342903397d0" +dependencies = [ + "log", +] + +[[package]] +name = "env_logger" +version = "0.11.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dcaee3d8e3cfc3fd92428d477bc97fc29ec8716d180c0d74c643bb26166660e0" +dependencies = [ + "anstream", + "anstyle", + "env_filter", + "log", +] + +[[package]] +name = "equivalent" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" + +[[package]] +name = "errno" +version = "0.3.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33d852cb9b869c2a9b3df2f71a3074817f01e1844f839a144f5fcef059a4eb5d" +dependencies = [ + "libc", + "windows-sys 0.59.0", +] + +[[package]] +name = "fallible-iterator" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2acce4a10f12dc2fb14a218589d4f1f62ef011b2d0cc4b3cb1bba8e94da14649" + +[[package]] +name = "fastrand" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" + +[[package]] +name = "fixedbitset" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80" + +[[package]] +name = "flatbuffers" +version = "24.12.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4f1baf0dbf96932ec9a3038d57900329c015b0bfb7b63d904f3bc27e2b02a096" +dependencies = [ + "bitflags 1.3.2", + "rustc_version", +] + +[[package]] +name = "flate2" +version = "1.0.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c936bfdafb507ebbf50b8074c54fa31c5be9a1e7e5f467dd659697041407d07c" +dependencies = [ + "crc32fast", + "miniz_oxide", +] + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "foreign-types" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" +dependencies = [ + "foreign-types-shared", +] + +[[package]] +name = "foreign-types-shared" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" + +[[package]] +name = "form_urlencoded" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13624c2627564efccf4934284bdd98cbaa14e79b0b5a141218e507b3a823456" +dependencies = [ + "percent-encoding", +] + +[[package]] +name = "fundu" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ce12752fc64f35be3d53e0a57017cd30970f0cffd73f62c791837d8845badbd" +dependencies = [ + "fundu-core", +] + +[[package]] +name = "fundu-core" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e463452e2d8b7600d38dcea1ed819773a57f0d710691bfc78db3961bd3f4c3ba" + +[[package]] +name = "futures" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65bc07b1a8bc7c85c5f2e110c476c7389b4554ba72af57d8445ea63a576b0876" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-channel" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2dff15bf788c671c1934e366d07e30c1814a8ef514e1af724a602e8a2fbe1b10" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05f29059c0c2090612e8d742178b0580d2dc940c837851ad723096f87af6663e" + +[[package]] +name = "futures-executor" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e28d1d997f585e54aebc3f97d39e72338912123a67330d723fdbb564d646c9f" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-io" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e5c1b78ca4aae1ac06c48a526a655760685149f0d465d21f37abfe57ce075c6" + +[[package]] +name = "futures-macro" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "futures-sink" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e575fab7d1e0dcb8d0c7bcf9a63ee213816ab51902e6d244a95819acacf1d4f7" + +[[package]] +name = "futures-task" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f90f7dce0722e95104fcb095585910c0977252f286e354b5e3bd38902cd99988" + +[[package]] +name = "futures-timer" +version = "3.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f288b0a4f20f9a56b5d1da57e2227c661b7b16168e2f72365f57b63326e29b24" + +[[package]] +name = "futures-util" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9fa08315bb612088cc391249efdc3bc77536f16c91f6cf495e6fbe85b20a4a81" +dependencies = [ + "futures-channel", + "futures-core", + "futures-io", + "futures-macro", + "futures-sink", + "futures-task", + "memchr", + "pin-project-lite", + "pin-utils", + "slab", +] + +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "geo-types" +version = "0.7.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3bd1157f0f936bf0cd68dec91e8f7c311afe60295574d62b70d4861a1bfdf2d9" +dependencies = [ + "approx", + "num-traits", + "serde", +] + +[[package]] +name = "geojson" +version = "0.24.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5d728c1df1fbf328d74151efe6cb0586f79ee813346ea981add69bd22c9241b" +dependencies = [ + "log", + "serde", + "serde_json", + "thiserror 1.0.69", +] + +[[package]] +name = "geozero" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5f28f34864745eb2f123c990c6ffd92c1584bd39439b3f27ff2a0f4ea5b309b" +dependencies = [ + "geo-types", + "geojson", + "log", + "scroll", + "serde_json", + "thiserror 1.0.69", + "wkt", +] + +[[package]] +name = "getrandom" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + +[[package]] +name = "gimli" +version = "0.31.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07e28edb80900c19c28f1072f2e8aeca7fa06b23cd4169cefe1af5aa3260783f" + +[[package]] +name = "glob" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8d1add55171497b4705a648c6b583acafb01d58050a51727785f0b2c8e0a2b2" + +[[package]] +name = "h2" +version = "0.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccae279728d634d083c00f6099cb58f01cc99c145b84b8be2f6c74618d79922e" +dependencies = [ + "atomic-waker", + "bytes", + "fnv", + "futures-core", + "futures-sink", + "http", + "indexmap 2.7.0", + "slab", + "tokio", + "tokio-util", + "tracing", +] + +[[package]] +name = "half" +version = "2.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dd08c532ae367adf81c312a4580bc67f1d0fe8bc9c460520283f4c0ff277888" +dependencies = [ + "cfg-if", + "crunchy", + "num-traits", +] + +[[package]] +name = "hashbrown" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" + +[[package]] +name = "hashbrown" +version = "0.14.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" +dependencies = [ + "ahash", + "allocator-api2", +] + +[[package]] +name = "hashbrown" +version = "0.15.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf151400ff0baff5465007dd2f3e717f3fe502074ca563069ce3a6629d07b289" + +[[package]] +name = "heck" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + +[[package]] +name = "hostname" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c731c3e10504cc8ed35cfe2f1db4c9274c3d35fa486e3b31df46f068ef3e867" +dependencies = [ + "libc", + "match_cfg", + "winapi", +] + +[[package]] +name = "http" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f16ca2af56261c99fba8bac40a10251ce8188205a4c448fbb745a2e4daa76fea" +dependencies = [ + "bytes", + "fnv", + "itoa", +] + +[[package]] +name = "http-body" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" +dependencies = [ + "bytes", + "http", +] + +[[package]] +name = "http-body-util" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "793429d76616a256bcb62c2a2ec2bed781c8307e797e2598c50010f2bee2544f" +dependencies = [ + "bytes", + "futures-util", + "http", + "http-body", + "pin-project-lite", +] + +[[package]] +name = "httparse" +version = "1.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d71d3574edd2771538b901e6549113b4006ece66150fb69c0fb6d9a2adae946" + +[[package]] +name = "httpdate" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" + +[[package]] +name = "humantime" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" + +[[package]] +name = "hyper" +version = "1.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "256fb8d4bd6413123cc9d91832d78325c48ff41677595be797d90f42969beae0" +dependencies = [ + "bytes", + "futures-channel", + "futures-util", + "h2", + "http", + "http-body", + "httparse", + "httpdate", + "itoa", + "pin-project-lite", + "smallvec", + "tokio", + "want", +] + +[[package]] +name = "hyper-named-pipe" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73b7d8abf35697b81a825e386fc151e0d503e8cb5fcb93cc8669c376dfd6f278" +dependencies = [ + "hex", + "hyper", + "hyper-util", + "pin-project-lite", + "tokio", + "tower-service", + "winapi", +] + +[[package]] +name = "hyper-rustls" +version = "0.27.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d191583f3da1305256f22463b9bb0471acad48a4e534a5218b9963e9c1f59b2" +dependencies = [ + "futures-util", + "http", + "hyper", + "hyper-util", + "rustls", + "rustls-pki-types", + "tokio", + "tokio-rustls", + "tower-service", +] + +[[package]] +name = "hyper-timeout" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b90d566bffbce6a75bd8b09a05aa8c2cb1fabb6cb348f8840c9e4c90a0d83b0" +dependencies = [ + "hyper", + "hyper-util", + "pin-project-lite", + "tokio", + "tower-service", +] + +[[package]] +name = "hyper-tls" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70206fc6890eaca9fde8a0bf71caa2ddfc9fe045ac9e5c70df101a7dbde866e0" +dependencies = [ + "bytes", + "http-body-util", + "hyper", + "hyper-util", + "native-tls", + "tokio", + "tokio-native-tls", + "tower-service", +] + +[[package]] +name = "hyper-util" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df2dcfbe0677734ab2f3ffa7fa7bfd4706bfdc1ef393f2ee30184aed67e631b4" +dependencies = [ + "bytes", + "futures-channel", + "futures-util", + "http", + "http-body", + "hyper", + "pin-project-lite", + "socket2", + "tokio", + "tower-service", + "tracing", +] + +[[package]] +name = "hyperlocal" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "986c5ce3b994526b3cd75578e62554abd09f0899d6206de48b3e96ab34ccc8c7" +dependencies = [ + "hex", + "http-body-util", + "hyper", + "hyper-util", + "pin-project-lite", + "tokio", + "tower-service", +] + +[[package]] +name = "iana-time-zone" +version = "0.1.61" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "235e081f3925a06703c2d0117ea8b91f042756fd6e7a6e5d901e8ca1a996b220" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "wasm-bindgen", + "windows-core", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + +[[package]] +name = "icu_collections" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db2fa452206ebee18c4b5c2274dbf1de17008e874b4dc4f0aea9d01ca79e4526" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_locid" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13acbb8371917fc971be86fc8057c41a64b521c184808a698c02acc242dbf637" +dependencies = [ + "displaydoc", + "litemap", + "tinystr", + "writeable", + "zerovec", +] + +[[package]] +name = "icu_locid_transform" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "01d11ac35de8e40fdeda00d9e1e9d92525f3f9d887cdd7aa81d727596788b54e" +dependencies = [ + "displaydoc", + "icu_locid", + "icu_locid_transform_data", + "icu_provider", + "tinystr", + "zerovec", +] + +[[package]] +name = "icu_locid_transform_data" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fdc8ff3388f852bede6b579ad4e978ab004f139284d7b28715f773507b946f6e" + +[[package]] +name = "icu_normalizer" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19ce3e0da2ec68599d193c93d088142efd7f9c5d6fc9b803774855747dc6a84f" +dependencies = [ + "displaydoc", + "icu_collections", + "icu_normalizer_data", + "icu_properties", + "icu_provider", + "smallvec", + "utf16_iter", + "utf8_iter", + "write16", + "zerovec", +] + +[[package]] +name = "icu_normalizer_data" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8cafbf7aa791e9b22bec55a167906f9e1215fd475cd22adfcf660e03e989516" + +[[package]] +name = "icu_properties" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93d6020766cfc6302c15dbbc9c8778c37e62c14427cb7f6e601d849e092aeef5" +dependencies = [ + "displaydoc", + "icu_collections", + "icu_locid_transform", + "icu_properties_data", + "icu_provider", + "tinystr", + "zerovec", +] + +[[package]] +name = "icu_properties_data" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67a8effbc3dd3e4ba1afa8ad918d5684b8868b3b26500753effea8d2eed19569" + +[[package]] +name = "icu_provider" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ed421c8a8ef78d3e2dbc98a973be2f3770cb42b606e3ab18d6237c4dfde68d9" +dependencies = [ + "displaydoc", + "icu_locid", + "icu_provider_macros", + "stable_deref_trait", + "tinystr", + "writeable", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_provider_macros" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ec89e9337638ecdc08744df490b221a7399bf8d164eb52a665454e60e075ad6" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "ident_case" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" + +[[package]] +name = "idna" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d20d6b07bfbc108882d88ed8e37d39636dcc260e15e30c45e6ba089610b917c" +dependencies = [ + "unicode-bidi", + "unicode-normalization", +] + +[[package]] +name = "idna" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "686f825264d630750a544639377bae737628043f20d38bbc029e8f29ea968a7e" +dependencies = [ + "idna_adapter", + "smallvec", + "utf8_iter", +] + +[[package]] +name = "idna_adapter" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "daca1df1c957320b2cf139ac61e7bd64fed304c5040df000a745aa1de3b4ef71" +dependencies = [ + "icu_normalizer", + "icu_properties", +] + +[[package]] +name = "indexmap" +version = "1.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" +dependencies = [ + "autocfg", + "hashbrown 0.12.3", + "serde", +] + +[[package]] +name = "indexmap" +version = "2.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62f822373a4fe84d4bb149bf54e584a7f4abec90e072ed49cda0edea5b95471f" +dependencies = [ + "equivalent", + "hashbrown 0.15.2", + "serde", +] + +[[package]] +name = "inherent" +version = "1.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0122b7114117e64a63ac49f752a5ca4624d534c7b1c7de796ac196381cd2d947" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "insta" +version = "1.42.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6513e4067e16e69ed1db5ab56048ed65db32d10ba5fc1217f5393f8f17d8b5a5" +dependencies = [ + "console", + "linked-hash-map", + "once_cell", + "regex", + "similar", +] + +[[package]] +name = "integer-encoding" +version = "3.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8bb03732005da905c88227371639bf1ad885cc712789c011c31c5fb3ab3ccf02" + +[[package]] +name = "ipconfig" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b58db92f96b720de98181bbbe63c831e87005ab460c1bf306eb2622b4707997f" +dependencies = [ + "socket2", + "widestring", + "windows-sys 0.48.0", + "winreg", +] + +[[package]] +name = "ipnet" +version = "2.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddc24109865250148c2e0f3d25d4f0f479571723792d3802153c60922a4fb708" + +[[package]] +name = "is_terminal_polyfill" +version = "1.70.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf" + +[[package]] +name = "itertools" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186" +dependencies = [ + "either", +] + +[[package]] +name = "itertools" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d75a2a4b1b190afb6f5425f10f6a8f959d2ea0b9c2b1d79553551850539e4674" + +[[package]] +name = "jni" +version = "0.21.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a87aa2bb7d2af34197c04845522473242e1aa17c12f4935d5856491a7fb8c97" +dependencies = [ + "cesu8", + "cfg-if", + "combine", + "jni-sys", + "log", + "thiserror 1.0.69", + "walkdir", + "windows-sys 0.45.0", +] + +[[package]] +name = "jni-sys" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8eaf4bc02d17cbdd7ff4c7438cafcdf7fb9a4613313ad11b4f8fefe7d3fa0130" + +[[package]] +name = "jobserver" +version = "0.1.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48d1dbcbbeb6a7fec7e059840aa538bd62aaccf972c7346c4d9d2059312853d0" +dependencies = [ + "libc", +] + +[[package]] +name = "js-sys" +version = "0.3.77" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1cfaf33c695fc6e08064efbc1f72ec937429614f25eef83af942d0e227c3a28f" +dependencies = [ + "once_cell", + "wasm-bindgen", +] + +[[package]] +name = "lazy_static" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" + +[[package]] +name = "lexical-core" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b765c31809609075565a70b4b71402281283aeda7ecaf4818ac14a7b2ade8958" +dependencies = [ + "lexical-parse-float", + "lexical-parse-integer", + "lexical-util", + "lexical-write-float", + "lexical-write-integer", +] + +[[package]] +name = "lexical-parse-float" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de6f9cb01fb0b08060209a057c048fcbab8717b4c1ecd2eac66ebfe39a65b0f2" +dependencies = [ + "lexical-parse-integer", + "lexical-util", + "static_assertions", +] + +[[package]] +name = "lexical-parse-integer" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72207aae22fc0a121ba7b6d479e42cbfea549af1479c3f3a4f12c70dd66df12e" +dependencies = [ + "lexical-util", + "static_assertions", +] + +[[package]] +name = "lexical-util" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a82e24bf537fd24c177ffbbdc6ebcc8d54732c35b50a3f28cc3f4e4c949a0b3" +dependencies = [ + "static_assertions", +] + +[[package]] +name = "lexical-write-float" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c5afc668a27f460fb45a81a757b6bf2f43c2d7e30cb5a2dcd3abf294c78d62bd" +dependencies = [ + "lexical-util", + "lexical-write-integer", + "static_assertions", +] + +[[package]] +name = "lexical-write-integer" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "629ddff1a914a836fb245616a7888b62903aae58fa771e1d83943035efa0f978" +dependencies = [ + "lexical-util", + "static_assertions", +] + +[[package]] +name = "libc" +version = "0.2.169" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5aba8db14291edd000dfcc4d620c7ebfb122c613afb886ca8803fa4e128a20a" + +[[package]] +name = "libm" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8355be11b20d696c8f18f6cc018c4e372165b1fa8126cef092399c9951984ffa" + +[[package]] +name = "linked-hash-map" +version = "0.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0717cef1bc8b636c6e1c1bbdefc09e6322da8a9321966e8928ef80d20f7f770f" + +[[package]] +name = "linux-raw-sys" +version = "0.4.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d26c52dbd32dccf2d10cac7725f8eae5296885fb5703b261f7d0a0739ec807ab" + +[[package]] +name = "litemap" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ee93343901ab17bd981295f2cf0026d4ad018c7c31ba84549a4ddbb47a45104" + +[[package]] +name = "lock_api" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07af8b9cdd281b7915f413fa73f29ebd5d55d0d3f0155584dade1ff18cea1b17" +dependencies = [ + "autocfg", + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04cbf5b083de1c7e0222a7a51dbfdba1cbe1c6ab0b15e29fff3f6c077fd9cd9f" + +[[package]] +name = "lru-cache" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "31e24f1ad8321ca0e8a1e0ac13f23cb668e6f5466c2c57319f6a5cf1cc8e3b1c" +dependencies = [ + "linked-hash-map", +] + +[[package]] +name = "lz4_flex" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75761162ae2b0e580d7e7c390558127e5f01b4194debd6221fd8c207fc80e3f5" +dependencies = [ + "twox-hash", +] + +[[package]] +name = "lzma-sys" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fda04ab3764e6cde78b9974eec4f779acaba7c4e84b36eca3cf77c581b85d27" +dependencies = [ + "cc", + "libc", + "pkg-config", +] + +[[package]] +name = "match_cfg" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ffbee8634e0d45d258acb448e7eaab3fce7a0a467395d4d9f228e3c1f01fb2e4" + +[[package]] +name = "matchers" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8263075bb86c5a1b1427b5ae862e8889656f126e9f77c484496e8b47cf5c5558" +dependencies = [ + "regex-automata 0.1.10", +] + +[[package]] +name = "matchit" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e7465ac9959cc2b1404e8e2367b43684a6d13790fe23056cc8c6c5a6b7bcb94" + +[[package]] +name = "md-5" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf" +dependencies = [ + "cfg-if", + "digest", +] + +[[package]] +name = "memchr" +version = "2.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" + +[[package]] +name = "mime" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" + +[[package]] +name = "miniz_oxide" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8402cab7aefae129c6977bb0ff1b8fd9a04eb5b51efc50a70bea51cda0c7924" +dependencies = [ + "adler2", +] + +[[package]] +name = "mio" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2886843bf800fba2e3377cff24abf6379b4c4d5c6681eaf9ea5b0d15090450bd" +dependencies = [ + "libc", + "wasi", + "windows-sys 0.52.0", +] + +[[package]] +name = "native-tls" +version = "0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8614eb2c83d59d1c8cc974dd3f920198647674a0a035e1af1fa58707e317466" +dependencies = [ + "libc", + "log", + "openssl", + "openssl-probe", + "openssl-sys", + "schannel", + "security-framework 2.11.1", + "security-framework-sys", + "tempfile", +] + +[[package]] +name = "nu-ansi-term" +version = "0.46.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77a8165726e8236064dbb45459242600304b42a5ea24ee2948e18e023bf7ba84" +dependencies = [ + "overload", + "winapi", +] + +[[package]] +name = "num" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35bd024e8b2ff75562e5f34e7f4905839deb4b22955ef5e73d2fea1b9813cb23" +dependencies = [ + "num-bigint", + "num-complex", + "num-integer", + "num-iter", + "num-rational", + "num-traits", +] + +[[package]] +name = "num-bigint" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5e44f723f1133c9deac646763579fdb3ac745e418f2a7af9cd0c431da1f20b9" +dependencies = [ + "num-integer", + "num-traits", +] + +[[package]] +name = "num-complex" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-conv" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9" + +[[package]] +name = "num-integer" +version = "0.1.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-iter" +version = "0.1.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1429034a0490724d0075ebb2bc9e875d6503c3cf69e235a8941aa757d83ef5bf" +dependencies = [ + "autocfg", + "num-integer", + "num-traits", +] + +[[package]] +name = "num-rational" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f83d14da390562dca69fc84082e73e548e1ad308d24accdedd2720017cb37824" +dependencies = [ + "num-bigint", + "num-integer", + "num-traits", +] + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", + "libm", +] + +[[package]] +name = "object" +version = "0.36.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62948e14d923ea95ea2c7c86c71013138b66525b86bdc08d2dcc262bdb497b87" +dependencies = [ + "memchr", +] + +[[package]] +name = "object_store" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3cfccb68961a56facde1163f9319e0d15743352344e7808a11795fb99698dcaf" +dependencies = [ + "async-trait", + "bytes", + "chrono", + "futures", + "humantime", + "itertools 0.13.0", + "parking_lot", + "percent-encoding", + "snafu", + "tokio", + "tracing", + "url", + "walkdir", +] + +[[package]] +name = "once_cell" +version = "1.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1261fe7e33c73b354eab43b1273a57c8f967d0391e80353e51f764ac02cf6775" + +[[package]] +name = "openssl" +version = "0.10.68" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6174bc48f102d208783c2c84bf931bb75927a617866870de8a4ea85597f871f5" +dependencies = [ + "bitflags 2.8.0", + "cfg-if", + "foreign-types", + "libc", + "once_cell", + "openssl-macros", + "openssl-sys", +] + +[[package]] +name = "openssl-macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "openssl-probe" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" + +[[package]] +name = "openssl-sys" +version = "0.9.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "45abf306cbf99debc8195b66b7346498d7b10c210de50418b5ccd7ceba08c741" +dependencies = [ + "cc", + "libc", + "pkg-config", + "vcpkg", +] + +[[package]] +name = "ordered-float" +version = "2.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68f19d67e5a2795c94e73e0bb1cc1a7edeb2e28efd39e2e1c9b7a40c1108b11c" +dependencies = [ + "num-traits", +] + +[[package]] +name = "overload" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39" + +[[package]] +name = "parking_lot" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1bf18183cf54e8d6059647fc3063646a1801cf30896933ec2311622cc4b9a27" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e401f977ab385c9e4e3ab30627d6f26d00e2c73eef317493c4ec6d468726cf8" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-targets 0.52.6", +] + +[[package]] +name = "parquet" +version = "53.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b449890367085eb65d7d3321540abc3d7babbd179ce31df0016e90719114191" +dependencies = [ + "ahash", + "arrow-array", + "arrow-buffer", + "arrow-cast", + "arrow-data", + "arrow-ipc", + "arrow-schema", + "arrow-select", + "base64", + "brotli", + "bytes", + "chrono", + "flate2", + "futures", + "half", + "hashbrown 0.15.2", + "lz4_flex", + "num", + "num-bigint", + "object_store", + "paste", + "seq-macro", + "snap", + "thrift", + "tokio", + "twox-hash", + "zstd", + "zstd-sys", +] + +[[package]] +name = "parse-zoneinfo" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f2a05b18d44e2957b88f96ba460715e295bc1d7510468a2f3d3b44535d26c24" +dependencies = [ + "regex", +] + +[[package]] +name = "paste" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" + +[[package]] +name = "percent-encoding" +version = "2.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" + +[[package]] +name = "petgraph" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4c5cc86750666a3ed20bdaf5ca2a0344f9c67674cae0515bec2da16fbaa47db" +dependencies = [ + "fixedbitset", + "indexmap 2.7.0", +] + +[[package]] +name = "phf" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd6780a80ae0c52cc120a26a1a42c1ae51b247a253e4e06113d23d2c2edd078" +dependencies = [ + "phf_shared", +] + +[[package]] +name = "phf_codegen" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aef8048c789fa5e851558d709946d6d79a8ff88c0440c587967f8e94bfb1216a" +dependencies = [ + "phf_generator", + "phf_shared", +] + +[[package]] +name = "phf_generator" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c80231409c20246a13fddb31776fb942c38553c51e871f8cbd687a4cfb5843d" +dependencies = [ + "phf_shared", + "rand", +] + +[[package]] +name = "phf_shared" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67eabc2ef2a60eb7faa00097bd1ffdb5bd28e62bf39990626a582201b7a754e5" +dependencies = [ + "siphasher", +] + +[[package]] +name = "pin-project" +version = "1.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e2ec53ad785f4d35dac0adea7f7dc6f1bb277ad84a680c7afefeae05d1f5916" +dependencies = [ + "pin-project-internal", +] + +[[package]] +name = "pin-project-internal" +version = "1.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d56a66c0c55993aa927429d0f8a0abfd74f084e4d9c192cffed01e418d83eefb" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "pin-project-lite" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b3cff922bd51709b605d9ead9aa71031d81447142d828eb4a6eba76fe619f9b" + +[[package]] +name = "pin-utils" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" + +[[package]] +name = "pkg-config" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "953ec861398dccce10c670dfeaf3ec4911ca479e9c02154b3a215178c5f566f2" + +[[package]] +name = "powerfmt" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" + +[[package]] +name = "ppv-lite86" +version = "0.2.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77957b295656769bb8ad2b6a6b09d897d94f05c41b069aede1fcdaa675eaea04" +dependencies = [ + "zerocopy", +] + +[[package]] +name = "proc-macro-crate" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecf48c7ca261d60b74ab1a7b20da18bede46776b2e55535cb958eb595c5fa7b" +dependencies = [ + "toml_edit", +] + +[[package]] +name = "proc-macro2" +version = "1.0.93" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60946a68e5f9d28b0dc1c21bb8a97ee7d018a8b322fa57838ba31cc878e22d99" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "prost" +version = "0.13.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c0fef6c4230e4ccf618a35c59d7ede15dea37de8427500f50aff708806e42ec" +dependencies = [ + "bytes", + "prost-derive", +] + +[[package]] +name = "prost-derive" +version = "0.13.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "157c5a9d7ea5c2ed2d9fb8f495b64759f7816c7eaea54ba3978f0d63000162e3" +dependencies = [ + "anyhow", + "itertools 0.13.0", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "prost-types" +version = "0.13.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc2f1e56baa61e93533aebc21af4d2134b70f66275e0fcdf3cbe43d77ff7e8fc" +dependencies = [ + "prost", +] + +[[package]] +name = "psm" +version = "0.1.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "200b9ff220857e53e184257720a14553b2f4aa02577d2ed9842d45d4b9654810" +dependencies = [ + "cc", +] + +[[package]] +name = "quick-error" +version = "1.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" + +[[package]] +name = "quote" +version = "1.0.38" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e4dccaaaf89514f546c693ddc140f729f958c247918a13380cccc6078391acc" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "rand" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" +dependencies = [ + "libc", + "rand_chacha", + "rand_core", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom", +] + +[[package]] +name = "recursive" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0786a43debb760f491b1bc0269fe5e84155353c67482b9e60d0cfb596054b43e" +dependencies = [ + "recursive-proc-macro-impl", + "stacker", +] + +[[package]] +name = "recursive-proc-macro-impl" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76009fbe0614077fc1a2ce255e3a1881a2e3a3527097d5dc6d8212c585e7e38b" +dependencies = [ + "quote", + "syn", +] + +[[package]] +name = "redox_syscall" +version = "0.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03a862b389f93e68874fbf580b9de08dd02facb9a788ebadaf4a3fd33cf58834" +dependencies = [ + "bitflags 2.8.0", +] + +[[package]] +name = "regex" +version = "1.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b544ef1b4eac5dc2db33ea63606ae9ffcfac26c1416a2806ae0bf5f56b201191" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata 0.4.9", + "regex-syntax 0.8.5", +] + +[[package]] +name = "regex-automata" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c230d73fb8d8c1b9c0b3135c5142a8acee3a0558fb8db5cf1cb65f8d7862132" +dependencies = [ + "regex-syntax 0.6.29", +] + +[[package]] +name = "regex-automata" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "809e8dc61f6de73b46c85f4c96486310fe304c434cfa43669d7b40f711150908" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax 0.8.5", +] + +[[package]] +name = "regex-syntax" +version = "0.6.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1" + +[[package]] +name = "regex-syntax" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c" + +[[package]] +name = "relative-path" +version = "1.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba39f3699c378cd8970968dcbff9c43159ea4cfbd88d43c00b22f2ef10a435d2" + +[[package]] +name = "reqwest" +version = "0.12.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43e734407157c3c2034e0258f5e4473ddb361b1e85f95a66690d67264d7cd1da" +dependencies = [ + "base64", + "bytes", + "encoding_rs", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-rustls", + "hyper-tls", + "hyper-util", + "ipnet", + "js-sys", + "log", + "mime", + "native-tls", + "once_cell", + "percent-encoding", + "pin-project-lite", + "rustls-pemfile", + "serde", + "serde_json", + "serde_urlencoded", + "sync_wrapper", + "system-configuration", + "tokio", + "tokio-native-tls", + "tower 0.5.2", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", + "windows-registry", +] + +[[package]] +name = "resolv-conf" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52e44394d2086d010551b14b53b1f24e31647570cd1deb0379e2c21b329aba00" +dependencies = [ + "hostname", + "quick-error", +] + +[[package]] +name = "ring" +version = "0.17.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c17fa4cb658e3583423e915b9f3acc01cceaee1860e33d59ebae66adc3a2dc0d" +dependencies = [ + "cc", + "cfg-if", + "getrandom", + "libc", + "spin", + "untrusted", + "windows-sys 0.52.0", +] + +[[package]] +name = "rstest" +version = "0.24.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03e905296805ab93e13c1ec3a03f4b6c4f35e9498a3d5fa96dc626d22c03cd89" +dependencies = [ + "futures-timer", + "futures-util", + "rstest_macros", + "rustc_version", +] + +[[package]] +name = "rstest_macros" +version = "0.24.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef0053bbffce09062bee4bcc499b0fbe7a57b879f1efe088d6d8d4c7adcdef9b" +dependencies = [ + "cfg-if", + "glob", + "proc-macro-crate", + "proc-macro2", + "quote", + "regex", + "relative-path", + "rustc_version", + "syn", + "unicode-ident", +] + +[[package]] +name = "rust_decimal" +version = "1.36.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b082d80e3e3cc52b2ed634388d436fe1f4de6af5786cc2de9ba9737527bdf555" +dependencies = [ + "arrayvec", + "num-traits", +] + +[[package]] +name = "rustc-demangle" +version = "0.1.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "719b953e2095829ee67db738b3bfa9fa368c94900df327b3f07fe6e794d2fe1f" + +[[package]] +name = "rustc_version" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" +dependencies = [ + "semver", +] + +[[package]] +name = "rustix" +version = "0.38.43" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a78891ee6bf2340288408954ac787aa063d8e8817e9f53abb37c695c6d834ef6" +dependencies = [ + "bitflags 2.8.0", + "errno", + "libc", + "linux-raw-sys", + "windows-sys 0.59.0", +] + +[[package]] +name = "rustls" +version = "0.23.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f287924602bf649d949c63dc8ac8b235fa5387d394020705b80c4eb597ce5b8" +dependencies = [ + "log", + "once_cell", + "ring", + "rustls-pki-types", + "rustls-webpki", + "subtle", + "zeroize", +] + +[[package]] +name = "rustls-native-certs" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fcff2dd52b58a8d98a70243663a0d234c4e2b79235637849d15913394a247d3" +dependencies = [ + "openssl-probe", + "rustls-pki-types", + "schannel", + "security-framework 3.2.0", +] + +[[package]] +name = "rustls-pemfile" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dce314e5fee3f39953d46bb63bb8a46d40c2f8fb7cc5a3b6cab2bde9721d6e50" +dependencies = [ + "rustls-pki-types", +] + +[[package]] +name = "rustls-pki-types" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2bf47e6ff922db3825eb750c4e2ff784c6ff8fb9e13046ef6a1d1c5401b0b37" + +[[package]] +name = "rustls-webpki" +version = "0.102.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "64ca1bc8749bd4cf37b5ce386cc146580777b4e8572c7b97baf22c83f444bee9" +dependencies = [ + "ring", + "rustls-pki-types", + "untrusted", +] + +[[package]] +name = "rustversion" +version = "1.0.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7c45b9784283f1b2e7fb61b42047c2fd678ef0960d4f6f1eba131594cc369d4" + +[[package]] +name = "ryu" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f" + +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "schannel" +version = "0.1.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f29ebaa345f945cec9fbbc532eb307f0fdad8161f281b6369539c8d84876b3d" +dependencies = [ + "windows-sys 0.59.0", +] + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + +[[package]] +name = "scroll" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04c565b551bafbef4157586fa379538366e4385d42082f255bfd96e4fe8519da" + +[[package]] +name = "sea-query" +version = "0.32.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "085e94f7d7271c0393ac2d164a39994b1dff1b06bc40cd9a0da04f3d672b0fee" +dependencies = [ + "bigdecimal", + "chrono", + "inherent", + "rust_decimal", + "sea-query-derive", + "time", +] + +[[package]] +name = "sea-query-derive" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9834af2c4bd8c5162f00c89f1701fb6886119a88062cf76fe842ea9e232b9839" +dependencies = [ + "darling", + "heck 0.4.1", + "proc-macro2", + "quote", + "syn", + "thiserror 1.0.69", +] + +[[package]] +name = "secrecy" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9bd1c54ea06cfd2f6b63219704de0b9b4f72dcc2b8fdef820be6cd799780e91e" +dependencies = [ + "zeroize", +] + +[[package]] +name = "security-framework" +version = "2.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "897b2245f0b511c87893af39b033e5ca9cce68824c4d7e7630b5a1d339658d02" +dependencies = [ + "bitflags 2.8.0", + "core-foundation 0.9.4", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "271720403f46ca04f7ba6f55d438f8bd878d6b8ca0a1046e8228c4145bcbb316" +dependencies = [ + "bitflags 2.8.0", + "core-foundation 0.10.0", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49db231d56a190491cb4aeda9527f1ad45345af50b0851622a7adb8c03b01c32" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "semver" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3cb6eb87a131f756572d7fb904f6e7b68633f09cca868c5df1c4b8d1a694bbba" + +[[package]] +name = "seq-macro" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a3f0bf26fd526d2a95683cd0f87bf103b8539e2ca1ef48ce002d67aad59aa0b4" + +[[package]] +name = "serde" +version = "1.0.217" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02fc4265df13d6fa1d00ecff087228cc0a2b5f3c0e87e258d8b94a156e984c70" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.217" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a9bf7cf98d04a2b28aead066b7496853d4779c9cc183c440dbac457641e19a0" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.135" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b0d7ba2887406110130a978386c4e1befb98c674b4fba677954e4db976630d9" +dependencies = [ + "itoa", + "memchr", + "ryu", + "serde", +] + +[[package]] +name = "serde_repr" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c64451ba24fc7a6a2d60fc75dd9c83c90903b19028d4eff35e88fc1e86564e9" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_urlencoded" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" +dependencies = [ + "form_urlencoded", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "serde_with" +version = "3.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6b6f7f2fcb69f747921f79f3926bd1e203fce4fef62c268dd3abfb6d86029aa" +dependencies = [ + "base64", + "chrono", + "hex", + "indexmap 1.9.3", + "indexmap 2.7.0", + "serde", + "serde_derive", + "serde_json", + "time", +] + +[[package]] +name = "sha2" +version = "0.10.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "793db75ad2bcafc3ffa7c68b215fee268f537982cd901d132f89c6343f3a3dc8" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + +[[package]] +name = "sharded-slab" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6" +dependencies = [ + "lazy_static", +] + +[[package]] +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + +[[package]] +name = "signal-hook-registry" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9e9e0b4211b72e7b8b6e85c807d36c212bdb33ea8587f7569562a84df5465b1" +dependencies = [ + "libc", +] + +[[package]] +name = "similar" +version = "2.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbbb5d9659141646ae647b42fe094daf6c6192d1620870b449d9557f748b2daa" + +[[package]] +name = "siphasher" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56199f7ddabf13fe5074ce809e7d3f42b42ae711800501b5b16ea82ad029c39d" + +[[package]] +name = "slab" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f92a496fb766b417c996b9c5e57daf2f7ad3b0bebe1ccfca4856390e3d3bb67" +dependencies = [ + "autocfg", +] + +[[package]] +name = "smallvec" +version = "1.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67" + +[[package]] +name = "snafu" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "223891c85e2a29c3fe8fb900c1fae5e69c2e42415e3177752e8718475efa5019" +dependencies = [ + "snafu-derive", +] + +[[package]] +name = "snafu-derive" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03c3c6b7927ffe7ecaa769ee0e3994da3b8cafc8f444578982c83ecb161af917" +dependencies = [ + "heck 0.5.0", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "snap" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b6b67fb9a61334225b5b790716f609cd58395f895b3fe8b328786812a40bc3b" + +[[package]] +name = "socket2" +version = "0.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c970269d99b64e60ec3bd6ad27270092a5394c4e309314b18ae3fe575695fbe8" +dependencies = [ + "libc", + "windows-sys 0.52.0", +] + +[[package]] +name = "spin" +version = "0.9.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" + +[[package]] +name = "sqlparser" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05a528114c392209b3264855ad491fcce534b94a38771b0a0b97a79379275ce8" +dependencies = [ + "log", + "sqlparser_derive", +] + +[[package]] +name = "sqlparser_derive" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da5fc6819faabb412da764b99d3b713bb55083c11e7e0c00144d386cd6a1939c" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "stable_deref_trait" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" + +[[package]] +name = "stacker" +version = "0.1.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "799c883d55abdb5e98af1a7b3f23b9b6de8ecada0ecac058672d7635eb48ca7b" +dependencies = [ + "cc", + "cfg-if", + "libc", + "psm", + "windows-sys 0.59.0", +] + +[[package]] +name = "static_assertions" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" + +[[package]] +name = "strum" +version = "0.26.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fec0f0aef304996cf250b31b5a10dee7980c85da9d759361292b8bca5a18f06" + +[[package]] +name = "strum_macros" +version = "0.26.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c6bee85a5a24955dc440386795aa378cd9cf82acd5f764469152d2270e581be" +dependencies = [ + "heck 0.5.0", + "proc-macro2", + "quote", + "rustversion", + "syn", +] + +[[package]] +name = "subtle" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" + +[[package]] +name = "syn" +version = "2.0.96" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d5d0adab1ae378d7f53bdebc67a39f1f151407ef230f0ce2883572f5d8985c80" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "sync_wrapper" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" +dependencies = [ + "futures-core", +] + +[[package]] +name = "synstructure" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8af7666ab7b6390ab78131fb5b0fce11d6b7a6951602017c35fa82800708971" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "system-configuration" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c879d448e9d986b661742763247d3693ed13609438cf3d006f51f5368a5ba6b" +dependencies = [ + "bitflags 2.8.0", + "core-foundation 0.9.4", + "system-configuration-sys", +] + +[[package]] +name = "system-configuration-sys" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e1d1b10ced5ca923a1fcb8d03e96b8d3268065d724548c0211415ff6ac6bac4" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "tempfile" +version = "3.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a8a559c81686f576e8cd0290cd2a24a2a9ad80c98b3478856500fcbd7acd704" +dependencies = [ + "cfg-if", + "fastrand", + "getrandom", + "once_cell", + "rustix", + "windows-sys 0.59.0", +] + +[[package]] +name = "test-log" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7f46083d221181166e5b6f6b1e5f1d499f3a76888826e6cb1d057554157cd0f" +dependencies = [ + "env_logger", + "test-log-macros", + "tracing-subscriber", +] + +[[package]] +name = "test-log-macros" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "888d0c3c6db53c0fdab160d2ed5e12ba745383d3e85813f2ea0f2b1475ab553f" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "thiserror" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" +dependencies = [ + "thiserror-impl 1.0.69", +] + +[[package]] +name = "thiserror" +version = "2.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d452f284b73e6d76dd36758a0c8684b1d5be31f92b89d07fd5822175732206fc" +dependencies = [ + "thiserror-impl 2.0.11", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26afc1baea8a989337eeb52b6e72a039780ce45c3edfcc9c5b9d112feeb173c2" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "thread_local" +version = "1.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b9ef9bad013ada3808854ceac7b46812a6465ba368859a37e2100283d2d719c" +dependencies = [ + "cfg-if", + "once_cell", +] + +[[package]] +name = "thrift" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e54bc85fc7faa8bc175c4bab5b92ba8d9a3ce893d0e9f42cc455c8ab16a9e09" +dependencies = [ + "byteorder", + "integer-encoding", + "ordered-float", +] + +[[package]] +name = "time" +version = "0.3.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35e7868883861bd0e56d9ac6efcaaca0d6d5d82a2a7ec8209ff492c07cf37b21" +dependencies = [ + "deranged", + "itoa", + "num-conv", + "powerfmt", + "serde", + "time-core", + "time-macros", +] + +[[package]] +name = "time-core" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef927ca75afb808a4d64dd374f00a2adf8d0fcff8e7b184af886c3c87ec4a3f3" + +[[package]] +name = "time-macros" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2834e6017e3e5e4b9834939793b282bc03b37a3336245fa820e35e233e2a85de" +dependencies = [ + "num-conv", + "time-core", +] + +[[package]] +name = "tiny-keccak" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c9d3793400a45f954c52e73d068316d76b6f4e36977e3fcebb13a2721e80237" +dependencies = [ + "crunchy", +] + +[[package]] +name = "tinystr" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9117f5d4db391c1cf6927e7bea3db74b9a1c1add8f7eda9ffd5364f40f57b82f" +dependencies = [ + "displaydoc", + "zerovec", +] + +[[package]] +name = "tinyvec" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "022db8904dfa342efe721985167e9fcd16c29b226db4397ed752a761cfce81e8" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" + +[[package]] +name = "tokio" +version = "1.43.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d61fa4ffa3de412bfea335c6ecff681de2b609ba3c77ef3e00e521813a9ed9e" +dependencies = [ + "backtrace", + "bytes", + "libc", + "mio", + "parking_lot", + "pin-project-lite", + "signal-hook-registry", + "socket2", + "tokio-macros", + "windows-sys 0.52.0", +] + +[[package]] +name = "tokio-macros" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e06d43f1345a3bcd39f6a56dbb7dcab2ba47e68e8ac134855e7e2bdbaf8cab8" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tokio-native-tls" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2" +dependencies = [ + "native-tls", + "tokio", +] + +[[package]] +name = "tokio-rustls" +version = "0.26.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f6d0975eaace0cf0fcadee4e4aaa5da15b5c079146f2cffb67c113be122bf37" +dependencies = [ + "rustls", + "tokio", +] + +[[package]] +name = "tokio-stream" +version = "0.1.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eca58d7bba4a75707817a2c44174253f9236b2d5fbd055602e9d5c07c139a047" +dependencies = [ + "futures-core", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "tokio-util" +version = "0.7.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7fcaa8d55a2bdd6b83ace262b016eca0d79ee02818c5c1bcdf0305114081078" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "toml_datetime" +version = "0.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0dd7358ecb8fc2f8d014bf86f6f638ce72ba252a2c3a2572f2a795f1d23efb41" + +[[package]] +name = "toml_edit" +version = "0.22.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ae48d6208a266e853d946088ed816055e556cc6028c5e8e2b84d9fa5dd7c7f5" +dependencies = [ + "indexmap 2.7.0", + "toml_datetime", + "winnow", +] + +[[package]] +name = "tonic" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877c5b330756d856ffcc4553ab34a5684481ade925ecc54bcd1bf02b1d0d4d52" +dependencies = [ + "async-stream", + "async-trait", + "axum", + "base64", + "bytes", + "h2", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-timeout", + "hyper-util", + "percent-encoding", + "pin-project", + "prost", + "rustls-native-certs", + "rustls-pemfile", + "socket2", + "tokio", + "tokio-rustls", + "tokio-stream", + "tower 0.4.13", + "tower-layer", + "tower-service", + "tracing", + "webpki-roots", +] + +[[package]] +name = "tower" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8fa9be0de6cf49e536ce1851f987bd21a43b771b09473c3549a6c853db37c1c" +dependencies = [ + "futures-core", + "futures-util", + "indexmap 1.9.3", + "pin-project", + "pin-project-lite", + "rand", + "slab", + "tokio", + "tokio-util", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tower" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d039ad9159c98b70ecfd540b2573b97f7f52c3e8d9f8ad57a24b916a536975f9" +dependencies = [ + "futures-core", + "futures-util", + "pin-project-lite", + "sync_wrapper", + "tokio", + "tower-layer", + "tower-service", +] + +[[package]] +name = "tower-layer" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" + +[[package]] +name = "tower-service" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" + +[[package]] +name = "tracing" +version = "0.1.41" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "784e0ac535deb450455cbfa28a6f0df145ea1bb7ae51b821cf5e7927fdcfbdd0" +dependencies = [ + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "395ae124c09f9e6918a2310af6038fba074bcf474ac352496d5910dd59a2226d" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tracing-core" +version = "0.1.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e672c95779cf947c5311f83787af4fa8fffd12fb27e4993211a84bdfd9610f9c" +dependencies = [ + "once_cell", + "valuable", +] + +[[package]] +name = "tracing-log" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3" +dependencies = [ + "log", + "once_cell", + "tracing-core", +] + +[[package]] +name = "tracing-subscriber" +version = "0.3.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8189decb5ac0fa7bc8b96b7cb9b2701d60d48805aca84a238004d665fcc4008" +dependencies = [ + "matchers", + "nu-ansi-term", + "once_cell", + "regex", + "sharded-slab", + "smallvec", + "thread_local", + "tracing", + "tracing-core", + "tracing-log", +] + +[[package]] +name = "trust-dns-proto" +version = "0.23.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3119112651c157f4488931a01e586aa459736e9d6046d3bd9105ffb69352d374" +dependencies = [ + "async-trait", + "cfg-if", + "data-encoding", + "enum-as-inner", + "futures-channel", + "futures-io", + "futures-util", + "idna 0.4.0", + "ipnet", + "once_cell", + "rand", + "smallvec", + "thiserror 1.0.69", + "tinyvec", + "tokio", + "tracing", + "url", +] + +[[package]] +name = "trust-dns-resolver" +version = "0.23.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10a3e6c3aff1718b3c73e395d1f35202ba2ffa847c6a62eea0db8fb4cfe30be6" +dependencies = [ + "cfg-if", + "futures-util", + "ipconfig", + "lru-cache", + "once_cell", + "parking_lot", + "rand", + "resolv-conf", + "smallvec", + "thiserror 1.0.69", + "tokio", + "tracing", + "trust-dns-proto", +] + +[[package]] +name = "try-lock" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" + +[[package]] +name = "twox-hash" +version = "1.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97fee6b57c6a41524a810daee9286c02d7752c4253064d0b05472833a438f675" +dependencies = [ + "cfg-if", + "static_assertions", +] + +[[package]] +name = "typenum" +version = "1.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825" + +[[package]] +name = "unicode-bidi" +version = "0.3.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c1cb5db39152898a79168971543b1cb5020dff7fe43c8dc468b0885f5e29df5" + +[[package]] +name = "unicode-ident" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "adb9e6ca4f869e1180728b7950e35922a7fc6397f7b641499e8f3ef06e50dc83" + +[[package]] +name = "unicode-normalization" +version = "0.1.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5033c97c4262335cded6d6fc3e5c18ab755e1a3dc96376350f3d8e9f009ad956" +dependencies = [ + "tinyvec", +] + +[[package]] +name = "unicode-segmentation" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6ccf251212114b54433ec949fd6a7841275f9ada20dddd2f29e9ceea4501493" + +[[package]] +name = "unicode-width" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fc81956842c57dac11422a97c3b8195a1ff727f06e85c84ed2e8aa277c9a0fd" + +[[package]] +name = "untrusted" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" + +[[package]] +name = "url" +version = "2.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32f8b686cadd1473f4bd0117a5d28d36b1ade384ea9b5069a1c40aefed7fda60" +dependencies = [ + "form_urlencoded", + "idna 1.0.3", + "percent-encoding", +] + +[[package]] +name = "utf16_iter" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8232dd3cdaed5356e0f716d285e4b40b932ac434100fe9b7e0e8e935b9e6246" + +[[package]] +name = "utf8_iter" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" + +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + +[[package]] +name = "uuid" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "744018581f9a3454a9e15beb8a33b017183f1e7c0cd170232a2d1453b23a51c4" +dependencies = [ + "getrandom", +] + +[[package]] +name = "valuable" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" + +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "walkdir" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +dependencies = [ + "same-file", + "winapi-util", +] + +[[package]] +name = "want" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" +dependencies = [ + "try-lock", +] + +[[package]] +name = "wasi" +version = "0.11.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" + +[[package]] +name = "wasm-bindgen" +version = "0.2.100" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1edc8929d7499fc4e8f0be2262a241556cfc54a0bea223790e71446f2aab1ef5" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", +] + +[[package]] +name = "wasm-bindgen-backend" +version = "0.2.100" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f0a0651a5c2bc21487bde11ee802ccaf4c51935d0d3d42a6101f98161700bc6" +dependencies = [ + "bumpalo", + "log", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.50" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "555d470ec0bc3bb57890405e5d4322cc9ea83cebb085523ced7be4144dac1e61" +dependencies = [ + "cfg-if", + "js-sys", + "once_cell", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.100" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fe63fc6d09ed3792bd0897b314f53de8e16568c2b3f7982f468c0bf9bd0b407" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.100" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ae87ea40c9f689fc23f209965b6fb8a99ad69aeeb0231408be24920604395de" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-backend", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.100" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a05d73b933a847d6cccdda8f838a22ff101ad9bf93e33684f39c1f5f0eece3d" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "web-sys" +version = "0.3.77" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33b6dd2ef9186f1f2072e409e99cd22a975331a6b3591b12c764e0e55c60d5d2" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "web-time" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "webpki-roots" +version = "0.26.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d642ff16b7e79272ae451b7322067cdc17cadf68c23264be9d94a32319efe7e" +dependencies = [ + "rustls-pki-types", +] + +[[package]] +name = "widestring" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7219d36b6eac893fa81e84ebe06485e7dcbb616177469b142df14f1f4deb1311" + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-util" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf221c93e13a30d793f7645a0e7762c55d169dbb0a49671918a2319d289b10bb" +dependencies = [ + "windows-sys 0.59.0", +] + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "windows-core" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33ab640c8d7e35bf8ba19b884ba838ceb4fba93a4e8c65a9059d08afcfc683d9" +dependencies = [ + "windows-targets 0.52.6", +] + +[[package]] +name = "windows-registry" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e400001bb720a623c1c69032f8e3e4cf09984deec740f007dd2b03ec864804b0" +dependencies = [ + "windows-result", + "windows-strings", + "windows-targets 0.52.6", +] + +[[package]] +name = "windows-result" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d1043d8214f791817bab27572aaa8af63732e11bf84aa21a45a78d6c317ae0e" +dependencies = [ + "windows-targets 0.52.6", +] + +[[package]] +name = "windows-strings" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cd9b125c486025df0eabcb585e62173c6c9eddcec5d117d3b6e8c30e2ee4d10" +dependencies = [ + "windows-result", + "windows-targets 0.52.6", +] + +[[package]] +name = "windows-sys" +version = "0.45.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75283be5efb2831d37ea142365f009c02ec203cd29a3ebecbc093d52315b66d0" +dependencies = [ + "windows-targets 0.42.2", +] + +[[package]] +name = "windows-sys" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" +dependencies = [ + "windows-targets 0.48.5", +] + +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets 0.52.6", +] + +[[package]] +name = "windows-sys" +version = "0.59.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" +dependencies = [ + "windows-targets 0.52.6", +] + +[[package]] +name = "windows-targets" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e5180c00cd44c9b1c88adb3693291f1cd93605ded80c250a75d472756b4d071" +dependencies = [ + "windows_aarch64_gnullvm 0.42.2", + "windows_aarch64_msvc 0.42.2", + "windows_i686_gnu 0.42.2", + "windows_i686_msvc 0.42.2", + "windows_x86_64_gnu 0.42.2", + "windows_x86_64_gnullvm 0.42.2", + "windows_x86_64_msvc 0.42.2", +] + +[[package]] +name = "windows-targets" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c" +dependencies = [ + "windows_aarch64_gnullvm 0.48.5", + "windows_aarch64_msvc 0.48.5", + "windows_i686_gnu 0.48.5", + "windows_i686_msvc 0.48.5", + "windows_x86_64_gnu 0.48.5", + "windows_x86_64_gnullvm 0.48.5", + "windows_x86_64_msvc 0.48.5", +] + +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm 0.52.6", + "windows_aarch64_msvc 0.52.6", + "windows_i686_gnu 0.52.6", + "windows_i686_gnullvm", + "windows_i686_msvc 0.52.6", + "windows_x86_64_gnu 0.52.6", + "windows_x86_64_gnullvm 0.52.6", + "windows_x86_64_msvc 0.52.6", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "597a5118570b68bc08d8d59125332c54f1ba9d9adeedeef5b99b02ba2b0698f8" + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e08e8864a60f06ef0d0ff4ba04124db8b0fb3be5776a5cd47641e942e58c4d43" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_i686_gnu" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c61d927d8da41da96a81f029489353e68739737d3beca43145c8afec9a31a84f" + +[[package]] +name = "windows_i686_gnu" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44d840b6ec649f480a41c8d80f9c65108b92d89345dd94027bfe06ac444d1060" + +[[package]] +name = "windows_i686_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8de912b8b8feb55c064867cf047dda097f92d51efad5b491dfb98f6bbb70cb36" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26d41b46a36d453748aedef1486d5c7a85db22e56aff34643984ea85514e94a3" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "winnow" +version = "0.6.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8d71a593cc5c42ad7876e2c1fda56f314f3754c084128833e64f1345ff8a03a" +dependencies = [ + "memchr", +] + +[[package]] +name = "winreg" +version = "0.50.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "524e57b2c537c0f9b1e69f1965311ec12182b4122e45035b1508cd24d2adadb1" +dependencies = [ + "cfg-if", + "windows-sys 0.48.0", +] + +[[package]] +name = "wkt" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "54f7f1ff4ea4c18936d6cd26a6fd24f0003af37e951a8e0e8b9e9a2d0bd0a46d" +dependencies = [ + "geo-types", + "log", + "num-traits", + "thiserror 1.0.69", +] + +[[package]] +name = "write16" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1890f4022759daae28ed4fe62859b1236caebfc61ede2f63ed4e695f3f6d936" + +[[package]] +name = "writeable" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e9df38ee2d2c3c5948ea468a8406ff0db0b29ae1ffde1bcf20ef305bcc95c51" + +[[package]] +name = "xz2" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "388c44dc09d76f1536602ead6d325eb532f5c122f17782bd57fb47baeeb767e2" +dependencies = [ + "lzma-sys", +] + +[[package]] +name = "yoke" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "120e6aef9aa629e3d4f52dc8cc43a015c7724194c97dfaf45180d2daf2b77f40" +dependencies = [ + "serde", + "stable_deref_trait", + "yoke-derive", + "zerofrom", +] + +[[package]] +name = "yoke-derive" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2380878cad4ac9aac1e2435f3eb4020e8374b5f13c296cb75b4620ff8e229154" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + +[[package]] +name = "zerocopy" +version = "0.7.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b9b4fd18abc82b8136838da5d50bae7bdea537c574d8dc1a34ed098d6c166f0" +dependencies = [ + "byteorder", + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.7.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "zerofrom" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cff3ee08c995dee1859d998dea82f7374f2826091dd9cd47def953cae446cd2e" +dependencies = [ + "zerofrom-derive", +] + +[[package]] +name = "zerofrom-derive" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "595eed982f7d355beb85837f651fa22e90b3c044842dc7f2c2842c086f295808" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + +[[package]] +name = "zeroize" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ced3678a2879b30306d323f4542626697a464a97c0a07c9aebf7ebca65cd4dde" + +[[package]] +name = "zerovec" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa2b893d79df23bfb12d5461018d408ea19dfafe76c2c7ef6d4eba614f8ff079" +dependencies = [ + "yoke", + "zerofrom", + "zerovec-derive", +] + +[[package]] +name = "zerovec-derive" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6eafa6dfb17584ea3e2bd6e76e0cc15ad7af12b09abdd1ca55961bed9b1063c6" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "zstd" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fcf2b778a664581e31e389454a7072dab1647606d44f7feea22cd5abb9c9f3f9" +dependencies = [ + "zstd-safe", +] + +[[package]] +name = "zstd-safe" +version = "7.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "54a3ab4db68cea366acc5c897c7b4d4d1b8994a9cd6e6f841f8964566a419059" +dependencies = [ + "zstd-sys", +] + +[[package]] +name = "zstd-sys" +version = "2.0.13+zstd.1.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38ff0f21cfee8f97d94cef41359e0c89aa6113028ab0291aa8ca0038995a95aa" +dependencies = [ + "cc", + "pkg-config", +] diff --git a/libs/datafusion/jni/Cargo.toml b/libs/datafusion/jni/Cargo.toml new file mode 100644 index 0000000000000..df28efe109707 --- /dev/null +++ b/libs/datafusion/jni/Cargo.toml @@ -0,0 +1,33 @@ +[package] +name = "datafusion_jni" +version = "0.1.0" +edition = "2021" + +[dependencies] +jni = "0.21.1" +datafusion = "44.0.0" +datafusion-table-providers = { version = "0.2.2", features = ["flight"] } +tokio = { version = "1.41.0", features = ["full"] } +arrow = { version = "53.1.0", features = ["ffi" ] } +arrow-flight = "53.0.0" +futures = "0.3" +serde_json = "1.0.132" +tonic = "0.12.3" +bytes = "1.8.0" +async-trait = "0.1" + +[dev-dependencies] +anyhow = "1.0" +bollard = "0.18.1" +geozero = { version = "0.14.0", features = ["with-wkb"] } +insta = { version = "1.42.0", features = ["filters"] } +prost = { version = "0.13" } +rand = "0.8" +reqwest = "0.12" +rstest = "0.24.0" +test-log = { version = "0.2", features = ["trace"] } +tokio-stream = { version = "0.1", features = ["net"] } +tracing-subscriber = { version = "0.3", features = ["env-filter"] } + +[lib] +crate_type = ["cdylib"] \ No newline at end of file diff --git a/libs/datafusion/jni/src/lib.rs b/libs/datafusion/jni/src/lib.rs new file mode 100644 index 0000000000000..d8965029a9475 --- /dev/null +++ b/libs/datafusion/jni/src/lib.rs @@ -0,0 +1,398 @@ +use std::error::Error; +use std::io::Cursor; +use std::ptr::addr_of_mut; +use std::time::Duration; + +use arrow::array::ffi::{FFI_ArrowArray, FFI_ArrowSchema}; +use arrow::array::{Array, StructArray}; +use arrow::ipc::writer::FileWriter; +use bytes::Bytes; +use datafusion::execution::SendableRecordBatchStream; +use datafusion::prelude::{DataFrame, SessionConfig, SessionContext}; +use futures::stream::TryStreamExt; +use jni::objects::{JByteArray, JClass, JObject, JString}; +use jni::sys::jlong; +use jni::JNIEnv; +use std::io::BufWriter; +use tokio::runtime::Runtime; +mod provider; + +#[no_mangle] +pub extern "system" fn Java_org_opensearch_datafusion_DataFusion_query( + mut env: JNIEnv, + _class: JClass, + runtime: jlong, + ctx: jlong, + ticket: JByteArray, + callback: JObject, +) { + let input = env.convert_byte_array(&ticket).unwrap(); + let context = unsafe { &mut *(ctx as *mut SessionContext) }; + let runtime = unsafe { &mut *(runtime as *mut Runtime) }; + + runtime.block_on(async { + let result = provider::read_aggs(context.clone(), Bytes::from(input)).await; + let addr = result.map(|df| Box::into_raw(Box::new(df))); + set_object_result(&mut env, callback, addr); + }); +} + +#[no_mangle] +pub extern "system" fn Java_org_opensearch_datafusion_DataFusion_agg( + mut env: JNIEnv, + _class: JClass, + runtime: jlong, + ctx: jlong, + ticket: JByteArray, + callback: JObject, +) { + let input = env.convert_byte_array(&ticket).unwrap(); + let context = unsafe { &mut *(ctx as *mut SessionContext) }; + let runtime = unsafe { &mut *(runtime as *mut Runtime) }; + + runtime.block_on(async { + let result = provider::read_aggs(context.clone(), Bytes::from(input)).await; + let addr = result.map(|df| Box::into_raw(Box::new(df))); + set_object_result(&mut env, callback, addr); + }); +} + +#[no_mangle] +pub extern "system" fn Java_org_opensearch_datafusion_DataFusion_join( + mut env: JNIEnv, + _class: JClass, + runtime: jlong, + ctx: jlong, + join_field: JString, + left_ticket: JByteArray, + right_ticket: JByteArray, + callback: JObject, +) { + let left = Bytes::from(env.convert_byte_array(&left_ticket).unwrap()); + let right = Bytes::from(env.convert_byte_array(&right_ticket).unwrap()); + + let field = env + .get_string(&join_field) + .expect("Error fetching join field") + .into(); + + let context = unsafe { &mut *(ctx as *mut SessionContext) }; + let runtime = unsafe { &mut *(runtime as *mut Runtime) }; + + runtime.block_on(async { + let result = provider::join(context.clone(), left, right, field).await; + let addr = result.map(|df| Box::into_raw(Box::new(df))); + set_object_result(&mut env, callback, addr); + }); +} + +// fn unwrap_tickets<'local>(env: &mut JNIEnv, tickets: JObject) -> Vec { +// let list: JList<'_, '_, '_> = env.get_list(&tickets).unwrap(); + +// let size: usize = list.size(env).unwrap() as usize; +// let mut tickets: Vec = Vec::with_capacity(size); + +// for i in 0..size { +// let byte_array: JByteArray = list.get(env, i as i32).unwrap().unwrap().into(); +// let input = env.convert_byte_array(&byte_array).unwrap(); +// let ticket = Bytes::from(input); +// tickets.push(ticket); +// } +// tickets +// } + +#[no_mangle] +pub extern "system" fn Java_org_opensearch_datafusion_DataFusion_collect( + mut env: JNIEnv, + _class: JClass, + runtime: jlong, + df: jlong, + callback: JObject, +) { + let runtime = unsafe { &mut *(runtime as *mut Runtime) }; + let dataframe = unsafe { &mut *(df as *mut DataFrame) }; + + let schema = dataframe.schema().into(); + runtime.block_on(async { + let batches = dataframe + .clone() + .collect() + .await + .expect("failed to collect dataframe"); + + let mut buff = Cursor::new(vec![0; 0]); + + { + let mut writer = FileWriter::try_new(BufWriter::new(&mut buff), &schema) + .expect("failed to create writer"); + for batch in batches { + writer.write(&batch).expect("failed to write batch"); + } + writer.finish().expect("failed to finish"); + } + + let err_message = env + .new_string("".to_string()) + .expect("Couldn't create java string!"); + + let ba = env + .byte_array_from_slice(buff.get_ref()) + .expect("cannot create empty byte array"); + + env.call_method( + callback, + "accept", + "(Ljava/lang/Object;Ljava/lang/Object;)V", + &[(&err_message).into(), (&ba).into()], + ) + .expect("failed to call method"); + }); +} + +#[no_mangle] +pub extern "system" fn Java_org_opensearch_datafusion_SessionContext_destroySessionContext( + _env: JNIEnv, + _class: JClass, + pointer: jlong, +) { + let _ = unsafe { Box::from_raw(pointer as *mut SessionContext) }; +} + +#[no_mangle] +pub extern "system" fn Java_org_opensearch_datafusion_SessionContext_createSessionContext( + _env: JNIEnv, + _class: JClass, +) -> jlong { + let config = SessionConfig::new(); + let context = SessionContext::new_with_config(config); + Box::into_raw(Box::new(context)) as jlong +} + +#[no_mangle] +pub extern "system" fn Java_org_opensearch_datafusion_SessionContext_createRuntime( + _env: JNIEnv, + _class: JClass, +) -> jlong { + if let Ok(runtime) = Runtime::new() { + // println!("successfully created tokio runtime"); + Box::into_raw(Box::new(runtime)) as jlong + } else { + // TODO error handling + -1 + } +} + +#[no_mangle] +pub extern "system" fn Java_org_opensearch_datafusion_SessionContext_destroyRuntime( + _env: JNIEnv, + _class: JClass, + pointer: jlong, +) { + let runtime = unsafe { Box::from_raw(pointer as *mut Runtime) }; + runtime.shutdown_timeout(Duration::from_millis(100)); + // println!("successfully shutdown tokio runtime"); +} + +#[cfg(test)] +mod tests { + use std::sync::Arc; + + use arrow::{array::{Int32Array, RecordBatch, StringArray}, datatypes::{DataType, Field, Schema}}; + use datafusion::{common::JoinType, prelude::{col, SessionContext}}; + + + #[tokio::test] + async fn test_unwrap_tickets() { + let ctx = SessionContext::new(); + + // Define schemas + let employee_schema = Arc::new(Schema::new(vec![ + Field::new("id", DataType::Int32, false), + Field::new("name", DataType::Utf8, false), + Field::new("age", DataType::Int32, false), + ])); + + let department_schema = Arc::new(Schema::new(vec![ + Field::new("id", DataType::Int32, false), + Field::new("department_name", DataType::Utf8, false), + ])); + + // Create employee RecordBatch + let employee_batch = RecordBatch::try_new( + employee_schema.clone(), + vec![ + Arc::new(Int32Array::from(vec![1, 2, 3])), + Arc::new(StringArray::from(vec!["John", "Jane", "Bob"])), + Arc::new(Int32Array::from(vec![30, 25, 35])), + ], + ).unwrap(); + + // Create department RecordBatch + let department_batch = RecordBatch::try_new( + department_schema.clone(), + vec![ + Arc::new(Int32Array::from(vec![1, 2, 3])), + Arc::new(StringArray::from(vec!["Engineering", "Marketing", "Sales"])), + ], + ).unwrap(); + + // Create DataFrames from RecordBatches + let employee_df = ctx.read_batch(employee_batch).unwrap(); + let department_df = ctx.read_batch(department_batch).unwrap().select(vec![ + col("id").alias("dept_Id"), + col("department_name") + ]).unwrap(); + + // Perform join using DataFrame API + let joined_df = employee_df + .join( + department_df, + JoinType::Inner, + &["id"], // join columns from left DataFrame + &[r#""dept_Id""#], // join columns from right DataFrame + None, // additional join filter + ).unwrap(); + + joined_df.show().await; + // Execute and collect results + // let results = joined_df.collect().await.unwrap(); + + // // Print results + // println!("Join Results:"); + // for batch in results { + // println!("{:?}", batch); + // } + + + } +} + + + +pub fn set_object_result( + env: &mut JNIEnv, + callback: JObject, + address: Result<*mut T, Err>, +) { + match address { + Ok(address) => set_object_result_ok(env, callback, address), + Err(err) => set_object_result_error(env, callback, &err), + }; +} + +/// Set success result by calling an ObjectResultCallback +pub fn set_object_result_ok(env: &mut JNIEnv, callback: JObject, address: *mut T) { + let err_message = JObject::null(); + env.call_method( + callback, + "callback", + "(Ljava/lang/String;J)V", + &[(&err_message).into(), (address as jlong).into()], + ) + .expect("Failed to call object result callback with address"); +} + +/// Set error result by calling an ObjectResultCallback +pub fn set_object_result_error(env: &mut JNIEnv, callback: JObject, error: &T) { + let err_message = env + .new_string(error.to_string()) + .expect("Couldn't create java string for error message"); + let address = -1 as jlong; + env.call_method( + callback, + "callback", + "(Ljava/lang/String;J)V", + &[(&err_message).into(), address.into()], + ) + .expect("Failed to call object result callback with error"); +} + +#[no_mangle] +pub extern "system" fn Java_org_opensearch_datafusion_DataFusion_executeStream( + mut env: JNIEnv, + _class: JClass, + runtime: jlong, + dataframe: jlong, + callback: JObject, +) { + let runtime = unsafe { &mut *(runtime as *mut Runtime) }; + let dataframe = unsafe { &mut *(dataframe as *mut DataFrame) }; + runtime.block_on(async { + let stream_result = dataframe.clone().execute_stream().await; + set_object_result( + &mut env, + callback, + stream_result.map(|stream| Box::into_raw(Box::new(stream))), + ); + }); +} + +#[no_mangle] +pub extern "system" fn Java_org_opensearch_datafusion_RecordBatchStream_next( + mut env: JNIEnv, + _class: JClass, + runtime: jlong, + stream: jlong, + callback: JObject, +) { + let runtime = unsafe { &mut *(runtime as *mut Runtime) }; + let stream = unsafe { &mut *(stream as *mut SendableRecordBatchStream) }; + runtime.block_on(async { + let next: Result, datafusion::error::DataFusionError> = stream.try_next().await; + match next { + Ok(Some(batch)) => { + // Convert to struct array for compatibility with FFI + let struct_array: StructArray = batch.into(); + let array_data = struct_array.into_data(); + let mut ffi_array = FFI_ArrowArray::new(&array_data); + // ffi_array must remain alive until after the callback is called + set_object_result_ok(&mut env, callback, addr_of_mut!(ffi_array)); + } + Ok(None) => { + set_object_result_ok(&mut env, callback, 0 as *mut FFI_ArrowSchema); + } + Err(err) => { + set_object_result_error(&mut env, callback, &err); + } + } + }); +} + +#[no_mangle] +pub extern "system" fn Java_org_opensearch_datafusion_RecordBatchStream_destroy( + mut env: JNIEnv, + _class: JClass, + pointer: jlong +) { + let _ = unsafe { Box::from_raw(pointer as *mut SendableRecordBatchStream) }; +} + +#[no_mangle] +pub extern "system" fn Java_org_opensearch_datafusion_RecordBatchStream_getSchema( + mut env: JNIEnv, + _class: JClass, + stream: jlong, + callback: JObject, +) { + let stream = unsafe { &mut *(stream as *mut SendableRecordBatchStream) }; + let schema = stream.schema(); + let ffi_schema = FFI_ArrowSchema::try_from(&*schema); + match ffi_schema { + Ok(mut ffi_schema) => { + // ffi_schema must remain alive until after the callback is called + set_object_result_ok(&mut env, callback, addr_of_mut!(ffi_schema)); + } + Err(err) => { + set_object_result_error(&mut env, callback, &err); + } + } +} + +#[no_mangle] +pub extern "system" fn Java_org_opensearch_datafusion_DataFrame_destroyDataFrame( + _env: JNIEnv, + _class: JClass, + pointer: jlong, +) { + let _ = unsafe { Box::from_raw(pointer as *mut DataFrame) }; +} diff --git a/libs/datafusion/jni/src/provider.rs b/libs/datafusion/jni/src/provider.rs new file mode 100644 index 0000000000000..472288c9aa050 --- /dev/null +++ b/libs/datafusion/jni/src/provider.rs @@ -0,0 +1,216 @@ +use arrow_flight::FlightEndpoint; +use arrow_flight::{flight_service_client::FlightServiceClient, FlightDescriptor, FlightInfo}; +use bytes::Bytes; +use datafusion::catalog::TableProvider; +use datafusion::common::JoinType; +use datafusion::common::Result; +use datafusion::error::DataFusionError; +use datafusion::functions_aggregate::expr_fn::sum; +use datafusion::prelude::Expr; +use futures::TryFutureExt; +use std::{collections::HashMap, sync::Arc}; + +use datafusion::prelude::SessionContext; +use datafusion::prelude::{col, DataFrame}; +use datafusion_table_providers::flight::{ + FlightDriver, FlightMetadata, FlightProperties, FlightTableFactory, FlightTable +}; +use datafusion::logical_expr::test::function_stub::count; +use futures::future::try_join_all; +use tonic::async_trait; +use tonic::transport::Channel; +mod test; + +// Returns a DataFrame that represents a query on a single index. +// Each ticket should correlate to a single shard so that this executes shard fan-out +pub async fn query( + ctx: SessionContext, + ticket: Bytes, +) -> datafusion::common::Result { + let df = dataframe_for_index(&ctx, "theIndex".to_owned(), ticket).await?; + + df.sort(vec![col("score").sort(false, true)]) + .map_err(|e| DataFusionError::Execution(format!("Failed to sort DataFrame: {}", e))) +} + +pub async fn read_aggs( + ctx: SessionContext, + ticket: Bytes, +) -> datafusion::common::Result { + let df = dataframe_for_index(&ctx, "theIndex".to_owned(), ticket).await?; + df.filter(col("ord").is_not_null())? + .aggregate(vec![col("ord")], vec![sum(col("count")).alias("count")]) +} + +// inner join two tables together, returning a single DataFrame that can be consumed +// represents a join query against two indices +pub async fn join( + ctx: SessionContext, + left: Bytes, + right: Bytes, + join_field: String, +) -> datafusion::common::Result { + + let select_cols = vec![col(r#""docId""#), col(r#""shardId""#), col(&join_field)]; + + let left_df = dataframe_for_index(&ctx, "left".to_owned(), left).await?.select( select_cols.clone())?; + + let alias = format!("right.{}", &join_field); + let right_df = dataframe_for_index(&ctx, "right".to_owned(), right).await?.select( + vec![col(r#""docId""#).alias("right_docId"), col(r#""shardId""#).alias("right_shardId"), col("instance_id").alias("right_instance_id")] + )?; + + println!("LEFT FRAME:"); + // left_df.clone().show().await; + println!("RIGHT FRAME:"); + // right_df.clone().show().await; + + // // select all the cols returned by right but alias the join field + // let select_cols: Vec = right_df.schema() + // .fields() + // .iter() + // .map(|field| { + // if field.name() == &join_field { + // col(&join_field).alias("right.join_field") + // } else { + // col(field.name()) + // } + // }) + // .collect(); + + return left_df + .join( + right_df, + JoinType::Inner, + &[&join_field], + &["right_instance_id"], + None, + ) + .map_err(|e| DataFusionError::Execution(format!("Join operation failed: {}", e))); + + // let df2 = frame.as_ref().unwrap().clone().show().await; + // return frame; +} + +pub async fn aggregate( + ctx: SessionContext, + ticket: Bytes, +) -> datafusion::common::Result { + let df = dataframe_for_index(&ctx, "theIndex".to_owned(), ticket).await?; + df.aggregate(vec![col("")], vec![count(col("a"))]) + .map_err(|e| DataFusionError::Execution(format!("Failed to sort DataFrame: {}", e))) +} + + +async fn dataframe_for_index( + ctx: &SessionContext, + prefix: String, + ticket: Bytes +) -> Result { + let table_name = format!("{}-s", prefix); + get_dataframe_for_tickets(ctx, table_name, ticket.clone()).await +} + +// Return a single dataframe for an entire index. +// Each ticket in tickets represents a single shard. +// async fn dataframe_for_index( +// ctx: &SessionContext, +// prefix: String, +// tickets: Vec, +// ) -> Result { +// println!("UNION"); +// let inner_futures = tickets +// .into_iter() +// .enumerate() +// .map(|(j, bytes)| { +// let table_name = format!("{}-s-{}", prefix, j); +// get_dataframe_for_tickets(ctx, table_name, vec![bytes.clone()]) +// }) +// .collect::>(); + +// let frames = try_join_all(inner_futures) +// .await +// .map_err(|e| DataFusionError::Execution(format!("Failed to join futures: {}", e)))?; +// frames[0].clone().show().await?; +// union_df(frames) +// } + +// // Union a list of DataFrames +// fn union_df(frames: Vec) -> Result { +// Ok(frames +// .into_iter() +// .reduce(|acc, df| match acc.union(df) { +// Ok(unioned_df) => unioned_df, +// Err(e) => panic!("Failed to union DataFrames: {}", e), +// }) +// .ok_or_else(|| DataFusionError::Execution("No frames to union".to_string()))?) +// } + +// registers a single table from the list of given tickets, then reads it immediately returning a dataframe. +// intended to be used to register and get a df for a single shard. +async fn get_dataframe_for_tickets( + ctx: &SessionContext, + name: String, + ticket: Bytes, +) -> Result { + println!("Register"); + register_table(ctx, name.clone(), ticket) + .and_then(|_| ctx.table(&name)) + .await +} +// registers a single table with datafusion using DataFusion TableProviders. +// Uses a TicketedFlightDriver to register the table with the list of given tickets. +async fn register_table(ctx: &SessionContext, name: String, ticket: Bytes) -> Result<()> { + let driver: TicketedFlightDriver = TicketedFlightDriver { ticket }; + let table_factory: FlightTableFactory = FlightTableFactory::new(Arc::new(driver)); + let table: FlightTable = table_factory + .open_table(format!("http://localhost:{}", "8815"), HashMap::new()) + .await + .map_err(|e| DataFusionError::Execution(format!("Error creating table: {}", e)))?; + println!("Registering table {:?}", table); + ctx.register_table(name, Arc::new(table)) + .map_err(|e| DataFusionError::Execution(format!("Error registering table: {}", e)))?; + // let df = ctx.sql("SHOW TABLES").await?; + // df.show().await?; + Ok(()) +} + +#[derive(Clone, Debug, Default)] +pub struct TicketedFlightDriver { + ticket: Bytes, +} + +#[async_trait] +impl FlightDriver for TicketedFlightDriver { + // this doesn't work - we don't have schema data... + async fn metadata( + &self, + channel: Channel, + _options: &HashMap, + ) -> arrow_flight::error::Result { + println!("DRIVER: options: {:?}", _options); + + let mut client: FlightServiceClient = FlightServiceClient::new(channel.clone()); + + println!("DRIVER: Using ticket: {:?}", self.ticket); + + let descriptor = FlightDescriptor::new_cmd(self.ticket.clone()); + println!("DRIVER: Created descriptor: {:?}", descriptor); + + let request = tonic::Request::new(descriptor); + println!("DRIVER: Sending get_flight_info request"); + + match client.get_flight_info(request).await { + Ok(info) => { + println!("DRIVER: Received flight info response"); + let info = info.into_inner(); + println!("DRIVER: Flight info: {:?}", info); + FlightMetadata::try_new(info, FlightProperties::default()) + } + Err(status) => { + println!("DRIVER: Error getting flight info: {:?}", status); + Err(status.into()) + } + } + } +} diff --git a/libs/datafusion/jni/src/provider/test/mod.rs b/libs/datafusion/jni/src/provider/test/mod.rs new file mode 100644 index 0000000000000..182ff94e3ba5a --- /dev/null +++ b/libs/datafusion/jni/src/provider/test/mod.rs @@ -0,0 +1,261 @@ +use std::collections::HashMap; +use std::net::SocketAddr; +use std::pin::Pin; +use std::sync::Arc; +use std::time::Duration; + +use arrow::array::{StringArray, StructArray}; +use arrow::ipc; +use arrow_flight::encode::FlightDataEncoderBuilder; +use arrow_flight::flight_service_server::{FlightService, FlightServiceServer}; +use arrow_flight::sql::server::FlightSqlService; +use arrow_flight::sql::{CommandStatementQuery, ProstMessageExt, SqlInfo, TicketStatementQuery}; +use arrow_flight::{ + Action, ActionType, Criteria, Empty, FlightData, FlightDescriptor, FlightEndpoint, FlightInfo, HandshakeRequest, HandshakeResponse, PollInfo, PutResult, SchemaResult, Ticket +}; +use async_trait::async_trait; +use datafusion::arrow::array::{Array, Float32Array, Int64Array, Int8Array, RecordBatch}; +use datafusion::arrow::datatypes::{DataType, Field, Schema}; +use datafusion::prelude::{DataFrame, SessionConfig, SessionContext}; +use futures::{stream, Stream, TryStreamExt}; +use tokio::net::TcpListener; +use tokio::sync::oneshot::{channel, Receiver, Sender}; +use tokio_stream::wrappers::TcpListenerStream; +use tonic::codegen::http::HeaderMap; +use tonic::codegen::tokio_stream; +use tonic::metadata::MetadataMap; +use tonic::transport::{Channel, Server}; +use tonic::{Extensions, Request, Response, Status, Streaming}; +use datafusion_table_providers::flight::sql::FlightSqlDriver; +use datafusion_table_providers::flight::{FlightProperties, FlightTableFactory}; +use bytes::Bytes; +use crate::provider; + +const AUTH_HEADER: &str = "authorization"; +const BEARER_TOKEN: &str = "Bearer flight-sql-token"; + +struct TestFlightService { + flight_info: FlightInfo, + partition_data: RecordBatch, + shutdown_sender: Option>, +} + +impl TestFlightService { + async fn run_in_background(self, rx: Receiver<()>) -> SocketAddr { + let addr = SocketAddr::from(([127, 0, 0, 1], 8815)); + let listener = TcpListener::bind(addr).await.unwrap(); + let addr = listener.local_addr().unwrap(); + let service = FlightServiceServer::new(self); + #[allow(clippy::disallowed_methods)] // spawn allowed only in tests + tokio::spawn(async move { + Server::builder() + .timeout(Duration::from_secs(1)) + .add_service(service) + .serve_with_incoming_shutdown(TcpListenerStream::new(listener), async { + rx.await.ok(); + }) + .await + .unwrap(); + }); + tokio::time::sleep(Duration::from_millis(25)).await; + addr + } +} + +impl Drop for TestFlightService { + fn drop(&mut self) { + if let Some(tx) = self.shutdown_sender.take() { + tx.send(()).ok(); + } + } +} + +#[async_trait] +impl FlightService for TestFlightService { + + type HandshakeStream = Pin> + Send>>; + type ListFlightsStream = Pin> + Send>>; + type DoGetStream = Pin> + Send>>; + type DoPutStream = Pin> + Send>>; + type DoActionStream = Pin> + Send>>; + type ListActionsStream = Pin> + Send>>; + type DoExchangeStream = Pin> + Send>>; + + async fn get_flight_info( + &self, + request: Request, + ) -> Result, Status> { + println!("TEST SERVER: get_flight_info called"); + println!("TEST SERVER: descriptor: {:?}", request.get_ref()); + + // Accept any ticket and return the flight info + Ok(Response::new(self.flight_info.clone())) + } + + async fn do_get( + &self, + request: Request, + ) -> Result, Status> { + println!("TEST SERVER: do_get called"); + println!("TEST SERVER: ticket: {:?}", request.get_ref()); + + let data = self.partition_data.clone(); + let rb = async move { Ok(data) }; + let stream = FlightDataEncoderBuilder::default() + .with_schema(self.partition_data.schema()) + .build(stream::once(rb)) + .map_err(|e| Status::internal(e.to_string())); + + Ok(Response::new(Box::pin(stream))) + } + + // Implement other required methods with default behaviors + async fn handshake( + &self, + _request: Request>, + ) -> Result, Status> { + Ok(Response::new(Box::pin(tokio_stream::empty()))) + } + + async fn list_flights( + &self, + _request: Request, + ) -> Result, Status> { + Ok(Response::new(Box::pin(tokio_stream::empty()))) + } + + async fn do_put( + &self, + _request: Request>, + ) -> Result, Status> { + Ok(Response::new(Box::pin(tokio_stream::empty()))) + } + + async fn do_action( + &self, + _request: Request, + ) -> Result, Status> { + Ok(Response::new(Box::pin(tokio_stream::empty()))) + } + + async fn list_actions( + &self, + _request: Request, + ) -> Result, Status> { + Ok(Response::new(Box::pin(tokio_stream::empty()))) + } + + async fn do_exchange( + &self, + _request: Request>, + ) -> Result, Status> { + Ok(Response::new(Box::pin(tokio_stream::empty()))) + } + + async fn get_schema( + &self, + request: Request, + ) -> Result, Status> { + println!("TEST SERVER: get_schema called"); + println!("TEST SERVER: descriptor: {:?}", request.get_ref()); + // let schema = self.partition_data.schema(); + // let schema_bytes = ipc::writer::serialize_schema(schema.as_ref()) + // .map_err(|e| Status::internal(format!("Failed to serialize schema: {}", e)))?; + Ok(Response::new(SchemaResult { schema: self.flight_info.schema.clone() })) + } + + async fn poll_flight_info( + &self, + request: Request, + ) -> Result, Status> { + println!("TEST SERVER: poll_flight_info called"); + println!("TEST SERVER: descriptor: {:?}", request.get_ref()); + + Ok(Response::new(PollInfo {info:Some(__self.flight_info.clone()),expiration_time:std::option::Option::None, flight_descriptor: todo!(), progress: todo!() })) + } +} + +#[tokio::test] + async fn test_read_aggs() -> datafusion::common::Result<()> { + // Create test data + let partition_data: RecordBatch = RecordBatch::try_new( + Arc::new(Schema::new([ + Arc::new(Field::new("ord", DataType::Utf8, false)), + Arc::new(Field::new("count", DataType::Int64, false)), + ])), + vec![ + Arc::new(StringArray::from(vec!["A", "B", "A", "C"])), + Arc::new(Int64Array::from(vec![10, 20, 30, 40])), + ], + )?; + + // Set up flight endpoint + let endpoint = FlightEndpoint::default().with_ticket(Ticket::new("bytes".as_bytes())); + let flight_info = FlightInfo::default() + .try_with_schema(partition_data.schema().as_ref())? + .with_endpoint(endpoint); + + // Set up test service + let (tx, rx) = channel(); + let service = TestFlightService { + flight_info, + partition_data, + shutdown_sender: Some(tx), + }; + + let port = service.run_in_background(rx).await.port(); + println!("TEST: Server started on port {}", port); + let channel = Channel::from_shared(format!("http://127.0.0.1:{}", port)) + .unwrap() + .connect() + .await; + + println!("TEST: Channel connection result: {:?}", channel); + // Set up session context + let config = SessionConfig::new().with_batch_size(1); + let ctx = SessionContext::new_with_config(config); + // let props_template = FlightProperties::new().with_reusable_flight_info(true); + // let driver = FlightSqlDriver::new().with_properties_template(props_template); + // ctx.state_ref().write().table_factories_mut().insert( + // "FLIGHT_SQL".into(), + // Arc::new(FlightTableFactory::new(Arc::new(driver))), + // ); + + // Create test bytes for ticket + let test_bytes = Bytes::from("test_ticket"); + + // Execute read_aggs + let result_df = provider::read_aggs(ctx.clone(), test_bytes).await.map_err(|e| { + println!("TEST: Error in read_aggs: {:?}", e); + e + })?; + + let result = result_df.execute_stream().await?; + let mut stream = result; + while let Some(batch) = stream.try_next().await? { + println!("\nRecord batch:"); + println!("Number of rows: {}", batch.num_rows()); + println!("Number of columns: {}", batch.num_columns()); + + let ord_array = batch.column(0) + .as_any() + .downcast_ref::() + .expect("Expected StringArray for ord column"); + + println!("ord_array contents:"); + for i in 0..ord_array.len() { + println!(" index {}: {:?}", i, ord_array.value(i)); + } + + let count_array = batch.column(1) + .as_any() + .downcast_ref::() + .expect("Expected Int64Array for count column"); + + println!("count_array contents:"); + for i in 0..count_array.len() { + println!(" index {}: {:?}", i, count_array.value(i)); + } + } + Ok(()) + } \ No newline at end of file diff --git a/libs/datafusion/src/main/java/org.opensearch.datafusion/DataFrame.java b/libs/datafusion/src/main/java/org.opensearch.datafusion/DataFrame.java new file mode 100644 index 0000000000000..6285ae06c4995 --- /dev/null +++ b/libs/datafusion/src/main/java/org.opensearch.datafusion/DataFrame.java @@ -0,0 +1,73 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.datafusion; + +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.vector.ipc.ArrowFileReader; +import org.apache.arrow.vector.ipc.ArrowReader; +import org.apache.arrow.vector.util.ByteArrayReadableSeekableByteChannel; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.util.concurrent.CompletableFuture; + +/** + * Represents a DataFrame, which is a result of a query. + * This class provides methods to collect the DataFrame from the DataFusion runtime. + */ +public class DataFrame implements AutoCloseable { + + public static Logger logger = LogManager.getLogger(DataFrame.class); + private final SessionContext ctx; + + long ptr; + + static native void destroyDataFrame(long pointer); + + public DataFrame(SessionContext ctx, long ptr) { + this.ctx = ctx; + this.ptr = ptr; + } + + public CompletableFuture collect(BufferAllocator allocator) { + CompletableFuture result = new CompletableFuture<>(); + DataFusion.collect(ctx.getRuntime(), ptr, (err, obj) -> { + if (err != null && err.isEmpty() == false) { + result.completeExceptionally(new RuntimeException(err)); + } else { + result.complete(new ArrowFileReader(new ByteArrayReadableSeekableByteChannel(obj), allocator)); + } + }); + return result; + } + + // return a stream over the dataframe + public CompletableFuture getStream(BufferAllocator allocator) { + CompletableFuture result = new CompletableFuture<>(); + long runtimePointer = ctx.getRuntime(); + DataFusion.executeStream(runtimePointer, ptr, (String errString, long streamId) -> { + if (errString != null && errString.isEmpty() == false) { + result.completeExceptionally(new RuntimeException(errString)); + } else { + result.complete(new RecordBatchStream(ctx, streamId, allocator)); + } + }); + return result; + } + + public SessionContext context() { + return ctx; + } + + @Override + public void close() throws Exception { + destroyDataFrame(ptr); + ctx.close(); + } +} diff --git a/libs/datafusion/src/main/java/org.opensearch.datafusion/DataFrameStreamProducer.java b/libs/datafusion/src/main/java/org.opensearch.datafusion/DataFrameStreamProducer.java new file mode 100644 index 0000000000000..3602a1e58628a --- /dev/null +++ b/libs/datafusion/src/main/java/org.opensearch.datafusion/DataFrameStreamProducer.java @@ -0,0 +1,132 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.datafusion; + +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.vector.VectorSchemaRoot; +import org.apache.arrow.vector.types.pojo.ArrowType; +import org.apache.arrow.vector.types.pojo.Field; +import org.apache.arrow.vector.types.pojo.FieldType; +import org.apache.arrow.vector.types.pojo.Schema; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.arrow.spi.PartitionedStreamProducer; +import org.opensearch.arrow.spi.StreamProducer; +import org.opensearch.arrow.spi.StreamTicket; + +import java.io.IOException; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.CompletableFuture; +import java.util.function.Function; +import java.util.function.Supplier; + +/** + * + */ +public class DataFrameStreamProducer implements PartitionedStreamProducer { + + public static Logger logger = LogManager.getLogger(DataFrameStreamProducer.class); + private final Function> frameSupplier; + + private StreamTicket rootTicket; + private Set partitions; + + public DataFrameStreamProducer(Set partitions, Function> frameSupplier) { + logger.info("Constructed DataFrameFlightProducer"); + this.frameSupplier = frameSupplier; + this.partitions = partitions; + } + + @Override + public VectorSchemaRoot createRoot(BufferAllocator allocator) { + // TODO: Read this from one of the producers? + Map arrowFields = new HashMap<>(); + Field countField = new Field("count", FieldType.nullable(new ArrowType.Int(64, false)), null); + arrowFields.put("count", countField); + arrowFields.put("ord", new Field("ord", FieldType.nullable(new ArrowType.Utf8()), null)); + Schema schema = new Schema(arrowFields.values()); + return VectorSchemaRoot.create(schema, allocator); + } + + @Override + public BatchedJob createJob(BufferAllocator allocator) { + assert rootTicket != null; + return new BatchedJob() { + + private DataFrame df; + private RecordBatchStream recordBatchStream; + + @Override + public void run(VectorSchemaRoot root, FlushSignal flushSignal) { + try { + df = frameSupplier.apply(rootTicket).join(); + recordBatchStream = df.getStream(allocator).get(); + while (recordBatchStream.loadNextBatch().join()) { + logger.info(recordBatchStream.getVectorSchemaRoot().getRowCount()); + // wait for a signal to load the next batch + flushSignal.awaitConsumption(1000); + } + close(); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + @Override + public void onCancel() { + try { + close(); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + @Override + public boolean isCancelled() { + return false; + } + + void close() throws Exception { + if (recordBatchStream != null) { + recordBatchStream.close(); + } + ; + df.close(); + } + }; + } + + @Override + public int estimatedRowCount() { + return 0; + } + + @Override + public String getAction() { + return ""; + } + + @Override + public void close() throws IOException { + + } + + @Override + public Set partitions() { + return partitions; + } + + @Override + public void setRootTicket(StreamTicket ticket) { + this.rootTicket = ticket; + } +} diff --git a/libs/datafusion/src/main/java/org.opensearch.datafusion/DataFusion.java b/libs/datafusion/src/main/java/org.opensearch.datafusion/DataFusion.java new file mode 100644 index 0000000000000..958203d7f42c6 --- /dev/null +++ b/libs/datafusion/src/main/java/org.opensearch.datafusion/DataFusion.java @@ -0,0 +1,83 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.datafusion; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.util.Set; +import java.util.concurrent.CompletableFuture; +import java.util.function.BiConsumer; + +/** + * Main DataFusion Entrypoint. + */ +public class DataFusion { + + static { + System.loadLibrary("datafusion_jni"); + } + + // create a DataFrame from a list of tickets. + static native void query(long runtime, long ctx, byte[] ticket, ObjectResultCallback callback); + + static native void agg(long runtime, long ctx, byte[] ticket, ObjectResultCallback callback); + + static native void join(long runtime, long ctx, String joinField, byte[] left, byte[] right, ObjectResultCallback callback); + + // collect the DataFrame + static native void collect(long runtime, long df, BiConsumer callback); + + static native void executeStream(long runtime, long dataframe, ObjectResultCallback callback); + + public static CompletableFuture query(byte[] ticket) { + SessionContext ctx = new SessionContext(); + CompletableFuture future = new CompletableFuture<>(); + DataFusion.query(ctx.getRuntime(), ctx.getPointer(), ticket, (err, ptr) -> { + if (err != null) { + future.completeExceptionally(new RuntimeException(err)); + } else { + DataFrame df = new DataFrame(ctx, ptr); + future.complete(df); + } + }); + return future; + } + + public static Logger logger = LogManager.getLogger(DataFusion.class); + + public static CompletableFuture agg(byte[] ticket) { + SessionContext ctx = new SessionContext(); + CompletableFuture future = new CompletableFuture<>(); + DataFusion.agg(ctx.getRuntime(), ctx.getPointer(), ticket, (err, ptr) -> { + if (err != null) { + future.completeExceptionally(new RuntimeException(err)); + } else { + DataFrame df = new DataFrame(ctx, ptr); + logger.info("Returning DataFrame ref from jni"); + future.complete(df); + } + }); + return future; + } + + public static CompletableFuture join(byte[] left, byte[] right, String joinField) { + SessionContext ctx = new SessionContext(); + CompletableFuture future = new CompletableFuture<>(); + DataFusion.join(ctx.getRuntime(), ctx.getPointer(), joinField, left, right, (err, ptr) -> { + if (err != null) { + future.completeExceptionally(new RuntimeException(err)); + } else { + DataFrame df = new DataFrame(ctx, ptr); + future.complete(df); + } + }); + return future; + } +} diff --git a/libs/datafusion/src/main/java/org.opensearch.datafusion/ObjectResultCallback.java b/libs/datafusion/src/main/java/org.opensearch.datafusion/ObjectResultCallback.java new file mode 100644 index 0000000000000..3c6c16f0914f4 --- /dev/null +++ b/libs/datafusion/src/main/java/org.opensearch.datafusion/ObjectResultCallback.java @@ -0,0 +1,16 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.datafusion; + +/** + * Callback to point to the result of a query + */ +public interface ObjectResultCallback { + void callback(String errMessage, long value); +} diff --git a/libs/datafusion/src/main/java/org.opensearch.datafusion/RecordBatchStream.java b/libs/datafusion/src/main/java/org.opensearch.datafusion/RecordBatchStream.java new file mode 100644 index 0000000000000..9dae2a2a0ff91 --- /dev/null +++ b/libs/datafusion/src/main/java/org.opensearch.datafusion/RecordBatchStream.java @@ -0,0 +1,122 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.datafusion; + +import org.apache.arrow.c.ArrowArray; +import org.apache.arrow.c.ArrowSchema; +import org.apache.arrow.c.CDataDictionaryProvider; +import org.apache.arrow.c.Data; +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.vector.VectorSchemaRoot; +import org.apache.arrow.vector.types.pojo.Schema; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.util.concurrent.CompletableFuture; + +public class RecordBatchStream implements AutoCloseable { + + private final SessionContext context; + private final long ptr; + private final BufferAllocator allocator; + private final CDataDictionaryProvider dictionaryProvider; + + public RecordBatchStream(SessionContext ctx, long streamId, BufferAllocator allocator) { + this.context = ctx; + this.ptr = streamId; + this.allocator = allocator; + this.dictionaryProvider = new CDataDictionaryProvider(); + } + + private static native void destroy(long pointer); + + + @Override + public void close() throws Exception { + destroy(ptr); + dictionaryProvider.close(); + if (initialized) { + vectorSchemaRoot.close(); + } + } + + private static native void next(long runtime, long pointer, ObjectResultCallback callback); + public static Logger logger = LogManager.getLogger(RecordBatchStream.class); + + public CompletableFuture loadNextBatch() { + ensureInitialized(); + long runtimePointer = context.getRuntime(); + CompletableFuture result = new CompletableFuture<>(); + next( + runtimePointer, + ptr, + (String errString, long arrowArrayAddress) -> { + if (errString != null && errString.isEmpty() == false) { + logger.error("ERROR IS ", errString); + result.completeExceptionally(new RuntimeException(errString)); + } else if (arrowArrayAddress == 0) { + // Reached end of stream + logger.info("Reached end of stream"); + result.complete(false); + } else { + try { + ArrowArray arrowArray = ArrowArray.wrap(arrowArrayAddress); + Data.importIntoVectorSchemaRoot( + allocator, arrowArray, vectorSchemaRoot, dictionaryProvider); + logger.info("Sending batch of stream"); + result.complete(true); + } catch (Exception e) { + logger.error("lol wtf", e); + result.completeExceptionally(e); + } + } + }); + return result; + } + + private boolean initialized = false; + private VectorSchemaRoot vectorSchemaRoot = null; + + public VectorSchemaRoot getVectorSchemaRoot() { + ensureInitialized(); + return vectorSchemaRoot; + } + + private void ensureInitialized() { + if (!initialized) { + Schema schema = getSchema(); + this.vectorSchemaRoot = VectorSchemaRoot.create(schema, allocator); + } + initialized = true; + } + + private Schema getSchema() { + // Native method is not async, but use a future to store the result for convenience + CompletableFuture result = new CompletableFuture<>(); + getSchema( + ptr, + (errString, arrowSchemaAddress) -> { + if (errString != null && errString.isEmpty() == false) { + result.completeExceptionally(new RuntimeException(errString)); + } else { + try { + ArrowSchema arrowSchema = ArrowSchema.wrap(arrowSchemaAddress); + Schema schema = Data.importSchema(allocator, arrowSchema, dictionaryProvider); + result.complete(schema); + // The FFI schema will be released from rust when it is dropped + } catch (Exception e) { + result.completeExceptionally(e); + } + } + }); + return result.join(); + } + + private static native void getSchema(long pointer, ObjectResultCallback callback); +} diff --git a/libs/datafusion/src/main/java/org.opensearch.datafusion/SessionContext.java b/libs/datafusion/src/main/java/org.opensearch.datafusion/SessionContext.java new file mode 100644 index 0000000000000..83a9648e2ea45 --- /dev/null +++ b/libs/datafusion/src/main/java/org.opensearch.datafusion/SessionContext.java @@ -0,0 +1,50 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.datafusion; + +/** + * Session context for datafusion + */ +public class SessionContext implements AutoCloseable { + + static { + System.loadLibrary("datafusion_jni"); + } + + // ptr to context in df + private final long ptr; + private final long runtime; + + static native void destroySessionContext(long pointer); + + static native long createSessionContext(); + + static native long createRuntime(); + + static native void destroyRuntime(long pointer); + + public long getRuntime() { + return runtime; + } + + public long getPointer() { + return ptr; + } + + public SessionContext() { + this.ptr = createSessionContext(); + this.runtime = createRuntime(); + } + + @Override + public void close() throws Exception { + destroySessionContext(this.ptr); + destroyRuntime(this.runtime); + } +} diff --git a/libs/datafusion/src/main/java/org.opensearch.datafusion/package-info.java b/libs/datafusion/src/main/java/org.opensearch.datafusion/package-info.java new file mode 100644 index 0000000000000..41d7e3d10a455 --- /dev/null +++ b/libs/datafusion/src/main/java/org.opensearch.datafusion/package-info.java @@ -0,0 +1,12 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +/** + * Contains DataFusion classes + */ +package org.opensearch.datafusion; diff --git a/plugins/arrow-flight-rpc/src/main/java/org/opensearch/arrow/flight/impl/BaseFlightProducer.java b/plugins/arrow-flight-rpc/src/main/java/org/opensearch/arrow/flight/impl/BaseFlightProducer.java index 64a04ed46ff19..dbe6676a2f278 100644 --- a/plugins/arrow-flight-rpc/src/main/java/org/opensearch/arrow/flight/impl/BaseFlightProducer.java +++ b/plugins/arrow-flight-rpc/src/main/java/org/opensearch/arrow/flight/impl/BaseFlightProducer.java @@ -22,10 +22,14 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.opensearch.arrow.flight.bootstrap.FlightClientManager; +import org.opensearch.arrow.spi.PartitionedStreamProducer; import org.opensearch.arrow.spi.StreamProducer; import org.opensearch.arrow.spi.StreamTicket; +import java.util.ArrayList; import java.util.Collections; +import java.util.List; +import java.util.Set; /** * BaseFlightProducer extends NoOpFlightProducer to provide stream management functionality @@ -132,21 +136,30 @@ public FlightInfo getFlightInfo(CallContext context, FlightDescriptor descriptor // TODO: this api should only be used internally StreamTicket streamTicket = streamManager.getStreamTicketFactory().fromBytes(descriptor.getCommand()); FlightStreamManager.StreamProducerHolder streamProducerHolder; + List endpoints = new ArrayList<>(); if (streamTicket.getNodeId().equals(flightClientManager.getLocalNodeId())) { streamProducerHolder = streamManager.getStreamProducer(streamTicket); if (streamProducerHolder == null) { throw CallStatus.NOT_FOUND.withDescription("FlightInfo not found").toRuntimeException(); } - Location location = flightClientManager.getFlightClientLocation(streamTicket.getNodeId()); - if (location == null) { - throw CallStatus.UNAVAILABLE.withDescription("Internal error while determining location information from ticket.") - .toRuntimeException(); + StreamProducer producer = streamProducerHolder.getProducer(); + if (producer instanceof PartitionedStreamProducer) { + Set partitions = ((PartitionedStreamProducer) producer).partitions(); + for (StreamTicket partition : partitions) { + Location location = flightClientManager.getFlightClientLocation(streamTicket.getNodeId()); + if (location == null) { + throw CallStatus.UNAVAILABLE.withDescription("Internal error while determining location information from ticket.") + .toRuntimeException(); + } + endpoints.add(new FlightEndpoint(new Ticket(partition.toBytes()), location)); + } } - FlightEndpoint endpoint = new FlightEndpoint(new Ticket(descriptor.getCommand()), location); +// Location location = flightClientManager.getFlightClientLocation(streamTicket.getNodeId()); +// FlightEndpoint endpoint = new FlightEndpoint(new Ticket(descriptor.getCommand()), location); FlightInfo.Builder infoBuilder = FlightInfo.builder( streamProducerHolder.getRoot().getSchema(), descriptor, - Collections.singletonList(endpoint) + endpoints ).setRecords(streamProducerHolder.getProducer().estimatedRowCount()); return infoBuilder.build(); } else { diff --git a/plugins/arrow-flight-rpc/src/main/java/org/opensearch/arrow/flight/impl/FlightStreamManager.java b/plugins/arrow-flight-rpc/src/main/java/org/opensearch/arrow/flight/impl/FlightStreamManager.java index 30ecbaecabf30..8cec73249a9da 100644 --- a/plugins/arrow-flight-rpc/src/main/java/org/opensearch/arrow/flight/impl/FlightStreamManager.java +++ b/plugins/arrow-flight-rpc/src/main/java/org/opensearch/arrow/flight/impl/FlightStreamManager.java @@ -13,6 +13,7 @@ import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.vector.VectorSchemaRoot; import org.opensearch.arrow.flight.bootstrap.FlightClientManager; +import org.opensearch.arrow.spi.PartitionedStreamProducer; import org.opensearch.arrow.spi.StreamManager; import org.opensearch.arrow.spi.StreamProducer; import org.opensearch.arrow.spi.StreamReader; @@ -22,8 +23,11 @@ import org.opensearch.common.cache.Cache; import org.opensearch.common.cache.CacheBuilder; import org.opensearch.common.unit.TimeValue; +import org.opensearch.common.util.LongObjectPagedHashMap; import org.opensearch.core.tasks.TaskId; +import java.util.Set; +import java.util.function.Function; import java.util.function.Supplier; /** @@ -75,6 +79,9 @@ public void setClientManager(FlightClientManager clientManager) { public StreamTicket registerStream(StreamProducer provider, TaskId parentTaskId) { StreamTicket ticket = ticketFactory.newTicket(); streamProducers.put(ticket.getTicketId(), new StreamProducerHolder(provider, allocatorSupplier.get())); + if (provider instanceof PartitionedStreamProducer) { + ((PartitionedStreamProducer) provider).setRootTicket(ticket); + } return ticket; } diff --git a/server/build.gradle b/server/build.gradle index bb2fa0b36b341..d2b55c4fb09bf 100644 --- a/server/build.gradle +++ b/server/build.gradle @@ -71,6 +71,8 @@ dependencies { api project(":libs:opensearch-task-commons") implementation project(':libs:opensearch-arrow-spi') + implementation project(':libs:opensearch-datafusion') + compileOnly project(':libs:opensearch-plugin-classloader') testRuntimeOnly project(':libs:opensearch-plugin-classloader') diff --git a/server/src/main/java/org/opensearch/action/search/FetchSearchPhase.java b/server/src/main/java/org/opensearch/action/search/FetchSearchPhase.java index ef99c55ba1eca..f7a1197781312 100644 --- a/server/src/main/java/org/opensearch/action/search/FetchSearchPhase.java +++ b/server/src/main/java/org/opensearch/action/search/FetchSearchPhase.java @@ -47,6 +47,7 @@ import org.opensearch.search.internal.ShardSearchContextId; import org.opensearch.search.internal.ShardSearchRequest; import org.opensearch.search.query.QuerySearchResult; +import org.opensearch.search.stream.StreamSearchResult; import org.opensearch.transport.Transport; import java.util.List; @@ -141,11 +142,11 @@ private void innerRun() throws Exception { reducedQueryPhase, queryAndFetchOptimization ? queryResults : fetchResults.getAtomicArray() ); - if (queryAndFetchOptimization) { - // assert phaseResults.isEmpty() || phaseResults.get(0).fetchResult() != null : "phaseResults empty [" - // + phaseResults.isEmpty() - // + "], single result: " - // + phaseResults.get(0).fetchResult(); + if (queryAndFetchOptimization && queryResults.get(0) instanceof StreamSearchResult == false) { + assert phaseResults.isEmpty() || phaseResults.get(0).fetchResult() != null : "phaseResults empty [" + + phaseResults.isEmpty() + + "], single result: " + + phaseResults.get(0).fetchResult(); // query AND fetch optimization finishPhase.run(); } else { diff --git a/server/src/main/java/org/opensearch/action/search/QueryPhaseResultConsumer.java b/server/src/main/java/org/opensearch/action/search/QueryPhaseResultConsumer.java index 39d527f345da4..35e1c33081c26 100644 --- a/server/src/main/java/org/opensearch/action/search/QueryPhaseResultConsumer.java +++ b/server/src/main/java/org/opensearch/action/search/QueryPhaseResultConsumer.java @@ -128,13 +128,9 @@ public void close() { @Override public void consumeResult(SearchPhaseResult result, Runnable next) { super.consumeResult(result, () -> {}); - if (result instanceof StreamSearchResult) { - next.run(); - } else { - QuerySearchResult querySearchResult = result.queryResult(); - progressListener.notifyQueryResult(querySearchResult.getShardIndex()); - pendingMerges.consume(querySearchResult, next); - } + QuerySearchResult querySearchResult = result.queryResult(); + progressListener.notifyQueryResult(querySearchResult.getShardIndex()); + pendingMerges.consume(querySearchResult, next); } @Override @@ -150,12 +146,8 @@ public SearchPhaseController.ReducedQueryPhase reduce() throws Exception { SearchPhaseController.ReducedQueryPhase reducePhase = null; if (results.get(0) instanceof StreamSearchResult) { - reducePhase = controller.reducedFromStream( - results.asList().stream().map(r -> (StreamSearchResult) r).collect(Collectors.toList()), - aggReduceContextBuilder, - performFinalReduce - ); - logger.info("Will reduce results for {}", results.get(0)); + reducePhase = controller.reducedAggsFromStream(results.asList() + .stream().map(r -> (StreamSearchResult) r).collect(Collectors.toList())); } else { final SearchPhaseController.TopDocsStats topDocsStats = pendingMerges.consumeTopDocsStats(); final List topDocsList = pendingMerges.consumeTopDocs(); @@ -343,7 +335,8 @@ long ramBytesUsedQueryResult(QuerySearchResult result) { if (hasAggs == false) { return 0; } - return result.aggregations().asSerialized(InternalAggregations::readFrom, namedWriteableRegistry).ramBytesUsed(); + return 0; +// return result.aggregations().asSerialized(InternalAggregations::readFrom, namedWriteableRegistry).ramBytesUsed(); } /** diff --git a/server/src/main/java/org/opensearch/action/search/SearchPhaseController.java b/server/src/main/java/org/opensearch/action/search/SearchPhaseController.java index 06588a1463072..a447e55deb65f 100644 --- a/server/src/main/java/org/opensearch/action/search/SearchPhaseController.java +++ b/server/src/main/java/org/opensearch/action/search/SearchPhaseController.java @@ -32,6 +32,7 @@ package org.opensearch.action.search; +import org.apache.arrow.memory.RootAllocator; import org.apache.arrow.vector.BigIntVector; import org.apache.arrow.vector.BitVector; import org.apache.arrow.vector.FieldVector; @@ -41,6 +42,9 @@ import org.apache.arrow.vector.UInt8Vector; import org.apache.arrow.vector.VarCharVector; import org.apache.arrow.vector.VectorSchemaRoot; +import org.apache.arrow.vector.ipc.ArrowReader; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.apache.lucene.index.Term; import org.apache.lucene.search.CollectionStatistics; import org.apache.lucene.search.FieldDoc; @@ -61,13 +65,18 @@ import org.opensearch.common.lucene.search.TopDocsAndMaxScore; import org.opensearch.core.common.breaker.CircuitBreaker; import org.opensearch.core.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.index.shard.ShardId; +import org.opensearch.core.tasks.TaskId; +import org.opensearch.datafusion.DataFrame; +import org.opensearch.datafusion.DataFrameStreamProducer; +import org.opensearch.datafusion.DataFusion; +import org.opensearch.datafusion.SessionContext; import org.opensearch.index.fielddata.IndexFieldData; import org.opensearch.search.DocValueFormat; import org.opensearch.search.SearchHit; import org.opensearch.search.SearchHits; import org.opensearch.search.SearchPhaseResult; import org.opensearch.search.SearchService; -import org.opensearch.search.aggregations.BucketOrder; import org.opensearch.search.aggregations.InternalAggregation; import org.opensearch.search.aggregations.InternalAggregation.ReduceContext; import org.opensearch.search.aggregations.InternalAggregations; @@ -90,12 +99,16 @@ import org.opensearch.search.suggest.Suggest.Suggestion; import org.opensearch.search.suggest.completion.CompletionSuggestion; +import java.io.IOException; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Set; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutionException; import java.util.concurrent.Executor; import java.util.function.Consumer; import java.util.function.Function; @@ -386,7 +399,9 @@ private SearchHits getHits( } // clean the fetch counter for (SearchPhaseResult entry : fetchResults) { - entry.fetchResult().initCounter(); + if (entry.fetchResult() != null) { + entry.fetchResult().initCounter(); + } } int from = ignoreFrom ? 0 : reducedQueryPhase.from; int numSearchHits = (int) Math.min(reducedQueryPhase.fetchHits - from, reducedQueryPhase.size); @@ -706,113 +721,169 @@ static int getTopDocsSize(SearchRequest request) { : source.from()); } - public ReducedQueryPhase reducedFromStream( - List list, - InternalAggregation.ReduceContextBuilder aggReduceContextBuilder, - boolean performFinalReduce - ) { - System.out.println("will try to reduce from stream here"); - try { + public static Logger logger = LogManager.getLogger(SearchPhaseController.class); + + public ReducedQueryPhase reducedAggsFromStream(List list) throws Exception { + + List tickets = list.stream().flatMap(r -> r.getFlightTickets().stream()) + .map(OSTicket::getBytes) + .collect(Collectors.toList()); + + Set streamTickets = tickets.stream().map(t -> streamManager.getStreamTicketFactory().fromBytes(t)).collect(Collectors.toSet()); + StreamTicket ticket = streamManager.registerStream(new DataFrameStreamProducer(streamTickets, (t) -> DataFusion.agg(t.toBytes())), TaskId.EMPTY_TASK_ID); + + logger.info("Register stream at coordinator"); + StreamReader streamIterator = streamManager.getStreamReader(ticket); + logger.info("Finished register stream at coordinator"); + + int totalRows = 0; + List scoreDocs = new ArrayList<>(); + List aggs = new ArrayList<>(); + VectorSchemaRoot root = streamIterator.getRoot(); + List buckets = new ArrayList<>(); + logger.info("Starting iteration at coordinator"); + while (streamIterator.next()) { + logger.info(root.getRowCount()); + int rowCount = root.getRowCount(); + totalRows+= rowCount; + logger.info("AT COORD Record Batch with " + rowCount + " rows: total {}", totalRows); + + // Iterate through rows + for (int row = 0; row < rowCount; row++) { + FieldVector ordKey = root.getVector("ord"); + String ordName = (String) getValue(ordKey, row); + UInt8Vector count = (UInt8Vector) root.getVector("count"); + + Long bucketCount = (Long) getValue(count, row); + buckets.add(new StringTerms.Bucket(new BytesRef(ordName.getBytes()), bucketCount.longValue(), new InternalAggregations(List.of()), false, 0, DocValueFormat.RAW)); + } + } +// recordBatchStream.close(); +// dataFrame.close(); +// while (streamIterator.next()) { +// int rowCount = root.getRowCount(); +// totalRows+= rowCount; +// logger.info("AT COORD Record Batch with " + rowCount + " rows:"); +// +// // Iterate through rows +// for (int row = 0; row < rowCount; row++) { +// FieldVector ordKey = root.getVector("ord"); +// String ordName = (String) getValue(ordKey, row); +// Float8Vector count = (Float8Vector) root.getVector("count"); +// +// Double bucketCount = (Double) getValue(count, row); +// logger.info("Ord: " + ordName + " Bucket Count" + bucketCount + "NodeID: "); +// buckets.add(new StringTerms.Bucket(new BytesRef(ordName.getBytes()), bucketCount.longValue(), new InternalAggregations(List.of()), false, 0, DocValueFormat.RAW)); +// } +// } + aggs.add(new StringTerms( + root.getSchema().getCustomMetadata().get("name"), + InternalOrder.key(true), + InternalOrder.key(true), + null, + DocValueFormat.RAW, + 1, + false, + 0L, + buckets, + 0, + new TermsAggregator.BucketCountThresholds(0, 0, 0, 0) + )); + logger.info("End stream iterations?"); - List tickets = list.stream() - .flatMap(r -> r.getFlightTickets().stream()) + InternalAggregations aggregations = new InternalAggregations(aggs); + + TotalHits totalHits = new TotalHits(totalRows, Relation.EQUAL_TO); + return new ReducedQueryPhase( + totalHits, + totalRows, + 1.0f, + false, + false, + null, + aggregations, + null, + new SortedTopDocs(scoreDocs.toArray(ScoreDoc[]::new), false, null, null, null), + null, + 1, + totalRows, + 0, + totalRows == 0, + list.stream().flatMap(ssr -> ssr.getFlightTickets().stream()).collect(Collectors.toList()) + ); + } + + public ReducedQueryPhase reducedFromStream(List list) { + + + try (SessionContext context = new SessionContext()) { + + List tickets = list.stream().flatMap(r -> r.getFlightTickets().stream()) .map(OSTicket::getBytes) .collect(Collectors.toList()); + + // execute the query and get a dataframe + CompletableFuture frame = DataFusion.query(tickets.get(0)); + + DataFrame dataFrame = null; + ArrowReader arrowReader = null; + try { + dataFrame = frame.get(); + arrowReader = dataFrame.collect(new RootAllocator()).get(); + } catch (InterruptedException | ExecutionException e) { + throw new RuntimeException(e); + } + int totalRows = 0; List scoreDocs = new ArrayList<>(); - TotalHits totalHits = new TotalHits(totalRows, Relation.EQUAL_TO); - List aggs = new ArrayList<>(); - Map bucketMap = new HashMap(); - - for (byte[] ticket : tickets) { - StreamTicket streamTicket = streamManager.getStreamTicketFactory().fromBytes(ticket); - StreamReader streamIterator = streamManager.getStreamReader(streamTicket); - while (streamIterator.next()) { - VectorSchemaRoot root = streamIterator.getRoot(); + try { + while (arrowReader.loadNextBatch()) { + VectorSchemaRoot root = arrowReader.getVectorSchemaRoot(); int rowCount = root.getRowCount(); - totalRows += rowCount; + totalRows+= rowCount; System.out.println("Record Batch with " + rowCount + " rows:"); // Iterate through rows for (int row = 0; row < rowCount; row++) { - FieldVector ord = root.getVector("ord"); - FieldVector count = root.getVector("count"); - long countValue = (long) getValue(count, row); - String ordValue = (String) getValue(ord, row); - bucketMap.put(ordValue, bucketMap.getOrDefault(ordValue, 0L) + countValue); + FieldVector docID = root.getVector("docID"); + Float4Vector score = (Float4Vector) root.getVector("score"); + FieldVector shardID = root.getVector("shardID"); + FieldVector nodeID = root.getVector("nodeID"); + + ShardId sid = ShardId.fromString((String) getValue(shardID, row)); + int value = (int) getValue(docID, row); + System.out.println("DocID: " + value + " ShardID" + sid + "NodeID: " + getValue(nodeID, row)); + scoreDocs.add(new ScoreDoc(value, score.get(row), sid.id())); } } - // List orders = new ArrayList<>(); - // orders.add(BucketOrder.count(false)); - // aggs.add(new StringTerms( - // "categories", - // InternalOrder.key(true), - // InternalOrder.compound(orders), - // null, - // DocValueFormat.RAW, - // 25, - // false, - // 0L, - // buckets, - // 0, - // new TermsAggregator.BucketCountThresholds(1, 0, 10, 25) - // )); - // InternalAggregations agg = InternalAggregations.reduce(List.of(InternalAggregations.from(aggs)), - // aggReduceContextBuilder.forFinalReduction()); - // finalAggs.add(agg); - } - List orders = new ArrayList<>(); - orders.add(BucketOrder.count(false)); - List buckets = new ArrayList<>(); - bucketMap.entrySet().stream().sorted(Map.Entry.comparingByValue().reversed()).limit(500).forEach(entry -> { - buckets.add( - new StringTerms.Bucket( - new BytesRef(entry.getKey()), - entry.getValue(), - new InternalAggregations(List.of()), - false, - 0, - DocValueFormat.RAW - ) - ); - }); - aggs.add( - new StringTerms( - "categories", - InternalOrder.key(true), - InternalOrder.compound(orders), - null, - DocValueFormat.RAW, - 25, + TotalHits totalHits = new TotalHits(totalRows, Relation.EQUAL_TO); + return new ReducedQueryPhase( + totalHits, + totalRows, + 1.0f, false, - 0L, - buckets, + false, + null, + null, + null, + new SortedTopDocs(scoreDocs.toArray(ScoreDoc[]::new), false, null, null, null), + null, + 1, + totalRows, 0, - new TermsAggregator.BucketCountThresholds(1, 0, 10, 25) - ) - ); - - // InternalAggregations finalReduce = reduceAggs(aggReduceContextBuilder, performFinalReduce, - // List.of(InternalAggregations.from(aggs))); - - return new ReducedQueryPhase( - totalHits, - totalRows, - 1.0f, - false, - false, - null, - InternalAggregations.from(aggs), - null, - new SortedTopDocs(scoreDocs.toArray(ScoreDoc[]::new), false, null, null, null), - null, - 1, - 500, - 0, - totalRows == 0, - list.stream().flatMap(ssr -> ssr.getFlightTickets().stream()).collect(Collectors.toList()) - ); + totalRows == 0, + list.stream().flatMap(ssr -> ssr.getFlightTickets().stream()).collect(Collectors.toList()) + ); + } catch (IOException e) { + throw new RuntimeException(e); + } finally { + try { + arrowReader.close(); + } catch (IOException e) { + throw new RuntimeException(e); + } + } } catch (Exception e) { throw new RuntimeException(e); } @@ -823,6 +894,9 @@ private static Object getValue(FieldVector vector, int index) { return "null"; } + if (vector instanceof UInt8Vector) { + return ((UInt8Vector) vector).get(index); + } else if (vector instanceof IntVector) { return ((IntVector) vector).get(index); } else if (vector instanceof BigIntVector) { diff --git a/server/src/main/java/org/opensearch/action/search/StreamAsyncAction.java b/server/src/main/java/org/opensearch/action/search/StreamAsyncAction.java index 890eb63ecce5d..0c8f4334bace7 100644 --- a/server/src/main/java/org/opensearch/action/search/StreamAsyncAction.java +++ b/server/src/main/java/org/opensearch/action/search/StreamAsyncAction.java @@ -147,6 +147,7 @@ protected void doRun() throws Exception { tickets.addAll(((StreamSearchResult) entry).getFlightTickets()); } } + List targets = StreamAsyncAction.this.results.getAtomicArray() .asList() .stream() diff --git a/server/src/main/java/org/opensearch/arrow/custom/StreamManagerWrapper.java b/server/src/main/java/org/opensearch/arrow/custom/StreamManagerWrapper.java index 0ce4c5ac0fda8..fc109d0dbb782 100644 --- a/server/src/main/java/org/opensearch/arrow/custom/StreamManagerWrapper.java +++ b/server/src/main/java/org/opensearch/arrow/custom/StreamManagerWrapper.java @@ -25,6 +25,8 @@ import org.opensearch.tasks.TaskManager; import java.io.IOException; +import java.util.Set; +import java.util.function.Function; import java.util.function.Supplier; /** diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/BucketsAggregator.java b/server/src/main/java/org/opensearch/search/aggregations/bucket/BucketsAggregator.java index 7c2e79e074498..092a3cb39f650 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/BucketsAggregator.java +++ b/server/src/main/java/org/opensearch/search/aggregations/bucket/BucketsAggregator.java @@ -230,6 +230,10 @@ protected final InternalAggregations[] buildSubAggsForBuckets(long[] bucketOrdsT } InternalAggregations[] result = new InternalAggregations[bucketOrdsToCollect.length]; for (int ord = 0; ord < bucketOrdsToCollect.length; ord++) { + InternalAggregation[] slice = new InternalAggregation[subAggregators.length]; + for (int i = 0; i < subAggregators.length; i++) { + slice[i] = aggregations[i][ord]; + } final int thisOrd = ord; result[ord] = InternalAggregations.from(new AbstractList() { @Override diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/GlobalOrdinalsStringTermsAggregator.java b/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/GlobalOrdinalsStringTermsAggregator.java index 971bd58454a0d..af339b5df8d68 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/GlobalOrdinalsStringTermsAggregator.java +++ b/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/GlobalOrdinalsStringTermsAggregator.java @@ -230,18 +230,18 @@ public LeafBucketCollector getLeafCollector(LeafReaderContext ctx, LeafBucketCol SortedSetDocValues globalOrds = valuesSource.globalOrdinalsValues(ctx); collectionStrategy.globalOrdsReady(globalOrds); - // if (collectionStrategy instanceof DenseGlobalOrds - // && this.resultStrategy instanceof StandardTermsResults - // && sub == LeafBucketCollector.NO_OP_COLLECTOR) { - // LeafBucketCollector termDocFreqCollector = termDocFreqCollector( - // ctx, - // globalOrds, - // (ord, docCount) -> incrementBucketDocCount(collectionStrategy.globalOrdToBucketOrd(0, ord), docCount) - // ); - // if (termDocFreqCollector != null) { - // return termDocFreqCollector; - // } - // } +// if (collectionStrategy instanceof DenseGlobalOrds +// && this.resultStrategy instanceof StandardTermsResults +// && sub == LeafBucketCollector.NO_OP_COLLECTOR) { +// LeafBucketCollector termDocFreqCollector = termDocFreqCollector( +// ctx, +// globalOrds, +// (ord, docCount) -> incrementBucketDocCount(collectionStrategy.globalOrdToBucketOrd(0, ord), docCount) +// ); +// if (termDocFreqCollector != null) { +// return termDocFreqCollector; +// } +// } SortedDocValues singleValues = DocValues.unwrapSingleton(globalOrds); if (singleValues != null) { diff --git a/server/src/main/java/org/opensearch/search/aggregations/support/StreamingAggregator.java b/server/src/main/java/org/opensearch/search/aggregations/support/StreamingAggregator.java index 05604174b5a8b..059a1ee76e976 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/support/StreamingAggregator.java +++ b/server/src/main/java/org/opensearch/search/aggregations/support/StreamingAggregator.java @@ -9,28 +9,40 @@ package org.opensearch.search.aggregations.support; import org.apache.arrow.vector.FieldVector; +import org.apache.arrow.vector.Float4Vector; +import org.apache.arrow.vector.Float8Vector; +import org.apache.arrow.vector.IntVector; import org.apache.arrow.vector.UInt8Vector; import org.apache.arrow.vector.VarCharVector; import org.apache.arrow.vector.VectorSchemaRoot; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.search.Collector; import org.apache.lucene.search.FilterCollector; import org.apache.lucene.search.LeafCollector; import org.apache.lucene.search.Scorable; +import org.opensearch.action.search.SearchPhaseController; import org.opensearch.arrow.spi.StreamProducer; import org.opensearch.core.index.shard.ShardId; import org.opensearch.search.aggregations.Aggregation; import org.opensearch.search.aggregations.Aggregations; import org.opensearch.search.aggregations.Aggregator; +import org.opensearch.search.aggregations.BucketCollectorProcessor; import org.opensearch.search.aggregations.InternalAggregation; +import org.opensearch.search.aggregations.InternalAggregations; import org.opensearch.search.aggregations.LeafBucketCollector; +import org.opensearch.search.aggregations.LeafBucketCollectorBase; import org.opensearch.search.aggregations.bucket.terms.InternalMappedTerms; import org.opensearch.search.aggregations.bucket.terms.InternalTerms; +import org.opensearch.search.aggregations.metrics.InternalCardinality; import org.opensearch.search.internal.SearchContext; import java.io.IOException; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.function.Supplier; public class StreamingAggregator extends FilterCollector { @@ -40,7 +52,6 @@ public class StreamingAggregator extends FilterCollector { private final StreamProducer.FlushSignal flushSignal; private final int batchSize; private final ShardId shardId; - /** * Sole constructor. * @@ -62,16 +73,17 @@ public StreamingAggregator( this.flushSignal = flushSignal; this.shardId = shardId; } + public static Logger logger = LogManager.getLogger(StreamingAggregator.class); @Override public LeafCollector getLeafCollector(LeafReaderContext context) throws IOException { - Map vectors = new HashMap<>(); vectors.put("ord", root.getVector("ord")); vectors.put("count", root.getVector("count")); - final int[] currentRow = { 0 }; + final int[] currentRow = {0}; + final LeafBucketCollector leaf = aggregator.getLeafCollector(context); return new LeafBucketCollector() { - final LeafBucketCollector leaf = aggregator.getLeafCollector(context); + @Override public void setScorer(Scorable scorer) throws IOException { @@ -82,51 +94,78 @@ public void setScorer(Scorable scorer) throws IOException { public void collect(int doc, long owningBucketOrd) throws IOException { leaf.collect(doc); currentRow[0]++; - if (currentRow[0] >= batchSize) { + if (currentRow[0] == batchSize) { + flushBatch(); + flushSignal.awaitConsumption(10000000); + } + } + + @Override + public void finish() throws IOException { + if (currentRow[0] > 0) { flushBatch(); + flushSignal.awaitConsumption(10000000); + logger.info("Flushed last batch for segment {}", context.toString()); } } private void flushBatch() throws IOException { - int bucketCount = 0; - InternalAggregation agg = aggregator.buildAggregations(new long[] { 0 })[0]; + InternalAggregation agg = aggregator.buildAggregations(new long[]{0})[0]; + + int bucketNum = 0; + FieldVector termVector = vectors.get("ord"); + FieldVector countVector = vectors.get("count"); if (agg instanceof InternalMappedTerms) { - InternalMappedTerms terms = (InternalMappedTerms) agg; + InternalMappedTerms terms = (InternalMappedTerms) agg; List buckets = terms.getBuckets(); - for (int i = 0; i < buckets.size(); i++) { + for (InternalTerms.Bucket bucket : buckets) { // Get key/value info - String key = buckets.get(i).getKeyAsString(); - long docCount = buckets.get(i).getDocCount(); - - Aggregations aggregations = buckets.get(i).getAggregations(); - for (Aggregation aggregation : aggregations) { - // TODO: subs - } - - FieldVector termVector = vectors.get("ord"); - FieldVector countVector = vectors.get("count"); - ((VarCharVector) termVector).setSafe(i, key.getBytes()); - ((UInt8Vector) countVector).setSafe(i, docCount); - bucketCount++; + String key = bucket.getKeyAsString(); + + long docCount = bucket.getDocCount(); +// Aggregations aggregations = bucket.getAggregations(); +// for (Aggregation aggregation : aggregations) { +// // TODO: subs +// } + + // Write to vector storage + // e.g., for term and count vectors: +// VarCharVector keyVector = (VarCharVector) vectors.get("key"); +// keyVector.setSafe(i, key.getBytes()); +// logger.info("KEY {} VAL {}", ke) + + ((VarCharVector) termVector).setSafe(bucketNum, key.getBytes()); + ((UInt8Vector) countVector).setSafe(bucketNum, docCount); +// logger.info("Writing {} for {}", docCount, key); + + // Add the values... + bucketNum++; } - aggregator.reset(); + + // Also access high-level statistics +// long otherDocCount = terms.getSumOfOtherDocCounts(); +// long docCountError = terms.getDocCountError(); + } else if (agg instanceof InternalCardinality) { + InternalCardinality ic = (InternalCardinality) agg; + String key = ic.getName(); + long count = ic.getValue(); + ((VarCharVector) termVector).setSafe(bucketNum, key.getBytes()); + ((UInt8Vector) countVector).setSafe(bucketNum, count); + logger.info("Writing cardinality {} for {}", count, key); + bucketNum++; } - System.out.println("Row count " + bucketCount); // Reset for next batch - root.setRowCount(bucketCount); - flushSignal.awaitConsumption(100000); + logger.info("flush after {} docs with {} buckets", currentRow[0], bucketNum); + root.setRowCount(bucketNum); + logger.info("Awaiting consumption at data node"); + flushSignal.awaitConsumption(10000000); + logger.info("Consumed batch at data node"); + aggregator.reset(); currentRow[0] = 0; } - - @Override - public void finish() throws IOException { - if (currentRow[0] > 0) { - flushBatch(); - } - } }; } }