From 50b1b9ae983b63787792ed4defdafa386d386ea0 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Thu, 2 Jul 2026 08:00:47 -0600 Subject: [PATCH 01/11] chore(deps): track DataFusion git main and port to its APIs Pin the datafusion* crates to a git rev of apache/datafusion main instead of the 54 crates.io release so Ballista tracks unreleased DataFusion and surfaces breaking changes between releases. Bump arrow/arrow-flight to 59 to match. Port to the DataFusion main APIs: - TableReference moved from datafusion::sql to datafusion::common. - PhysicalExtensionCodec::try_decode/try_encode gained a PhysicalProtoConverterExtension argument. - MetricValue gained a PeakMemoryUsage variant; serialize it as a named gauge. - Partitioning gained a Range variant; format it in the DOT graph. - EnforceSorting and EnforceDistribution were unified into EnsureRequirements; use it for post-stage-split re-optimization. - ExecutionPlan::partition_statistics is deprecated in favour of statistics_with_args(&StatisticsArgs); migrate all call sites and operator overrides. This is a correctness fix: DataFusion operators that now override only statistics_with_args return unknown stats through the deprecated method, which would silently disable Ballista's broadcast-threshold checks. - ListingOptions dropped its target_partitions/collect_stat fields; build it via the builder (target_partitions comes from the session config). - Distribution::HashPartitioned is deprecated; use KeyPartitioned. Disable dynamic filter pushdown in the AQE mock session config to match the Ballista production default so join-input swaps during re-optimization remain legal. Refresh the EXPLAIN ANALYZE and AQE plan snapshots for DataFusion's dropped peak_mem_used metric and its new RightSemi join lowering. --- Cargo.lock | 275 +++++++----------- Cargo.toml | 18 +- ballista/client/tests/context_checks.rs | 34 +-- ballista/client/tests/context_setup.rs | 13 +- .../core/src/execution_plans/chaos_exec.rs | 5 +- .../src/execution_plans/shuffle_reader.rs | 23 +- .../src/execution_plans/shuffle_writer.rs | 6 +- .../execution_plans/sort_shuffle/writer.rs | 6 +- ballista/core/src/planner.rs | 4 +- ballista/core/src/serde/mod.rs | 88 ++++-- ballista/core/src/serde/scheduler/to_proto.rs | 8 + ballista/executor/src/collect.rs | 6 +- .../src/physical_optimizer/join_selection.rs | 7 +- ballista/scheduler/src/planner.rs | 19 +- .../state/aqe/execution_plan/dynamic_join.rs | 11 +- .../aqe/optimizer_rule/join_selection.rs | 5 +- .../aqe/optimizer_rule/propagate_empty.rs | 7 +- ballista/scheduler/src/state/aqe/test/mod.rs | 6 +- .../src/state/aqe/test/plan_to_stages.rs | 2 +- .../src/state/execution_graph_dot.rs | 3 + benchmarks/src/bin/tpch.rs | 11 +- 21 files changed, 294 insertions(+), 263 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 369a967188..df6ca6dd7f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -179,9 +179,9 @@ checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" [[package]] name = "arrow" -version = "58.3.0" +version = "59.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "378530e55cd479eda3c14eb345310799717e6f76d0c332041e8487022166b471" +checksum = "b952ca5a8046ad741b60f142d6eca4aeebcad615694202bc64c5341f23e32c5b" dependencies = [ "arrow-arith", "arrow-array", @@ -200,9 +200,9 @@ dependencies = [ [[package]] name = "arrow-arith" -version = "58.3.0" +version = "59.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a0ab212d2c1886e802f51c5212d78ebbcbb0bec980fff9dadc1eb8d45cd0b738" +checksum = "64a13b8d3008c4e9063c597a08f46446fe3fd5789277127672d6c0bdbb43b1ff" dependencies = [ "arrow-array", "arrow-buffer", @@ -214,9 +214,9 @@ dependencies = [ [[package]] name = "arrow-array" -version = "58.3.0" +version = "59.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cfd33d3e92f207444098c75b42de99d329562be0cf686b307b097cc52b4e999e" +checksum = "9486151b2f0785bafc6fa04fc5c99fcb4495455662e58787ea32eaaed33c4192" dependencies = [ "ahash", "arrow-buffer", @@ -233,9 +233,9 @@ dependencies = [ [[package]] name = "arrow-avro" -version = "58.3.0" +version = "59.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "049230728cd6e093088c8d231b4beede184e35cad7777c1505c0d5a8571f4376" +checksum = "2e4f9b23a0d7b613acb59fa20bdbe0f80ffdae6411498378340b3915e45f5b84" dependencies = [ "arrow-array", "arrow-buffer", @@ -257,9 +257,9 @@ dependencies = [ [[package]] name = "arrow-buffer" -version = "58.3.0" +version = "59.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c6cd424c2693bcdbc150d843dc9d4d137dd2de4782ce6df491ad11a3a0416c0" +checksum = "c4776577a87794bfdf0b4e90e2ea12454fa7738ea2823c4be5b9d1851da7b434" dependencies = [ "bytes", "half", @@ -269,9 +269,9 @@ dependencies = [ [[package]] name = "arrow-cast" -version = "58.3.0" +version = "59.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c5aefb56a2c02e9e2b30746241058b85f8983f0fcff2ba0c6d09006e1cded7f" +checksum = "a9ad451ce4f98710828a455b96991b8f031deb2e67f5fcad6773f017e4a69c3a" dependencies = [ "arrow-array", "arrow-buffer", @@ -291,9 +291,9 @@ dependencies = [ [[package]] name = "arrow-csv" -version = "58.3.0" +version = "59.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e94e8cf7e517657a52b91ea1263acf38c4ca62a84655d72458a3359b12ab97de" +checksum = "8aa7bf96d6141a7bcca2eed57c7c9767d2a2175281857b8a7b68308992864784" dependencies = [ "arrow-array", "arrow-cast", @@ -306,9 +306,9 @@ dependencies = [ [[package]] name = "arrow-data" -version = "58.3.0" +version = "59.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c88210023a2bfee1896af366309a3028fc3bcbd6515fa29a7990ee1baa08ee0" +checksum = "b38fe43e2e8704360f1464e6e8cc4fc381ef02cc4fb0192afa8df1aaa0115c66" dependencies = [ "arrow-buffer", "arrow-schema", @@ -319,9 +319,9 @@ dependencies = [ [[package]] name = "arrow-flight" -version = "58.3.0" +version = "59.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28abfe8bf9f124e5fc83b334af4fa58f8d0323ad25312ccb2d1da50178415704" +checksum = "42115e09dbb694b5955da998912121451c6910b338228cb80a5701370dba43ff" dependencies = [ "arrow-arith", "arrow-array", @@ -347,9 +347,9 @@ dependencies = [ [[package]] name = "arrow-ipc" -version = "58.3.0" +version = "59.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "238438f0834483703d88896db6fe5a7138b2230debc31b34c0336c2996e3c64f" +checksum = "29dac499fcbc6ba74ee0324057821d381929a48526a3966bd9dffb44aa06d98c" dependencies = [ "arrow-array", "arrow-buffer", @@ -363,9 +363,9 @@ dependencies = [ [[package]] name = "arrow-json" -version = "58.3.0" +version = "59.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "205ca2119e6d679d5c133c6f30e68f027738d95ed948cf77677ea69c7800036b" +checksum = "0fe05e916ddc50f4c7a363cd69c0ef5894fcee063517e9a0b8582f0c56746af6" dependencies = [ "arrow-array", "arrow-buffer", @@ -388,9 +388,9 @@ dependencies = [ [[package]] name = "arrow-ord" -version = "58.3.0" +version = "59.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1bffd8fd2579286a5d63bac898159873e5094a79009940bcb42bbfce4f19f1d0" +checksum = "0e13dbdc2a9c053c10c7baa6e30faee04a180aa7ce88e471835850ce37abd20b" dependencies = [ "arrow-array", "arrow-buffer", @@ -401,9 +401,9 @@ dependencies = [ [[package]] name = "arrow-row" -version = "58.3.0" +version = "59.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bab5994731204603c73ba69267616c50f80780774c6bb0476f1f830625115e0c" +checksum = "4d5a1f8c733d15260b305683472ee8ad89c62cbd706703ca873b90d051b41592" dependencies = [ "arrow-array", "arrow-buffer", @@ -414,9 +414,9 @@ dependencies = [ [[package]] name = "arrow-schema" -version = "58.3.0" +version = "59.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f633dbfdf39c039ada1bf9e34c694816eb71fbb7dc78f613993b7245e078a1ed" +checksum = "d9e4969dc350d571766247143ab36a5187d095d3d3690970408bc630d47c69e5" dependencies = [ "serde_core", "serde_json", @@ -424,9 +424,9 @@ dependencies = [ [[package]] name = "arrow-select" -version = "58.3.0" +version = "59.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8cd065c54172ac787cf3f2f8d4107e0d3fdc26edba76fdf4f4cc170258942222" +checksum = "402770dba90865359d98d1ef92ef16e23d75c0cca9c2c880c8a05468b7743bf9" dependencies = [ "ahash", "arrow-array", @@ -438,9 +438,9 @@ dependencies = [ [[package]] name = "arrow-string" -version = "58.3.0" +version = "59.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29dd7cda3ab9692f43a2e4acc444d760cc17b12bb6d8232ddf64e9bab7c06b42" +checksum = "a2b0afbb8b9016700938291123df30838b89decc3213dba00852021988b170d3" dependencies = [ "arrow-array", "arrow-buffer", @@ -1499,12 +1499,6 @@ version = "1.25.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c8efb64bd706a16a1bdde310ae86b351e4d21550d98d056f22f8a7f7a2183fec" -[[package]] -name = "byteorder" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" - [[package]] name = "bytes" version = "1.11.1" @@ -2196,8 +2190,7 @@ dependencies = [ [[package]] name = "datafusion" version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "997a31e15872606a49478e670c58302094c97cb96abb0a7d60720f8e92170040" +source = "git+https://github.com/apache/datafusion.git?rev=b79d3965b7a2c29853e83def04656408d9d7ee7f#b79d3965b7a2c29853e83def04656408d9d7ee7f" dependencies = [ "arrow", "arrow-schema", @@ -2233,7 +2226,7 @@ dependencies = [ "flate2", "futures", "indexmap 2.14.0", - "itertools 0.14.0", + "itertools 0.15.0", "liblzma", "log", "object_store", @@ -2250,8 +2243,7 @@ dependencies = [ [[package]] name = "datafusion-catalog" version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7dd61161508f8f5fa1107774ea687bd753c22d83a32eebf963549f89de14139" +source = "git+https://github.com/apache/datafusion.git?rev=b79d3965b7a2c29853e83def04656408d9d7ee7f#b79d3965b7a2c29853e83def04656408d9d7ee7f" dependencies = [ "arrow", "async-trait", @@ -2265,7 +2257,7 @@ dependencies = [ "datafusion-physical-plan", "datafusion-session", "futures", - "itertools 0.14.0", + "itertools 0.15.0", "log", "object_store", "parking_lot", @@ -2275,8 +2267,7 @@ dependencies = [ [[package]] name = "datafusion-catalog-listing" version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "897c70f871277f9ce99aa38347be0d679bbe3e617156c4d2a8378cec8a2a0891" +source = "git+https://github.com/apache/datafusion.git?rev=b79d3965b7a2c29853e83def04656408d9d7ee7f#b79d3965b7a2c29853e83def04656408d9d7ee7f" dependencies = [ "arrow", "async-trait", @@ -2290,7 +2281,7 @@ dependencies = [ "datafusion-physical-expr-common", "datafusion-physical-plan", "futures", - "itertools 0.14.0", + "itertools 0.15.0", "log", "object_store", ] @@ -2298,8 +2289,7 @@ dependencies = [ [[package]] name = "datafusion-cli" version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a9533845ab888508718c2ad540cff205d1111525a9952def8b0e584b6453784" +source = "git+https://github.com/apache/datafusion.git?rev=b79d3965b7a2c29853e83def04656408d9d7ee7f#b79d3965b7a2c29853e83def04656408d9d7ee7f" dependencies = [ "arrow", "async-trait", @@ -2326,8 +2316,7 @@ dependencies = [ [[package]] name = "datafusion-common" version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "121c9ded5d87d9172319e006f2afdb9928d72dbacd6a90a458d8acb1e3b43a65" +source = "git+https://github.com/apache/datafusion.git?rev=b79d3965b7a2c29853e83def04656408d9d7ee7f#b79d3965b7a2c29853e83def04656408d9d7ee7f" dependencies = [ "arrow", "arrow-ipc", @@ -2338,9 +2327,10 @@ dependencies = [ "hashbrown 0.17.1", "hex", "indexmap 2.14.0", - "itertools 0.14.0", + "itertools 0.15.0", "libc", "log", + "num-traits", "object_store", "parquet", "recursive", @@ -2353,8 +2343,7 @@ dependencies = [ [[package]] name = "datafusion-common-runtime" version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "981b9dae74f78ee3d9f714fb49b01919eab975461b56149510c3ba9ea11287d1" +source = "git+https://github.com/apache/datafusion.git?rev=b79d3965b7a2c29853e83def04656408d9d7ee7f#b79d3965b7a2c29853e83def04656408d9d7ee7f" dependencies = [ "futures", "log", @@ -2364,8 +2353,7 @@ dependencies = [ [[package]] name = "datafusion-datasource" version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ffd7d295b2ec7c00d8a56562f41ed41062cf0af75549ed891c12a0a09eddfefe" +source = "git+https://github.com/apache/datafusion.git?rev=b79d3965b7a2c29853e83def04656408d9d7ee7f#b79d3965b7a2c29853e83def04656408d9d7ee7f" dependencies = [ "arrow", "async-compression", @@ -2385,7 +2373,7 @@ dependencies = [ "flate2", "futures", "glob", - "itertools 0.14.0", + "itertools 0.15.0", "liblzma", "log", "object_store", @@ -2400,8 +2388,7 @@ dependencies = [ [[package]] name = "datafusion-datasource-arrow" version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "552b0b3f342f7ec41b3fbd70f6339dc82a30cfd0349e7f280e7852528085349f" +source = "git+https://github.com/apache/datafusion.git?rev=b79d3965b7a2c29853e83def04656408d9d7ee7f#b79d3965b7a2c29853e83def04656408d9d7ee7f" dependencies = [ "arrow", "arrow-ipc", @@ -2416,7 +2403,7 @@ dependencies = [ "datafusion-physical-plan", "datafusion-session", "futures", - "itertools 0.14.0", + "itertools 0.15.0", "object_store", "tokio", ] @@ -2424,8 +2411,7 @@ dependencies = [ [[package]] name = "datafusion-datasource-avro" version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb517d08967d536284ce70afb5fe8583133779249f2d7b90587d339741a7f195" +source = "git+https://github.com/apache/datafusion.git?rev=b79d3965b7a2c29853e83def04656408d9d7ee7f#b79d3965b7a2c29853e83def04656408d9d7ee7f" dependencies = [ "arrow", "arrow-avro", @@ -2443,8 +2429,7 @@ dependencies = [ [[package]] name = "datafusion-datasource-csv" version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68850aa426b897e879c8b87e512ea8124f1d0a2869a4e51808ddaaddf1bc0ada" +source = "git+https://github.com/apache/datafusion.git?rev=b79d3965b7a2c29853e83def04656408d9d7ee7f#b79d3965b7a2c29853e83def04656408d9d7ee7f" dependencies = [ "arrow", "async-trait", @@ -2466,8 +2451,7 @@ dependencies = [ [[package]] name = "datafusion-datasource-json" version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "402f93242ae08ef99139ee2c528a49d087efe88d5c7b2c3ff5480855a40ce54f" +source = "git+https://github.com/apache/datafusion.git?rev=b79d3965b7a2c29853e83def04656408d9d7ee7f#b79d3965b7a2c29853e83def04656408d9d7ee7f" dependencies = [ "arrow", "async-trait", @@ -2489,10 +2473,10 @@ dependencies = [ [[package]] name = "datafusion-datasource-parquet" version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ffd2499c1bee0eeccf6a57156105700eeeb17bc701899ac719183c4e74231450" +source = "git+https://github.com/apache/datafusion.git?rev=b79d3965b7a2c29853e83def04656408d9d7ee7f#b79d3965b7a2c29853e83def04656408d9d7ee7f" dependencies = [ "arrow", + "arrow-schema", "async-trait", "bytes", "datafusion-common", @@ -2509,7 +2493,7 @@ dependencies = [ "datafusion-pruning", "datafusion-session", "futures", - "itertools 0.14.0", + "itertools 0.15.0", "log", "object_store", "parking_lot", @@ -2520,18 +2504,17 @@ dependencies = [ [[package]] name = "datafusion-doc" version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb9e7e5d11130c48c8bd4e80c79a9772dd28ce6dc330baca9246205d245b9e2e" +source = "git+https://github.com/apache/datafusion.git?rev=b79d3965b7a2c29853e83def04656408d9d7ee7f#b79d3965b7a2c29853e83def04656408d9d7ee7f" [[package]] name = "datafusion-execution" version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37a8643ab852eb68864e1b72ae789e8066282dce48eea6347ffb0aee33d1ccc0" +source = "git+https://github.com/apache/datafusion.git?rev=b79d3965b7a2c29853e83def04656408d9d7ee7f#b79d3965b7a2c29853e83def04656408d9d7ee7f" dependencies = [ "arrow", "arrow-buffer", "async-trait", + "bytes", "dashmap", "datafusion-common", "datafusion-expr", @@ -2543,14 +2526,15 @@ dependencies = [ "parquet", "rand 0.9.4", "tempfile", + "tokio", + "tokio-util", "url", ] [[package]] name = "datafusion-expr" version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6932f4d71eed9c8d9341476a2b845aadfabde5495d08dbcd8fc23881f49fa7a0" +source = "git+https://github.com/apache/datafusion.git?rev=b79d3965b7a2c29853e83def04656408d9d7ee7f#b79d3965b7a2c29853e83def04656408d9d7ee7f" dependencies = [ "arrow", "arrow-schema", @@ -2563,7 +2547,7 @@ dependencies = [ "datafusion-functions-window-common", "datafusion-physical-expr-common", "indexmap 2.14.0", - "itertools 0.14.0", + "itertools 0.15.0", "recursive", "serde_json", "sqlparser", @@ -2572,20 +2556,18 @@ dependencies = [ [[package]] name = "datafusion-expr-common" version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0225491839a31b1f7d2cb8092c2d50792e2fe1c1724e4e6d08e011f5feaf4ed2" +source = "git+https://github.com/apache/datafusion.git?rev=b79d3965b7a2c29853e83def04656408d9d7ee7f#b79d3965b7a2c29853e83def04656408d9d7ee7f" dependencies = [ "arrow", "datafusion-common", "indexmap 2.14.0", - "itertools 0.14.0", + "itertools 0.15.0", ] [[package]] name = "datafusion-functions" version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "14872c47bfc3d21e53ec82f57074e6987a15941c1e2f43cde4ac6ae2746634e3" +source = "git+https://github.com/apache/datafusion.git?rev=b79d3965b7a2c29853e83def04656408d9d7ee7f#b79d3965b7a2c29853e83def04656408d9d7ee7f" dependencies = [ "arrow", "arrow-buffer", @@ -2602,7 +2584,7 @@ dependencies = [ "datafusion-macros", "datafusion-physical-expr-common", "hex", - "itertools 0.14.0", + "itertools 0.15.0", "log", "md-5 0.11.0", "memchr", @@ -2616,8 +2598,7 @@ dependencies = [ [[package]] name = "datafusion-functions-aggregate" version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75a2ca14e1b609be21e657e2d3130b2f446456b08393b377bb721a33952d2e09" +source = "git+https://github.com/apache/datafusion.git?rev=b79d3965b7a2c29853e83def04656408d9d7ee7f#b79d3965b7a2c29853e83def04656408d9d7ee7f" dependencies = [ "arrow", "datafusion-common", @@ -2628,7 +2609,6 @@ dependencies = [ "datafusion-macros", "datafusion-physical-expr", "datafusion-physical-expr-common", - "foldhash 0.2.0", "half", "log", "num-traits", @@ -2637,8 +2617,7 @@ dependencies = [ [[package]] name = "datafusion-functions-aggregate-common" version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ece74ba09092d2ef9c9b54a38445450aea292a1f8b04faf531936b723a24b3c" +source = "git+https://github.com/apache/datafusion.git?rev=b79d3965b7a2c29853e83def04656408d9d7ee7f#b79d3965b7a2c29853e83def04656408d9d7ee7f" dependencies = [ "arrow", "datafusion-common", @@ -2649,8 +2628,7 @@ dependencies = [ [[package]] name = "datafusion-functions-nested" version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f3e3f9ee8ca59bf70518802107de6f1b88a9509efdc629fadc5de9d6b2d5ef5" +source = "git+https://github.com/apache/datafusion.git?rev=b79d3965b7a2c29853e83def04656408d9d7ee7f#b79d3965b7a2c29853e83def04656408d9d7ee7f" dependencies = [ "arrow", "arrow-ord", @@ -2665,7 +2643,7 @@ dependencies = [ "datafusion-macros", "datafusion-physical-expr-common", "hashbrown 0.17.1", - "itertools 0.14.0", + "itertools 0.15.0", "itoa", "log", "memchr", @@ -2674,8 +2652,7 @@ dependencies = [ [[package]] name = "datafusion-functions-table" version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89161dffc22cf2b50f9f4b1bee83b5221d3b4ed7c2e37fd7aa2b22a5297b3a26" +source = "git+https://github.com/apache/datafusion.git?rev=b79d3965b7a2c29853e83def04656408d9d7ee7f#b79d3965b7a2c29853e83def04656408d9d7ee7f" dependencies = [ "arrow", "async-trait", @@ -2690,8 +2667,7 @@ dependencies = [ [[package]] name = "datafusion-functions-window" version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7339345b226b3874037708bf5023ba1c2de705128f8457a095aae5ae9cb9c78" +source = "git+https://github.com/apache/datafusion.git?rev=b79d3965b7a2c29853e83def04656408d9d7ee7f#b79d3965b7a2c29853e83def04656408d9d7ee7f" dependencies = [ "arrow", "datafusion-common", @@ -2707,8 +2683,7 @@ dependencies = [ [[package]] name = "datafusion-functions-window-common" version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa84836dc2392df6f43d6a29d37fb56a8ebdc8b3f4e10ae8dc15861fd20278fb" +source = "git+https://github.com/apache/datafusion.git?rev=b79d3965b7a2c29853e83def04656408d9d7ee7f#b79d3965b7a2c29853e83def04656408d9d7ee7f" dependencies = [ "datafusion-common", "datafusion-physical-expr-common", @@ -2717,8 +2692,7 @@ dependencies = [ [[package]] name = "datafusion-macros" version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "587164e03ad68732aa9e7bfe5686e3f25970d4c64fd4bd80790749840892dae5" +source = "git+https://github.com/apache/datafusion.git?rev=b79d3965b7a2c29853e83def04656408d9d7ee7f#b79d3965b7a2c29853e83def04656408d9d7ee7f" dependencies = [ "datafusion-doc", "quote", @@ -2728,8 +2702,7 @@ dependencies = [ [[package]] name = "datafusion-optimizer" version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77f20e8cf9e8654d92f4c16b24c487353ee5bf153ffc12d5772cd399ab8cd281" +source = "git+https://github.com/apache/datafusion.git?rev=b79d3965b7a2c29853e83def04656408d9d7ee7f#b79d3965b7a2c29853e83def04656408d9d7ee7f" dependencies = [ "arrow", "chrono", @@ -2738,7 +2711,7 @@ dependencies = [ "datafusion-expr-common", "datafusion-physical-expr", "indexmap 2.14.0", - "itertools 0.14.0", + "itertools 0.15.0", "log", "recursive", "regex", @@ -2748,8 +2721,7 @@ dependencies = [ [[package]] name = "datafusion-physical-expr" version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f015a4a82f6f7ff7e1d8d4bf3870a936752fa38b17705dfcc14adef95aa8922c" +source = "git+https://github.com/apache/datafusion.git?rev=b79d3965b7a2c29853e83def04656408d9d7ee7f#b79d3965b7a2c29853e83def04656408d9d7ee7f" dependencies = [ "arrow", "datafusion-common", @@ -2757,10 +2729,11 @@ dependencies = [ "datafusion-expr-common", "datafusion-functions-aggregate-common", "datafusion-physical-expr-common", + "datafusion-proto-models", "half", "hashbrown 0.17.1", "indexmap 2.14.0", - "itertools 0.14.0", + "itertools 0.15.0", "parking_lot", "petgraph", "recursive", @@ -2770,8 +2743,7 @@ dependencies = [ [[package]] name = "datafusion-physical-expr-adapter" version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "51e6ffff8acdfe54e0ea15ccf38115c4a9184433b0439f42907637928d00a235" +source = "git+https://github.com/apache/datafusion.git?rev=b79d3965b7a2c29853e83def04656408d9d7ee7f#b79d3965b7a2c29853e83def04656408d9d7ee7f" dependencies = [ "arrow", "datafusion-common", @@ -2779,22 +2751,22 @@ dependencies = [ "datafusion-functions", "datafusion-physical-expr", "datafusion-physical-expr-common", - "itertools 0.14.0", + "itertools 0.15.0", ] [[package]] name = "datafusion-physical-expr-common" version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7967a3e171c6a4bf09474b3f7a14f1a3db13ed1714ba12156f33fcce2bba54e8" +source = "git+https://github.com/apache/datafusion.git?rev=b79d3965b7a2c29853e83def04656408d9d7ee7f#b79d3965b7a2c29853e83def04656408d9d7ee7f" dependencies = [ "arrow", "chrono", "datafusion-common", "datafusion-expr-common", + "datafusion-proto-models", "hashbrown 0.17.1", "indexmap 2.14.0", - "itertools 0.14.0", + "itertools 0.15.0", "parking_lot", "pin-project", ] @@ -2802,8 +2774,7 @@ dependencies = [ [[package]] name = "datafusion-physical-optimizer" version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59ff803e2a96054cb6d83f35f9e60fd4f42eac515e1932bd1b2dbc91d5fcbf36" +source = "git+https://github.com/apache/datafusion.git?rev=b79d3965b7a2c29853e83def04656408d9d7ee7f#b79d3965b7a2c29853e83def04656408d9d7ee7f" dependencies = [ "arrow", "datafusion-common", @@ -2814,15 +2785,14 @@ dependencies = [ "datafusion-physical-expr-common", "datafusion-physical-plan", "datafusion-pruning", - "itertools 0.14.0", + "itertools 0.15.0", "recursive", ] [[package]] name = "datafusion-physical-plan" version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "776ee54d47d15bdb126452f9ca17b03761e3b004682914beaedd3f86eb507fbc" +source = "git+https://github.com/apache/datafusion.git?rev=b79d3965b7a2c29853e83def04656408d9d7ee7f#b79d3965b7a2c29853e83def04656408d9d7ee7f" dependencies = [ "arrow", "arrow-data", @@ -2830,6 +2800,7 @@ dependencies = [ "arrow-ord", "arrow-schema", "async-trait", + "bytes", "datafusion-common", "datafusion-common-runtime", "datafusion-execution", @@ -2839,23 +2810,25 @@ dependencies = [ "datafusion-functions-window-common", "datafusion-physical-expr", "datafusion-physical-expr-common", + "datafusion-proto-common", + "datafusion-proto-models", "futures", "half", "hashbrown 0.17.1", "indexmap 2.14.0", - "itertools 0.14.0", + "itertools 0.15.0", "log", "num-traits", "parking_lot", "pin-project-lite", + "serde_json", "tokio", ] [[package]] name = "datafusion-proto" version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9dd15a1ba5d3af93808241065c6c44dbca8296a189845e8a587c45c07bf0ffae" +source = "git+https://github.com/apache/datafusion.git?rev=b79d3965b7a2c29853e83def04656408d9d7ee7f#b79d3965b7a2c29853e83def04656408d9d7ee7f" dependencies = [ "arrow", "chrono", @@ -2874,6 +2847,7 @@ dependencies = [ "datafusion-physical-expr-common", "datafusion-physical-plan", "datafusion-proto-common", + "datafusion-proto-models", "object_store", "prost", ] @@ -2881,19 +2855,26 @@ dependencies = [ [[package]] name = "datafusion-proto-common" version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90042982cf9462eb06a0b81f92efa4188dae871e7ea3ab8dc61aa9c9349b2530" +source = "git+https://github.com/apache/datafusion.git?rev=b79d3965b7a2c29853e83def04656408d9d7ee7f#b79d3965b7a2c29853e83def04656408d9d7ee7f" dependencies = [ "arrow", "datafusion-common", "prost", ] +[[package]] +name = "datafusion-proto-models" +version = "54.0.0" +source = "git+https://github.com/apache/datafusion.git?rev=b79d3965b7a2c29853e83def04656408d9d7ee7f#b79d3965b7a2c29853e83def04656408d9d7ee7f" +dependencies = [ + "datafusion-proto-common", + "prost", +] + [[package]] name = "datafusion-pruning" version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d5fb9e5774660aa69c3ba93c610f175f75b65cb8c3776edb3626de8f3a4f4ee3" +source = "git+https://github.com/apache/datafusion.git?rev=b79d3965b7a2c29853e83def04656408d9d7ee7f#b79d3965b7a2c29853e83def04656408d9d7ee7f" dependencies = [ "arrow", "datafusion-common", @@ -2908,8 +2889,7 @@ dependencies = [ [[package]] name = "datafusion-session" version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "15ce715fa2a61f4623cc234bcc14a3ef6a91f189128d5b14b468a6a17cdfc417" +source = "git+https://github.com/apache/datafusion.git?rev=b79d3965b7a2c29853e83def04656408d9d7ee7f#b79d3965b7a2c29853e83def04656408d9d7ee7f" dependencies = [ "async-trait", "datafusion-common", @@ -2922,8 +2902,7 @@ dependencies = [ [[package]] name = "datafusion-spark" version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "390bb0bf37cb2b95ffd65eacd66f60df50793d1f94097799e416f39477a51957" +source = "git+https://github.com/apache/datafusion.git?rev=b79d3965b7a2c29853e83def04656408d9d7ee7f#b79d3965b7a2c29853e83def04656408d9d7ee7f" dependencies = [ "arrow", "bigdecimal", @@ -2952,8 +2931,7 @@ dependencies = [ [[package]] name = "datafusion-sql" version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6094ad36a3ed6d7ac87b20b479b2d0b118250f66cf997603829fdc65b44a7099" +source = "git+https://github.com/apache/datafusion.git?rev=b79d3965b7a2c29853e83def04656408d9d7ee7f#b79d3965b7a2c29853e83def04656408d9d7ee7f" dependencies = [ "arrow", "bigdecimal", @@ -2966,20 +2944,20 @@ dependencies = [ "recursive", "regex", "sqlparser", + "stacker", ] [[package]] name = "datafusion-substrait" version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b22c8f8c72d317e54fad6f85c0ef6d1e1da53cc7faadc7eea8daf0f8d86d4f2" +source = "git+https://github.com/apache/datafusion.git?rev=b79d3965b7a2c29853e83def04656408d9d7ee7f#b79d3965b7a2c29853e83def04656408d9d7ee7f" dependencies = [ "async-recursion", "async-trait", "chrono", "datafusion", "half", - "itertools 0.14.0", + "itertools 0.15.0", "object_store", "pbjson-types", "prost", @@ -4102,12 +4080,6 @@ dependencies = [ "syn 2.0.117", ] -[[package]] -name = "integer-encoding" -version = "3.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8bb03732005da905c88227371639bf1ad885cc712789c011c31c5fb3ab3ccf02" - [[package]] name = "into-attr" version = "0.1.1" @@ -4912,15 +4884,6 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d" -[[package]] -name = "ordered-float" -version = "2.10.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68f19d67e5a2795c94e73e0bb1cc1a7edeb2e28efd39e2e1c9b7a40c1108b11c" -dependencies = [ - "num-traits", -] - [[package]] name = "ordered-float" version = "4.6.0" @@ -4995,9 +4958,9 @@ dependencies = [ [[package]] name = "parquet" -version = "58.3.0" +version = "59.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5dafa7d01085b62a47dd0c1829550a0a36710ea9c4fe358a05a85477cec8a908" +checksum = "5302d4da74d6596a1f11f9928767995b53bca657cbeea1e4e8c5074f8a1157dd" dependencies = [ "ahash", "arrow-array", @@ -5024,7 +4987,6 @@ dependencies = [ "seq-macro", "simdutf8", "snap", - "thrift", "tokio", "twox-hash", "zstd", @@ -6992,7 +6954,7 @@ dependencies = [ "nix 0.29.0", "num-derive", "num-traits", - "ordered-float 4.6.0", + "ordered-float", "pest", "pest_derive", "phf 0.11.3", @@ -7112,17 +7074,6 @@ dependencies = [ "cfg-if", ] -[[package]] -name = "thrift" -version = "0.17.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e54bc85fc7faa8bc175c4bab5b92ba8d9a3ce893d0e9f42cc455c8ab16a9e09" -dependencies = [ - "byteorder", - "integer-encoding", - "ordered-float 2.10.1", -] - [[package]] name = "time" version = "0.3.47" @@ -7995,7 +7946,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5f2ab60e120fd6eaa68d9567f3226e876684639d22a4219b313ff69ec0ccd5ac" dependencies = [ "log", - "ordered-float 4.6.0", + "ordered-float", "strsim", "thiserror 1.0.69", "wezterm-dynamic-derive", diff --git a/Cargo.toml b/Cargo.toml index 5719b9b16c..aba3f4574e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -26,16 +26,18 @@ edition = "2024" rust-version = "1.88.0" [workspace.dependencies] -arrow = { version = "58.3", features = ["ipc_compression"] } -arrow-flight = { version = "58.3", features = ["flight-sql-experimental"] } +arrow = { version = "59.1.0", features = ["ipc_compression"] } +arrow-flight = { version = "59.1.0", features = ["flight-sql-experimental"] } clap = { version = "4.5", features = ["derive", "cargo"] } -datafusion = "54" -datafusion-cli = "54" -datafusion-proto = "54" -datafusion-proto-common = "54" -datafusion-spark = "54" -datafusion-substrait = "54" +# DataFusion crates are pinned to a git rev of apache/datafusion main rather than +# a crates.io release. Bump this rev to advance to newer DataFusion commits. +datafusion = { git = "https://github.com/apache/datafusion.git", rev = "b79d3965b7a2c29853e83def04656408d9d7ee7f" } +datafusion-cli = { git = "https://github.com/apache/datafusion.git", rev = "b79d3965b7a2c29853e83def04656408d9d7ee7f" } +datafusion-proto = { git = "https://github.com/apache/datafusion.git", rev = "b79d3965b7a2c29853e83def04656408d9d7ee7f" } +datafusion-proto-common = { git = "https://github.com/apache/datafusion.git", rev = "b79d3965b7a2c29853e83def04656408d9d7ee7f" } +datafusion-spark = { git = "https://github.com/apache/datafusion.git", rev = "b79d3965b7a2c29853e83def04656408d9d7ee7f" } +datafusion-substrait = { git = "https://github.com/apache/datafusion.git", rev = "b79d3965b7a2c29853e83def04656408d9d7ee7f" } insta = "1.47" itertools = "0.15" diff --git a/ballista/client/tests/context_checks.rs b/ballista/client/tests/context_checks.rs index 75d17f37ff..2a0212939a 100644 --- a/ballista/client/tests/context_checks.rs +++ b/ballista/client/tests/context_checks.rs @@ -1180,23 +1180,23 @@ mod supported { )?; let expected = [ - "+-------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+", - "| plan_type | plan |", - "+-------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+", - "| Plan with Metrics | =========SuccessfulStage[stage_id=1, partitions=1]========= |", - "| | SortShuffleWriterExec: partitioning=Hash([id@0], 16), metrics=[output_rows=..., input_rows=..., spill_bytes=..., spill_count=..., repart_time=..., spill_time=..., write_time=...] |", - "| | AggregateExec: mode=Partial, gby=[id@0 as id], aggr=[count(Int64(1))], metrics=[output_rows=..., elapsed_compute=..., output_bytes=..., output_batches=..., spill_count=..., spilled_bytes=..., spilled_rows=..., skipped_aggregation_rows=..., peak_mem_used=..., aggregate_arguments_time=..., aggregation_time=..., emitting_time=..., time_calculating_group_ids=..., reduction_factor=...] |", - "| | ProjectionExec: expr=[__unnest_placeholder(make_array(Int64(1),Int64(2),Int64(3),Int64(4),Int64(5)),depth=1)@0 as id], metrics=[output_rows=..., elapsed_compute=..., output_bytes=..., output_batches=..., expr_0_eval_time=...] |", - "| | UnnestExec, metrics=[output_rows=..., elapsed_compute=..., output_bytes=..., output_batches=..., input_batches=..., input_rows=...] |", - "| | ProjectionExec: expr=[[1, 2, 3, 4, 5] as __unnest_placeholder(make_array(Int64(1),Int64(2),Int64(3),Int64(4),Int64(5)))], metrics=[output_rows=..., elapsed_compute=..., output_bytes=..., output_batches=..., expr_0_eval_time=...] |", - "| | PlaceholderRowExec, metrics=[...] |", - "| | |", - "| | =========SuccessfulStage[stage_id=2, partitions=16]========= |", - "| | ShuffleWriterExec: partitioning: None, metrics=[output_rows=..., input_rows=..., repart_time=..., write_time=...] |", - "| | ProjectionExec: expr=[count(Int64(1))@1 as count(*), id@0 as id], metrics=[output_rows=..., elapsed_compute=..., output_bytes=..., output_batches=..., expr_0_eval_time=..., expr_1_eval_time=...] |", - "| | AggregateExec: mode=FinalPartitioned, gby=[id@0 as id], aggr=[count(Int64(1))], metrics=[output_rows=..., elapsed_compute=..., output_bytes=..., output_batches=..., spill_count=..., spilled_bytes=..., spilled_rows=..., peak_mem_used=..., aggregate_arguments_time=..., aggregation_time=..., emitting_time=..., time_calculating_group_ids=...] |", - "| | ShuffleReaderExec: upstream_stage: 1, partitioning: Hash([id@0], 16), metrics=[output_rows=..., elapsed_compute=..., output_bytes=..., output_batches=...] |", - "+-------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+", + "+-------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+", + "| plan_type | plan |", + "+-------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+", + "| Plan with Metrics | =========SuccessfulStage[stage_id=1, partitions=1]========= |", + "| | SortShuffleWriterExec: partitioning=Hash([id@0], 16), metrics=[output_rows=..., input_rows=..., spill_bytes=..., spill_count=..., repart_time=..., spill_time=..., write_time=...] |", + "| | AggregateExec: mode=Partial, gby=[id@0 as id], aggr=[count(Int64(1))], metrics=[output_rows=..., elapsed_compute=..., output_bytes=..., output_batches=..., spill_count=..., spilled_bytes=..., spilled_rows=..., skipped_aggregation_rows=..., aggregate_arguments_time=..., aggregation_time=..., emitting_time=..., time_calculating_group_ids=..., reduction_factor=...] |", + "| | ProjectionExec: expr=[__unnest_placeholder(make_array(Int64(1),Int64(2),Int64(3),Int64(4),Int64(5)),depth=1)@0 as id], metrics=[output_rows=..., elapsed_compute=..., output_bytes=..., output_batches=..., expr_0_eval_time=...] |", + "| | UnnestExec, metrics=[output_rows=..., elapsed_compute=..., output_bytes=..., output_batches=..., input_batches=..., input_rows=...] |", + "| | ProjectionExec: expr=[[1, 2, 3, 4, 5] as __unnest_placeholder(make_array(Int64(1),Int64(2),Int64(3),Int64(4),Int64(5)))], metrics=[output_rows=..., elapsed_compute=..., output_bytes=..., output_batches=..., expr_0_eval_time=...] |", + "| | PlaceholderRowExec, metrics=[...] |", + "| | |", + "| | =========SuccessfulStage[stage_id=2, partitions=16]========= |", + "| | ShuffleWriterExec: partitioning: None, metrics=[output_rows=..., input_rows=..., repart_time=..., write_time=...] |", + "| | ProjectionExec: expr=[count(Int64(1))@1 as count(*), id@0 as id], metrics=[output_rows=..., elapsed_compute=..., output_bytes=..., output_batches=..., expr_0_eval_time=..., expr_1_eval_time=...] |", + "| | AggregateExec: mode=FinalPartitioned, gby=[id@0 as id], aggr=[count(Int64(1))], metrics=[output_rows=..., elapsed_compute=..., output_bytes=..., output_batches=..., spill_count=..., spilled_bytes=..., spilled_rows=..., aggregate_arguments_time=..., aggregation_time=..., emitting_time=..., time_calculating_group_ids=...] |", + "| | ShuffleReaderExec: upstream_stage: 1, partitioning: Hash([id@0], 16), metrics=[output_rows=..., elapsed_compute=..., output_bytes=..., output_batches=...] |", + "+-------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+", ]; assert_batches_eq!(expected, &[sanitized]); diff --git a/ballista/client/tests/context_setup.rs b/ballista/client/tests/context_setup.rs index df999c06e4..37eaa8040a 100644 --- a/ballista/client/tests/context_setup.rs +++ b/ballista/client/tests/context_setup.rs @@ -118,7 +118,8 @@ mod standalone { prelude::{SessionConfig, SessionContext}, }; use datafusion_proto::{ - logical_plan::LogicalExtensionCodec, physical_plan::PhysicalExtensionCodec, + logical_plan::LogicalExtensionCodec, + physical_plan::{PhysicalExtensionCodec, PhysicalProtoConverterExtension}, }; #[tokio::test] @@ -299,7 +300,7 @@ mod standalone { fn try_decode_table_provider( &self, _buf: &[u8], - _table_ref: &datafusion::sql::TableReference, + _table_ref: &datafusion::common::TableReference, _schema: datafusion::arrow::datatypes::SchemaRef, _ctx: &TaskContext, ) -> datafusion::error::Result< @@ -312,7 +313,7 @@ mod standalone { fn try_encode_table_provider( &self, - _table_ref: &datafusion::sql::TableReference, + _table_ref: &datafusion::common::TableReference, _node: std::sync::Arc, _buf: &mut Vec, ) -> datafusion::error::Result<()> { @@ -366,21 +367,23 @@ mod standalone { buf: &[u8], inputs: &[Arc], ctx: &TaskContext, + proto_converter: &dyn PhysicalProtoConverterExtension, ) -> datafusion::error::Result> { self.invoked .store(true, std::sync::atomic::Ordering::Relaxed); - self.codec.try_decode(buf, inputs, ctx) + self.codec.try_decode(buf, inputs, ctx, proto_converter) } fn try_encode( &self, node: Arc, buf: &mut Vec, + proto_converter: &dyn PhysicalProtoConverterExtension, ) -> datafusion::error::Result<()> { self.invoked .store(true, std::sync::atomic::Ordering::Relaxed); - self.codec.try_encode(node, buf) + self.codec.try_encode(node, buf, proto_converter) } } diff --git a/ballista/core/src/execution_plans/chaos_exec.rs b/ballista/core/src/execution_plans/chaos_exec.rs index 569f4da724..3bacab34e7 100644 --- a/ballista/core/src/execution_plans/chaos_exec.rs +++ b/ballista/core/src/execution_plans/chaos_exec.rs @@ -32,6 +32,7 @@ use datafusion::common::{DataFusionError, Result, Statistics, internal_err}; use datafusion::config::ConfigOptions; use datafusion::execution::TaskContext; +use datafusion::physical_plan::StatisticsArgs; use datafusion::physical_plan::execution_plan::CardinalityEffect; use datafusion::physical_plan::metrics::MetricsSet; use datafusion::physical_plan::stream::RecordBatchStreamAdapter; @@ -213,8 +214,8 @@ impl ExecutionPlan for ChaosExec { self.input.metrics() } - fn partition_statistics(&self, partition: Option) -> Result> { - self.input.partition_statistics(partition) + fn statistics_with_args(&self, args: &StatisticsArgs) -> Result> { + args.compute_child_statistics(&self.input, args.partition()) } fn supports_limit_pushdown(&self) -> bool { diff --git a/ballista/core/src/execution_plans/shuffle_reader.rs b/ballista/core/src/execution_plans/shuffle_reader.rs index a9955ac5f5..f0a9115728 100644 --- a/ballista/core/src/execution_plans/shuffle_reader.rs +++ b/ballista/core/src/execution_plans/shuffle_reader.rs @@ -988,6 +988,7 @@ mod tests { use datafusion::datasource::memory::MemorySourceConfig; use datafusion::datasource::source::DataSourceExec; use datafusion::physical_expr::expressions::Column; + use datafusion::physical_plan::StatisticsArgs; use datafusion::physical_plan::common; use datafusion::prelude::SessionContext; use tempfile::{TempDir, tempdir}; @@ -1161,7 +1162,7 @@ mod tests { Partitioning::UnknownPartitioning(4), )?; - let stats = shuffle_reader_exec.partition_statistics(None)?; + let stats = shuffle_reader_exec.statistics_with_args(&StatisticsArgs::new())?; assert_eq!(8, *stats.num_rows.get_value().unwrap()); assert_eq!(80, *stats.total_byte_size.get_value().unwrap()); @@ -1212,7 +1213,8 @@ mod tests { Partitioning::UnknownPartitioning(4), )?; - let stats = shuffle_reader_exec.partition_statistics(Some(3))?; + let stats = shuffle_reader_exec + .statistics_with_args(&StatisticsArgs::new().with_partition(Some(3)))?; assert_eq!(2, *stats.num_rows.get_value().unwrap()); assert_eq!(20, *stats.total_byte_size.get_value().unwrap()); @@ -1264,7 +1266,8 @@ mod tests { Partitioning::UnknownPartitioning(4), )?; - let stats = shuffle_reader_exec.partition_statistics(Some(4)); + let stats = shuffle_reader_exec + .statistics_with_args(&StatisticsArgs::new().with_partition(Some(4))); assert!(stats.is_err()); Ok(()) @@ -1739,7 +1742,9 @@ mod tests { assert_eq!(reader.properties().partitioning.partition_count(), 1); assert_eq!(reader.partition[0].len(), 3); - let stats = reader.partition_statistics(Some(0)).unwrap(); + let stats = reader + .statistics_with_args(&StatisticsArgs::new().with_partition(Some(0))) + .unwrap(); assert_eq!(stats.num_rows.get_value().copied(), Some(300)); assert_eq!(stats.total_byte_size.get_value().copied(), Some(3072)); } @@ -1748,7 +1753,9 @@ mod tests { fn broadcast_reader_rejects_out_of_range_partition_index() { let schema = Arc::new(Schema::new(vec![Field::new("c", DataType::Int32, false)])); let reader = ShuffleReaderExec::try_new_broadcast(7, vec![], schema, 3).unwrap(); - let err = reader.partition_statistics(Some(1)).unwrap_err(); + let err = reader + .statistics_with_args(&StatisticsArgs::new().with_partition(Some(1))) + .unwrap_err(); let msg = err.to_string(); assert!( msg.contains("invalid partition index 1"), @@ -1835,11 +1842,13 @@ mod tests { )?; // partition[0] = upstream [0,1,2] -> 10+20+30 = 60 bytes, 1+2+3 = 6 rows - let stats0 = exec.partition_statistics(Some(0))?; + let stats0 = + exec.statistics_with_args(&StatisticsArgs::new().with_partition(Some(0)))?; assert_eq!(60, *stats0.total_byte_size.get_value().unwrap()); assert_eq!(6, *stats0.num_rows.get_value().unwrap()); // partition[1] = upstream [3,4] -> 40+50 = 90 bytes, 4+5 = 9 rows - let stats1 = exec.partition_statistics(Some(1))?; + let stats1 = + exec.statistics_with_args(&StatisticsArgs::new().with_partition(Some(1)))?; assert_eq!(90, *stats1.total_byte_size.get_value().unwrap()); assert_eq!(9, *stats1.num_rows.get_value().unwrap()); Ok(()) diff --git a/ballista/core/src/execution_plans/shuffle_writer.rs b/ballista/core/src/execution_plans/shuffle_writer.rs index 3398be2275..61980ea316 100644 --- a/ballista/core/src/execution_plans/shuffle_writer.rs +++ b/ballista/core/src/execution_plans/shuffle_writer.rs @@ -55,7 +55,7 @@ use datafusion::physical_plan::metrics::{ use datafusion::physical_plan::{ DisplayAs, DisplayFormatType, ExecutionPlan, Partitioning, PlanProperties, - SendableRecordBatchStream, Statistics, displayable, + SendableRecordBatchStream, Statistics, StatisticsArgs, displayable, }; use futures::{StreamExt, TryFutureExt, TryStreamExt}; @@ -556,8 +556,8 @@ impl ExecutionPlan for ShuffleWriterExec { Some(self.metrics.clone_inner()) } - fn partition_statistics(&self, partition: Option) -> Result> { - self.plan.partition_statistics(partition) + fn statistics_with_args(&self, args: &StatisticsArgs) -> Result> { + args.compute_child_statistics(&self.plan, args.partition()) } } diff --git a/ballista/core/src/execution_plans/sort_shuffle/writer.rs b/ballista/core/src/execution_plans/sort_shuffle/writer.rs index 632b5aca07..83c864dd72 100644 --- a/ballista/core/src/execution_plans/sort_shuffle/writer.rs +++ b/ballista/core/src/execution_plans/sort_shuffle/writer.rs @@ -59,7 +59,7 @@ use datafusion::physical_plan::repartition::REPARTITION_RANDOM_STATE; use datafusion::physical_plan::stream::RecordBatchStreamAdapter; use datafusion::physical_plan::{ DisplayAs, DisplayFormatType, ExecutionPlan, Partitioning, PlanProperties, - SendableRecordBatchStream, Statistics, displayable, + SendableRecordBatchStream, Statistics, StatisticsArgs, displayable, }; use futures::{StreamExt, TryFutureExt, TryStreamExt}; use log::debug; @@ -658,8 +658,8 @@ impl ExecutionPlan for SortShuffleWriterExec { Some(self.metrics.clone_inner()) } - fn partition_statistics(&self, partition: Option) -> Result> { - self.plan.partition_statistics(partition) + fn statistics_with_args(&self, args: &StatisticsArgs) -> Result> { + args.compute_child_statistics(&self.plan, args.partition()) } } diff --git a/ballista/core/src/planner.rs b/ballista/core/src/planner.rs index ecf41733b8..1f25bb5548 100644 --- a/ballista/core/src/planner.rs +++ b/ballista/core/src/planner.rs @@ -181,8 +181,8 @@ impl<'n> TreeNodeVisitor<'n> for LocalRun { ) -> datafusion::error::Result { match node { LogicalPlan::TableScan(TableScan { table_name, .. }) => match table_name { - datafusion::sql::TableReference::Partial { schema, .. } - | datafusion::sql::TableReference::Full { schema, .. } + datafusion::common::TableReference::Partial { schema, .. } + | datafusion::common::TableReference::Full { schema, .. } if schema.as_ref() == "information_schema" => { self.can_be_local = true; diff --git a/ballista/core/src/serde/mod.rs b/ballista/core/src/serde/mod.rs index 0206afe5e8..2786e93d22 100644 --- a/ballista/core/src/serde/mod.rs +++ b/ballista/core/src/serde/mod.rs @@ -36,7 +36,7 @@ use datafusion_proto::physical_plan::from_proto::parse_protobuf_partitioning; use datafusion_proto::physical_plan::to_proto::serialize_partitioning; use datafusion_proto::physical_plan::{ DefaultPhysicalExtensionCodec, DefaultPhysicalProtoConverter, - PhysicalPlanDecodeContext, + PhysicalPlanDecodeContext, PhysicalProtoConverterExtension, }; use datafusion_proto::protobuf::proto_error; use datafusion_proto::protobuf::{LogicalPlanNode, PhysicalPlanNode}; @@ -292,7 +292,7 @@ impl LogicalExtensionCodec for BallistaLogicalExtensionCodec { fn try_decode_table_provider( &self, buf: &[u8], - table_ref: &datafusion::sql::TableReference, + table_ref: &datafusion::common::TableReference, schema: datafusion::arrow::datatypes::SchemaRef, ctx: &TaskContext, ) -> Result> { @@ -302,7 +302,7 @@ impl LogicalExtensionCodec for BallistaLogicalExtensionCodec { fn try_encode_table_provider( &self, - table_ref: &datafusion::sql::TableReference, + table_ref: &datafusion::common::TableReference, node: Arc, buf: &mut Vec, ) -> Result<()> { @@ -367,6 +367,7 @@ impl PhysicalExtensionCodec for BallistaPhysicalExtensionCodec { buf: &[u8], inputs: &[Arc], ctx: &TaskContext, + _proto_converter: &dyn PhysicalProtoConverterExtension, ) -> Result, DataFusionError> { let ballista_plan: protobuf::BallistaPhysicalPlanNode = protobuf::BallistaPhysicalPlanNode::decode(buf).map_err(|e| { @@ -554,6 +555,7 @@ impl PhysicalExtensionCodec for BallistaPhysicalExtensionCodec { &self, node: Arc, buf: &mut Vec, + _proto_converter: &dyn PhysicalProtoConverterExtension, ) -> Result<(), DataFusionError> { if let Some(exec) = node.downcast_ref::() { // note that we use shuffle_output_partitioning() rather than output_partitioning() @@ -874,11 +876,17 @@ mod test { let codec = BallistaPhysicalExtensionCodec::default(); let mut buf: Vec = vec![]; codec - .try_encode(Arc::new(original_exec.clone()), &mut buf) + .try_encode( + Arc::new(original_exec.clone()), + &mut buf, + &DefaultPhysicalProtoConverter {}, + ) .unwrap(); let ctx = SessionContext::new().task_ctx(); - let decoded_plan = codec.try_decode(&buf, &[], &ctx).unwrap(); + let decoded_plan = codec + .try_decode(&buf, &[], &ctx, &DefaultPhysicalProtoConverter {}) + .unwrap(); let decoded_exec = decoded_plan .downcast_ref::() @@ -910,11 +918,17 @@ mod test { let codec = BallistaPhysicalExtensionCodec::default(); let mut buf: Vec = vec![]; codec - .try_encode(Arc::new(original_exec.clone()), &mut buf) + .try_encode( + Arc::new(original_exec.clone()), + &mut buf, + &DefaultPhysicalProtoConverter {}, + ) .unwrap(); let ctx = SessionContext::new().task_ctx(); - let decoded_plan = codec.try_decode(&buf, &[], &ctx).unwrap(); + let decoded_plan = codec + .try_decode(&buf, &[], &ctx, &DefaultPhysicalProtoConverter {}) + .unwrap(); let decoded_exec = decoded_plan .downcast_ref::() @@ -953,11 +967,17 @@ mod test { let codec = BallistaPhysicalExtensionCodec::default(); let mut buf: Vec = vec![]; codec - .try_encode(Arc::new(original_exec.clone()), &mut buf) + .try_encode( + Arc::new(original_exec.clone()), + &mut buf, + &DefaultPhysicalProtoConverter {}, + ) .unwrap(); let ctx = SessionContext::new().task_ctx(); - let decoded_plan = codec.try_decode(&buf, &[], &ctx).unwrap(); + let decoded_plan = codec + .try_decode(&buf, &[], &ctx, &DefaultPhysicalProtoConverter {}) + .unwrap(); let decoded_exec = decoded_plan .downcast_ref::() .expect("Expected ShuffleReaderExec"); @@ -1006,11 +1026,17 @@ mod test { let codec = BallistaPhysicalExtensionCodec::default(); let mut buf: Vec = vec![]; codec - .try_encode(Arc::new(original_exec.clone()), &mut buf) + .try_encode( + Arc::new(original_exec.clone()), + &mut buf, + &DefaultPhysicalProtoConverter {}, + ) .unwrap(); let ctx = SessionContext::new().task_ctx(); - let decoded_plan = codec.try_decode(&buf, &[], &ctx).unwrap(); + let decoded_plan = codec + .try_decode(&buf, &[], &ctx, &DefaultPhysicalProtoConverter {}) + .unwrap(); let decoded_exec = decoded_plan .downcast_ref::() .expect("Expected ShuffleReaderExec"); @@ -1051,11 +1077,17 @@ mod test { let codec = BallistaPhysicalExtensionCodec::default(); let mut buf: Vec = vec![]; codec - .try_encode(Arc::new(original_exec.clone()), &mut buf) + .try_encode( + Arc::new(original_exec.clone()), + &mut buf, + &DefaultPhysicalProtoConverter {}, + ) .unwrap(); let ctx = SessionContext::new().task_ctx(); - let decoded_plan = codec.try_decode(&buf, &[], &ctx).unwrap(); + let decoded_plan = codec + .try_decode(&buf, &[], &ctx, &DefaultPhysicalProtoConverter {}) + .unwrap(); let decoded_exec = decoded_plan .downcast_ref::() .expect("Expected UnresolvedShuffleExec"); @@ -1101,11 +1133,17 @@ mod test { let codec = BallistaPhysicalExtensionCodec::default(); let mut buf: Vec = vec![]; codec - .try_encode(Arc::new(original_exec.clone()), &mut buf) + .try_encode( + Arc::new(original_exec.clone()), + &mut buf, + &DefaultPhysicalProtoConverter {}, + ) .unwrap(); let ctx = SessionContext::new().task_ctx(); - let decoded_plan = codec.try_decode(&buf, &[], &ctx).unwrap(); + let decoded_plan = codec + .try_decode(&buf, &[], &ctx, &DefaultPhysicalProtoConverter {}) + .unwrap(); let decoded_exec = decoded_plan .downcast_ref::() .expect("Expected ShuffleReaderExec"); @@ -1197,11 +1235,17 @@ mod test { let codec = BallistaPhysicalExtensionCodec::default(); let mut buf: Vec = vec![]; codec - .try_encode(Arc::new(original_exec.clone()), &mut buf) + .try_encode( + Arc::new(original_exec.clone()), + &mut buf, + &DefaultPhysicalProtoConverter {}, + ) .unwrap(); let ctx = SessionContext::new().task_ctx(); - let decoded_plan = codec.try_decode(&buf, &[], &ctx).unwrap(); + let decoded_plan = codec + .try_decode(&buf, &[], &ctx, &DefaultPhysicalProtoConverter {}) + .unwrap(); let decoded_exec = decoded_plan .downcast_ref::() @@ -1223,11 +1267,17 @@ mod test { let codec = BallistaPhysicalExtensionCodec::default(); let mut buf: Vec = vec![]; codec - .try_encode(Arc::new(original_exec.clone()), &mut buf) + .try_encode( + Arc::new(original_exec.clone()), + &mut buf, + &DefaultPhysicalProtoConverter {}, + ) .unwrap(); let ctx = SessionContext::new().task_ctx(); - let decoded_plan = codec.try_decode(&buf, &[], &ctx).unwrap(); + let decoded_plan = codec + .try_decode(&buf, &[], &ctx, &DefaultPhysicalProtoConverter {}) + .unwrap(); let decoded_exec = decoded_plan .downcast_ref::() diff --git a/ballista/core/src/serde/scheduler/to_proto.rs b/ballista/core/src/serde/scheduler/to_proto.rs index 2f804a8ce0..8542565877 100644 --- a/ballista/core/src/serde/scheduler/to_proto.rs +++ b/ballista/core/src/serde/scheduler/to_proto.rs @@ -161,6 +161,14 @@ impl TryInto for &MetricValue { value: gauge.value() as u64, })), }), + MetricValue::PeakMemoryUsage { name, gauge } => { + Ok(protobuf::OperatorMetric { + metric: Some(operator_metric::Metric::Gauge(NamedGauge { + name: name.to_string(), + value: gauge.value() as u64, + })), + }) + } MetricValue::Time { name, time } => Ok(protobuf::OperatorMetric { metric: Some(operator_metric::Metric::Time(NamedTime { name: name.to_string(), diff --git a/ballista/executor/src/collect.rs b/ballista/executor/src/collect.rs index 31d2bc28ca..010f4fb852 100644 --- a/ballista/executor/src/collect.rs +++ b/ballista/executor/src/collect.rs @@ -27,7 +27,7 @@ use datafusion::error::DataFusionError; use datafusion::execution::context::TaskContext; use datafusion::physical_plan::{ DisplayAs, DisplayFormatType, ExecutionPlan, Partitioning, PlanProperties, - SendableRecordBatchStream, Statistics, + SendableRecordBatchStream, Statistics, StatisticsArgs, }; use datafusion::{error::Result, physical_plan::RecordBatchStream}; use futures::Stream; @@ -121,8 +121,8 @@ impl ExecutionPlan for CollectExec { })) } - fn partition_statistics(&self, partition: Option) -> Result> { - self.plan.partition_statistics(partition) + fn statistics_with_args(&self, args: &StatisticsArgs) -> Result> { + args.compute_child_statistics(&self.plan, args.partition()) } } diff --git a/ballista/scheduler/src/physical_optimizer/join_selection.rs b/ballista/scheduler/src/physical_optimizer/join_selection.rs index 20d4649283..b43117edac 100644 --- a/ballista/scheduler/src/physical_optimizer/join_selection.rs +++ b/ballista/scheduler/src/physical_optimizer/join_selection.rs @@ -34,6 +34,7 @@ use std::sync::Arc; use datafusion::common::config::ConfigOptions; use datafusion::common::error::Result; +use datafusion::physical_plan::StatisticsArgs; use datafusion::common::tree_node::{Transformed, TransformedResult, TreeNode}; use datafusion::common::{JoinSide, JoinType, internal_err}; @@ -72,8 +73,8 @@ pub(crate) fn should_swap_join_order( // Get the left and right table's total bytes // If both the left and right tables contain total_byte_size statistics, // use `total_byte_size` to determine `should_swap_join_order`, else use `num_rows` - let left_stats = left.partition_statistics(None)?; - let right_stats = right.partition_statistics(None)?; + let left_stats = left.statistics_with_args(&StatisticsArgs::new())?; + let right_stats = right.statistics_with_args(&StatisticsArgs::new())?; // First compare `total_byte_size` of left and right side, // if information in this field is insufficient fallback to the `num_rows` match ( @@ -98,7 +99,7 @@ fn supports_collect_by_thresholds( ) -> bool { // Currently we do not trust the 0 value from stats, due to stats collection might have bug // TODO check the logic in datasource::get_statistics_with_limit() - let Ok(stats) = plan.partition_statistics(None) else { + let Ok(stats) = plan.statistics_with_args(&StatisticsArgs::new()) else { return false; }; diff --git a/ballista/scheduler/src/planner.rs b/ballista/scheduler/src/planner.rs index 6538cb45bb..6586777c8a 100644 --- a/ballista/scheduler/src/planner.rs +++ b/ballista/scheduler/src/planner.rs @@ -35,7 +35,8 @@ use ballista_core::{ use datafusion::arrow::datatypes::DataType; use datafusion::config::ConfigOptions; use datafusion::physical_optimizer::PhysicalOptimizerRule; -use datafusion::physical_optimizer::enforce_sorting::EnforceSorting; +use datafusion::physical_optimizer::ensure_requirements::EnsureRequirements; +use datafusion::physical_plan::StatisticsArgs; use datafusion::physical_plan::coalesce_partitions::CoalescePartitionsExec; use datafusion::physical_plan::joins::{HashJoinExec, PartitionMode}; use datafusion::physical_plan::repartition::RepartitionExec; @@ -75,8 +76,9 @@ pub trait DistributedPlanner { pub struct DefaultDistributedPlanner { /// Counter for generating unique stage IDs. next_stage_id: usize, - /// Optimizer rule for enforcing sort requirements after stage splitting. - optimizer_enforce_sorting: EnforceSorting, + /// Optimizer rule for re-enforcing distribution and sort requirements after + /// stage splitting. + optimizer_ensure_requirements: EnsureRequirements, } impl DefaultDistributedPlanner { @@ -86,8 +88,8 @@ impl DefaultDistributedPlanner { next_stage_id: 0, // when plan is broken into stages some sorting information may get lost in the process // thus stage re-optimisation is needed to adjust sort information - optimizer_enforce_sorting: - datafusion::physical_optimizer::enforce_sorting::EnforceSorting::default(), + optimizer_ensure_requirements: + datafusion::physical_optimizer::ensure_requirements::EnsureRequirements::default(), } } } @@ -194,7 +196,7 @@ impl DefaultDistributedPlanner { if let Some(_coalesce) = execution_plan.downcast_ref::() { let input = children[0].clone(); - let input = self.optimizer_enforce_sorting.optimize(input, config)?; + let input = self.optimizer_ensure_requirements.optimize(input, config)?; let shuffle_writer = create_shuffle_writer_with_config( job_id, self.next_stage_id(), @@ -229,7 +231,8 @@ impl DefaultDistributedPlanner { match repart.properties().output_partitioning() { Partitioning::Hash(_, _) => { let input = children[0].clone(); - let input = self.optimizer_enforce_sorting.optimize(input, config)?; + let input = + self.optimizer_ensure_requirements.optimize(input, config)?; let shuffle_writer = create_shuffle_writer_with_config( job_id, @@ -322,7 +325,7 @@ impl DefaultDistributedPlanner { let right = hash_join.right(); fn under(plan: &dyn ExecutionPlan, threshold: usize) -> bool { - let Ok(stats) = plan.partition_statistics(None) else { + let Ok(stats) = plan.statistics_with_args(&StatisticsArgs::new()) else { debug!( "broadcast check: partition_statistics returned error for {}", plan.name() diff --git a/ballista/scheduler/src/state/aqe/execution_plan/dynamic_join.rs b/ballista/scheduler/src/state/aqe/execution_plan/dynamic_join.rs index c7d22ae8aa..f1ed7b3ef6 100644 --- a/ballista/scheduler/src/state/aqe/execution_plan/dynamic_join.rs +++ b/ballista/scheduler/src/state/aqe/execution_plan/dynamic_join.rs @@ -15,6 +15,7 @@ // specific language governing permissions and limitations // under the License. +use datafusion::physical_plan::StatisticsArgs; use datafusion::{ arrow::compute::SortOptions, common::{JoinType, NullEquality, Result, exec_err, internal_err}, @@ -204,8 +205,8 @@ impl DynamicJoinSelectionExec { .map(|(l, r)| (Arc::clone(l), Arc::clone(r))) .unzip(); vec![ - Distribution::HashPartitioned(left_expr), - Distribution::HashPartitioned(right_expr), + Distribution::KeyPartitioned(left_expr), + Distribution::KeyPartitioned(right_expr), ] } @@ -252,8 +253,8 @@ impl DynamicJoinSelectionExec { PartitionMode::Partitioned }; - let stats_left = self.left.partition_statistics(None)?; - let stats_right = self.right.partition_statistics(None)?; + let stats_left = self.left.statistics_with_args(&StatisticsArgs::new())?; + let stats_right = self.right.statistics_with_args(&StatisticsArgs::new())?; debug!( "to_actual_join - plan_id: {}, decision: {:?} left: ({:?} | {:?}), right: ({:?} | {:?})", @@ -345,7 +346,7 @@ impl DynamicJoinSelectionExec { ) -> bool { // Currently we do not trust the 0 value from stats, due to stats collection might have bug // TODO check the logic in datasource::get_statistics_with_limit() - let Ok(stats) = plan.partition_statistics(None) else { + let Ok(stats) = plan.statistics_with_args(&StatisticsArgs::new()) else { return false; }; diff --git a/ballista/scheduler/src/state/aqe/optimizer_rule/join_selection.rs b/ballista/scheduler/src/state/aqe/optimizer_rule/join_selection.rs index 53f44dae61..92d8c13038 100644 --- a/ballista/scheduler/src/state/aqe/optimizer_rule/join_selection.rs +++ b/ballista/scheduler/src/state/aqe/optimizer_rule/join_selection.rs @@ -16,6 +16,7 @@ // under the License. use ballista_core::config::BallistaConfig; +use datafusion::physical_plan::StatisticsArgs; use datafusion::{ catalog::memory::DataSourceExec, common::tree_node::{Transformed, TreeNode}, @@ -110,8 +111,8 @@ impl SelectJoinRule { // Get the left and right table's total bytes // If both the left and right tables contain total_byte_size statistics, // use `total_byte_size` to determine `should_swap_join_order`, else use `num_rows` - let left_stats = left.partition_statistics(None)?; - let right_stats = right.partition_statistics(None)?; + let left_stats = left.statistics_with_args(&StatisticsArgs::new())?; + let right_stats = right.statistics_with_args(&StatisticsArgs::new())?; // First compare `total_byte_size` of left and right side, // if information in this field is insufficient fallback to the `num_rows` match ( diff --git a/ballista/scheduler/src/state/aqe/optimizer_rule/propagate_empty.rs b/ballista/scheduler/src/state/aqe/optimizer_rule/propagate_empty.rs index 87d52a5bd8..2c0e049fb2 100644 --- a/ballista/scheduler/src/state/aqe/optimizer_rule/propagate_empty.rs +++ b/ballista/scheduler/src/state/aqe/optimizer_rule/propagate_empty.rs @@ -26,6 +26,7 @@ use datafusion::physical_expr::PhysicalExpr; use datafusion::physical_expr::expressions::{Column, Literal}; use datafusion::physical_optimizer::PhysicalOptimizerRule; use datafusion::physical_plan::ExecutionPlan; +use datafusion::physical_plan::StatisticsArgs; use datafusion::physical_plan::aggregates::AggregateExec; #[allow(deprecated)] use datafusion::physical_plan::coalesce_batches::CoalesceBatchesExec; @@ -181,7 +182,7 @@ impl PropagateEmptyExecRule { _ => Ok(Transformed::no(plan)), } } else if let Some(exchange) = plan.downcast_ref::() { - let stats = exchange.partition_statistics(None)?; + let stats = exchange.statistics_with_args(&StatisticsArgs::new())?; match stats.num_rows { Precision::Exact(0) => empty_exec!(plan), _ => Ok(Transformed::no(plan)), @@ -240,7 +241,7 @@ pub fn as_join(plan: &Arc) -> Option> { /// Note that when we have [Precision::Absent] it means we know nothing about the value /// Hence we can't decide whether it is empty pub fn is_guaranteed_empty(plan: &Arc) -> bool { - let Ok(stats) = plan.partition_statistics(None) else { + let Ok(stats) = plan.statistics_with_args(&StatisticsArgs::new()) else { return false; }; @@ -255,7 +256,7 @@ pub fn is_guaranteed_empty(plan: &Arc) -> bool { /// Returns true only when we have exact stats confirming the plan is non-empty. /// Precision::Absent means we know nothing — we cannot claim non-empty. pub fn is_guaranteed_non_empty(plan: &Arc) -> bool { - let Ok(stats) = plan.partition_statistics(None) else { + let Ok(stats) = plan.statistics_with_args(&StatisticsArgs::new()) else { return false; }; diff --git a/ballista/scheduler/src/state/aqe/test/mod.rs b/ballista/scheduler/src/state/aqe/test/mod.rs index c1237f216d..5ba8b9427b 100644 --- a/ballista/scheduler/src/state/aqe/test/mod.rs +++ b/ballista/scheduler/src/state/aqe/test/mod.rs @@ -120,7 +120,11 @@ pub(crate) fn mock_memory_table() -> Arc { pub(crate) fn mock_context() -> SessionContext { let config = SessionConfig::new() .with_target_partitions(2) - .with_round_robin_repartition(false); + .with_round_robin_repartition(false) + // Ballista disables dynamic filter pushdown in its session defaults + // because dynamic filters cannot cross stage boundaries; mirror that + // here so join-input swaps during AQE re-optimization are legal. + .set_bool("datafusion.optimizer.enable_dynamic_filter_pushdown", false); let state = SessionStateBuilder::new() .with_config(config) diff --git a/ballista/scheduler/src/state/aqe/test/plan_to_stages.rs b/ballista/scheduler/src/state/aqe/test/plan_to_stages.rs index b115f273c5..f36e1325ec 100644 --- a/ballista/scheduler/src/state/aqe/test/plan_to_stages.rs +++ b/ballista/scheduler/src/state/aqe/test/plan_to_stages.rs @@ -195,7 +195,7 @@ async fn should_create_initial_plan() -> datafusion::error::Result<()> { AggregateExec: mode=FinalPartitioned, gby=[c0@0 as c0], aggr=[sum(t0.c0)] RepartitionExec: partitioning=Hash([c0@0], 2), input_partitions=2 AggregateExec: mode=Partial, gby=[c0@0 as c0], aggr=[sum(t0.c0)] - HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(p2@0, c2@1)], projection=[c0@1] + HashJoinExec: mode=CollectLeft, join_type=RightSemi, on=[(p2@0, c2@1)], projection=[c0@0] CoalescePartitionsExec ProjectionExec: expr=[c@0 as p2] AggregateExec: mode=FinalPartitioned, gby=[c@0 as c], aggr=[] diff --git a/ballista/scheduler/src/state/execution_graph_dot.rs b/ballista/scheduler/src/state/execution_graph_dot.rs index eaef4dc5e9..3f92a91c45 100644 --- a/ballista/scheduler/src/state/execution_graph_dot.rs +++ b/ballista/scheduler/src/state/execution_graph_dot.rs @@ -363,6 +363,9 @@ fn format_partitioning(x: Partitioning) -> String { Partitioning::Hash(expr, n) => { format!("{} partitions, expr={}", n, format_expr_list(&expr)) } + Partitioning::Range(range) => { + format!("{} partitions, range", range.partition_count()) + } } } diff --git a/benchmarks/src/bin/tpch.rs b/benchmarks/src/bin/tpch.rs index 282eb241f2..df01159824 100644 --- a/benchmarks/src/bin/tpch.rs +++ b/benchmarks/src/bin/tpch.rs @@ -976,7 +976,7 @@ async fn get_table( path: &str, table: &str, table_format: &str, - target_partitions: usize, + _target_partitions: usize, ) -> Result> { let (format, path, extension, schema): ( Arc, @@ -1028,14 +1028,7 @@ async fn get_table( } }; - let options = ListingOptions { - format, - file_extension: extension.to_owned(), - target_partitions, - collect_stat: true, - table_partition_cols: vec![], - file_sort_order: vec![], - }; + let options = ListingOptions::new(format).with_file_extension(extension.to_owned()); let url = ListingTableUrl::parse(path)?; let config = ListingTableConfig::new(url).with_listing_options(options); From 25f9cbd278fd59fabefa792736fc7009c4b7752c Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Fri, 24 Jul 2026 12:48:55 -0600 Subject: [PATCH 02/11] chore: cargo fmt --- .../core/src/execution_plans/chaos_exec.rs | 2 +- .../src/execution_plans/shuffle_reader.rs | 27 ++++-- ballista/core/src/serde/mod.rs | 84 +++++++++++++++---- .../src/physical_optimizer/join_selection.rs | 2 +- ballista/scheduler/src/planner.rs | 6 +- .../state/aqe/execution_plan/dynamic_join.rs | 11 ++- .../aqe/optimizer_rule/join_selection.rs | 8 +- .../aqe/optimizer_rule/propagate_empty.rs | 13 ++- 8 files changed, 115 insertions(+), 38 deletions(-) diff --git a/ballista/core/src/execution_plans/chaos_exec.rs b/ballista/core/src/execution_plans/chaos_exec.rs index 50159c15e6..c56b0035c1 100644 --- a/ballista/core/src/execution_plans/chaos_exec.rs +++ b/ballista/core/src/execution_plans/chaos_exec.rs @@ -33,9 +33,9 @@ use datafusion::common::{DataFusionError, Result, Statistics, internal_err}; use datafusion::config::ConfigOptions; use datafusion::execution::TaskContext; use datafusion::physical_plan::StatisticsArgs; -use datafusion::physical_plan::statistics::ChildStats; use datafusion::physical_plan::execution_plan::CardinalityEffect; use datafusion::physical_plan::metrics::MetricsSet; +use datafusion::physical_plan::statistics::ChildStats; use datafusion::physical_plan::stream::RecordBatchStreamAdapter; use datafusion::physical_plan::{ DisplayAs, DisplayFormatType, ExecutionPlan, PlanProperties, diff --git a/ballista/core/src/execution_plans/shuffle_reader.rs b/ballista/core/src/execution_plans/shuffle_reader.rs index 717061882c..ebf33d7e80 100644 --- a/ballista/core/src/execution_plans/shuffle_reader.rs +++ b/ballista/core/src/execution_plans/shuffle_reader.rs @@ -1462,7 +1462,8 @@ mod tests { Partitioning::UnknownPartitioning(4), )?; - let stats = StatisticsContext::new().compute(&shuffle_reader_exec, &StatisticsArgs::new())?; + let stats = StatisticsContext::new() + .compute(&shuffle_reader_exec, &StatisticsArgs::new())?; assert_eq!(8, *stats.num_rows.get_value().unwrap()); assert_eq!(80, *stats.total_byte_size.get_value().unwrap()); @@ -1513,7 +1514,10 @@ mod tests { Partitioning::UnknownPartitioning(4), )?; - let stats = StatisticsContext::new().compute(&shuffle_reader_exec, &StatisticsArgs::new().with_partition(Some(3)))?; + let stats = StatisticsContext::new().compute( + &shuffle_reader_exec, + &StatisticsArgs::new().with_partition(Some(3)), + )?; assert_eq!(2, *stats.num_rows.get_value().unwrap()); assert_eq!(20, *stats.total_byte_size.get_value().unwrap()); @@ -1565,7 +1569,10 @@ mod tests { Partitioning::UnknownPartitioning(4), )?; - let stats = StatisticsContext::new().compute(&shuffle_reader_exec, &StatisticsArgs::new().with_partition(Some(4))); + let stats = StatisticsContext::new().compute( + &shuffle_reader_exec, + &StatisticsArgs::new().with_partition(Some(4)), + ); assert!(stats.is_err()); Ok(()) @@ -2220,7 +2227,8 @@ mod tests { assert_eq!(reader.properties().partitioning.partition_count(), 1); assert_eq!(reader.partition[0].len(), 3); - let stats = StatisticsContext::new().compute(&reader, &StatisticsArgs::new().with_partition(Some(0))) + let stats = StatisticsContext::new() + .compute(&reader, &StatisticsArgs::new().with_partition(Some(0))) .unwrap(); assert_eq!(stats.num_rows.get_value().copied(), Some(300)); assert_eq!(stats.total_byte_size.get_value().copied(), Some(3072)); @@ -2230,7 +2238,8 @@ mod tests { fn broadcast_reader_rejects_out_of_range_partition_index() { let schema = Arc::new(Schema::new(vec![Field::new("c", DataType::Int32, false)])); let reader = ShuffleReaderExec::try_new_broadcast(7, vec![], schema, 3).unwrap(); - let err = StatisticsContext::new().compute(&reader, &StatisticsArgs::new().with_partition(Some(1))) + let err = StatisticsContext::new() + .compute(&reader, &StatisticsArgs::new().with_partition(Some(1))) .unwrap_err(); let msg = err.to_string(); assert!( @@ -2318,13 +2327,13 @@ mod tests { )?; // partition[0] = upstream [0,1,2] -> 10+20+30 = 60 bytes, 1+2+3 = 6 rows - let stats0 = - StatisticsContext::new().compute(&exec, &StatisticsArgs::new().with_partition(Some(0)))?; + let stats0 = StatisticsContext::new() + .compute(&exec, &StatisticsArgs::new().with_partition(Some(0)))?; assert_eq!(60, *stats0.total_byte_size.get_value().unwrap()); assert_eq!(6, *stats0.num_rows.get_value().unwrap()); // partition[1] = upstream [3,4] -> 40+50 = 90 bytes, 4+5 = 9 rows - let stats1 = - StatisticsContext::new().compute(&exec, &StatisticsArgs::new().with_partition(Some(1)))?; + let stats1 = StatisticsContext::new() + .compute(&exec, &StatisticsArgs::new().with_partition(Some(1)))?; assert_eq!(90, *stats1.total_byte_size.get_value().unwrap()); assert_eq!(9, *stats1.num_rows.get_value().unwrap()); Ok(()) diff --git a/ballista/core/src/serde/mod.rs b/ballista/core/src/serde/mod.rs index 9c299df370..e19ee1c0ff 100644 --- a/ballista/core/src/serde/mod.rs +++ b/ballista/core/src/serde/mod.rs @@ -1114,10 +1114,18 @@ mod test { let codec = BallistaPhysicalExtensionCodec::default(); let mut buf: Vec = vec![]; - codec.try_encode(Arc::new(original_exec), &mut buf, &DefaultPhysicalProtoConverter {}).unwrap(); + codec + .try_encode( + Arc::new(original_exec), + &mut buf, + &DefaultPhysicalProtoConverter {}, + ) + .unwrap(); let ctx = SessionContext::new().task_ctx(); - let decoded_plan = codec.try_decode(&buf, &[input], &ctx, &DefaultPhysicalProtoConverter {}).unwrap(); + let decoded_plan = codec + .try_decode(&buf, &[input], &ctx, &DefaultPhysicalProtoConverter {}) + .unwrap(); let decoded_exec = decoded_plan .downcast_ref::() .expect("Expected SortShuffleWriterExec"); @@ -1205,10 +1213,18 @@ mod test { let codec = BallistaPhysicalExtensionCodec::default(); let mut buf: Vec = vec![]; - codec.try_encode(Arc::new(original), &mut buf, &DefaultPhysicalProtoConverter {}).unwrap(); + codec + .try_encode( + Arc::new(original), + &mut buf, + &DefaultPhysicalProtoConverter {}, + ) + .unwrap(); let ctx = SessionContext::new().task_ctx(); - let decoded = codec.try_decode(&buf, &[input], &ctx, &DefaultPhysicalProtoConverter {}).unwrap(); + let decoded = codec + .try_decode(&buf, &[input], &ctx, &DefaultPhysicalProtoConverter {}) + .unwrap(); let decoded = decoded .downcast_ref::() .expect("Expected SortShuffleWriterExec"); @@ -1612,10 +1628,18 @@ mod test { let codec = BallistaPhysicalExtensionCodec::default(); let mut buf: Vec = vec![]; - codec.try_encode(Arc::new(original), &mut buf, &DefaultPhysicalProtoConverter {}).unwrap(); + codec + .try_encode( + Arc::new(original), + &mut buf, + &DefaultPhysicalProtoConverter {}, + ) + .unwrap(); let ctx = SessionContext::new().task_ctx(); - let decoded_plan = codec.try_decode(&buf, &[input], &ctx, &DefaultPhysicalProtoConverter {}).unwrap(); + let decoded_plan = codec + .try_decode(&buf, &[input], &ctx, &DefaultPhysicalProtoConverter {}) + .unwrap(); let decoded = decoded_plan .downcast_ref::() @@ -1658,10 +1682,18 @@ mod test { let codec = BallistaPhysicalExtensionCodec::default(); let mut buf: Vec = vec![]; - codec.try_encode(Arc::new(original), &mut buf, &DefaultPhysicalProtoConverter {}).unwrap(); + codec + .try_encode( + Arc::new(original), + &mut buf, + &DefaultPhysicalProtoConverter {}, + ) + .unwrap(); let ctx = SessionContext::new().task_ctx(); - let decoded_plan = codec.try_decode(&buf, &[input], &ctx, &DefaultPhysicalProtoConverter {}).unwrap(); + let decoded_plan = codec + .try_decode(&buf, &[input], &ctx, &DefaultPhysicalProtoConverter {}) + .unwrap(); let decoded = decoded_plan .downcast_ref::() .expect("Expected RuntimeStatsExec"); @@ -1769,10 +1801,18 @@ mod test { let codec = BallistaPhysicalExtensionCodec::default(); let mut buf: Vec = vec![]; - codec.try_encode(Arc::new(original), &mut buf, &DefaultPhysicalProtoConverter {}).unwrap(); + codec + .try_encode( + Arc::new(original), + &mut buf, + &DefaultPhysicalProtoConverter {}, + ) + .unwrap(); let ctx = SessionContext::new().task_ctx(); - let decoded_plan = codec.try_decode(&buf, &[input], &ctx, &DefaultPhysicalProtoConverter {}).unwrap(); + let decoded_plan = codec + .try_decode(&buf, &[input], &ctx, &DefaultPhysicalProtoConverter {}) + .unwrap(); let decoded = decoded_plan .downcast_ref::() .expect("Expected BufferExec"); @@ -1810,10 +1850,18 @@ mod test { let codec = BallistaPhysicalExtensionCodec::default(); let mut buf: Vec = vec![]; - codec.try_encode(Arc::new(original), &mut buf, &DefaultPhysicalProtoConverter {}).unwrap(); + codec + .try_encode( + Arc::new(original), + &mut buf, + &DefaultPhysicalProtoConverter {}, + ) + .unwrap(); let ctx = SessionContext::new().task_ctx(); - let decoded_plan = codec.try_decode(&buf, &[input], &ctx, &DefaultPhysicalProtoConverter {}).unwrap(); + let decoded_plan = codec + .try_decode(&buf, &[input], &ctx, &DefaultPhysicalProtoConverter {}) + .unwrap(); let decoded = decoded_plan .downcast_ref::() @@ -1863,10 +1911,18 @@ mod test { let codec = BallistaPhysicalExtensionCodec::default(); let mut buf: Vec = vec![]; - codec.try_encode(Arc::new(original), &mut buf, &DefaultPhysicalProtoConverter {}).unwrap(); + codec + .try_encode( + Arc::new(original), + &mut buf, + &DefaultPhysicalProtoConverter {}, + ) + .unwrap(); let ctx = SessionContext::new().task_ctx(); - let decoded_plan = codec.try_decode(&buf, &[input], &ctx, &DefaultPhysicalProtoConverter {}).unwrap(); + let decoded_plan = codec + .try_decode(&buf, &[input], &ctx, &DefaultPhysicalProtoConverter {}) + .unwrap(); let decoded = decoded_plan .downcast_ref::() diff --git a/ballista/scheduler/src/physical_optimizer/join_selection.rs b/ballista/scheduler/src/physical_optimizer/join_selection.rs index c95be6e8e9..c48e0a49c4 100644 --- a/ballista/scheduler/src/physical_optimizer/join_selection.rs +++ b/ballista/scheduler/src/physical_optimizer/join_selection.rs @@ -32,10 +32,10 @@ // use std::sync::Arc; -use datafusion::physical_plan::statistics::StatisticsContext; use datafusion::common::config::ConfigOptions; use datafusion::common::error::Result; use datafusion::physical_plan::StatisticsArgs; +use datafusion::physical_plan::statistics::StatisticsContext; use datafusion::common::tree_node::{Transformed, TransformedResult, TreeNode}; use datafusion::common::{JoinSide, JoinType, internal_err}; diff --git a/ballista/scheduler/src/planner.rs b/ballista/scheduler/src/planner.rs index 8d1106cd47..7300e06de4 100644 --- a/ballista/scheduler/src/planner.rs +++ b/ballista/scheduler/src/planner.rs @@ -32,7 +32,6 @@ use ballista_core::{ }, serde::scheduler::PartitionLocation, }; -use datafusion::physical_plan::statistics::StatisticsContext; use datafusion::arrow::datatypes::DataType; use datafusion::common::tree_node::{Transformed, TreeNode}; use datafusion::config::ConfigOptions; @@ -45,6 +44,7 @@ use datafusion::physical_plan::joins::{HashJoinExec, PartitionMode}; use datafusion::physical_plan::projection::ProjectionExec; use datafusion::physical_plan::repartition::RepartitionExec; use datafusion::physical_plan::sorts::sort_preserving_merge::SortPreservingMergeExec; +use datafusion::physical_plan::statistics::StatisticsContext; use datafusion::physical_plan::{ ExecutionPlan, Partitioning, with_new_children_if_necessary, }; @@ -364,7 +364,9 @@ impl DefaultDistributedPlanner { let right = hash_join.right(); fn under(plan: &dyn ExecutionPlan, threshold: usize) -> bool { - let Ok(stats) = StatisticsContext::new().compute(plan, &StatisticsArgs::new()) else { + let Ok(stats) = + StatisticsContext::new().compute(plan, &StatisticsArgs::new()) + else { debug!( "broadcast check: partition_statistics returned error for {}", plan.name() diff --git a/ballista/scheduler/src/state/aqe/execution_plan/dynamic_join.rs b/ballista/scheduler/src/state/aqe/execution_plan/dynamic_join.rs index 482ddb620c..7b8b09cd1b 100644 --- a/ballista/scheduler/src/state/aqe/execution_plan/dynamic_join.rs +++ b/ballista/scheduler/src/state/aqe/execution_plan/dynamic_join.rs @@ -16,8 +16,8 @@ // under the License. use ballista_core::config::BallistaConfig; -use datafusion::physical_plan::statistics::StatisticsContext; use datafusion::physical_plan::StatisticsArgs; +use datafusion::physical_plan::statistics::StatisticsContext; use datafusion::{ arrow::compute::SortOptions, arrow::datatypes::{DataType, Schema}, @@ -293,8 +293,10 @@ impl DynamicJoinSelectionExec { PartitionMode::Partitioned }; - let stats_left = StatisticsContext::new().compute(self.left.as_ref(), &StatisticsArgs::new())?; - let stats_right = StatisticsContext::new().compute(self.right.as_ref(), &StatisticsArgs::new())?; + let stats_left = StatisticsContext::new() + .compute(self.left.as_ref(), &StatisticsArgs::new())?; + let stats_right = StatisticsContext::new() + .compute(self.right.as_ref(), &StatisticsArgs::new())?; let action = match (&self.selection_state, partition_mode) { (JoinInputState::Unknown, PartitionMode::CollectLeft) => self @@ -407,7 +409,8 @@ impl DynamicJoinSelectionExec { ) -> bool { // Currently we do not trust the 0 value from stats, due to stats collection might have bug // TODO check the logic in datasource::get_statistics_with_limit() - let Ok(stats) = StatisticsContext::new().compute(plan, &StatisticsArgs::new()) else { + let Ok(stats) = StatisticsContext::new().compute(plan, &StatisticsArgs::new()) + else { return false; }; diff --git a/ballista/scheduler/src/state/aqe/optimizer_rule/join_selection.rs b/ballista/scheduler/src/state/aqe/optimizer_rule/join_selection.rs index 60f589533c..6188d8dc9c 100644 --- a/ballista/scheduler/src/state/aqe/optimizer_rule/join_selection.rs +++ b/ballista/scheduler/src/state/aqe/optimizer_rule/join_selection.rs @@ -16,8 +16,8 @@ // under the License. use ballista_core::config::BallistaConfig; -use datafusion::physical_plan::statistics::StatisticsContext; use datafusion::physical_plan::StatisticsArgs; +use datafusion::physical_plan::statistics::StatisticsContext; use datafusion::{ catalog::memory::DataSourceExec, common::tree_node::{Transformed, TreeNode}, @@ -112,8 +112,10 @@ impl SelectJoinRule { // Get the left and right table's total bytes // If both the left and right tables contain total_byte_size statistics, // use `total_byte_size` to determine `should_swap_join_order`, else use `num_rows` - let left_stats = StatisticsContext::new().compute(left, &StatisticsArgs::new())?; - let right_stats = StatisticsContext::new().compute(right, &StatisticsArgs::new())?; + let left_stats = + StatisticsContext::new().compute(left, &StatisticsArgs::new())?; + let right_stats = + StatisticsContext::new().compute(right, &StatisticsArgs::new())?; // First compare `total_byte_size` of left and right side, // if information in this field is insufficient fallback to the `num_rows` match ( diff --git a/ballista/scheduler/src/state/aqe/optimizer_rule/propagate_empty.rs b/ballista/scheduler/src/state/aqe/optimizer_rule/propagate_empty.rs index bd1274cb75..6e4c223886 100644 --- a/ballista/scheduler/src/state/aqe/optimizer_rule/propagate_empty.rs +++ b/ballista/scheduler/src/state/aqe/optimizer_rule/propagate_empty.rs @@ -16,7 +16,6 @@ // under the License. use crate::state::aqe::execution_plan::ExchangeExec; -use datafusion::physical_plan::statistics::StatisticsContext; use datafusion::arrow::datatypes::SchemaRef; use datafusion::common::JoinType; use datafusion::common::ScalarValue; @@ -39,6 +38,7 @@ use datafusion::physical_plan::limit::{GlobalLimitExec, LocalLimitExec}; use datafusion::physical_plan::projection::ProjectionExec; use datafusion::physical_plan::repartition::RepartitionExec; use datafusion::physical_plan::sorts::sort_preserving_merge::SortPreservingMergeExec; +use datafusion::physical_plan::statistics::StatisticsContext; use std::sync::Arc; macro_rules! is_empty_exec { @@ -183,7 +183,8 @@ impl PropagateEmptyExecRule { _ => Ok(Transformed::no(plan)), } } else if let Some(exchange) = plan.downcast_ref::() { - let stats = StatisticsContext::new().compute(exchange, &StatisticsArgs::new())?; + let stats = + StatisticsContext::new().compute(exchange, &StatisticsArgs::new())?; match stats.num_rows { Precision::Exact(0) => empty_exec!(plan), _ => Ok(Transformed::no(plan)), @@ -242,7 +243,9 @@ pub fn as_join(plan: &Arc) -> Option> { /// Note that when we have [Precision::Absent] it means we know nothing about the value /// Hence we can't decide whether it is empty pub fn is_guaranteed_empty(plan: &Arc) -> bool { - let Ok(stats) = StatisticsContext::new().compute(plan.as_ref(), &StatisticsArgs::new()) else { + let Ok(stats) = + StatisticsContext::new().compute(plan.as_ref(), &StatisticsArgs::new()) + else { return false; }; @@ -257,7 +260,9 @@ pub fn is_guaranteed_empty(plan: &Arc) -> bool { /// Returns true only when we have exact stats confirming the plan is non-empty. /// Precision::Absent means we know nothing — we cannot claim non-empty. pub fn is_guaranteed_non_empty(plan: &Arc) -> bool { - let Ok(stats) = StatisticsContext::new().compute(plan.as_ref(), &StatisticsArgs::new()) else { + let Ok(stats) = + StatisticsContext::new().compute(plan.as_ref(), &StatisticsArgs::new()) + else { return false; }; From 6868a2b55c34c668ccf0bf86d27798695fe37e0d Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Fri, 24 Jul 2026 13:00:49 -0600 Subject: [PATCH 03/11] chore: sync vendored datafusion_common.proto and python Cargo.lock --- ballista/core/proto/datafusion_common.proto | 30 + python/Cargo.lock | 2041 ++++++++++++++----- 2 files changed, 1524 insertions(+), 547 deletions(-) diff --git a/ballista/core/proto/datafusion_common.proto b/ballista/core/proto/datafusion_common.proto index dd3588d134..7fff5b6b71 100644 --- a/ballista/core/proto/datafusion_common.proto +++ b/ballista/core/proto/datafusion_common.proto @@ -627,6 +627,10 @@ message ParquetOptions { uint64 max_predicate_cache_size = 33; } + oneof max_row_group_bytes_opt { + uint64 max_row_group_bytes = 37; + } + ParquetCdcOptions content_defined_chunking = 35; // Optional timezone applied to INT96-coerced timestamps when `coerce_int96` @@ -685,4 +689,30 @@ enum ExplainFormat { EXPLAIN_FORMAT_TREE = 1; EXPLAIN_FORMAT_PGJSON = 2; EXPLAIN_FORMAT_GRAPHVIZ = 3; +} + +// Verbosity level for `EXPLAIN ANALYZE`. Mirrors +// `datafusion_common::format::MetricType`. +enum MetricType { + METRIC_TYPE_SUMMARY = 0; + METRIC_TYPE_DEV = 1; +} + +// Category of an `EXPLAIN ANALYZE` metric. Mirrors +// `datafusion_common::format::MetricCategory`. +enum MetricCategory { + METRIC_CATEGORY_ROWS = 0; + METRIC_CATEGORY_BYTES = 1; + METRIC_CATEGORY_TIMING = 2; + METRIC_CATEGORY_UNCATEGORIZED = 3; +} + +// Wire encoding for `datafusion_common::format::ExplainAnalyzeCategories`. +// +// If `all` is true, every category is shown (the `only` list is ignored). +// If `all` is false, only the categories listed in `only` are shown — an +// empty `only` means "plan only", i.e. suppress all metrics. +message ExplainAnalyzeCategoriesNode { + bool all = 1; + repeated MetricCategory only = 2; } \ No newline at end of file diff --git a/python/Cargo.lock b/python/Cargo.lock index 25ad6095d3..18951e3e05 100644 --- a/python/Cargo.lock +++ b/python/Cargo.lock @@ -99,50 +99,104 @@ checksum = "d3fb67a6e08acf24fdeccbac2cb6ac4305825bd1f117462e0e6f2f193345ad56" [[package]] name = "arrow" -version = "58.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "378530e55cd479eda3c14eb345310799717e6f76d0c332041e8487022166b471" -dependencies = [ - "arrow-arith", - "arrow-array", - "arrow-buffer", - "arrow-cast", - "arrow-csv", - "arrow-data", - "arrow-ipc", - "arrow-json", - "arrow-ord", +version = "58.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6cfdd0833e32a9874d2b55089333ad310c0be208aafa277385ce2461dec90be3" +dependencies = [ + "arrow-arith 58.4.0", + "arrow-array 58.4.0", + "arrow-buffer 58.4.0", + "arrow-cast 58.4.0", + "arrow-csv 58.4.0", + "arrow-data 58.4.0", + "arrow-ipc 58.4.0", + "arrow-json 58.4.0", + "arrow-ord 58.4.0", "arrow-pyarrow", - "arrow-row", - "arrow-schema", - "arrow-select", - "arrow-string", + "arrow-row 58.4.0", + "arrow-schema 58.4.0", + "arrow-select 58.4.0", + "arrow-string 58.4.0", +] + +[[package]] +name = "arrow" +version = "59.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b952ca5a8046ad741b60f142d6eca4aeebcad615694202bc64c5341f23e32c5b" +dependencies = [ + "arrow-arith 59.1.0", + "arrow-array 59.1.0", + "arrow-buffer 59.1.0", + "arrow-cast 59.1.0", + "arrow-csv 59.1.0", + "arrow-data 59.1.0", + "arrow-ipc 59.1.0", + "arrow-json 59.1.0", + "arrow-ord 59.1.0", + "arrow-row 59.1.0", + "arrow-schema 59.1.0", + "arrow-select 59.1.0", + "arrow-string 59.1.0", ] [[package]] name = "arrow-arith" -version = "58.3.0" +version = "58.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0a41203398f0eaa6f7ec8e62c0da742a21abf282c148fc157f6c35c90e29981a" +dependencies = [ + "arrow-array 58.4.0", + "arrow-buffer 58.4.0", + "arrow-data 58.4.0", + "arrow-schema 58.4.0", + "chrono", + "num-traits", +] + +[[package]] +name = "arrow-arith" +version = "59.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "64a13b8d3008c4e9063c597a08f46446fe3fd5789277127672d6c0bdbb43b1ff" +dependencies = [ + "arrow-array 59.1.0", + "arrow-buffer 59.1.0", + "arrow-data 59.1.0", + "arrow-schema 59.1.0", + "chrono", + "num-traits", +] + +[[package]] +name = "arrow-array" +version = "58.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a0ab212d2c1886e802f51c5212d78ebbcbb0bec980fff9dadc1eb8d45cd0b738" +checksum = "ae33dad492b7df00a217563a7b0ef2874df68a0deea1b1a3acf628152f7f7a69" dependencies = [ - "arrow-array", - "arrow-buffer", - "arrow-data", - "arrow-schema", + "ahash", + "arrow-buffer 58.4.0", + "arrow-data 58.4.0", + "arrow-schema 58.4.0", "chrono", + "chrono-tz", + "half", + "hashbrown 0.17.1", + "num-complex", + "num-integer", "num-traits", ] [[package]] name = "arrow-array" -version = "58.3.0" +version = "59.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cfd33d3e92f207444098c75b42de99d329562be0cf686b307b097cc52b4e999e" +checksum = "9486151b2f0785bafc6fa04fc5c99fcb4495455662e58787ea32eaaed33c4192" dependencies = [ "ahash", - "arrow-buffer", - "arrow-data", - "arrow-schema", + "arrow-buffer 59.1.0", + "arrow-data 59.1.0", + "arrow-schema 59.1.0", "chrono", "chrono-tz", "half", @@ -154,13 +208,13 @@ dependencies = [ [[package]] name = "arrow-avro" -version = "58.3.0" +version = "58.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "049230728cd6e093088c8d231b4beede184e35cad7777c1505c0d5a8571f4376" +checksum = "f8108cbcae32141577bc55ab490449944c63cfc93d47384906b475d13d89d43d" dependencies = [ - "arrow-array", - "arrow-buffer", - "arrow-schema", + "arrow-array 58.4.0", + "arrow-buffer 58.4.0", + "arrow-schema 58.4.0", "bytes", "bzip2", "crc", @@ -178,9 +232,21 @@ dependencies = [ [[package]] name = "arrow-buffer" -version = "58.3.0" +version = "58.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c6cd424c2693bcdbc150d843dc9d4d137dd2de4782ce6df491ad11a3a0416c0" +checksum = "b9552f96391c005e6ab449fa941420935e7e062489b12b8b1b08879b2163f5b5" +dependencies = [ + "bytes", + "half", + "num-bigint", + "num-traits", +] + +[[package]] +name = "arrow-buffer" +version = "59.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4776577a87794bfdf0b4e90e2ea12454fa7738ea2823c4be5b9d1851da7b434" dependencies = [ "bytes", "half", @@ -190,16 +256,38 @@ dependencies = [ [[package]] name = "arrow-cast" -version = "58.3.0" +version = "58.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c5aefb56a2c02e9e2b30746241058b85f8983f0fcff2ba0c6d09006e1cded7f" +checksum = "3a8a327c9649f30d8406995f27642b68df354713cca3baaaf100f076f18d5f34" dependencies = [ - "arrow-array", - "arrow-buffer", - "arrow-data", - "arrow-ord", - "arrow-schema", - "arrow-select", + "arrow-array 58.4.0", + "arrow-buffer 58.4.0", + "arrow-data 58.4.0", + "arrow-ord 58.4.0", + "arrow-schema 58.4.0", + "arrow-select 58.4.0", + "atoi", + "base64", + "chrono", + "comfy-table", + "half", + "lexical-core", + "num-traits", + "ryu", +] + +[[package]] +name = "arrow-cast" +version = "59.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9ad451ce4f98710828a455b96991b8f031deb2e67f5fcad6773f017e4a69c3a" +dependencies = [ + "arrow-array 59.1.0", + "arrow-buffer 59.1.0", + "arrow-data 59.1.0", + "arrow-ord 59.1.0", + "arrow-schema 59.1.0", + "arrow-select 59.1.0", "atoi", "base64", "chrono", @@ -212,13 +300,28 @@ dependencies = [ [[package]] name = "arrow-csv" -version = "58.3.0" +version = "58.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e94e8cf7e517657a52b91ea1263acf38c4ca62a84655d72458a3359b12ab97de" +checksum = "af0dd6d90d1955e9f9a014c1e563ee8aeffc21909085d25623e1da44d96eca26" dependencies = [ - "arrow-array", - "arrow-cast", - "arrow-schema", + "arrow-array 58.4.0", + "arrow-cast 58.4.0", + "arrow-schema 58.4.0", + "chrono", + "csv", + "csv-core", + "regex", +] + +[[package]] +name = "arrow-csv" +version = "59.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8aa7bf96d6141a7bcca2eed57c7c9767d2a2175281857b8a7b68308992864784" +dependencies = [ + "arrow-array 59.1.0", + "arrow-cast 59.1.0", + "arrow-schema 59.1.0", "chrono", "csv", "csv-core", @@ -227,12 +330,25 @@ dependencies = [ [[package]] name = "arrow-data" -version = "58.3.0" +version = "58.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c88210023a2bfee1896af366309a3028fc3bcbd6515fa29a7990ee1baa08ee0" +checksum = "2b24852db04738907e06c04ea61e42fe7fda962a34513022dc0d0e754fb7976b" dependencies = [ - "arrow-buffer", - "arrow-schema", + "arrow-buffer 58.4.0", + "arrow-schema 58.4.0", + "half", + "num-integer", + "num-traits", +] + +[[package]] +name = "arrow-data" +version = "59.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b38fe43e2e8704360f1464e6e8cc4fc381ef02cc4fb0192afa8df1aaa0115c66" +dependencies = [ + "arrow-buffer 59.1.0", + "arrow-schema 59.1.0", "half", "num-integer", "num-traits", @@ -240,21 +356,21 @@ dependencies = [ [[package]] name = "arrow-flight" -version = "58.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28abfe8bf9f124e5fc83b334af4fa58f8d0323ad25312ccb2d1da50178415704" -dependencies = [ - "arrow-arith", - "arrow-array", - "arrow-buffer", - "arrow-cast", - "arrow-data", - "arrow-ipc", - "arrow-ord", - "arrow-row", - "arrow-schema", - "arrow-select", - "arrow-string", +version = "59.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42115e09dbb694b5955da998912121451c6910b338228cb80a5701370dba43ff" +dependencies = [ + "arrow-arith 59.1.0", + "arrow-array 59.1.0", + "arrow-buffer 59.1.0", + "arrow-cast 59.1.0", + "arrow-data 59.1.0", + "arrow-ipc 59.1.0", + "arrow-ord 59.1.0", + "arrow-row 59.1.0", + "arrow-schema 59.1.0", + "arrow-select 59.1.0", + "arrow-string 59.1.0", "base64", "bytes", "futures", @@ -268,15 +384,31 @@ dependencies = [ [[package]] name = "arrow-ipc" -version = "58.3.0" +version = "58.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29a908a11fcfb3fb2f6730f4ac15e367bc644e419155e96238f68cf3adde572b" +dependencies = [ + "arrow-array 58.4.0", + "arrow-buffer 58.4.0", + "arrow-data 58.4.0", + "arrow-schema 58.4.0", + "arrow-select 58.4.0", + "flatbuffers", + "lz4_flex", + "zstd", +] + +[[package]] +name = "arrow-ipc" +version = "59.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "238438f0834483703d88896db6fe5a7138b2230debc31b34c0336c2996e3c64f" +checksum = "29dac499fcbc6ba74ee0324057821d381929a48526a3966bd9dffb44aa06d98c" dependencies = [ - "arrow-array", - "arrow-buffer", - "arrow-data", - "arrow-schema", - "arrow-select", + "arrow-array 59.1.0", + "arrow-buffer 59.1.0", + "arrow-data 59.1.0", + "arrow-schema 59.1.0", + "arrow-select 59.1.0", "flatbuffers", "lz4_flex", "zstd", @@ -284,16 +416,16 @@ dependencies = [ [[package]] name = "arrow-json" -version = "58.3.0" +version = "58.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "205ca2119e6d679d5c133c6f30e68f027738d95ed948cf77677ea69c7800036b" +checksum = "b8a96aed3931c076adee39ec2a40d8219fc7f09e79bcdaca1df16272993e1e14" dependencies = [ - "arrow-array", - "arrow-buffer", - "arrow-cast", - "arrow-ord", - "arrow-schema", - "arrow-select", + "arrow-array 58.4.0", + "arrow-buffer 58.4.0", + "arrow-cast 58.4.0", + "arrow-ord 58.4.0", + "arrow-schema 58.4.0", + "arrow-select 58.4.0", "chrono", "half", "indexmap", @@ -307,80 +439,172 @@ dependencies = [ "simdutf8", ] +[[package]] +name = "arrow-json" +version = "59.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fe05e916ddc50f4c7a363cd69c0ef5894fcee063517e9a0b8582f0c56746af6" +dependencies = [ + "arrow-array 59.1.0", + "arrow-buffer 59.1.0", + "arrow-cast 59.1.0", + "arrow-ord 59.1.0", + "arrow-schema 59.1.0", + "arrow-select 59.1.0", + "chrono", + "half", + "indexmap", + "itoa", + "lexical-core", + "memchr", + "num-traits", + "ryu", + "serde_core", + "serde_json", + "simdutf8", +] + +[[package]] +name = "arrow-ord" +version = "58.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63a083ec750f5c043f02946b4baf05fcdbb55f4560a3277055caca5cc99f3eb0" +dependencies = [ + "arrow-array 58.4.0", + "arrow-buffer 58.4.0", + "arrow-data 58.4.0", + "arrow-schema 58.4.0", + "arrow-select 58.4.0", +] + [[package]] name = "arrow-ord" -version = "58.3.0" +version = "59.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1bffd8fd2579286a5d63bac898159873e5094a79009940bcb42bbfce4f19f1d0" +checksum = "0e13dbdc2a9c053c10c7baa6e30faee04a180aa7ce88e471835850ce37abd20b" dependencies = [ - "arrow-array", - "arrow-buffer", - "arrow-data", - "arrow-schema", - "arrow-select", + "arrow-array 59.1.0", + "arrow-buffer 59.1.0", + "arrow-data 59.1.0", + "arrow-schema 59.1.0", + "arrow-select 59.1.0", ] [[package]] name = "arrow-pyarrow" -version = "58.3.0" +version = "58.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d29abdf672a81c1aeb57fd2661457f9918964d49aed0e9f18932535f2a9e49ce" +checksum = "3ffb9be5a873590f825aef50df20e0f8dff5fd42a77058e28bbe7bd44bb53dec" dependencies = [ - "arrow-array", - "arrow-data", - "arrow-schema", + "arrow-array 58.4.0", + "arrow-data 58.4.0", + "arrow-schema 58.4.0", "pyo3", ] [[package]] name = "arrow-row" -version = "58.3.0" +version = "58.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bab5994731204603c73ba69267616c50f80780774c6bb0476f1f830625115e0c" +checksum = "514ba0ef0d4c5896202dae736251ce415abb43a950bed570fb7981b8716c0e4c" dependencies = [ - "arrow-array", - "arrow-buffer", - "arrow-data", - "arrow-schema", + "arrow-array 58.4.0", + "arrow-buffer 58.4.0", + "arrow-data 58.4.0", + "arrow-schema 58.4.0", + "half", +] + +[[package]] +name = "arrow-row" +version = "59.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4d5a1f8c733d15260b305683472ee8ad89c62cbd706703ca873b90d051b41592" +dependencies = [ + "arrow-array 59.1.0", + "arrow-buffer 59.1.0", + "arrow-data 59.1.0", + "arrow-schema 59.1.0", "half", ] [[package]] name = "arrow-schema" -version = "58.3.0" +version = "58.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f633dbfdf39c039ada1bf9e34c694816eb71fbb7dc78f613993b7245e078a1ed" +checksum = "21ca356ad6425cecb6eb7b28e4f659f1ee7880fbb1a16127de7dd62901efee9e" dependencies = [ "bitflags", "serde_core", "serde_json", ] +[[package]] +name = "arrow-schema" +version = "59.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9e4969dc350d571766247143ab36a5187d095d3d3690970408bc630d47c69e5" +dependencies = [ + "serde_core", + "serde_json", +] + +[[package]] +name = "arrow-select" +version = "58.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c58da39eb3d8350ad4a549e5c2bc49284dac554016c69829310350f1731b0aad" +dependencies = [ + "ahash", + "arrow-array 58.4.0", + "arrow-buffer 58.4.0", + "arrow-data 58.4.0", + "arrow-schema 58.4.0", + "num-traits", +] + [[package]] name = "arrow-select" -version = "58.3.0" +version = "59.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8cd065c54172ac787cf3f2f8d4107e0d3fdc26edba76fdf4f4cc170258942222" +checksum = "402770dba90865359d98d1ef92ef16e23d75c0cca9c2c880c8a05468b7743bf9" dependencies = [ "ahash", - "arrow-array", - "arrow-buffer", - "arrow-data", - "arrow-schema", + "arrow-array 59.1.0", + "arrow-buffer 59.1.0", + "arrow-data 59.1.0", + "arrow-schema 59.1.0", "num-traits", ] [[package]] name = "arrow-string" -version = "58.3.0" +version = "58.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29dd7cda3ab9692f43a2e4acc444d760cc17b12bb6d8232ddf64e9bab7c06b42" +checksum = "b6789b388467525e3271326b6b4915666ecfdf5142aef09779445c954b67543c" dependencies = [ - "arrow-array", - "arrow-buffer", - "arrow-data", - "arrow-schema", - "arrow-select", + "arrow-array 58.4.0", + "arrow-buffer 58.4.0", + "arrow-data 58.4.0", + "arrow-schema 58.4.0", + "arrow-select 58.4.0", + "memchr", + "num-traits", + "regex", + "regex-syntax", +] + +[[package]] +name = "arrow-string" +version = "59.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2b0afbb8b9016700938291123df30838b89decc3213dba00852021988b170d3" +dependencies = [ + "arrow-array 59.1.0", + "arrow-buffer 59.1.0", + "arrow-data 59.1.0", + "arrow-schema 59.1.0", + "arrow-select 59.1.0", "memchr", "num-traits", "regex", @@ -401,9 +625,9 @@ dependencies = [ [[package]] name = "async-ffi" -version = "0.5.0" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f4de21c0feef7e5a556e51af767c953f0501f7f300ba785cc99c47bdc8081a50" +checksum = "39cd9de47399986d5b216c6bef9434dfff1689ab61ba8d1e2720dc5fe5c84083" [[package]] name = "async-trait" @@ -413,7 +637,7 @@ checksum = "ae36dc4177970ef04fde5178d3e2429882def40e57a451f919c098f72baa6cec" dependencies = [ "proc-macro2", "quote", - "syn 3.0.0", + "syn 3.0.3", ] [[package]] @@ -497,7 +721,7 @@ dependencies = [ "ballista-core", "ballista-executor", "ballista-scheduler", - "datafusion", + "datafusion 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", "log", "tokio", "url", @@ -510,10 +734,10 @@ dependencies = [ "arrow-flight", "async-trait", "chrono", - "datafusion", - "datafusion-functions-aggregate-common", - "datafusion-proto", - "datafusion-proto-common", + "datafusion 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "datafusion-functions-aggregate-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "datafusion-proto 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "datafusion-proto-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", "futures", "itertools 0.15.0", "log", @@ -538,14 +762,14 @@ dependencies = [ name = "ballista-executor" version = "54.0.0" dependencies = [ - "arrow", + "arrow 59.1.0", "arrow-flight", "async-trait", "ballista-core", "bytesize", "dashmap", - "datafusion", - "datafusion-proto", + "datafusion 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "datafusion-proto 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", "futures", "libc", "log", @@ -571,8 +795,8 @@ dependencies = [ "axum", "ballista-core", "dashmap", - "datafusion", - "datafusion-proto", + "datafusion 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "datafusion-proto 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", "futures", "http", "insta", @@ -713,9 +937,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.3.0" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c89588d05638b5b4594a3348a2d6c20277e43a7f5c5202b05cc56888475a47b8" +checksum = "5add81bb678e6cb321aff7fa0dc7689ad82b112dbc032cea19f91d6b8e3582b9" dependencies = [ "find-msvc-tools", "jobserver", @@ -969,41 +1193,41 @@ dependencies = [ [[package]] name = "datafusion" -version = "54.0.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "997a31e15872606a49478e670c58302094c97cb96abb0a7d60720f8e92170040" +checksum = "754ef4e8f073922a26f5b23133b9db4829342362b09be0bc94309cf261c2f098" dependencies = [ - "arrow", - "arrow-schema", + "arrow 58.4.0", + "arrow-schema 58.4.0", "async-trait", "bzip2", "chrono", - "datafusion-catalog", - "datafusion-catalog-listing", - "datafusion-common", - "datafusion-common-runtime", - "datafusion-datasource", - "datafusion-datasource-arrow", + "datafusion-catalog 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-catalog-listing 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-common 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-common-runtime 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-datasource 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-datasource-arrow 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "datafusion-datasource-avro", - "datafusion-datasource-csv", - "datafusion-datasource-json", - "datafusion-datasource-parquet", - "datafusion-execution", - "datafusion-expr", - "datafusion-expr-common", - "datafusion-functions", - "datafusion-functions-aggregate", - "datafusion-functions-nested", - "datafusion-functions-table", - "datafusion-functions-window", - "datafusion-optimizer", - "datafusion-physical-expr", - "datafusion-physical-expr-adapter", - "datafusion-physical-expr-common", - "datafusion-physical-optimizer", - "datafusion-physical-plan", - "datafusion-session", - "datafusion-sql", + "datafusion-datasource-csv 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-datasource-json 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-datasource-parquet 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-execution 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-expr 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-expr-common 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-functions 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-functions-aggregate 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-functions-nested 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-functions-table 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-functions-window 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-optimizer 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-physical-expr 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-physical-expr-adapter 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-physical-expr-common 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-physical-optimizer 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-physical-plan 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-session 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-sql 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "flate2", "futures", "indexmap", @@ -1012,7 +1236,59 @@ dependencies = [ "log", "object_store", "parking_lot", - "parquet", + "parquet 58.4.0", + "sqlparser", + "tempfile", + "tokio", + "url", + "uuid", + "zstd", +] + +[[package]] +name = "datafusion" +version = "54.1.0" +source = "git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a#efa84e8309ced6614e62d1607899c47ed0b6b08a" +dependencies = [ + "arrow 59.1.0", + "arrow-schema 59.1.0", + "async-trait", + "bzip2", + "chrono", + "datafusion-catalog 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "datafusion-catalog-listing 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "datafusion-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "datafusion-common-runtime 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "datafusion-datasource 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "datafusion-datasource-arrow 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "datafusion-datasource-csv 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "datafusion-datasource-json 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "datafusion-datasource-parquet 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "datafusion-execution 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "datafusion-expr 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "datafusion-expr-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "datafusion-functions 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "datafusion-functions-aggregate 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "datafusion-functions-nested 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "datafusion-functions-table 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "datafusion-functions-window 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "datafusion-optimizer 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "datafusion-physical-expr 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "datafusion-physical-expr-adapter 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "datafusion-physical-expr-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "datafusion-physical-optimizer 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "datafusion-physical-plan 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "datafusion-session 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "datafusion-sql 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "flate2", + "futures", + "indexmap", + "itertools 0.15.0", + "liblzma", + "log", + "object_store", + "parking_lot", + "parquet 59.1.0", "sqlparser", "tempfile", "tokio", @@ -1023,21 +1299,21 @@ dependencies = [ [[package]] name = "datafusion-catalog" -version = "54.0.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7dd61161508f8f5fa1107774ea687bd753c22d83a32eebf963549f89de14139" +checksum = "06afd1e38dd27bbb1258685a1fc6524df6aff4e07b25b393a47de59635178d99" dependencies = [ - "arrow", + "arrow 58.4.0", "async-trait", "dashmap", - "datafusion-common", - "datafusion-common-runtime", - "datafusion-datasource", - "datafusion-execution", - "datafusion-expr", - "datafusion-physical-expr", - "datafusion-physical-plan", - "datafusion-session", + "datafusion-common 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-common-runtime 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-datasource 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-execution 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-expr 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-physical-expr 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-physical-plan 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-session 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "futures", "itertools 0.14.0", "log", @@ -1046,38 +1322,85 @@ dependencies = [ "tokio", ] +[[package]] +name = "datafusion-catalog" +version = "54.1.0" +source = "git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a#efa84e8309ced6614e62d1607899c47ed0b6b08a" +dependencies = [ + "arrow 59.1.0", + "async-trait", + "dashmap", + "datafusion-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "datafusion-common-runtime 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "datafusion-datasource 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "datafusion-execution 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "datafusion-expr 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "datafusion-physical-expr 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "datafusion-physical-plan 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "datafusion-session 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "futures", + "itertools 0.15.0", + "log", + "object_store", + "parking_lot", + "tokio", +] + [[package]] name = "datafusion-catalog-listing" -version = "54.0.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "897c70f871277f9ce99aa38347be0d679bbe3e617156c4d2a8378cec8a2a0891" +checksum = "f0668fb32c12065ec242be0e5b4bc62bd7a06a0be3ecd83791ef877e4be67e02" dependencies = [ - "arrow", + "arrow 58.4.0", "async-trait", - "datafusion-catalog", - "datafusion-common", - "datafusion-datasource", - "datafusion-execution", - "datafusion-expr", - "datafusion-physical-expr", - "datafusion-physical-expr-adapter", - "datafusion-physical-expr-common", - "datafusion-physical-plan", + "datafusion-catalog 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-common 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-datasource 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-execution 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-expr 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-physical-expr 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-physical-expr-adapter 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-physical-expr-common 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-physical-plan 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "futures", "itertools 0.14.0", "log", "object_store", ] +[[package]] +name = "datafusion-catalog-listing" +version = "54.1.0" +source = "git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a#efa84e8309ced6614e62d1607899c47ed0b6b08a" +dependencies = [ + "arrow 59.1.0", + "async-trait", + "datafusion-catalog 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "datafusion-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "datafusion-datasource 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "datafusion-execution 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "datafusion-expr 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "datafusion-physical-expr 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "datafusion-physical-expr-adapter 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "datafusion-physical-expr-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "datafusion-physical-plan 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "futures", + "itertools 0.15.0", + "log", + "object_store", + "percent-encoding", +] + [[package]] name = "datafusion-common" -version = "54.0.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "121c9ded5d87d9172319e006f2afdb9928d72dbacd6a90a458d8acb1e3b43a65" +checksum = "ca43b263cdff57042cfa8fb817fb3469f4878933380dccff25f5e793580abbf9" dependencies = [ - "arrow", - "arrow-ipc", - "arrow-schema", + "arrow 58.4.0", + "arrow-ipc 58.4.0", + "arrow-schema 58.4.0", "chrono", "foldhash 0.2.0", "half", @@ -1087,7 +1410,33 @@ dependencies = [ "libc", "log", "object_store", - "parquet", + "parquet 58.4.0", + "recursive", + "sqlparser", + "tokio", + "uuid", + "web-time", +] + +[[package]] +name = "datafusion-common" +version = "54.1.0" +source = "git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a#efa84e8309ced6614e62d1607899c47ed0b6b08a" +dependencies = [ + "arrow 59.1.0", + "arrow-ipc 59.1.0", + "arrow-schema 59.1.0", + "chrono", + "foldhash 0.2.0", + "half", + "hashbrown 0.17.1", + "indexmap", + "itertools 0.15.0", + "libc", + "log", + "num-traits", + "object_store", + "parquet 59.1.0", "recursive", "sqlparser", "tokio", @@ -1097,9 +1446,19 @@ dependencies = [ [[package]] name = "datafusion-common-runtime" -version = "54.0.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "981b9dae74f78ee3d9f714fb49b01919eab975461b56149510c3ba9ea11287d1" +checksum = "05f0ba2b864792bdca4d76c59a1de0ab6e1b61946596b9936888dbd6360035f2" +dependencies = [ + "futures", + "log", + "tokio", +] + +[[package]] +name = "datafusion-common-runtime" +version = "54.1.0" +source = "git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a#efa84e8309ced6614e62d1607899c47ed0b6b08a" dependencies = [ "futures", "log", @@ -1108,25 +1467,25 @@ dependencies = [ [[package]] name = "datafusion-datasource" -version = "54.0.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ffd7d295b2ec7c00d8a56562f41ed41062cf0af75549ed891c12a0a09eddfefe" +checksum = "b840a8bce0bcbf5afad02946d438591e7c373f7afccaf3d874c04485772514dd" dependencies = [ - "arrow", + "arrow 58.4.0", "async-compression", "async-trait", "bytes", "bzip2", "chrono", - "datafusion-common", - "datafusion-common-runtime", - "datafusion-execution", - "datafusion-expr", - "datafusion-physical-expr", - "datafusion-physical-expr-adapter", - "datafusion-physical-expr-common", - "datafusion-physical-plan", - "datafusion-session", + "datafusion-common 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-common-runtime 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-execution 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-expr 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-physical-expr 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-physical-expr-adapter 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-physical-expr-common 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-physical-plan 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-session 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "flate2", "futures", "glob", @@ -1142,66 +1501,146 @@ dependencies = [ "zstd", ] +[[package]] +name = "datafusion-datasource" +version = "54.1.0" +source = "git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a#efa84e8309ced6614e62d1607899c47ed0b6b08a" +dependencies = [ + "arrow 59.1.0", + "async-compression", + "async-trait", + "bytes", + "bzip2", + "chrono", + "datafusion-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "datafusion-common-runtime 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "datafusion-execution 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "datafusion-expr 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "datafusion-physical-expr 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "datafusion-physical-expr-adapter 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "datafusion-physical-expr-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "datafusion-physical-plan 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "datafusion-session 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "flate2", + "futures", + "glob", + "itertools 0.15.0", + "liblzma", + "log", + "object_store", + "parking_lot", + "rand 0.9.5", + "tokio", + "tokio-util", + "url", + "zstd", +] + [[package]] name = "datafusion-datasource-arrow" -version = "54.0.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "552b0b3f342f7ec41b3fbd70f6339dc82a30cfd0349e7f280e7852528085349f" +checksum = "a24cc0b9cf6e367f27f27406eff13abf48a11b72446aaa40b3105c0ded5c17d9" dependencies = [ - "arrow", - "arrow-ipc", + "arrow 58.4.0", + "arrow-ipc 58.4.0", "async-trait", "bytes", - "datafusion-common", - "datafusion-common-runtime", - "datafusion-datasource", - "datafusion-execution", - "datafusion-expr", - "datafusion-physical-expr-common", - "datafusion-physical-plan", - "datafusion-session", + "datafusion-common 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-common-runtime 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-datasource 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-execution 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-expr 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-physical-expr-common 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-physical-plan 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-session 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "futures", "itertools 0.14.0", "object_store", "tokio", ] +[[package]] +name = "datafusion-datasource-arrow" +version = "54.1.0" +source = "git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a#efa84e8309ced6614e62d1607899c47ed0b6b08a" +dependencies = [ + "arrow 59.1.0", + "arrow-ipc 59.1.0", + "async-trait", + "bytes", + "datafusion-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "datafusion-common-runtime 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "datafusion-datasource 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "datafusion-execution 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "datafusion-expr 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "datafusion-physical-expr-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "datafusion-physical-plan 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "datafusion-session 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "futures", + "itertools 0.15.0", + "object_store", + "tokio", +] + [[package]] name = "datafusion-datasource-avro" -version = "54.0.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb517d08967d536284ce70afb5fe8583133779249f2d7b90587d339741a7f195" +checksum = "f26a312bb06528c17b4bb86b798b0c4321e7656d85e0ef65c2e9adf07b0a518d" dependencies = [ - "arrow", + "arrow 58.4.0", "arrow-avro", "async-trait", "bytes", - "datafusion-common", - "datafusion-datasource", - "datafusion-physical-expr-adapter", - "datafusion-physical-plan", - "datafusion-session", + "datafusion-common 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-datasource 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-physical-expr-adapter 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-physical-plan 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-session 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "futures", "object_store", ] [[package]] name = "datafusion-datasource-csv" -version = "54.0.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68850aa426b897e879c8b87e512ea8124f1d0a2869a4e51808ddaaddf1bc0ada" +checksum = "e1abe56b2a7a2d1d6de5117dd1a203181e267f28529faa5da546947621b697d7" dependencies = [ - "arrow", + "arrow 58.4.0", "async-trait", "bytes", - "datafusion-common", - "datafusion-common-runtime", - "datafusion-datasource", - "datafusion-execution", - "datafusion-expr", - "datafusion-physical-expr-common", - "datafusion-physical-plan", - "datafusion-session", + "datafusion-common 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-common-runtime 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-datasource 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-execution 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-expr 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-physical-expr-common 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-physical-plan 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-session 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "futures", + "object_store", + "regex", + "tokio", +] + +[[package]] +name = "datafusion-datasource-csv" +version = "54.1.0" +source = "git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a#efa84e8309ced6614e62d1607899c47ed0b6b08a" +dependencies = [ + "arrow 59.1.0", + "async-trait", + "bytes", + "datafusion-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "datafusion-common-runtime 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "datafusion-datasource 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "datafusion-execution 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "datafusion-expr 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "datafusion-physical-expr-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "datafusion-physical-plan 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "datafusion-session 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", "futures", "object_store", "regex", @@ -1210,21 +1649,43 @@ dependencies = [ [[package]] name = "datafusion-datasource-json" -version = "54.0.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "402f93242ae08ef99139ee2c528a49d087efe88d5c7b2c3ff5480855a40ce54f" +checksum = "9c3e467f0611ad7bdd5aad17c63c9bb6182d04e5282e5496d897ea2b49c024ba" dependencies = [ - "arrow", + "arrow 58.4.0", "async-trait", "bytes", - "datafusion-common", - "datafusion-common-runtime", - "datafusion-datasource", - "datafusion-execution", - "datafusion-expr", - "datafusion-physical-expr-common", - "datafusion-physical-plan", - "datafusion-session", + "datafusion-common 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-common-runtime 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-datasource 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-execution 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-expr 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-physical-expr-common 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-physical-plan 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-session 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "futures", + "object_store", + "tokio", + "tokio-stream", +] + +[[package]] +name = "datafusion-datasource-json" +version = "54.1.0" +source = "git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a#efa84e8309ced6614e62d1607899c47ed0b6b08a" +dependencies = [ + "arrow 59.1.0", + "async-trait", + "bytes", + "datafusion-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "datafusion-common-runtime 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "datafusion-datasource 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "datafusion-execution 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "datafusion-expr 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "datafusion-physical-expr-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "datafusion-physical-plan 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "datafusion-session 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", "futures", "object_store", "tokio", @@ -1233,54 +1694,90 @@ dependencies = [ [[package]] name = "datafusion-datasource-parquet" -version = "54.0.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ffd2499c1bee0eeccf6a57156105700eeeb17bc701899ac719183c4e74231450" +checksum = "4cc35b92cd560082155e80d9c826929c852d3c51543f4affd3a51c464a0aab3a" dependencies = [ - "arrow", + "arrow 58.4.0", "async-trait", "bytes", - "datafusion-common", - "datafusion-common-runtime", - "datafusion-datasource", - "datafusion-execution", - "datafusion-expr", - "datafusion-functions", - "datafusion-functions-aggregate-common", - "datafusion-physical-expr", - "datafusion-physical-expr-adapter", - "datafusion-physical-expr-common", - "datafusion-physical-plan", - "datafusion-pruning", - "datafusion-session", + "datafusion-common 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-common-runtime 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-datasource 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-execution 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-expr 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-functions 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-functions-aggregate-common 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-physical-expr 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-physical-expr-adapter 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-physical-expr-common 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-physical-plan 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-pruning 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-session 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "futures", "itertools 0.14.0", "log", "object_store", "parking_lot", - "parquet", + "parquet 58.4.0", + "tokio", +] + +[[package]] +name = "datafusion-datasource-parquet" +version = "54.1.0" +source = "git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a#efa84e8309ced6614e62d1607899c47ed0b6b08a" +dependencies = [ + "arrow 59.1.0", + "arrow-schema 59.1.0", + "async-trait", + "bytes", + "datafusion-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "datafusion-common-runtime 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "datafusion-datasource 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "datafusion-execution 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "datafusion-expr 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "datafusion-functions 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "datafusion-functions-aggregate-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "datafusion-physical-expr 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "datafusion-physical-expr-adapter 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "datafusion-physical-expr-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "datafusion-physical-plan 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "datafusion-pruning 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "datafusion-session 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "futures", + "itertools 0.15.0", + "log", + "object_store", + "parking_lot", + "parquet 59.1.0", "tokio", ] [[package]] name = "datafusion-doc" -version = "54.0.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb9e7e5d11130c48c8bd4e80c79a9772dd28ce6dc330baca9246205d245b9e2e" +checksum = "d69bb69d8769e34f76839c960dbde24c1ac0c885a79b6c3c2287bdc56ec67891" + +[[package]] +name = "datafusion-doc" +version = "54.1.0" +source = "git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a#efa84e8309ced6614e62d1607899c47ed0b6b08a" [[package]] name = "datafusion-execution" -version = "54.0.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37a8643ab852eb68864e1b72ae789e8066282dce48eea6347ffb0aee33d1ccc0" +checksum = "d8eac0a09bc8d263f52025cad9e001da4d8138d633fa288edda4d06b1772eae6" dependencies = [ - "arrow", - "arrow-buffer", + "arrow 58.4.0", + "arrow-buffer 58.4.0", "async-trait", "dashmap", - "datafusion-common", - "datafusion-expr", - "datafusion-physical-expr-common", + "datafusion-common 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-expr 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-physical-expr-common 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "futures", "log", "object_store", @@ -1290,24 +1787,71 @@ dependencies = [ "url", ] +[[package]] +name = "datafusion-execution" +version = "54.1.0" +source = "git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a#efa84e8309ced6614e62d1607899c47ed0b6b08a" +dependencies = [ + "arrow 59.1.0", + "arrow-buffer 59.1.0", + "async-trait", + "bytes", + "dashmap", + "datafusion-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "datafusion-expr 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "datafusion-physical-expr-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "futures", + "log", + "object_store", + "parking_lot", + "pin-project-lite", + "rand 0.9.5", + "tempfile", + "tokio", + "tokio-util", + "url", +] + +[[package]] +name = "datafusion-expr" +version = "54.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eeb14d374767ee0fc62dc79a5ba8bcf8a63c14e993c7d992d0e63adfa23d77d3" +dependencies = [ + "arrow 58.4.0", + "arrow-schema 58.4.0", + "async-trait", + "chrono", + "datafusion-common 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-doc 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-expr-common 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-functions-aggregate-common 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-functions-window-common 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-physical-expr-common 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "indexmap", + "itertools 0.14.0", + "recursive", + "serde_json", + "sqlparser", +] + [[package]] name = "datafusion-expr" -version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6932f4d71eed9c8d9341476a2b845aadfabde5495d08dbcd8fc23881f49fa7a0" +version = "54.1.0" +source = "git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a#efa84e8309ced6614e62d1607899c47ed0b6b08a" dependencies = [ - "arrow", - "arrow-schema", + "arrow 59.1.0", + "arrow-schema 59.1.0", "async-trait", "chrono", - "datafusion-common", - "datafusion-doc", - "datafusion-expr-common", - "datafusion-functions-aggregate-common", - "datafusion-functions-window-common", - "datafusion-physical-expr-common", + "datafusion-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "datafusion-doc 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "datafusion-expr-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "datafusion-functions-aggregate-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "datafusion-functions-window-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "datafusion-physical-expr-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", "indexmap", - "itertools 0.14.0", + "itertools 0.15.0", "recursive", "serde_json", "sqlparser", @@ -1315,40 +1859,51 @@ dependencies = [ [[package]] name = "datafusion-expr-common" -version = "54.0.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0225491839a31b1f7d2cb8092c2d50792e2fe1c1724e4e6d08e011f5feaf4ed2" +checksum = "b7b19a8c95522bee8cbb313d74263b85e355d2b52f42e67ef5694bf5de9e9356" dependencies = [ - "arrow", - "datafusion-common", + "arrow 58.4.0", + "datafusion-common 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "indexmap", "itertools 0.14.0", ] +[[package]] +name = "datafusion-expr-common" +version = "54.1.0" +source = "git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a#efa84e8309ced6614e62d1607899c47ed0b6b08a" +dependencies = [ + "arrow 59.1.0", + "datafusion-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "indexmap", + "itertools 0.15.0", +] + [[package]] name = "datafusion-ffi" -version = "54.0.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5660e8fa79fd51e29ce46f3026b67317ef738ebd633e106beb1a1907a406152" +checksum = "d9a4a09f24f1387408810acca070207a9cf6c7fc2368122982a8e7671f643d1e" dependencies = [ - "arrow", - "arrow-schema", + "arrow 58.4.0", + "arrow-schema 58.4.0", "async-ffi", "async-trait", "chrono", - "datafusion-catalog", - "datafusion-common", - "datafusion-datasource", - "datafusion-execution", - "datafusion-expr", - "datafusion-functions-aggregate-common", - "datafusion-physical-expr", - "datafusion-physical-expr-common", - "datafusion-physical-optimizer", - "datafusion-physical-plan", - "datafusion-proto", - "datafusion-proto-common", - "datafusion-session", + "datafusion-catalog 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-common 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-datasource 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-execution 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-expr 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-functions-aggregate-common 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-physical-expr 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-physical-expr-common 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-physical-optimizer 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-physical-plan 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-proto 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-proto-common 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-session 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "futures", "libloading", "log", @@ -1360,24 +1915,24 @@ dependencies = [ [[package]] name = "datafusion-functions" -version = "54.0.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "14872c47bfc3d21e53ec82f57074e6987a15941c1e2f43cde4ac6ae2746634e3" +checksum = "5f64c983bbbdcb729d921a2b2ac3375598719b5cc0c30345ad664936f3176fc7" dependencies = [ - "arrow", - "arrow-buffer", + "arrow 58.4.0", + "arrow-buffer 58.4.0", "base64", "blake2", "blake3", "chrono", "chrono-tz", - "datafusion-common", - "datafusion-doc", - "datafusion-execution", - "datafusion-expr", - "datafusion-expr-common", - "datafusion-macros", - "datafusion-physical-expr-common", + "datafusion-common 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-doc 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-execution 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-expr 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-expr-common 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-macros 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-physical-expr-common 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "hex", "itertools 0.14.0", "log", @@ -1390,57 +1945,118 @@ dependencies = [ "uuid", ] +[[package]] +name = "datafusion-functions" +version = "54.1.0" +source = "git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a#efa84e8309ced6614e62d1607899c47ed0b6b08a" +dependencies = [ + "arrow 59.1.0", + "arrow-buffer 59.1.0", + "base64", + "blake2", + "blake3", + "chrono", + "chrono-tz", + "datafusion-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "datafusion-doc 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "datafusion-execution 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "datafusion-expr 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "datafusion-expr-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "datafusion-macros 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "datafusion-physical-expr-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "hex", + "itertools 0.15.0", + "log", + "md-5 0.11.0", + "memchr", + "num-traits", + "rand 0.9.5", + "regex", + "sha2", + "uuid", +] + [[package]] name = "datafusion-functions-aggregate" -version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75a2ca14e1b609be21e657e2d3130b2f446456b08393b377bb721a33952d2e09" -dependencies = [ - "arrow", - "datafusion-common", - "datafusion-doc", - "datafusion-execution", - "datafusion-expr", - "datafusion-functions-aggregate-common", - "datafusion-macros", - "datafusion-physical-expr", - "datafusion-physical-expr-common", +version = "54.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89bc17041e424a47ed062f43df24d84aab8b57c4c3221e5c1a5eef46d6c5718b" +dependencies = [ + "arrow 58.4.0", + "datafusion-common 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-doc 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-execution 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-expr 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-functions-aggregate-common 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-macros 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-physical-expr 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-physical-expr-common 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "foldhash 0.2.0", "half", "log", "num-traits", ] +[[package]] +name = "datafusion-functions-aggregate" +version = "54.1.0" +source = "git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a#efa84e8309ced6614e62d1607899c47ed0b6b08a" +dependencies = [ + "arrow 59.1.0", + "datafusion-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "datafusion-doc 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "datafusion-execution 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "datafusion-expr 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "datafusion-functions-aggregate-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "datafusion-macros 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "datafusion-physical-expr 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "datafusion-physical-expr-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "half", + "log", + "num-traits", +] + [[package]] name = "datafusion-functions-aggregate-common" -version = "54.0.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ece74ba09092d2ef9c9b54a38445450aea292a1f8b04faf531936b723a24b3c" +checksum = "97dd2a9e865c6108059f5b37b77934f84b50bfb108f837bd0e5c9536e03f0545" +dependencies = [ + "arrow 58.4.0", + "datafusion-common 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-expr-common 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-physical-expr-common 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "datafusion-functions-aggregate-common" +version = "54.1.0" +source = "git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a#efa84e8309ced6614e62d1607899c47ed0b6b08a" dependencies = [ - "arrow", - "datafusion-common", - "datafusion-expr-common", - "datafusion-physical-expr-common", + "arrow 59.1.0", + "datafusion-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "datafusion-expr-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "datafusion-physical-expr-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", ] [[package]] name = "datafusion-functions-nested" -version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f3e3f9ee8ca59bf70518802107de6f1b88a9509efdc629fadc5de9d6b2d5ef5" -dependencies = [ - "arrow", - "arrow-ord", - "datafusion-common", - "datafusion-doc", - "datafusion-execution", - "datafusion-expr", - "datafusion-expr-common", - "datafusion-functions", - "datafusion-functions-aggregate", - "datafusion-functions-aggregate-common", - "datafusion-macros", - "datafusion-physical-expr-common", +version = "54.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75f0bdfeef16d96417b9632ef855645376b242e9006a126dfd0bedfc54a93f5f" +dependencies = [ + "arrow 58.4.0", + "arrow-ord 58.4.0", + "datafusion-common 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-doc 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-execution 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-expr 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-expr-common 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-functions 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-functions-aggregate 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-functions-aggregate-common 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-macros 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-physical-expr-common 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "hashbrown 0.17.1", "itertools 0.14.0", "itoa", @@ -1448,72 +2064,146 @@ dependencies = [ "memchr", ] +[[package]] +name = "datafusion-functions-nested" +version = "54.1.0" +source = "git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a#efa84e8309ced6614e62d1607899c47ed0b6b08a" +dependencies = [ + "arrow 59.1.0", + "arrow-ord 59.1.0", + "datafusion-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "datafusion-doc 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "datafusion-execution 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "datafusion-expr 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "datafusion-expr-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "datafusion-functions 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "datafusion-functions-aggregate 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "datafusion-functions-aggregate-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "datafusion-macros 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "datafusion-physical-expr-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "hashbrown 0.17.1", + "itertools 0.15.0", + "itoa", + "log", + "memchr", +] + [[package]] name = "datafusion-functions-table" -version = "54.0.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89161dffc22cf2b50f9f4b1bee83b5221d3b4ed7c2e37fd7aa2b22a5297b3a26" +checksum = "f4e4941673c917819616877e9993da4503e4f4739812be0bc32c5356184c6383" dependencies = [ - "arrow", + "arrow 58.4.0", "async-trait", - "datafusion-catalog", - "datafusion-common", - "datafusion-expr", - "datafusion-physical-expr", - "datafusion-physical-plan", + "datafusion-catalog 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-common 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-expr 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-physical-expr 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-physical-plan 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "parking_lot", ] [[package]] -name = "datafusion-functions-window" -version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7339345b226b3874037708bf5023ba1c2de705128f8457a095aae5ae9cb9c78" +name = "datafusion-functions-table" +version = "54.1.0" +source = "git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a#efa84e8309ced6614e62d1607899c47ed0b6b08a" dependencies = [ - "arrow", - "datafusion-common", - "datafusion-doc", - "datafusion-expr", - "datafusion-functions-window-common", - "datafusion-macros", - "datafusion-physical-expr", - "datafusion-physical-expr-common", + "arrow 59.1.0", + "async-trait", + "datafusion-catalog 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "datafusion-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "datafusion-expr 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "datafusion-physical-expr 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "datafusion-physical-plan 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "parking_lot", +] + +[[package]] +name = "datafusion-functions-window" +version = "54.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12dd2e16c12b84b6f6b41b19f55b366dd1c46876bb35b86896c6349067379e8d" +dependencies = [ + "arrow 58.4.0", + "datafusion-common 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-doc 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-expr 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-functions-window-common 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-macros 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-physical-expr 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-physical-expr-common 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "log", +] + +[[package]] +name = "datafusion-functions-window" +version = "54.1.0" +source = "git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a#efa84e8309ced6614e62d1607899c47ed0b6b08a" +dependencies = [ + "arrow 59.1.0", + "datafusion-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "datafusion-doc 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "datafusion-expr 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "datafusion-functions-window-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "datafusion-macros 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "datafusion-physical-expr 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "datafusion-physical-expr-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", "log", ] [[package]] name = "datafusion-functions-window-common" -version = "54.0.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa84836dc2392df6f43d6a29d37fb56a8ebdc8b3f4e10ae8dc15861fd20278fb" +checksum = "4cdc5e4b6f8b6ef823cc1c761f85088ad4c884fe8df64df3cbcc6b2b84698441" +dependencies = [ + "datafusion-common 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-physical-expr-common 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "datafusion-functions-window-common" +version = "54.1.0" +source = "git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a#efa84e8309ced6614e62d1607899c47ed0b6b08a" dependencies = [ - "datafusion-common", - "datafusion-physical-expr-common", + "datafusion-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "datafusion-physical-expr-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", ] [[package]] name = "datafusion-macros" -version = "54.0.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "587164e03ad68732aa9e7bfe5686e3f25970d4c64fd4bd80790749840892dae5" +checksum = "1a3614234dd93578c92428cb4f408e020874f0d2b7e6c90c928d9d28b5df2ceb" +dependencies = [ + "datafusion-doc 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "datafusion-macros" +version = "54.1.0" +source = "git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a#efa84e8309ced6614e62d1607899c47ed0b6b08a" dependencies = [ - "datafusion-doc", + "datafusion-doc 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", "quote", "syn 2.0.119", ] [[package]] name = "datafusion-optimizer" -version = "54.0.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77f20e8cf9e8654d92f4c16b24c487353ee5bf153ffc12d5772cd399ab8cd281" +checksum = "a0635620b050b81bb92764e99250868f654e2cd5ad1bece413283b3f73c83179" dependencies = [ - "arrow", + "arrow 58.4.0", "chrono", - "datafusion-common", - "datafusion-expr", - "datafusion-expr-common", - "datafusion-physical-expr", + "datafusion-common 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-expr 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-expr-common 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-physical-expr 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "indexmap", "itertools 0.14.0", "log", @@ -1522,18 +2212,37 @@ dependencies = [ "regex-syntax", ] +[[package]] +name = "datafusion-optimizer" +version = "54.1.0" +source = "git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a#efa84e8309ced6614e62d1607899c47ed0b6b08a" +dependencies = [ + "arrow 59.1.0", + "chrono", + "datafusion-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "datafusion-expr 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "datafusion-expr-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "datafusion-physical-expr 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "indexmap", + "itertools 0.15.0", + "log", + "recursive", + "regex", + "regex-syntax", +] + [[package]] name = "datafusion-physical-expr" -version = "54.0.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f015a4a82f6f7ff7e1d8d4bf3870a936752fa38b17705dfcc14adef95aa8922c" +checksum = "8cabf7a86eb70b816729e33c81bf7767c936ee1226f607a114f5dac2decac8d0" dependencies = [ - "arrow", - "datafusion-common", - "datafusion-expr", - "datafusion-expr-common", - "datafusion-functions-aggregate-common", - "datafusion-physical-expr-common", + "arrow 58.4.0", + "datafusion-common 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-expr 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-expr-common 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-functions-aggregate-common 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-physical-expr-common 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "half", "hashbrown 0.17.1", "indexmap", @@ -1544,31 +2253,67 @@ dependencies = [ "tokio", ] +[[package]] +name = "datafusion-physical-expr" +version = "54.1.0" +source = "git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a#efa84e8309ced6614e62d1607899c47ed0b6b08a" +dependencies = [ + "arrow 59.1.0", + "datafusion-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "datafusion-expr 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "datafusion-expr-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "datafusion-functions-aggregate-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "datafusion-physical-expr-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "datafusion-proto-models", + "half", + "hashbrown 0.17.1", + "indexmap", + "itertools 0.15.0", + "parking_lot", + "petgraph", + "recursive", + "tokio", +] + [[package]] name = "datafusion-physical-expr-adapter" -version = "54.0.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "51e6ffff8acdfe54e0ea15ccf38115c4a9184433b0439f42907637928d00a235" +checksum = "de222e04f7e6744501555a54ab0abe26bfdfebee380af79a9bdc175704246859" dependencies = [ - "arrow", - "datafusion-common", - "datafusion-expr", - "datafusion-functions", - "datafusion-physical-expr", - "datafusion-physical-expr-common", + "arrow 58.4.0", + "datafusion-common 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-expr 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-functions 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-physical-expr 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-physical-expr-common 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "itertools 0.14.0", ] +[[package]] +name = "datafusion-physical-expr-adapter" +version = "54.1.0" +source = "git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a#efa84e8309ced6614e62d1607899c47ed0b6b08a" +dependencies = [ + "arrow 59.1.0", + "datafusion-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "datafusion-expr 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "datafusion-functions 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "datafusion-physical-expr 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "datafusion-physical-expr-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "itertools 0.15.0", +] + [[package]] name = "datafusion-physical-expr-common" -version = "54.0.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7967a3e171c6a4bf09474b3f7a14f1a3db13ed1714ba12156f33fcce2bba54e8" +checksum = "72d0d0057fc5a502d45c870cb6d47c66eb7bdd5edb1bd71ad6f3f724975ac2a8" dependencies = [ - "arrow", + "arrow 58.4.0", "chrono", - "datafusion-common", - "datafusion-expr-common", + "datafusion-common 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-expr-common 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "hashbrown 0.17.1", "indexmap", "itertools 0.14.0", @@ -1576,46 +2321,81 @@ dependencies = [ "pin-project", ] +[[package]] +name = "datafusion-physical-expr-common" +version = "54.1.0" +source = "git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a#efa84e8309ced6614e62d1607899c47ed0b6b08a" +dependencies = [ + "arrow 59.1.0", + "chrono", + "datafusion-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "datafusion-expr-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "datafusion-proto-models", + "hashbrown 0.17.1", + "indexmap", + "itertools 0.15.0", + "parking_lot", + "pin-project", +] + [[package]] name = "datafusion-physical-optimizer" -version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59ff803e2a96054cb6d83f35f9e60fd4f42eac515e1932bd1b2dbc91d5fcbf36" -dependencies = [ - "arrow", - "datafusion-common", - "datafusion-execution", - "datafusion-expr", - "datafusion-expr-common", - "datafusion-physical-expr", - "datafusion-physical-expr-common", - "datafusion-physical-plan", - "datafusion-pruning", +version = "54.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "86046eed10950c5f9aaed9acfd148e9bd2e1dfdfe4f9aef607d1447b271e4183" +dependencies = [ + "arrow 58.4.0", + "datafusion-common 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-execution 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-expr 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-expr-common 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-physical-expr 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-physical-expr-common 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-physical-plan 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-pruning 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "itertools 0.14.0", "recursive", ] +[[package]] +name = "datafusion-physical-optimizer" +version = "54.1.0" +source = "git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a#efa84e8309ced6614e62d1607899c47ed0b6b08a" +dependencies = [ + "arrow 59.1.0", + "datafusion-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "datafusion-execution 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "datafusion-expr 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "datafusion-expr-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "datafusion-physical-expr 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "datafusion-physical-expr-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "datafusion-physical-plan 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "datafusion-pruning 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "itertools 0.15.0", + "recursive", +] + [[package]] name = "datafusion-physical-plan" -version = "54.0.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "776ee54d47d15bdb126452f9ca17b03761e3b004682914beaedd3f86eb507fbc" +checksum = "9bc84da934c903407ba297971ebcc020c4c1a38aafd765d6c144c76eff3fa6a1" dependencies = [ - "arrow", - "arrow-data", - "arrow-ipc", - "arrow-ord", - "arrow-schema", + "arrow 58.4.0", + "arrow-data 58.4.0", + "arrow-ipc 58.4.0", + "arrow-ord 58.4.0", + "arrow-schema 58.4.0", "async-trait", - "datafusion-common", - "datafusion-common-runtime", - "datafusion-execution", - "datafusion-expr", - "datafusion-functions", - "datafusion-functions-aggregate-common", - "datafusion-functions-window-common", - "datafusion-physical-expr", - "datafusion-physical-expr-common", + "datafusion-common 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-common-runtime 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-execution 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-expr 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-functions 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-functions-aggregate-common 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-functions-window-common 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-physical-expr 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-physical-expr-common 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "futures", "half", "hashbrown 0.17.1", @@ -1628,57 +2408,154 @@ dependencies = [ "tokio", ] +[[package]] +name = "datafusion-physical-plan" +version = "54.1.0" +source = "git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a#efa84e8309ced6614e62d1607899c47ed0b6b08a" +dependencies = [ + "arrow 59.1.0", + "arrow-data 59.1.0", + "arrow-ipc 59.1.0", + "arrow-ord 59.1.0", + "arrow-schema 59.1.0", + "async-trait", + "bytes", + "datafusion-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "datafusion-common-runtime 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "datafusion-execution 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "datafusion-expr 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "datafusion-functions 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "datafusion-functions-aggregate-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "datafusion-functions-window-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "datafusion-physical-expr 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "datafusion-physical-expr-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "datafusion-proto-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "datafusion-proto-models", + "futures", + "half", + "hashbrown 0.17.1", + "indexmap", + "itertools 0.15.0", + "log", + "num-traits", + "parking_lot", + "pin-project-lite", + "serde_json", + "tokio", +] + [[package]] name = "datafusion-proto" -version = "54.0.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9dd15a1ba5d3af93808241065c6c44dbca8296a189845e8a587c45c07bf0ffae" +checksum = "67791dcfacd142a9d95f8f73c2a161094cc936d231d80c235f5017dacf24e84e" +dependencies = [ + "arrow 58.4.0", + "chrono", + "datafusion-catalog 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-catalog-listing 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-common 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-datasource 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-datasource-arrow 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-datasource-csv 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-datasource-json 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-datasource-parquet 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-execution 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-expr 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-functions-table 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-physical-expr 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-physical-expr-common 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-physical-plan 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-proto-common 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "object_store", + "prost", +] + +[[package]] +name = "datafusion-proto" +version = "54.1.0" +source = "git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a#efa84e8309ced6614e62d1607899c47ed0b6b08a" dependencies = [ - "arrow", + "arrow 59.1.0", "chrono", - "datafusion-catalog", - "datafusion-catalog-listing", - "datafusion-common", - "datafusion-datasource", - "datafusion-datasource-arrow", - "datafusion-datasource-csv", - "datafusion-datasource-json", - "datafusion-datasource-parquet", - "datafusion-execution", - "datafusion-expr", - "datafusion-functions-table", - "datafusion-physical-expr", - "datafusion-physical-expr-common", - "datafusion-physical-plan", - "datafusion-proto-common", + "datafusion-catalog 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "datafusion-catalog-listing 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "datafusion-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "datafusion-datasource 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "datafusion-datasource-arrow 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "datafusion-datasource-csv 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "datafusion-datasource-json 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "datafusion-datasource-parquet 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "datafusion-execution 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "datafusion-expr 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "datafusion-functions-table 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "datafusion-physical-expr 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "datafusion-physical-expr-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "datafusion-physical-plan 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "datafusion-proto-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "datafusion-proto-models", "object_store", "prost", ] [[package]] name = "datafusion-proto-common" -version = "54.0.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90042982cf9462eb06a0b81f92efa4188dae871e7ea3ab8dc61aa9c9349b2530" +checksum = "b8cd9e80d637891645d074db0f6c650b591117367247deb313fdfb78dff559cb" +dependencies = [ + "arrow 58.4.0", + "datafusion-common 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "prost", +] + +[[package]] +name = "datafusion-proto-common" +version = "54.1.0" +source = "git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a#efa84e8309ced6614e62d1607899c47ed0b6b08a" dependencies = [ - "arrow", - "datafusion-common", + "arrow 59.1.0", + "datafusion-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "prost", +] + +[[package]] +name = "datafusion-proto-models" +version = "54.1.0" +source = "git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a#efa84e8309ced6614e62d1607899c47ed0b6b08a" +dependencies = [ + "datafusion-proto-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", "prost", ] [[package]] name = "datafusion-pruning" -version = "54.0.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d5fb9e5774660aa69c3ba93c610f175f75b65cb8c3776edb3626de8f3a4f4ee3" +checksum = "eb63eeac6de19be40f487b65dd84e546195f783c5a9928618e0c4f2a3569b0d7" dependencies = [ - "arrow", - "datafusion-common", - "datafusion-datasource", - "datafusion-expr-common", - "datafusion-physical-expr", - "datafusion-physical-expr-common", - "datafusion-physical-plan", + "arrow 58.4.0", + "datafusion-common 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-datasource 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-expr-common 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-physical-expr 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-physical-expr-common 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-physical-plan 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "log", +] + +[[package]] +name = "datafusion-pruning" +version = "54.1.0" +source = "git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a#efa84e8309ced6614e62d1607899c47ed0b6b08a" +dependencies = [ + "arrow 59.1.0", + "datafusion-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "datafusion-datasource 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "datafusion-expr-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "datafusion-physical-expr 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "datafusion-physical-expr-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "datafusion-physical-plan 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", "log", ] @@ -1688,14 +2565,14 @@ version = "54.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4b5da6aab44da83d488e6fb05855f5cd4480c7ab6b44f164166b495cba9fbbb8" dependencies = [ - "arrow", - "arrow-select", + "arrow 58.4.0", + "arrow-select 58.4.0", "async-trait", "chrono", "cstr", - "datafusion", + "datafusion 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "datafusion-ffi", - "datafusion-proto", + "datafusion-proto 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "datafusion-python-util", "datafusion-spark", "futures", @@ -1720,10 +2597,10 @@ version = "54.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ba491c15adfebc9ae53aa11f65231814b5199627e9815e2fb62d2f81f4a17fb2" dependencies = [ - "arrow", - "datafusion", + "arrow 58.4.0", + "datafusion 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "datafusion-ffi", - "datafusion-proto", + "datafusion-proto 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "prost", "pyo3", "tokio", @@ -1731,36 +2608,50 @@ dependencies = [ [[package]] name = "datafusion-session" -version = "54.0.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "15ce715fa2a61f4623cc234bcc14a3ef6a91f189128d5b14b468a6a17cdfc417" +checksum = "5f961d209177f91bd014db5cbb2c33b7d28a2597b9003e77f17aeb712964315a" +dependencies = [ + "async-trait", + "datafusion-common 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-execution 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-expr 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-physical-plan 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "parking_lot", +] + +[[package]] +name = "datafusion-session" +version = "54.1.0" +source = "git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a#efa84e8309ced6614e62d1607899c47ed0b6b08a" dependencies = [ + "arrow-schema 59.1.0", "async-trait", - "datafusion-common", - "datafusion-execution", - "datafusion-expr", - "datafusion-physical-plan", + "datafusion-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "datafusion-execution 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "datafusion-expr 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "datafusion-physical-plan 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", "parking_lot", ] [[package]] name = "datafusion-spark" -version = "54.0.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "390bb0bf37cb2b95ffd65eacd66f60df50793d1f94097799e416f39477a51957" +checksum = "a1ccd16a6949503e56c084df1b90c8889db826ec9347d2f0f51a7837d6fa011e" dependencies = [ - "arrow", + "arrow 58.4.0", "bigdecimal", "chrono", "crc32fast", - "datafusion-catalog", - "datafusion-common", - "datafusion-execution", - "datafusion-expr", - "datafusion-functions", - "datafusion-functions-aggregate", - "datafusion-functions-aggregate-common", - "datafusion-functions-nested", + "datafusion-catalog 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-common 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-execution 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-expr 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-functions 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-functions-aggregate 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-functions-aggregate-common 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-functions-nested 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "log", "num-traits", "percent-encoding", @@ -1774,21 +2665,40 @@ dependencies = [ [[package]] name = "datafusion-sql" -version = "54.0.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6094ad36a3ed6d7ac87b20b479b2d0b118250f66cf997603829fdc65b44a7099" +checksum = "b1d71cb454da682b2af7488e1fc1ddd72ee1b28f19297b8ccad73f0a21ee9a69" +dependencies = [ + "arrow 58.4.0", + "bigdecimal", + "chrono", + "datafusion-common 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-expr 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-functions-nested 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "indexmap", + "log", + "recursive", + "regex", + "sqlparser", +] + +[[package]] +name = "datafusion-sql" +version = "54.1.0" +source = "git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a#efa84e8309ced6614e62d1607899c47ed0b6b08a" dependencies = [ - "arrow", + "arrow 59.1.0", "bigdecimal", "chrono", - "datafusion-common", - "datafusion-expr", - "datafusion-functions-nested", + "datafusion-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "datafusion-expr 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "datafusion-functions-nested 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", "indexmap", "log", "recursive", "regex", "sqlparser", + "stacker", ] [[package]] @@ -1826,9 +2736,9 @@ dependencies = [ [[package]] name = "either" -version = "1.16.0" +version = "1.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91622ff5e7162018101f2fea40d6ebf4a78bbe5a49736a2020649edf9693679e" +checksum = "9e5e8f6c15a24b9a3ee5efec809ccd006d3b30e8b3bb63c39af737c7f87daa1d" [[package]] name = "encode_unicode" @@ -1849,14 +2759,14 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] name = "fastrand" -version = "2.4.1" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f1f227452a390804cdb637b74a86990f2a7d7ba4b7d5693aac9b4dd6defd8d6" +checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" [[package]] name = "find-msvc-tools" @@ -2057,9 +2967,9 @@ dependencies = [ [[package]] name = "glob" -version = "0.3.3" +version = "0.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0cc23270f6e1808e30a928bdc84dea0b9b4136a8bc82338574f23baf47bbd280" +checksum = "e4eba85ea1d0a966a983acd07deee566e67395d2d96b6fb39e62b5a833f1eb0b" [[package]] name = "h2" @@ -2192,9 +3102,9 @@ dependencies = [ [[package]] name = "hyper" -version = "1.10.1" +version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "55281c53a1894c864990125767da440a4e630446785086f52523b20033b74498" +checksum = "d22053281f852e11534f5198498373cbb59295120a20771d90f7ed1897490a72" dependencies = [ "atomic-waker", "bytes", @@ -2535,9 +3445,9 @@ checksum = "34b357333733e8260735ba5894eb928c02ecc69c78715f01a8019e7fa7f2db4c" [[package]] name = "libc" -version = "0.2.186" +version = "0.2.189" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" [[package]] name = "libloading" @@ -2862,17 +3772,17 @@ dependencies = [ [[package]] name = "parquet" -version = "58.3.0" +version = "58.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5dafa7d01085b62a47dd0c1829550a0a36710ea9c4fe358a05a85477cec8a908" +checksum = "d298093b2dec60289dce0684c986d0f7679e9dd15771c2c65406e1aaf604a704" dependencies = [ "ahash", - "arrow-array", - "arrow-buffer", - "arrow-data", - "arrow-ipc", - "arrow-schema", - "arrow-select", + "arrow-array 58.4.0", + "arrow-buffer 58.4.0", + "arrow-data 58.4.0", + "arrow-ipc 58.4.0", + "arrow-schema 58.4.0", + "arrow-select 58.4.0", "base64", "brotli", "bytes", @@ -2896,6 +3806,41 @@ dependencies = [ "zstd", ] +[[package]] +name = "parquet" +version = "59.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5302d4da74d6596a1f11f9928767995b53bca657cbeea1e4e8c5074f8a1157dd" +dependencies = [ + "ahash", + "arrow-array 59.1.0", + "arrow-buffer 59.1.0", + "arrow-data 59.1.0", + "arrow-ipc 59.1.0", + "arrow-schema 59.1.0", + "arrow-select 59.1.0", + "base64", + "brotli", + "bytes", + "chrono", + "flate2", + "futures", + "half", + "hashbrown 0.17.1", + "lz4_flex", + "num-bigint", + "num-integer", + "num-traits", + "object_store", + "paste", + "seq-macro", + "simdutf8", + "snap", + "tokio", + "twox-hash", + "zstd", +] + [[package]] name = "paste" version = "1.0.15" @@ -3015,9 +3960,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.106" +version = "1.0.107" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" dependencies = [ "unicode-ident", ] @@ -3114,8 +4059,8 @@ dependencies = [ "ballista-core", "ballista-executor", "ballista-scheduler", - "datafusion", - "datafusion-proto", + "datafusion 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-proto 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "datafusion-python", "log", "pyo3", @@ -3290,14 +4235,14 @@ dependencies = [ "once_cell", "socket2", "tracing", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] name = "quote" -version = "1.0.46" +version = "1.0.47" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dfbc457d0c7a0759a614551b11a6409e5951f6c7537be1f1b7682b9ae9230368" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" dependencies = [ "proc-macro2", ] @@ -3508,7 +4453,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -3539,9 +4484,9 @@ dependencies = [ [[package]] name = "rustls-pki-types" -version = "1.15.0" +version = "1.15.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "764899a24af3980067ee14bc143654f297b22eaebfe3c7b6b211920a5a59b046" +checksum = "2f4925028c7eb5d1fcdaf196971378ed9d2c1c4efc7dc5d011256f76c99c0a96" dependencies = [ "web-time", "zeroize", @@ -3656,15 +4601,16 @@ checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" dependencies = [ "proc-macro2", "quote", - "syn 3.0.0", + "syn 3.0.3", ] [[package]] name = "serde_json" -version = "1.0.150" +version = "1.0.151" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8014e44b4736ed0538adeecded0fce2a272f22dc9578a7eb6b2d9993c74cfb9" +checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" dependencies = [ + "indexmap", "itoa", "memchr", "serde", @@ -3815,9 +4761,9 @@ dependencies = [ [[package]] name = "stabby" -version = "72.1.8" +version = "72.1.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7b834ec7ced12095fea1e4b07dcb7e8cf2b59b18afa3eac52494d835965a5ec" +checksum = "3d53d2428934c46277fafd2d41e39357595aa1e47954c75db2b14ed90632f3cc" dependencies = [ "rustversion", "stabby-abi", @@ -3825,9 +4771,9 @@ dependencies = [ [[package]] name = "stabby-abi" -version = "72.1.8" +version = "72.1.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff1a4f477858a5bdf927c9fab7f579899de9b13e39f8b3b3b300c89fbab632f4" +checksum = "f375eae680bb54203ee5e47d4cd2ae7b79c0a79ed90919279f38f500ad53f190" dependencies = [ "rustc_version", "rustversion", @@ -3837,9 +4783,9 @@ dependencies = [ [[package]] name = "stabby-macros" -version = "72.1.8" +version = "72.1.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b31c4b2434980b67ad83f300a58088ba14d59454dcd79ba3d87419bbd924d31e" +checksum = "ea664671a576c5f7e32fee291ac123d82af5e92b0689beb3555347c00c76eef1" dependencies = [ "proc-macro-crate", "proc-macro2", @@ -3897,9 +4843,9 @@ dependencies = [ [[package]] name = "syn" -version = "3.0.0" +version = "3.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2fac314a64dc9a36e61a9eb4261a5e9bbfbc922b27e518af97bc32b926cf967" +checksum = "53e9bae58849f64dfa4f5d5ae372c8341f7305f82a3868709269343628b659a3" dependencies = [ "proc-macro2", "quote", @@ -3956,7 +4902,7 @@ dependencies = [ "getrandom 0.4.3", "once_cell", "rustix", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -3976,7 +4922,7 @@ checksum = "43cbfe0cf76104d42a574802844187e84a305e531ed54455f11fbde0f10541cd" dependencies = [ "proc-macro2", "quote", - "syn 3.0.0", + "syn 3.0.3", ] [[package]] @@ -4076,13 +5022,14 @@ dependencies = [ [[package]] name = "tokio-util" -version = "0.7.18" +version = "0.7.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ae9cec805b01e8fc3fd2fe289f89149a9b66dd16786abd8b19cfa7b48cb0098" +checksum = "494815d09bf52b5548659851081238f0ca39ff638363907596da739561c62c52" dependencies = [ "bytes", "futures-core", "futures-sink", + "libc", "pin-project-lite", "tokio", ] @@ -4289,11 +5236,11 @@ checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" [[package]] name = "twox-hash" -version = "2.1.2" +version = "2.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ea3136b675547379c4bd395ca6b938e5ad3c3d20fad76e7fe85f9e0d011419c" +checksum = "8464ec13c3691491391d9fce00f6416c9a48e46972f72d7865688be2080192c9" dependencies = [ - "rand 0.9.5", + "rand 0.10.2", ] [[package]] @@ -4511,7 +5458,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -4758,18 +5705,18 @@ dependencies = [ [[package]] name = "zerocopy" -version = "0.8.54" +version = "0.8.55" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b7cbbc0a705a0fd05cc3676525980d2bf5a9bc4adac6d6475209a7887cf59d19" +checksum = "b5a105cd7b140f6eeec8acff2ea38135d3cab283ada58540f629fe51e46696eb" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.8.54" +version = "0.8.55" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2e817b7b52d0c7358d3246da9d69935ebb18116b2b102b4230dac079b4862f5" +checksum = "0fe976fb70c78cd64cccfe3a6fc142244e8a77b70959b30faf9d0ac37ee228eb" dependencies = [ "proc-macro2", "quote", From b1b5f48da189f59888a90bbfea81c8f2b948a529 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Fri, 24 Jul 2026 17:13:14 -0600 Subject: [PATCH 04/11] test: regenerate TPC-H plan-stability golden files for DataFusion main --- .../tests/tpch_plan_stability/approved/q1.txt | 10 +++++----- .../tests/tpch_plan_stability/approved/q10.txt | 8 ++++---- .../tests/tpch_plan_stability/approved/q12.txt | 4 ++-- .../tests/tpch_plan_stability/approved/q13.txt | 4 ++-- .../tests/tpch_plan_stability/approved/q14.txt | 6 +++--- .../tests/tpch_plan_stability/approved/q15.txt | 12 ++++++------ .../tests/tpch_plan_stability/approved/q16.txt | 6 +++--- .../tests/tpch_plan_stability/approved/q17.txt | 4 ++-- .../tests/tpch_plan_stability/approved/q18.txt | 2 +- .../tests/tpch_plan_stability/approved/q19.txt | 8 ++++---- .../tests/tpch_plan_stability/approved/q2.txt | 4 ++-- .../tests/tpch_plan_stability/approved/q20.txt | 2 +- .../tests/tpch_plan_stability/approved/q21.txt | 4 ++-- .../tests/tpch_plan_stability/approved/q22.txt | 6 +++--- .../tests/tpch_plan_stability/approved/q3.txt | 6 +++--- .../tests/tpch_plan_stability/approved/q4.txt | 4 ++-- .../tests/tpch_plan_stability/approved/q5.txt | 8 ++++---- .../tests/tpch_plan_stability/approved/q6.txt | 2 +- .../tests/tpch_plan_stability/approved/q7.txt | 6 +++--- .../tests/tpch_plan_stability/approved/q8.txt | 6 +++--- .../tests/tpch_plan_stability/approved/q9.txt | 6 +++--- 21 files changed, 59 insertions(+), 59 deletions(-) diff --git a/ballista/scheduler/tests/tpch_plan_stability/approved/q1.txt b/ballista/scheduler/tests/tpch_plan_stability/approved/q1.txt index b01d7f2aff..29efbf3766 100644 --- a/ballista/scheduler/tests/tpch_plan_stability/approved/q1.txt +++ b/ballista/scheduler/tests/tpch_plan_stability/approved/q1.txt @@ -1,15 +1,15 @@ === Stage 1 === SortShuffleWriterExec: partitioning=Hash([l_returnflag@0, l_linestatus@1], 16) - AggregateExec: mode=Partial, gby=[l_returnflag@5 as l_returnflag, l_linestatus@6 as l_linestatus], aggr=[sum(lineitem.l_quantity), sum(lineitem.l_extendedprice), sum(__common_expr_1) as sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount), sum(__common_expr_1 * Some(1),20,0 + lineitem.l_tax) as sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount * Int64(1) + lineitem.l_tax), avg(lineitem.l_quantity), avg(lineitem.l_extendedprice), avg(lineitem.l_discount), count(Int64(1))] - ProjectionExec: expr=[l_extendedprice@0 * (Some(1),20,0 - l_discount@1) as __common_expr_1, l_quantity@2 as l_quantity, l_extendedprice@0 as l_extendedprice, l_discount@1 as l_discount, l_tax@3 as l_tax, l_returnflag@4 as l_returnflag, l_linestatus@5 as l_linestatus] + AggregateExec: mode=Partial, gby=[l_returnflag@5 as l_returnflag, l_linestatus@6 as l_linestatus], aggr=[sum(lineitem.l_quantity), sum(lineitem.l_extendedprice), sum(__common_expr_1) as sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount), sum(__common_expr_1 * 1 + lineitem.l_tax) as sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount * Int64(1) + lineitem.l_tax), avg(lineitem.l_quantity), avg(lineitem.l_extendedprice), avg(lineitem.l_discount), count(Int64(1))] + ProjectionExec: expr=[l_extendedprice@0 * (1 - l_discount@1) as __common_expr_1, l_quantity@2 as l_quantity, l_extendedprice@0 as l_extendedprice, l_discount@1 as l_discount, l_tax@3 as l_tax, l_returnflag@4 as l_returnflag, l_linestatus@5 as l_linestatus] FilterExec: l_shipdate@6 <= 1998-09-02, projection=[l_extendedprice@1, l_discount@2, l_quantity@0, l_tax@3, l_returnflag@4, l_linestatus@5] StatsExec: rows=600037902 === Stage 2 === ShuffleWriterExec: partitioning: None - SortExec: expr=[l_returnflag@0 ASC NULLS LAST, l_linestatus@1 ASC NULLS LAST], preserve_partitioning=[true] - ProjectionExec: expr=[l_returnflag@0 as l_returnflag, l_linestatus@1 as l_linestatus, sum(lineitem.l_quantity)@2 as sum_qty, sum(lineitem.l_extendedprice)@3 as sum_base_price, sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)@4 as sum_disc_price, sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount * Int64(1) + lineitem.l_tax)@5 as sum_charge, avg(lineitem.l_quantity)@6 as avg_qty, avg(lineitem.l_extendedprice)@7 as avg_price, avg(lineitem.l_discount)@8 as avg_disc, count(Int64(1))@9 as count_order] - AggregateExec: mode=FinalPartitioned, gby=[l_returnflag@0 as l_returnflag, l_linestatus@1 as l_linestatus], aggr=[sum(lineitem.l_quantity), sum(lineitem.l_extendedprice), sum(__common_expr_1) as sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount), sum(__common_expr_1 * Some(1),20,0 + lineitem.l_tax) as sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount * Int64(1) + lineitem.l_tax), avg(lineitem.l_quantity), avg(lineitem.l_extendedprice), avg(lineitem.l_discount), count(Int64(1))] + ProjectionExec: expr=[l_returnflag@0 as l_returnflag, l_linestatus@1 as l_linestatus, sum(lineitem.l_quantity)@2 as sum_qty, sum(lineitem.l_extendedprice)@3 as sum_base_price, sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)@4 as sum_disc_price, sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount * Int64(1) + lineitem.l_tax)@5 as sum_charge, avg(lineitem.l_quantity)@6 as avg_qty, avg(lineitem.l_extendedprice)@7 as avg_price, avg(lineitem.l_discount)@8 as avg_disc, count(Int64(1))@9 as count_order] + SortExec: expr=[l_returnflag@0 ASC NULLS LAST, l_linestatus@1 ASC NULLS LAST], preserve_partitioning=[true] + AggregateExec: mode=FinalPartitioned, gby=[l_returnflag@0 as l_returnflag, l_linestatus@1 as l_linestatus], aggr=[sum(lineitem.l_quantity), sum(lineitem.l_extendedprice), sum(__common_expr_1) as sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount), sum(__common_expr_1 * 1 + lineitem.l_tax) as sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount * Int64(1) + lineitem.l_tax), avg(lineitem.l_quantity), avg(lineitem.l_extendedprice), avg(lineitem.l_discount), count(Int64(1))] UnresolvedShuffleExec: stage=1, partitioning: Hash([l_returnflag@0, l_linestatus@1], 16) === Stage 3 === diff --git a/ballista/scheduler/tests/tpch_plan_stability/approved/q10.txt b/ballista/scheduler/tests/tpch_plan_stability/approved/q10.txt index 7b12a86d23..b59c2f0c2f 100644 --- a/ballista/scheduler/tests/tpch_plan_stability/approved/q10.txt +++ b/ballista/scheduler/tests/tpch_plan_stability/approved/q10.txt @@ -36,7 +36,7 @@ SortShuffleWriterExec: partitioning=Hash([n_nationkey@0], 16) === Stage 7 === SortShuffleWriterExec: partitioning=Hash([c_custkey@0, c_name@1, c_acctbal@2, c_phone@3, n_name@4, c_address@5, c_comment@6], 16) - AggregateExec: mode=Partial, gby=[c_custkey@0 as c_custkey, c_name@1 as c_name, c_acctbal@4 as c_acctbal, c_phone@3 as c_phone, n_name@8 as n_name, c_address@2 as c_address, c_comment@5 as c_comment], aggr=[sum(lineitem.l_extendedprice * Some(1),20,0 - lineitem.l_discount) as sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)] + AggregateExec: mode=Partial, gby=[c_custkey@0 as c_custkey, c_name@1 as c_name, c_acctbal@4 as c_acctbal, c_phone@3 as c_phone, n_name@8 as n_name, c_address@2 as c_address, c_comment@5 as c_comment], aggr=[sum(lineitem.l_extendedprice * 1 - lineitem.l_discount) as sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)] ProjectionExec: expr=[c_custkey@0 as c_custkey, c_name@1 as c_name, c_address@2 as c_address, c_phone@4 as c_phone, c_acctbal@5 as c_acctbal, c_comment@6 as c_comment, l_extendedprice@7 as l_extendedprice, l_discount@8 as l_discount, n_name@10 as n_name] SortMergeJoinExec: join_type=Inner, on=[(c_nationkey@3, n_nationkey@0)] SortExec: expr=[c_nationkey@3 ASC], preserve_partitioning=[true] @@ -46,9 +46,9 @@ SortShuffleWriterExec: partitioning=Hash([c_custkey@0, c_name@1, c_acctbal@2, c_ === Stage 8 === ShuffleWriterExec: partitioning: None - SortExec: TopK(fetch=20), expr=[revenue@2 DESC], preserve_partitioning=[true] - ProjectionExec: expr=[c_custkey@0 as c_custkey, c_name@1 as c_name, sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)@7 as revenue, c_acctbal@2 as c_acctbal, n_name@4 as n_name, c_address@5 as c_address, c_phone@3 as c_phone, c_comment@6 as c_comment] - AggregateExec: mode=FinalPartitioned, gby=[c_custkey@0 as c_custkey, c_name@1 as c_name, c_acctbal@2 as c_acctbal, c_phone@3 as c_phone, n_name@4 as n_name, c_address@5 as c_address, c_comment@6 as c_comment], aggr=[sum(lineitem.l_extendedprice * Some(1),20,0 - lineitem.l_discount) as sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)] + ProjectionExec: expr=[c_custkey@0 as c_custkey, c_name@1 as c_name, sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)@7 as revenue, c_acctbal@2 as c_acctbal, n_name@4 as n_name, c_address@5 as c_address, c_phone@3 as c_phone, c_comment@6 as c_comment] + SortExec: TopK(fetch=20), expr=[sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)@7 DESC], preserve_partitioning=[true] + AggregateExec: mode=FinalPartitioned, gby=[c_custkey@0 as c_custkey, c_name@1 as c_name, c_acctbal@2 as c_acctbal, c_phone@3 as c_phone, n_name@4 as n_name, c_address@5 as c_address, c_comment@6 as c_comment], aggr=[sum(lineitem.l_extendedprice * 1 - lineitem.l_discount) as sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)] UnresolvedShuffleExec: stage=7, partitioning: Hash([c_custkey@0, c_name@1, c_acctbal@2, c_phone@3, n_name@4, c_address@5, c_comment@6], 16) === Stage 9 === diff --git a/ballista/scheduler/tests/tpch_plan_stability/approved/q12.txt b/ballista/scheduler/tests/tpch_plan_stability/approved/q12.txt index 3b86494e39..7707536367 100644 --- a/ballista/scheduler/tests/tpch_plan_stability/approved/q12.txt +++ b/ballista/scheduler/tests/tpch_plan_stability/approved/q12.txt @@ -19,8 +19,8 @@ SortShuffleWriterExec: partitioning=Hash([l_shipmode@0], 16) === Stage 4 === ShuffleWriterExec: partitioning: None - SortExec: expr=[l_shipmode@0 ASC NULLS LAST], preserve_partitioning=[true] - ProjectionExec: expr=[l_shipmode@0 as l_shipmode, sum(CASE WHEN orders.o_orderpriority = Utf8("1-URGENT") OR orders.o_orderpriority = Utf8("2-HIGH") THEN Int64(1) ELSE Int64(0) END)@1 as high_line_count, sum(CASE WHEN orders.o_orderpriority != Utf8("1-URGENT") AND orders.o_orderpriority != Utf8("2-HIGH") THEN Int64(1) ELSE Int64(0) END)@2 as low_line_count] + ProjectionExec: expr=[l_shipmode@0 as l_shipmode, sum(CASE WHEN orders.o_orderpriority = Utf8("1-URGENT") OR orders.o_orderpriority = Utf8("2-HIGH") THEN Int64(1) ELSE Int64(0) END)@1 as high_line_count, sum(CASE WHEN orders.o_orderpriority != Utf8("1-URGENT") AND orders.o_orderpriority != Utf8("2-HIGH") THEN Int64(1) ELSE Int64(0) END)@2 as low_line_count] + SortExec: expr=[l_shipmode@0 ASC NULLS LAST], preserve_partitioning=[true] AggregateExec: mode=FinalPartitioned, gby=[l_shipmode@0 as l_shipmode], aggr=[sum(CASE WHEN orders.o_orderpriority = Utf8("1-URGENT") OR orders.o_orderpriority = Utf8("2-HIGH") THEN Int64(1) ELSE Int64(0) END), sum(CASE WHEN orders.o_orderpriority != Utf8("1-URGENT") AND orders.o_orderpriority != Utf8("2-HIGH") THEN Int64(1) ELSE Int64(0) END)] UnresolvedShuffleExec: stage=3, partitioning: Hash([l_shipmode@0], 16) diff --git a/ballista/scheduler/tests/tpch_plan_stability/approved/q13.txt b/ballista/scheduler/tests/tpch_plan_stability/approved/q13.txt index 10bc255aeb..ca52111edf 100644 --- a/ballista/scheduler/tests/tpch_plan_stability/approved/q13.txt +++ b/ballista/scheduler/tests/tpch_plan_stability/approved/q13.txt @@ -21,8 +21,8 @@ SortShuffleWriterExec: partitioning=Hash([c_count@0], 16) === Stage 4 === ShuffleWriterExec: partitioning: None - SortExec: expr=[custdist@1 DESC, c_count@0 DESC], preserve_partitioning=[true] - ProjectionExec: expr=[c_count@0 as c_count, count(Int64(1))@1 as custdist] + ProjectionExec: expr=[c_count@0 as c_count, count(Int64(1))@1 as custdist] + SortExec: expr=[count(Int64(1))@1 DESC, c_count@0 DESC], preserve_partitioning=[true] AggregateExec: mode=FinalPartitioned, gby=[c_count@0 as c_count], aggr=[count(Int64(1))] UnresolvedShuffleExec: stage=3, partitioning: Hash([c_count@0], 16) diff --git a/ballista/scheduler/tests/tpch_plan_stability/approved/q14.txt b/ballista/scheduler/tests/tpch_plan_stability/approved/q14.txt index 5f8721f421..beae644299 100644 --- a/ballista/scheduler/tests/tpch_plan_stability/approved/q14.txt +++ b/ballista/scheduler/tests/tpch_plan_stability/approved/q14.txt @@ -9,8 +9,8 @@ SortShuffleWriterExec: partitioning=Hash([p_partkey@0], 16) === Stage 3 === ShuffleWriterExec: partitioning: None - AggregateExec: mode=Partial, gby=[], aggr=[sum(CASE WHEN part.p_type LIKE PROMO% THEN __common_expr_1 ELSE Some(0),38,4 END) as sum(CASE WHEN part.p_type LIKE Utf8("PROMO%") THEN lineitem.l_extendedprice * Int64(1) - lineitem.l_discount ELSE Int64(0) END), sum(__common_expr_1) as sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)] - ProjectionExec: expr=[l_extendedprice@1 * (Some(1),20,0 - l_discount@2) as __common_expr_1, p_type@4 as p_type] + AggregateExec: mode=Partial, gby=[], aggr=[sum(CASE WHEN part.p_type LIKE PROMO% THEN __common_expr_1 ELSE 0.0000 END) as sum(CASE WHEN part.p_type LIKE Utf8("PROMO%") THEN lineitem.l_extendedprice * Int64(1) - lineitem.l_discount ELSE Int64(0) END), sum(__common_expr_1) as sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)] + ProjectionExec: expr=[l_extendedprice@1 * (1 - l_discount@2) as __common_expr_1, p_type@4 as p_type] SortMergeJoinExec: join_type=Inner, on=[(l_partkey@0, p_partkey@0)] SortExec: expr=[l_partkey@0 ASC], preserve_partitioning=[true] UnresolvedShuffleExec: stage=1, partitioning: Hash([l_partkey@0], 16) @@ -20,6 +20,6 @@ ShuffleWriterExec: partitioning: None === Stage 4 === ShuffleWriterExec: partitioning: None ProjectionExec: expr=[100 * CAST(sum(CASE WHEN part.p_type LIKE Utf8("PROMO%") THEN lineitem.l_extendedprice * Int64(1) - lineitem.l_discount ELSE Int64(0) END)@0 AS Float64) / CAST(sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)@1 AS Float64) as promo_revenue] - AggregateExec: mode=Final, gby=[], aggr=[sum(CASE WHEN part.p_type LIKE PROMO% THEN __common_expr_1 ELSE Some(0),38,4 END) as sum(CASE WHEN part.p_type LIKE Utf8("PROMO%") THEN lineitem.l_extendedprice * Int64(1) - lineitem.l_discount ELSE Int64(0) END), sum(__common_expr_1) as sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)] + AggregateExec: mode=Final, gby=[], aggr=[sum(CASE WHEN part.p_type LIKE PROMO% THEN __common_expr_1 ELSE 0.0000 END) as sum(CASE WHEN part.p_type LIKE Utf8("PROMO%") THEN lineitem.l_extendedprice * Int64(1) - lineitem.l_discount ELSE Int64(0) END), sum(__common_expr_1) as sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)] CoalescePartitionsExec UnresolvedShuffleExec: stage=3, partitioning: Hash([p_partkey@3], 16) diff --git a/ballista/scheduler/tests/tpch_plan_stability/approved/q15.txt b/ballista/scheduler/tests/tpch_plan_stability/approved/q15.txt index caf8d0d9dd..c82ed09ab5 100644 --- a/ballista/scheduler/tests/tpch_plan_stability/approved/q15.txt +++ b/ballista/scheduler/tests/tpch_plan_stability/approved/q15.txt @@ -4,7 +4,7 @@ SortShuffleWriterExec: partitioning=Hash([s_suppkey@0], 16) === Stage 2 === SortShuffleWriterExec: partitioning=Hash([l_suppkey@0], 16) - AggregateExec: mode=Partial, gby=[l_suppkey@0 as l_suppkey], aggr=[sum(lineitem.l_extendedprice * Some(1),20,0 - lineitem.l_discount) as sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)] + AggregateExec: mode=Partial, gby=[l_suppkey@0 as l_suppkey], aggr=[sum(lineitem.l_extendedprice * 1 - lineitem.l_discount) as sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)] FilterExec: l_shipdate@3 >= 1996-01-01 AND l_shipdate@3 < 1996-04-01, projection=[l_suppkey@0, l_extendedprice@1, l_discount@2] StatsExec: rows=600037902 @@ -14,14 +14,14 @@ SortShuffleWriterExec: partitioning=Hash([total_revenue@4], 16) SortMergeJoinExec: join_type=Inner, on=[(s_suppkey@0, supplier_no@0)] SortExec: expr=[s_suppkey@0 ASC], preserve_partitioning=[true] UnresolvedShuffleExec: stage=1, partitioning: Hash([s_suppkey@0], 16) - SortExec: expr=[supplier_no@0 ASC], preserve_partitioning=[true] - ProjectionExec: expr=[l_suppkey@0 as supplier_no, sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)@1 as total_revenue] - AggregateExec: mode=FinalPartitioned, gby=[l_suppkey@0 as l_suppkey], aggr=[sum(lineitem.l_extendedprice * Some(1),20,0 - lineitem.l_discount) as sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)] + ProjectionExec: expr=[l_suppkey@0 as supplier_no, sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)@1 as total_revenue] + SortExec: expr=[l_suppkey@0 ASC], preserve_partitioning=[true] + AggregateExec: mode=FinalPartitioned, gby=[l_suppkey@0 as l_suppkey], aggr=[sum(lineitem.l_extendedprice * 1 - lineitem.l_discount) as sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)] UnresolvedShuffleExec: stage=2, partitioning: Hash([l_suppkey@0], 16) === Stage 4 === SortShuffleWriterExec: partitioning=Hash([l_suppkey@0], 16) - AggregateExec: mode=Partial, gby=[l_suppkey@0 as l_suppkey], aggr=[sum(lineitem.l_extendedprice * Some(1),20,0 - lineitem.l_discount) as sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)] + AggregateExec: mode=Partial, gby=[l_suppkey@0 as l_suppkey], aggr=[sum(lineitem.l_extendedprice * 1 - lineitem.l_discount) as sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)] FilterExec: l_shipdate@3 >= 1996-01-01 AND l_shipdate@3 < 1996-04-01, projection=[l_suppkey@0, l_extendedprice@1, l_discount@2] StatsExec: rows=600037902 @@ -29,7 +29,7 @@ SortShuffleWriterExec: partitioning=Hash([l_suppkey@0], 16) ShuffleWriterExec: partitioning: None AggregateExec: mode=Partial, gby=[], aggr=[max(revenue0.total_revenue)] ProjectionExec: expr=[sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)@1 as total_revenue] - AggregateExec: mode=FinalPartitioned, gby=[l_suppkey@0 as l_suppkey], aggr=[sum(lineitem.l_extendedprice * Some(1),20,0 - lineitem.l_discount) as sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)] + AggregateExec: mode=FinalPartitioned, gby=[l_suppkey@0 as l_suppkey], aggr=[sum(lineitem.l_extendedprice * 1 - lineitem.l_discount) as sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)] UnresolvedShuffleExec: stage=4, partitioning: Hash([l_suppkey@0], 16) === Stage 6 === diff --git a/ballista/scheduler/tests/tpch_plan_stability/approved/q16.txt b/ballista/scheduler/tests/tpch_plan_stability/approved/q16.txt index 2a62656215..bf85900ab3 100644 --- a/ballista/scheduler/tests/tpch_plan_stability/approved/q16.txt +++ b/ballista/scheduler/tests/tpch_plan_stability/approved/q16.txt @@ -4,7 +4,7 @@ SortShuffleWriterExec: partitioning=Hash([ps_partkey@0], 16) === Stage 2 === SortShuffleWriterExec: partitioning=Hash([p_partkey@0], 16) - FilterExec: p_brand@1 != Brand#45 AND p_type@2 NOT LIKE MEDIUM POLISHED% AND p_size@3 IN (SET) ([49, 14, 23, 45, 19, 3, 36, 9]) + FilterExec: p_brand@1 != Brand#45 AND p_size@3 IN (SET) ([49, 14, 23, 45, 19, 3, 36, 9]) AND p_type@2 NOT LIKE MEDIUM POLISHED% StatsExec: rows=20000000 === Stage 3 === @@ -33,8 +33,8 @@ SortShuffleWriterExec: partitioning=Hash([p_brand@0, p_type@1, p_size@2], 16) === Stage 6 === ShuffleWriterExec: partitioning: None - SortExec: expr=[supplier_cnt@3 DESC, p_brand@0 ASC NULLS LAST, p_type@1 ASC NULLS LAST, p_size@2 ASC NULLS LAST], preserve_partitioning=[true] - ProjectionExec: expr=[p_brand@0 as p_brand, p_type@1 as p_type, p_size@2 as p_size, count(alias1)@3 as supplier_cnt] + ProjectionExec: expr=[p_brand@0 as p_brand, p_type@1 as p_type, p_size@2 as p_size, count(alias1)@3 as supplier_cnt] + SortExec: expr=[count(alias1)@3 DESC, p_brand@0 ASC NULLS LAST, p_type@1 ASC NULLS LAST, p_size@2 ASC NULLS LAST], preserve_partitioning=[true] AggregateExec: mode=FinalPartitioned, gby=[p_brand@0 as p_brand, p_type@1 as p_type, p_size@2 as p_size], aggr=[count(alias1)] UnresolvedShuffleExec: stage=5, partitioning: Hash([p_brand@0, p_type@1, p_size@2], 16) diff --git a/ballista/scheduler/tests/tpch_plan_stability/approved/q17.txt b/ballista/scheduler/tests/tpch_plan_stability/approved/q17.txt index a74197f02d..20e64a1963 100644 --- a/ballista/scheduler/tests/tpch_plan_stability/approved/q17.txt +++ b/ballista/scheduler/tests/tpch_plan_stability/approved/q17.txt @@ -16,7 +16,7 @@ SortShuffleWriterExec: partitioning=Hash([l_partkey@0], 16) ShuffleWriterExec: partitioning: None AggregateExec: mode=Partial, gby=[], aggr=[sum(lineitem.l_extendedprice)] ProjectionExec: expr=[l_extendedprice@1 as l_extendedprice] - SortMergeJoinExec: join_type=Inner, on=[(p_partkey@2, l_partkey@1)], filter=CAST(l_quantity@0 AS Decimal128(30, 15)) < Float64(0.2) * avg(lineitem.l_quantity)@1 + SortMergeJoinExec: join_type=LeftSemi, on=[(p_partkey@2, l_partkey@1)], filter=CAST(l_quantity@0 AS Decimal128(30, 15)) < Float64(0.2) * avg(lineitem.l_quantity)@1 ProjectionExec: expr=[l_quantity@1 as l_quantity, l_extendedprice@2 as l_extendedprice, p_partkey@3 as p_partkey] SortMergeJoinExec: join_type=Inner, on=[(l_partkey@0, p_partkey@0)] SortExec: expr=[l_partkey@0 ASC], preserve_partitioning=[true] @@ -33,4 +33,4 @@ ShuffleWriterExec: partitioning: None ProjectionExec: expr=[CAST(sum(lineitem.l_extendedprice)@0 AS Float64) / 7 as avg_yearly] AggregateExec: mode=Final, gby=[], aggr=[sum(lineitem.l_extendedprice)] CoalescePartitionsExec - UnresolvedShuffleExec: stage=4, partitioning: Hash([l_partkey@4], 16) + UnresolvedShuffleExec: stage=4, partitioning: Hash([p_partkey@2], 16) diff --git a/ballista/scheduler/tests/tpch_plan_stability/approved/q18.txt b/ballista/scheduler/tests/tpch_plan_stability/approved/q18.txt index 7cfabb07ef..aca5a06ebf 100644 --- a/ballista/scheduler/tests/tpch_plan_stability/approved/q18.txt +++ b/ballista/scheduler/tests/tpch_plan_stability/approved/q18.txt @@ -36,7 +36,7 @@ ShuffleWriterExec: partitioning: None SortExec: expr=[l_orderkey@0 ASC], preserve_partitioning=[true] UnresolvedShuffleExec: stage=4, partitioning: Hash([l_orderkey@0], 16) SortExec: expr=[l_orderkey@0 ASC], preserve_partitioning=[true] - FilterExec: sum(lineitem.l_quantity)@1 > Some(30000),25,2, projection=[l_orderkey@0] + FilterExec: sum(lineitem.l_quantity)@1 > 300.00, projection=[l_orderkey@0] AggregateExec: mode=FinalPartitioned, gby=[l_orderkey@0 as l_orderkey], aggr=[sum(lineitem.l_quantity)] UnresolvedShuffleExec: stage=5, partitioning: Hash([l_orderkey@0], 16) diff --git a/ballista/scheduler/tests/tpch_plan_stability/approved/q19.txt b/ballista/scheduler/tests/tpch_plan_stability/approved/q19.txt index acd9d8cf14..fefda7d029 100644 --- a/ballista/scheduler/tests/tpch_plan_stability/approved/q19.txt +++ b/ballista/scheduler/tests/tpch_plan_stability/approved/q19.txt @@ -1,6 +1,6 @@ === Stage 1 === SortShuffleWriterExec: partitioning=Hash([l_partkey@0], 16) - FilterExec: (l_shipmode@5 = AIR OR l_shipmode@5 = AIR REG) AND l_shipinstruct@4 = DELIVER IN PERSON AND (l_quantity@1 >= Some(100),15,2 AND l_quantity@1 <= Some(1100),15,2 OR l_quantity@1 >= Some(1000),15,2 AND l_quantity@1 <= Some(2000),15,2 OR l_quantity@1 >= Some(2000),15,2 AND l_quantity@1 <= Some(3000),15,2), projection=[l_partkey@0, l_quantity@1, l_extendedprice@2, l_discount@3] + FilterExec: (l_shipmode@5 = AIR OR l_shipmode@5 = AIR REG) AND l_shipinstruct@4 = DELIVER IN PERSON AND (l_quantity@1 >= 1.00 AND l_quantity@1 <= 11.00 OR l_quantity@1 >= 10.00 AND l_quantity@1 <= 20.00 OR l_quantity@1 >= 20.00 AND l_quantity@1 <= 30.00), projection=[l_partkey@0, l_quantity@1, l_extendedprice@2, l_discount@3] StatsExec: rows=600037902 === Stage 2 === @@ -10,9 +10,9 @@ SortShuffleWriterExec: partitioning=Hash([p_partkey@0], 16) === Stage 3 === ShuffleWriterExec: partitioning: None - AggregateExec: mode=Partial, gby=[], aggr=[sum(lineitem.l_extendedprice * Some(1),20,0 - lineitem.l_discount) as sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)] + AggregateExec: mode=Partial, gby=[], aggr=[sum(lineitem.l_extendedprice * 1 - lineitem.l_discount) as sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)] ProjectionExec: expr=[l_extendedprice@2 as l_extendedprice, l_discount@3 as l_discount] - SortMergeJoinExec: join_type=Inner, on=[(l_partkey@0, p_partkey@0)], filter=p_brand@1 = Brand#12 AND p_container@3 IN (SET) ([SM CASE, SM BOX, SM PACK, SM PKG]) AND l_quantity@0 >= Some(100),15,2 AND l_quantity@0 <= Some(1100),15,2 AND p_size@2 <= 5 OR p_brand@1 = Brand#23 AND p_container@3 IN (SET) ([MED BAG, MED BOX, MED PKG, MED PACK]) AND l_quantity@0 >= Some(1000),15,2 AND l_quantity@0 <= Some(2000),15,2 AND p_size@2 <= 10 OR p_brand@1 = Brand#34 AND p_container@3 IN (SET) ([LG CASE, LG BOX, LG PACK, LG PKG]) AND l_quantity@0 >= Some(2000),15,2 AND l_quantity@0 <= Some(3000),15,2 AND p_size@2 <= 15 + SortMergeJoinExec: join_type=Inner, on=[(l_partkey@0, p_partkey@0)], filter=p_brand@1 = Brand#12 AND p_container@3 IN (SET) ([SM CASE, SM BOX, SM PACK, SM PKG]) AND l_quantity@0 >= 1.00 AND l_quantity@0 <= 11.00 AND p_size@2 <= 5 OR p_brand@1 = Brand#23 AND p_container@3 IN (SET) ([MED BAG, MED BOX, MED PKG, MED PACK]) AND l_quantity@0 >= 10.00 AND l_quantity@0 <= 20.00 AND p_size@2 <= 10 OR p_brand@1 = Brand#34 AND p_container@3 IN (SET) ([LG CASE, LG BOX, LG PACK, LG PKG]) AND l_quantity@0 >= 20.00 AND l_quantity@0 <= 30.00 AND p_size@2 <= 15 SortExec: expr=[l_partkey@0 ASC], preserve_partitioning=[true] UnresolvedShuffleExec: stage=1, partitioning: Hash([l_partkey@0], 16) SortExec: expr=[p_partkey@0 ASC], preserve_partitioning=[true] @@ -21,6 +21,6 @@ ShuffleWriterExec: partitioning: None === Stage 4 === ShuffleWriterExec: partitioning: None ProjectionExec: expr=[sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)@0 as revenue] - AggregateExec: mode=Final, gby=[], aggr=[sum(lineitem.l_extendedprice * Some(1),20,0 - lineitem.l_discount) as sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)] + AggregateExec: mode=Final, gby=[], aggr=[sum(lineitem.l_extendedprice * 1 - lineitem.l_discount) as sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)] CoalescePartitionsExec UnresolvedShuffleExec: stage=3, partitioning: Hash([p_partkey@4], 16) diff --git a/ballista/scheduler/tests/tpch_plan_stability/approved/q2.txt b/ballista/scheduler/tests/tpch_plan_stability/approved/q2.txt index ee10151895..0490884f6b 100644 --- a/ballista/scheduler/tests/tpch_plan_stability/approved/q2.txt +++ b/ballista/scheduler/tests/tpch_plan_stability/approved/q2.txt @@ -111,7 +111,7 @@ SortShuffleWriterExec: partitioning=Hash([ps_partkey@1, min(partsupp.ps_supplyco ShuffleWriterExec: partitioning: None SortExec: TopK(fetch=100), expr=[s_acctbal@0 DESC, n_name@2 ASC NULLS LAST, s_name@1 ASC NULLS LAST, p_partkey@3 ASC NULLS LAST], preserve_partitioning=[true] ProjectionExec: expr=[s_acctbal@5 as s_acctbal, s_name@2 as s_name, n_name@8 as n_name, p_partkey@0 as p_partkey, p_mfgr@1 as p_mfgr, s_address@3 as s_address, s_phone@4 as s_phone, s_comment@6 as s_comment] - SortMergeJoinExec: join_type=Inner, on=[(p_partkey@0, ps_partkey@1), (ps_supplycost@7, min(partsupp.ps_supplycost)@0)] + SortMergeJoinExec: join_type=LeftSemi, on=[(p_partkey@0, ps_partkey@1), (ps_supplycost@7, min(partsupp.ps_supplycost)@0)] SortExec: expr=[p_partkey@0 ASC, ps_supplycost@7 ASC], preserve_partitioning=[true] UnresolvedShuffleExec: stage=9, partitioning: Hash([p_partkey@0, ps_supplycost@7], 16) SortExec: expr=[ps_partkey@1 ASC, min(partsupp.ps_supplycost)@0 ASC], preserve_partitioning=[true] @@ -120,4 +120,4 @@ ShuffleWriterExec: partitioning: None === Stage 19 === ShuffleWriterExec: partitioning: None SortPreservingMergeExec: [s_acctbal@0 DESC, n_name@2 ASC NULLS LAST, s_name@1 ASC NULLS LAST, p_partkey@3 ASC NULLS LAST], fetch=100 - UnresolvedShuffleExec: stage=18, partitioning: Hash([p_partkey@3, min(partsupp.ps_supplycost)@9], 16) + UnresolvedShuffleExec: stage=18, partitioning: Hash([p_partkey@3, ps_supplycost@7], 16) diff --git a/ballista/scheduler/tests/tpch_plan_stability/approved/q20.txt b/ballista/scheduler/tests/tpch_plan_stability/approved/q20.txt index de78d20fb1..8d5118be1f 100644 --- a/ballista/scheduler/tests/tpch_plan_stability/approved/q20.txt +++ b/ballista/scheduler/tests/tpch_plan_stability/approved/q20.txt @@ -42,7 +42,7 @@ SortShuffleWriterExec: partitioning=Hash([l_partkey@0, l_suppkey@1], 16) === Stage 8 === SortShuffleWriterExec: partitioning=Hash([ps_suppkey@0], 16) ProjectionExec: expr=[ps_suppkey@1 as ps_suppkey] - SortMergeJoinExec: join_type=Inner, on=[(ps_partkey@0, l_partkey@1), (ps_suppkey@1, l_suppkey@2)], filter=CAST(ps_availqty@0 AS Float64) > Float64(0.5) * sum(lineitem.l_quantity)@1 + SortMergeJoinExec: join_type=LeftSemi, on=[(ps_partkey@0, l_partkey@1), (ps_suppkey@1, l_suppkey@2)], filter=CAST(ps_availqty@0 AS Float64) > Float64(0.5) * sum(lineitem.l_quantity)@1 SortExec: expr=[ps_partkey@0 ASC, ps_suppkey@1 ASC], preserve_partitioning=[true] UnresolvedShuffleExec: stage=6, partitioning: Hash([ps_partkey@0, ps_suppkey@1], 16) SortExec: expr=[l_partkey@1 ASC, l_suppkey@2 ASC], preserve_partitioning=[true] diff --git a/ballista/scheduler/tests/tpch_plan_stability/approved/q21.txt b/ballista/scheduler/tests/tpch_plan_stability/approved/q21.txt index 003c45c938..6d03dbe5ea 100644 --- a/ballista/scheduler/tests/tpch_plan_stability/approved/q21.txt +++ b/ballista/scheduler/tests/tpch_plan_stability/approved/q21.txt @@ -68,8 +68,8 @@ SortShuffleWriterExec: partitioning=Hash([s_name@0], 16) === Stage 11 === ShuffleWriterExec: partitioning: None - SortExec: TopK(fetch=100), expr=[numwait@1 DESC, s_name@0 ASC NULLS LAST], preserve_partitioning=[true] - ProjectionExec: expr=[s_name@0 as s_name, count(Int64(1))@1 as numwait] + ProjectionExec: expr=[s_name@0 as s_name, count(Int64(1))@1 as numwait] + SortExec: TopK(fetch=100), expr=[count(Int64(1))@1 DESC, s_name@0 ASC NULLS LAST], preserve_partitioning=[true] AggregateExec: mode=FinalPartitioned, gby=[s_name@0 as s_name], aggr=[count(Int64(1))] UnresolvedShuffleExec: stage=10, partitioning: Hash([s_name@0], 16) diff --git a/ballista/scheduler/tests/tpch_plan_stability/approved/q22.txt b/ballista/scheduler/tests/tpch_plan_stability/approved/q22.txt index 8255afee9a..3d01e97a48 100644 --- a/ballista/scheduler/tests/tpch_plan_stability/approved/q22.txt +++ b/ballista/scheduler/tests/tpch_plan_stability/approved/q22.txt @@ -13,7 +13,7 @@ SortShuffleWriterExec: partitioning=Hash([cntrycode@0], 16) ProjectionExec: expr=[substr(c_phone@1, 1, 2) as cntrycode, c_acctbal@2 as c_acctbal] NestedLoopJoinExec: join_type=Inner, filter=join_proj_push_down_1@1 > avg(customer.c_acctbal)@0, projection=[avg(customer.c_acctbal)@0, c_phone@1, c_acctbal@2] AggregateExec: mode=Single, gby=[], aggr=[avg(customer.c_acctbal)] - FilterExec: c_acctbal@1 > Some(0),15,2 AND substr(c_phone@0, 1, 2) IN (SET) ([13, 31, 23, 29, 30, 18, 17]), projection=[c_acctbal@1] + FilterExec: c_acctbal@1 > 0.00 AND substr(c_phone@0, 1, 2) IN (SET) ([13, 31, 23, 29, 30, 18, 17]), projection=[c_acctbal@1] StatsExec: rows=15000000 ProjectionExec: expr=[c_phone@1 as c_phone, c_acctbal@2 as c_acctbal, CAST(c_acctbal@2 AS Decimal128(19, 6)) as join_proj_push_down_1] SortMergeJoinExec: join_type=LeftAnti, on=[(c_custkey@0, o_custkey@0)] @@ -24,8 +24,8 @@ SortShuffleWriterExec: partitioning=Hash([cntrycode@0], 16) === Stage 4 === ShuffleWriterExec: partitioning: None - SortExec: expr=[cntrycode@0 ASC NULLS LAST], preserve_partitioning=[true] - ProjectionExec: expr=[cntrycode@0 as cntrycode, count(Int64(1))@1 as numcust, sum(custsale.c_acctbal)@2 as totacctbal] + ProjectionExec: expr=[cntrycode@0 as cntrycode, count(Int64(1))@1 as numcust, sum(custsale.c_acctbal)@2 as totacctbal] + SortExec: expr=[cntrycode@0 ASC NULLS LAST], preserve_partitioning=[true] AggregateExec: mode=FinalPartitioned, gby=[cntrycode@0 as cntrycode], aggr=[count(Int64(1)), sum(custsale.c_acctbal)] UnresolvedShuffleExec: stage=3, partitioning: Hash([cntrycode@0], 16) diff --git a/ballista/scheduler/tests/tpch_plan_stability/approved/q3.txt b/ballista/scheduler/tests/tpch_plan_stability/approved/q3.txt index ea275d184b..f8049b08ce 100644 --- a/ballista/scheduler/tests/tpch_plan_stability/approved/q3.txt +++ b/ballista/scheduler/tests/tpch_plan_stability/approved/q3.txt @@ -24,9 +24,9 @@ SortShuffleWriterExec: partitioning=Hash([l_orderkey@0], 16) === Stage 5 === ShuffleWriterExec: partitioning: None - SortExec: TopK(fetch=10), expr=[revenue@1 DESC, o_orderdate@2 ASC NULLS LAST], preserve_partitioning=[true] - ProjectionExec: expr=[l_orderkey@0 as l_orderkey, sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)@3 as revenue, o_orderdate@1 as o_orderdate, o_shippriority@2 as o_shippriority] - AggregateExec: mode=SinglePartitioned, gby=[l_orderkey@2 as l_orderkey, o_orderdate@0 as o_orderdate, o_shippriority@1 as o_shippriority], aggr=[sum(lineitem.l_extendedprice * Some(1),20,0 - lineitem.l_discount) as sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)], ordering_mode=PartiallySorted([0]) + ProjectionExec: expr=[l_orderkey@0 as l_orderkey, sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)@3 as revenue, o_orderdate@1 as o_orderdate, o_shippriority@2 as o_shippriority] + SortExec: TopK(fetch=10), expr=[sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)@3 DESC, o_orderdate@1 ASC NULLS LAST], preserve_partitioning=[true] + AggregateExec: mode=SinglePartitioned, gby=[l_orderkey@2 as l_orderkey, o_orderdate@0 as o_orderdate, o_shippriority@1 as o_shippriority], aggr=[sum(lineitem.l_extendedprice * 1 - lineitem.l_discount) as sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)], ordering_mode=PartiallySorted([0]) ProjectionExec: expr=[o_orderdate@1 as o_orderdate, o_shippriority@2 as o_shippriority, l_orderkey@3 as l_orderkey, l_extendedprice@4 as l_extendedprice, l_discount@5 as l_discount] SortMergeJoinExec: join_type=Inner, on=[(o_orderkey@0, l_orderkey@0)] SortExec: expr=[o_orderkey@0 ASC], preserve_partitioning=[true] diff --git a/ballista/scheduler/tests/tpch_plan_stability/approved/q4.txt b/ballista/scheduler/tests/tpch_plan_stability/approved/q4.txt index 5e05011fef..4df80b9342 100644 --- a/ballista/scheduler/tests/tpch_plan_stability/approved/q4.txt +++ b/ballista/scheduler/tests/tpch_plan_stability/approved/q4.txt @@ -20,8 +20,8 @@ SortShuffleWriterExec: partitioning=Hash([o_orderpriority@0], 16) === Stage 4 === ShuffleWriterExec: partitioning: None - SortExec: expr=[o_orderpriority@0 ASC NULLS LAST], preserve_partitioning=[true] - ProjectionExec: expr=[o_orderpriority@0 as o_orderpriority, count(Int64(1))@1 as order_count] + ProjectionExec: expr=[o_orderpriority@0 as o_orderpriority, count(Int64(1))@1 as order_count] + SortExec: expr=[o_orderpriority@0 ASC NULLS LAST], preserve_partitioning=[true] AggregateExec: mode=FinalPartitioned, gby=[o_orderpriority@0 as o_orderpriority], aggr=[count(Int64(1))] UnresolvedShuffleExec: stage=3, partitioning: Hash([o_orderpriority@0], 16) diff --git a/ballista/scheduler/tests/tpch_plan_stability/approved/q5.txt b/ballista/scheduler/tests/tpch_plan_stability/approved/q5.txt index 8849e6c0bb..c3dcae5ec2 100644 --- a/ballista/scheduler/tests/tpch_plan_stability/approved/q5.txt +++ b/ballista/scheduler/tests/tpch_plan_stability/approved/q5.txt @@ -62,7 +62,7 @@ SortShuffleWriterExec: partitioning=Hash([r_regionkey@0], 16) === Stage 11 === SortShuffleWriterExec: partitioning=Hash([n_name@0], 16) - AggregateExec: mode=Partial, gby=[n_name@2 as n_name], aggr=[sum(lineitem.l_extendedprice * Some(1),20,0 - lineitem.l_discount) as sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)] + AggregateExec: mode=Partial, gby=[n_name@2 as n_name], aggr=[sum(lineitem.l_extendedprice * 1 - lineitem.l_discount) as sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)] ProjectionExec: expr=[l_extendedprice@0 as l_extendedprice, l_discount@1 as l_discount, n_name@2 as n_name] SortMergeJoinExec: join_type=Inner, on=[(n_regionkey@3, r_regionkey@0)] SortExec: expr=[n_regionkey@3 ASC], preserve_partitioning=[true] @@ -72,9 +72,9 @@ SortShuffleWriterExec: partitioning=Hash([n_name@0], 16) === Stage 12 === ShuffleWriterExec: partitioning: None - SortExec: expr=[revenue@1 DESC], preserve_partitioning=[true] - ProjectionExec: expr=[n_name@0 as n_name, sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)@1 as revenue] - AggregateExec: mode=FinalPartitioned, gby=[n_name@0 as n_name], aggr=[sum(lineitem.l_extendedprice * Some(1),20,0 - lineitem.l_discount) as sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)] + ProjectionExec: expr=[n_name@0 as n_name, sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)@1 as revenue] + SortExec: expr=[sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)@1 DESC], preserve_partitioning=[true] + AggregateExec: mode=FinalPartitioned, gby=[n_name@0 as n_name], aggr=[sum(lineitem.l_extendedprice * 1 - lineitem.l_discount) as sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)] UnresolvedShuffleExec: stage=11, partitioning: Hash([n_name@0], 16) === Stage 13 === diff --git a/ballista/scheduler/tests/tpch_plan_stability/approved/q6.txt b/ballista/scheduler/tests/tpch_plan_stability/approved/q6.txt index 1da12a0cb1..3a70454e32 100644 --- a/ballista/scheduler/tests/tpch_plan_stability/approved/q6.txt +++ b/ballista/scheduler/tests/tpch_plan_stability/approved/q6.txt @@ -2,5 +2,5 @@ ShuffleWriterExec: partitioning: None ProjectionExec: expr=[sum(lineitem.l_extendedprice * lineitem.l_discount)@0 as revenue] AggregateExec: mode=Single, gby=[], aggr=[sum(lineitem.l_extendedprice * lineitem.l_discount)] - FilterExec: l_shipdate@3 >= 1994-01-01 AND l_shipdate@3 < 1995-01-01 AND l_discount@2 >= Some(5),15,2 AND l_discount@2 <= Some(7),15,2 AND l_quantity@0 < Some(2400),15,2, projection=[l_extendedprice@1, l_discount@2] + FilterExec: l_shipdate@3 >= 1994-01-01 AND l_shipdate@3 < 1995-01-01 AND l_discount@2 >= 0.05 AND l_discount@2 <= 0.07 AND l_quantity@0 < 24.00, projection=[l_extendedprice@1, l_discount@2] StatsExec: rows=600037902 diff --git a/ballista/scheduler/tests/tpch_plan_stability/approved/q7.txt b/ballista/scheduler/tests/tpch_plan_stability/approved/q7.txt index e2fbadd62c..8f877e6d4b 100644 --- a/ballista/scheduler/tests/tpch_plan_stability/approved/q7.txt +++ b/ballista/scheduler/tests/tpch_plan_stability/approved/q7.txt @@ -64,7 +64,7 @@ SortShuffleWriterExec: partitioning=Hash([n_nationkey@0], 16) === Stage 11 === SortShuffleWriterExec: partitioning=Hash([supp_nation@0, cust_nation@1, l_year@2], 16) AggregateExec: mode=Partial, gby=[supp_nation@0 as supp_nation, cust_nation@1 as cust_nation, l_year@2 as l_year], aggr=[sum(shipping.volume)] - ProjectionExec: expr=[n_name@4 as supp_nation, n_name@6 as cust_nation, date_part(YEAR, l_shipdate@2) as l_year, l_extendedprice@0 * (Some(1),20,0 - l_discount@1) as volume] + ProjectionExec: expr=[n_name@4 as supp_nation, n_name@6 as cust_nation, date_part(YEAR, l_shipdate@2) as l_year, l_extendedprice@0 * (1 - l_discount@1) as volume] SortMergeJoinExec: join_type=Inner, on=[(c_nationkey@3, n_nationkey@0)], filter=n_name@0 = FRANCE AND n_name@1 = GERMANY OR n_name@0 = GERMANY AND n_name@1 = FRANCE SortExec: expr=[c_nationkey@3 ASC], preserve_partitioning=[true] UnresolvedShuffleExec: stage=9, partitioning: Hash([c_nationkey@3], 16) @@ -73,8 +73,8 @@ SortShuffleWriterExec: partitioning=Hash([supp_nation@0, cust_nation@1, l_year@2 === Stage 12 === ShuffleWriterExec: partitioning: None - SortExec: expr=[supp_nation@0 ASC NULLS LAST, cust_nation@1 ASC NULLS LAST, l_year@2 ASC NULLS LAST], preserve_partitioning=[true] - ProjectionExec: expr=[supp_nation@0 as supp_nation, cust_nation@1 as cust_nation, l_year@2 as l_year, sum(shipping.volume)@3 as revenue] + ProjectionExec: expr=[supp_nation@0 as supp_nation, cust_nation@1 as cust_nation, l_year@2 as l_year, sum(shipping.volume)@3 as revenue] + SortExec: expr=[supp_nation@0 ASC NULLS LAST, cust_nation@1 ASC NULLS LAST, l_year@2 ASC NULLS LAST], preserve_partitioning=[true] AggregateExec: mode=FinalPartitioned, gby=[supp_nation@0 as supp_nation, cust_nation@1 as cust_nation, l_year@2 as l_year], aggr=[sum(shipping.volume)] UnresolvedShuffleExec: stage=11, partitioning: Hash([supp_nation@0, cust_nation@1, l_year@2], 16) diff --git a/ballista/scheduler/tests/tpch_plan_stability/approved/q8.txt b/ballista/scheduler/tests/tpch_plan_stability/approved/q8.txt index 77ca93cd28..a595e9fd17 100644 --- a/ballista/scheduler/tests/tpch_plan_stability/approved/q8.txt +++ b/ballista/scheduler/tests/tpch_plan_stability/approved/q8.txt @@ -89,8 +89,8 @@ SortShuffleWriterExec: partitioning=Hash([r_regionkey@0], 16) === Stage 15 === SortShuffleWriterExec: partitioning=Hash([o_year@0], 16) - AggregateExec: mode=Partial, gby=[o_year@0 as o_year], aggr=[sum(CASE WHEN all_nations.nation = BRAZIL THEN all_nations.volume ELSE Some(0),38,4 END) as sum(CASE WHEN all_nations.nation = Utf8("BRAZIL") THEN all_nations.volume ELSE Int64(0) END), sum(all_nations.volume)] - ProjectionExec: expr=[date_part(YEAR, o_orderdate@2) as o_year, l_extendedprice@0 * (Some(1),20,0 - l_discount@1) as volume, n_name@4 as nation] + AggregateExec: mode=Partial, gby=[o_year@0 as o_year], aggr=[sum(CASE WHEN all_nations.nation = BRAZIL THEN all_nations.volume ELSE 0.0000 END) as sum(CASE WHEN all_nations.nation = Utf8("BRAZIL") THEN all_nations.volume ELSE Int64(0) END), sum(all_nations.volume)] + ProjectionExec: expr=[date_part(YEAR, o_orderdate@2) as o_year, l_extendedprice@0 * (1 - l_discount@1) as volume, n_name@4 as nation] SortMergeJoinExec: join_type=Inner, on=[(n_regionkey@3, r_regionkey@0)] SortExec: expr=[n_regionkey@3 ASC], preserve_partitioning=[true] UnresolvedShuffleExec: stage=13, partitioning: Hash([n_regionkey@3], 16) @@ -101,7 +101,7 @@ SortShuffleWriterExec: partitioning=Hash([o_year@0], 16) ShuffleWriterExec: partitioning: None SortExec: expr=[o_year@0 ASC NULLS LAST], preserve_partitioning=[true] ProjectionExec: expr=[o_year@0 as o_year, sum(CASE WHEN all_nations.nation = Utf8("BRAZIL") THEN all_nations.volume ELSE Int64(0) END)@1 / sum(all_nations.volume)@2 as mkt_share] - AggregateExec: mode=FinalPartitioned, gby=[o_year@0 as o_year], aggr=[sum(CASE WHEN all_nations.nation = BRAZIL THEN all_nations.volume ELSE Some(0),38,4 END) as sum(CASE WHEN all_nations.nation = Utf8("BRAZIL") THEN all_nations.volume ELSE Int64(0) END), sum(all_nations.volume)] + AggregateExec: mode=FinalPartitioned, gby=[o_year@0 as o_year], aggr=[sum(CASE WHEN all_nations.nation = BRAZIL THEN all_nations.volume ELSE 0.0000 END) as sum(CASE WHEN all_nations.nation = Utf8("BRAZIL") THEN all_nations.volume ELSE Int64(0) END), sum(all_nations.volume)] UnresolvedShuffleExec: stage=15, partitioning: Hash([o_year@0], 16) === Stage 17 === diff --git a/ballista/scheduler/tests/tpch_plan_stability/approved/q9.txt b/ballista/scheduler/tests/tpch_plan_stability/approved/q9.txt index 48b15bca3a..95dd57e663 100644 --- a/ballista/scheduler/tests/tpch_plan_stability/approved/q9.txt +++ b/ballista/scheduler/tests/tpch_plan_stability/approved/q9.txt @@ -62,7 +62,7 @@ SortShuffleWriterExec: partitioning=Hash([n_nationkey@0], 16) === Stage 11 === SortShuffleWriterExec: partitioning=Hash([nation@0, o_year@1], 16) AggregateExec: mode=Partial, gby=[nation@0 as nation, o_year@1 as o_year], aggr=[sum(profit.amount)] - ProjectionExec: expr=[n_name@7 as nation, date_part(YEAR, o_orderdate@5) as o_year, l_extendedprice@1 * (Some(1),20,0 - l_discount@2) - ps_supplycost@4 * l_quantity@0 as amount] + ProjectionExec: expr=[n_name@7 as nation, date_part(YEAR, o_orderdate@5) as o_year, l_extendedprice@1 * (1 - l_discount@2) - ps_supplycost@4 * l_quantity@0 as amount] SortMergeJoinExec: join_type=Inner, on=[(s_nationkey@3, n_nationkey@0)] SortExec: expr=[s_nationkey@3 ASC], preserve_partitioning=[true] UnresolvedShuffleExec: stage=9, partitioning: Hash([s_nationkey@3], 16) @@ -71,8 +71,8 @@ SortShuffleWriterExec: partitioning=Hash([nation@0, o_year@1], 16) === Stage 12 === ShuffleWriterExec: partitioning: None - SortExec: expr=[nation@0 ASC NULLS LAST, o_year@1 DESC], preserve_partitioning=[true] - ProjectionExec: expr=[nation@0 as nation, o_year@1 as o_year, sum(profit.amount)@2 as sum_profit] + ProjectionExec: expr=[nation@0 as nation, o_year@1 as o_year, sum(profit.amount)@2 as sum_profit] + SortExec: expr=[nation@0 ASC NULLS LAST, o_year@1 DESC], preserve_partitioning=[true] AggregateExec: mode=FinalPartitioned, gby=[nation@0 as nation, o_year@1 as o_year], aggr=[sum(profit.amount)] UnresolvedShuffleExec: stage=11, partitioning: Hash([nation@0, o_year@1], 16) From bd6b63ce576a43a860f4cbff5072142a4136cfdf Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Thu, 6 Aug 2026 07:15:45 -0600 Subject: [PATCH 05/11] chore(deps): bump DataFusion rev to 70c26a06 and port to its APIs Advance the pinned apache/datafusion main rev from efa84e83 to 70c26a06. Port to the APIs that changed in between: - QueryPlanner::create_physical_plan takes &dyn Session instead of &SessionState. - datafusion_substrait::serializer::deserialize_bytes is now synchronous and takes &[u8]. Sync the vendored datafusion_common.proto for the new max_in_list_size field. --- Cargo.lock | 121 ++++++++++-------- Cargo.toml | 14 +- ballista/client/tests/context_setup.rs | 7 +- ballista/core/proto/datafusion_common.proto | 2 + .../src/execution_plans/shuffle_writer.rs | 2 +- ballista/core/src/planner.rs | 5 +- ballista/core/src/serde/mod.rs | 24 +++- .../scheduler/src/scheduler_server/grpc.rs | 2 +- ballista/scheduler/src/state/task_builder.rs | 2 +- 9 files changed, 103 insertions(+), 76 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d3b70a1a1a..1c76fe0c90 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -280,7 +280,7 @@ dependencies = [ "arrow-schema", "arrow-select", "atoi", - "base64", + "base64 0.22.1", "chrono", "comfy-table", "half", @@ -334,7 +334,7 @@ dependencies = [ "arrow-schema", "arrow-select", "arrow-string", - "base64", + "base64 0.22.1", "bytes", "futures", "once_cell", @@ -1250,6 +1250,12 @@ version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" +[[package]] +name = "base64" +version = "0.23.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac07cdecf99051d9a5238b80f35af32cdeba5b336e55d957b318b50137e18da5" + [[package]] name = "base64-simd" version = "0.8.0" @@ -1420,7 +1426,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ee04c4c84f1f811b017f2fbb7dd8815c976e7ca98593de9c1e2afad0f636bff4" dependencies = [ "async-stream", - "base64", + "base64 0.22.1", "bitflags 2.13.1", "bollard-buildkit-proto", "bollard-stubs", @@ -1477,7 +1483,7 @@ version = "1.52.1-rc.29.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0f0a8ca8799131c1837d1282c3f81f31e76ceb0ce426e04a7fe1ccee3287c066" dependencies = [ - "base64", + "base64 0.22.1", "bollard-buildkit-proto", "bytes", "prost", @@ -2191,7 +2197,7 @@ dependencies = [ [[package]] name = "datafusion" version = "54.1.0" -source = "git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a#efa84e8309ced6614e62d1607899c47ed0b6b08a" +source = "git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11#70c26a06716eb552f0fc1d2115957a4089504f11" dependencies = [ "arrow", "arrow-schema", @@ -2244,7 +2250,7 @@ dependencies = [ [[package]] name = "datafusion-catalog" version = "54.1.0" -source = "git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a#efa84e8309ced6614e62d1607899c47ed0b6b08a" +source = "git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11#70c26a06716eb552f0fc1d2115957a4089504f11" dependencies = [ "arrow", "async-trait", @@ -2268,7 +2274,7 @@ dependencies = [ [[package]] name = "datafusion-catalog-listing" version = "54.1.0" -source = "git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a#efa84e8309ced6614e62d1607899c47ed0b6b08a" +source = "git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11#70c26a06716eb552f0fc1d2115957a4089504f11" dependencies = [ "arrow", "async-trait", @@ -2291,7 +2297,7 @@ dependencies = [ [[package]] name = "datafusion-cli" version = "54.1.0" -source = "git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a#efa84e8309ced6614e62d1607899c47ed0b6b08a" +source = "git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11#70c26a06716eb552f0fc1d2115957a4089504f11" dependencies = [ "arrow", "async-trait", @@ -2318,7 +2324,7 @@ dependencies = [ [[package]] name = "datafusion-common" version = "54.1.0" -source = "git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a#efa84e8309ced6614e62d1607899c47ed0b6b08a" +source = "git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11#70c26a06716eb552f0fc1d2115957a4089504f11" dependencies = [ "arrow", "arrow-ipc", @@ -2345,7 +2351,7 @@ dependencies = [ [[package]] name = "datafusion-common-runtime" version = "54.1.0" -source = "git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a#efa84e8309ced6614e62d1607899c47ed0b6b08a" +source = "git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11#70c26a06716eb552f0fc1d2115957a4089504f11" dependencies = [ "futures", "log", @@ -2355,7 +2361,7 @@ dependencies = [ [[package]] name = "datafusion-datasource" version = "54.1.0" -source = "git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a#efa84e8309ced6614e62d1607899c47ed0b6b08a" +source = "git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11#70c26a06716eb552f0fc1d2115957a4089504f11" dependencies = [ "arrow", "async-compression", @@ -2371,6 +2377,7 @@ dependencies = [ "datafusion-physical-expr-adapter", "datafusion-physical-expr-common", "datafusion-physical-plan", + "datafusion-proto-models", "datafusion-session", "flate2", "futures", @@ -2390,7 +2397,7 @@ dependencies = [ [[package]] name = "datafusion-datasource-arrow" version = "54.1.0" -source = "git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a#efa84e8309ced6614e62d1607899c47ed0b6b08a" +source = "git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11#70c26a06716eb552f0fc1d2115957a4089504f11" dependencies = [ "arrow", "arrow-ipc", @@ -2413,7 +2420,7 @@ dependencies = [ [[package]] name = "datafusion-datasource-avro" version = "54.1.0" -source = "git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a#efa84e8309ced6614e62d1607899c47ed0b6b08a" +source = "git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11#70c26a06716eb552f0fc1d2115957a4089504f11" dependencies = [ "arrow", "arrow-avro", @@ -2431,7 +2438,7 @@ dependencies = [ [[package]] name = "datafusion-datasource-csv" version = "54.1.0" -source = "git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a#efa84e8309ced6614e62d1607899c47ed0b6b08a" +source = "git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11#70c26a06716eb552f0fc1d2115957a4089504f11" dependencies = [ "arrow", "async-trait", @@ -2453,7 +2460,7 @@ dependencies = [ [[package]] name = "datafusion-datasource-json" version = "54.1.0" -source = "git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a#efa84e8309ced6614e62d1607899c47ed0b6b08a" +source = "git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11#70c26a06716eb552f0fc1d2115957a4089504f11" dependencies = [ "arrow", "async-trait", @@ -2475,7 +2482,7 @@ dependencies = [ [[package]] name = "datafusion-datasource-parquet" version = "54.1.0" -source = "git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a#efa84e8309ced6614e62d1607899c47ed0b6b08a" +source = "git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11#70c26a06716eb552f0fc1d2115957a4089504f11" dependencies = [ "arrow", "arrow-schema", @@ -2506,12 +2513,12 @@ dependencies = [ [[package]] name = "datafusion-doc" version = "54.1.0" -source = "git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a#efa84e8309ced6614e62d1607899c47ed0b6b08a" +source = "git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11#70c26a06716eb552f0fc1d2115957a4089504f11" [[package]] name = "datafusion-execution" version = "54.1.0" -source = "git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a#efa84e8309ced6614e62d1607899c47ed0b6b08a" +source = "git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11#70c26a06716eb552f0fc1d2115957a4089504f11" dependencies = [ "arrow", "arrow-buffer", @@ -2537,7 +2544,7 @@ dependencies = [ [[package]] name = "datafusion-expr" version = "54.1.0" -source = "git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a#efa84e8309ced6614e62d1607899c47ed0b6b08a" +source = "git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11#70c26a06716eb552f0fc1d2115957a4089504f11" dependencies = [ "arrow", "arrow-schema", @@ -2559,7 +2566,7 @@ dependencies = [ [[package]] name = "datafusion-expr-common" version = "54.1.0" -source = "git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a#efa84e8309ced6614e62d1607899c47ed0b6b08a" +source = "git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11#70c26a06716eb552f0fc1d2115957a4089504f11" dependencies = [ "arrow", "datafusion-common", @@ -2570,11 +2577,11 @@ dependencies = [ [[package]] name = "datafusion-functions" version = "54.1.0" -source = "git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a#efa84e8309ced6614e62d1607899c47ed0b6b08a" +source = "git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11#70c26a06716eb552f0fc1d2115957a4089504f11" dependencies = [ "arrow", "arrow-buffer", - "base64", + "base64 0.23.1", "blake2", "blake3", "chrono", @@ -2601,7 +2608,7 @@ dependencies = [ [[package]] name = "datafusion-functions-aggregate" version = "54.1.0" -source = "git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a#efa84e8309ced6614e62d1607899c47ed0b6b08a" +source = "git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11#70c26a06716eb552f0fc1d2115957a4089504f11" dependencies = [ "arrow", "datafusion-common", @@ -2613,6 +2620,7 @@ dependencies = [ "datafusion-physical-expr", "datafusion-physical-expr-common", "half", + "hashbrown 0.17.1", "log", "num-traits", ] @@ -2620,7 +2628,7 @@ dependencies = [ [[package]] name = "datafusion-functions-aggregate-common" version = "54.1.0" -source = "git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a#efa84e8309ced6614e62d1607899c47ed0b6b08a" +source = "git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11#70c26a06716eb552f0fc1d2115957a4089504f11" dependencies = [ "arrow", "datafusion-common", @@ -2631,7 +2639,7 @@ dependencies = [ [[package]] name = "datafusion-functions-nested" version = "54.1.0" -source = "git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a#efa84e8309ced6614e62d1607899c47ed0b6b08a" +source = "git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11#70c26a06716eb552f0fc1d2115957a4089504f11" dependencies = [ "arrow", "arrow-ord", @@ -2655,7 +2663,7 @@ dependencies = [ [[package]] name = "datafusion-functions-table" version = "54.1.0" -source = "git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a#efa84e8309ced6614e62d1607899c47ed0b6b08a" +source = "git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11#70c26a06716eb552f0fc1d2115957a4089504f11" dependencies = [ "arrow", "async-trait", @@ -2670,7 +2678,7 @@ dependencies = [ [[package]] name = "datafusion-functions-window" version = "54.1.0" -source = "git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a#efa84e8309ced6614e62d1607899c47ed0b6b08a" +source = "git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11#70c26a06716eb552f0fc1d2115957a4089504f11" dependencies = [ "arrow", "datafusion-common", @@ -2686,7 +2694,7 @@ dependencies = [ [[package]] name = "datafusion-functions-window-common" version = "54.1.0" -source = "git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a#efa84e8309ced6614e62d1607899c47ed0b6b08a" +source = "git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11#70c26a06716eb552f0fc1d2115957a4089504f11" dependencies = [ "datafusion-common", "datafusion-physical-expr-common", @@ -2695,17 +2703,17 @@ dependencies = [ [[package]] name = "datafusion-macros" version = "54.1.0" -source = "git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a#efa84e8309ced6614e62d1607899c47ed0b6b08a" +source = "git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11#70c26a06716eb552f0fc1d2115957a4089504f11" dependencies = [ "datafusion-doc", "quote", - "syn 2.0.119", + "syn 3.0.3", ] [[package]] name = "datafusion-optimizer" version = "54.1.0" -source = "git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a#efa84e8309ced6614e62d1607899c47ed0b6b08a" +source = "git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11#70c26a06716eb552f0fc1d2115957a4089504f11" dependencies = [ "arrow", "chrono", @@ -2724,7 +2732,7 @@ dependencies = [ [[package]] name = "datafusion-physical-expr" version = "54.1.0" -source = "git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a#efa84e8309ced6614e62d1607899c47ed0b6b08a" +source = "git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11#70c26a06716eb552f0fc1d2115957a4089504f11" dependencies = [ "arrow", "datafusion-common", @@ -2746,7 +2754,7 @@ dependencies = [ [[package]] name = "datafusion-physical-expr-adapter" version = "54.1.0" -source = "git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a#efa84e8309ced6614e62d1607899c47ed0b6b08a" +source = "git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11#70c26a06716eb552f0fc1d2115957a4089504f11" dependencies = [ "arrow", "datafusion-common", @@ -2760,7 +2768,7 @@ dependencies = [ [[package]] name = "datafusion-physical-expr-common" version = "54.1.0" -source = "git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a#efa84e8309ced6614e62d1607899c47ed0b6b08a" +source = "git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11#70c26a06716eb552f0fc1d2115957a4089504f11" dependencies = [ "arrow", "chrono", @@ -2777,7 +2785,7 @@ dependencies = [ [[package]] name = "datafusion-physical-optimizer" version = "54.1.0" -source = "git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a#efa84e8309ced6614e62d1607899c47ed0b6b08a" +source = "git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11#70c26a06716eb552f0fc1d2115957a4089504f11" dependencies = [ "arrow", "datafusion-common", @@ -2788,6 +2796,7 @@ dependencies = [ "datafusion-physical-expr-common", "datafusion-physical-plan", "datafusion-pruning", + "datafusion-session", "itertools 0.15.0", "recursive", ] @@ -2795,7 +2804,7 @@ dependencies = [ [[package]] name = "datafusion-physical-plan" version = "54.1.0" -source = "git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a#efa84e8309ced6614e62d1607899c47ed0b6b08a" +source = "git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11#70c26a06716eb552f0fc1d2115957a4089504f11" dependencies = [ "arrow", "arrow-data", @@ -2831,7 +2840,7 @@ dependencies = [ [[package]] name = "datafusion-proto" version = "54.1.0" -source = "git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a#efa84e8309ced6614e62d1607899c47ed0b6b08a" +source = "git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11#70c26a06716eb552f0fc1d2115957a4089504f11" dependencies = [ "arrow", "chrono", @@ -2858,7 +2867,7 @@ dependencies = [ [[package]] name = "datafusion-proto-common" version = "54.1.0" -source = "git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a#efa84e8309ced6614e62d1607899c47ed0b6b08a" +source = "git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11#70c26a06716eb552f0fc1d2115957a4089504f11" dependencies = [ "arrow", "datafusion-common", @@ -2868,7 +2877,7 @@ dependencies = [ [[package]] name = "datafusion-proto-models" version = "54.1.0" -source = "git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a#efa84e8309ced6614e62d1607899c47ed0b6b08a" +source = "git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11#70c26a06716eb552f0fc1d2115957a4089504f11" dependencies = [ "datafusion-proto-common", "prost", @@ -2877,7 +2886,7 @@ dependencies = [ [[package]] name = "datafusion-pruning" version = "54.1.0" -source = "git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a#efa84e8309ced6614e62d1607899c47ed0b6b08a" +source = "git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11#70c26a06716eb552f0fc1d2115957a4089504f11" dependencies = [ "arrow", "datafusion-common", @@ -2892,7 +2901,7 @@ dependencies = [ [[package]] name = "datafusion-session" version = "54.1.0" -source = "git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a#efa84e8309ced6614e62d1607899c47ed0b6b08a" +source = "git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11#70c26a06716eb552f0fc1d2115957a4089504f11" dependencies = [ "arrow-schema", "async-trait", @@ -2906,7 +2915,7 @@ dependencies = [ [[package]] name = "datafusion-spark" version = "54.1.0" -source = "git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a#efa84e8309ced6614e62d1607899c47ed0b6b08a" +source = "git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11#70c26a06716eb552f0fc1d2115957a4089504f11" dependencies = [ "arrow", "bigdecimal", @@ -2935,7 +2944,7 @@ dependencies = [ [[package]] name = "datafusion-sql" version = "54.1.0" -source = "git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a#efa84e8309ced6614e62d1607899c47ed0b6b08a" +source = "git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11#70c26a06716eb552f0fc1d2115957a4089504f11" dependencies = [ "arrow", "bigdecimal", @@ -2954,7 +2963,7 @@ dependencies = [ [[package]] name = "datafusion-substrait" version = "54.1.0" -source = "git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a#efa84e8309ced6614e62d1607899c47ed0b6b08a" +source = "git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11#70c26a06716eb552f0fc1d2115957a4089504f11" dependencies = [ "async-recursion", "async-trait", @@ -3099,7 +3108,7 @@ version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "29547a1dc60885a552306986316bc9701ba120c1a8db6769fa68691529ad373d" dependencies = [ - "base64", + "base64 0.22.1", "serde", "serde_json", ] @@ -3834,7 +3843,7 @@ version = "0.1.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0" dependencies = [ - "base64", + "base64 0.22.1", "bytes", "futures-channel", "futures-util", @@ -4805,7 +4814,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "622acbc9100d3c10e2ee15804b0caa40e55c933d5aa53814cd520805b7958a49" dependencies = [ "async-trait", - "base64", + "base64 0.22.1", "bytes", "chrono", "form_urlencoded", @@ -4952,7 +4961,7 @@ dependencies = [ "arrow-ipc", "arrow-schema", "arrow-select", - "base64", + "base64 0.22.1", "brotli", "bytes", "chrono", @@ -5018,7 +5027,7 @@ version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "898bac3fa00d0ba57a4e8289837e965baa2dee8c3749f3b11d45a64b4223d9c3" dependencies = [ - "base64", + "base64 0.22.1", "serde", ] @@ -5931,7 +5940,7 @@ version = "0.12.28" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147" dependencies = [ - "base64", + "base64 0.22.1", "bytes", "futures-core", "futures-util", @@ -5973,7 +5982,7 @@ version = "0.13.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "219c5811de6525e5416c7d5d53bb656d3afdbc6c5af816e0802bcfa42dbdc1c3" dependencies = [ - "base64", + "base64 0.22.1", "bytes", "encoding_rs", "futures-core", @@ -6425,7 +6434,7 @@ version = "3.21.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "76a5c54c7310e7b8b9577c286d7e399ddd876c3e12b3ed917a8aabc4b96e9e8c" dependencies = [ - "base64", + "base64 0.22.1", "bs58", "chrono", "hex", @@ -6943,7 +6952,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4676b37242ccbd1aabf56edb093a4827dc49086c0ffd764a5705899e0f35f8f7" dependencies = [ "anyhow", - "base64", + "base64 0.22.1", "bitflags 2.13.1", "fancy-regex", "filedescriptor", @@ -7255,7 +7264,7 @@ checksum = "ac2a5518c70fa84342385732db33fb3f44bc4cc748936eb5833d2df34d6445ef" dependencies = [ "async-trait", "axum", - "base64", + "base64 0.22.1", "bytes", "h2", "http 1.4.2", @@ -7614,7 +7623,7 @@ version = "3.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dea7109cdcd5864d4eeb1b58a1648dc9bf520360d7af16ec26d0a9354bafcfc0" dependencies = [ - "base64", + "base64 0.22.1", "log", "percent-encoding", "rustls", @@ -7629,7 +7638,7 @@ version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e994ba84b0bd1b1b0cf92878b7ef898a5c1760108fe7b6010327e274917a808c" dependencies = [ - "base64", + "base64 0.22.1", "http 1.4.2", "httparse", "log", diff --git a/Cargo.toml b/Cargo.toml index 0fa9ecb69e..71e7c374d4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -41,13 +41,13 @@ clap = { version = "4.5", features = ["derive", "cargo"] } # DataFusion crates are pinned to a git rev of apache/datafusion main rather than # a crates.io release. Bump this rev to advance to newer DataFusion commits. -datafusion = { git = "https://github.com/apache/datafusion.git", rev = "efa84e8309ced6614e62d1607899c47ed0b6b08a" } -datafusion-cli = { git = "https://github.com/apache/datafusion.git", rev = "efa84e8309ced6614e62d1607899c47ed0b6b08a" } -datafusion-functions-aggregate-common = { git = "https://github.com/apache/datafusion.git", rev = "efa84e8309ced6614e62d1607899c47ed0b6b08a" } -datafusion-proto = { git = "https://github.com/apache/datafusion.git", rev = "efa84e8309ced6614e62d1607899c47ed0b6b08a" } -datafusion-proto-common = { git = "https://github.com/apache/datafusion.git", rev = "efa84e8309ced6614e62d1607899c47ed0b6b08a" } -datafusion-spark = { git = "https://github.com/apache/datafusion.git", rev = "efa84e8309ced6614e62d1607899c47ed0b6b08a" } -datafusion-substrait = { git = "https://github.com/apache/datafusion.git", rev = "efa84e8309ced6614e62d1607899c47ed0b6b08a" } +datafusion = { git = "https://github.com/apache/datafusion.git", rev = "70c26a06716eb552f0fc1d2115957a4089504f11" } +datafusion-cli = { git = "https://github.com/apache/datafusion.git", rev = "70c26a06716eb552f0fc1d2115957a4089504f11" } +datafusion-functions-aggregate-common = { git = "https://github.com/apache/datafusion.git", rev = "70c26a06716eb552f0fc1d2115957a4089504f11" } +datafusion-proto = { git = "https://github.com/apache/datafusion.git", rev = "70c26a06716eb552f0fc1d2115957a4089504f11" } +datafusion-proto-common = { git = "https://github.com/apache/datafusion.git", rev = "70c26a06716eb552f0fc1d2115957a4089504f11" } +datafusion-spark = { git = "https://github.com/apache/datafusion.git", rev = "70c26a06716eb552f0fc1d2115957a4089504f11" } +datafusion-substrait = { git = "https://github.com/apache/datafusion.git", rev = "70c26a06716eb552f0fc1d2115957a4089504f11" } ctor = { version = "1.0" } insta = "1.47" diff --git a/ballista/client/tests/context_setup.rs b/ballista/client/tests/context_setup.rs index 37eaa8040a..d6700c74c5 100644 --- a/ballista/client/tests/context_setup.rs +++ b/ballista/client/tests/context_setup.rs @@ -109,10 +109,9 @@ mod standalone { use ballista_core::serde::BallistaPhysicalExtensionCodec; use datafusion::{ assert_batches_eq, + catalog::Session, common::exec_err, - execution::{ - SessionState, SessionStateBuilder, TaskContext, context::QueryPlanner, - }, + execution::{SessionStateBuilder, TaskContext, context::QueryPlanner}, logical_expr::LogicalPlan, physical_plan::ExecutionPlan, prelude::{SessionConfig, SessionContext}, @@ -395,7 +394,7 @@ mod standalone { async fn create_physical_plan( &self, _logical_plan: &LogicalPlan, - _session_state: &SessionState, + _session_state: &dyn Session, ) -> datafusion::error::Result> { exec_err!("does not work") } diff --git a/ballista/core/proto/datafusion_common.proto b/ballista/core/proto/datafusion_common.proto index 7fff5b6b71..27d1101036 100644 --- a/ballista/core/proto/datafusion_common.proto +++ b/ballista/core/proto/datafusion_common.proto @@ -617,6 +617,8 @@ message ParquetOptions { uint64 max_row_group_size = 15; + uint64 max_in_list_size = 38; + string created_by = 16; oneof coerce_int96_opt { diff --git a/ballista/core/src/execution_plans/shuffle_writer.rs b/ballista/core/src/execution_plans/shuffle_writer.rs index a55eb0c082..7e4f4d1658 100644 --- a/ballista/core/src/execution_plans/shuffle_writer.rs +++ b/ballista/core/src/execution_plans/shuffle_writer.rs @@ -134,7 +134,7 @@ fn walk_child_partition_mapping( // Range-partitioning also freshly numbers its K outputs by // range bucket, so it forms a K-space just like Hash. Global // input ids from before the repartition are meaningless here. - return GlobalPartitionMap::HashSpace; + return GlobalPartitionMap::KSpace; } Partitioning::UnknownPartitioning(_) => { // RepartitionExec still exchanges rows and freshly numbers diff --git a/ballista/core/src/planner.rs b/ballista/core/src/planner.rs index 1f25bb5548..16cec95465 100644 --- a/ballista/core/src/planner.rs +++ b/ballista/core/src/planner.rs @@ -20,9 +20,10 @@ use crate::execution_plans::{DistributedExplainAnalyzeExec, DistributedQueryExec use crate::serde::BallistaLogicalExtensionCodec; use datafusion::arrow::datatypes::Schema; +use datafusion::catalog::Session; use datafusion::common::tree_node::{TreeNode, TreeNodeVisitor}; use datafusion::error::DataFusionError; -use datafusion::execution::context::{QueryPlanner, SessionState}; +use datafusion::execution::context::QueryPlanner; use datafusion::logical_expr::{LogicalPlan, TableScan}; use datafusion::physical_plan::ExecutionPlan; use datafusion::physical_plan::empty::EmptyExec; @@ -105,7 +106,7 @@ impl QueryPlanner for BallistaQueryPlanner { async fn create_physical_plan( &self, logical_plan: &LogicalPlan, - session_state: &SessionState, + session_state: &dyn Session, ) -> std::result::Result, DataFusionError> { log::debug!("create_physical_plan - plan: {:?}", logical_plan); // we inspect if plan scans local tables only, diff --git a/ballista/core/src/serde/mod.rs b/ballista/core/src/serde/mod.rs index 1a829e5f56..eb2ca41d15 100644 --- a/ballista/core/src/serde/mod.rs +++ b/ballista/core/src/serde/mod.rs @@ -1914,10 +1914,18 @@ mod test { let codec = BallistaPhysicalExtensionCodec::default(); let mut buf: Vec = vec![]; - codec.try_encode(Arc::new(original), &mut buf).unwrap(); + codec + .try_encode( + Arc::new(original), + &mut buf, + &DefaultPhysicalProtoConverter {}, + ) + .unwrap(); let ctx = SessionContext::new().task_ctx(); - let decoded_plan = codec.try_decode(&buf, &[input], &ctx).unwrap(); + let decoded_plan = codec + .try_decode(&buf, &[input], &ctx, &DefaultPhysicalProtoConverter {}) + .unwrap(); let decoded = decoded_plan .downcast_ref::() .expect("Expected PerPartitionFilterExec"); @@ -2115,10 +2123,18 @@ mod test { let codec = BallistaPhysicalExtensionCodec::default(); let mut buf: Vec = vec![]; - codec.try_encode(Arc::new(original), &mut buf).unwrap(); + codec + .try_encode( + Arc::new(original), + &mut buf, + &DefaultPhysicalProtoConverter {}, + ) + .unwrap(); let ctx = SessionContext::new().task_ctx(); - let decoded_plan = codec.try_decode(&buf, &[input], &ctx).unwrap(); + let decoded_plan = codec + .try_decode(&buf, &[input], &ctx, &DefaultPhysicalProtoConverter {}) + .unwrap(); let decoded = decoded_plan .downcast_ref::() diff --git a/ballista/scheduler/src/scheduler_server/grpc.rs b/ballista/scheduler/src/scheduler_server/grpc.rs index 3b41e918c0..2aaa0a9d06 100644 --- a/ballista/scheduler/src/scheduler_server/grpc.rs +++ b/ballista/scheduler/src/scheduler_server/grpc.rs @@ -856,7 +856,7 @@ impl SchedulerServer { - let plan = deserialize_bytes(bytes).await.map_err(|e| BallistaError::DataFusionError(e.into()))?; + let plan = deserialize_bytes(&bytes).map_err(|e| BallistaError::DataFusionError(e.into()))?; let ctx = session_ctx.clone(); from_substrait_plan(&ctx.state(), &plan) diff --git a/ballista/scheduler/src/state/task_builder.rs b/ballista/scheduler/src/state/task_builder.rs index 19bcb6f2a0..a3309f2772 100644 --- a/ballista/scheduler/src/state/task_builder.rs +++ b/ballista/scheduler/src/state/task_builder.rs @@ -637,7 +637,7 @@ mod tests { // Sanity: the operator really does declare its left input collected. assert!( matches!( - join.required_input_distribution().first(), + join.input_distribution_requirements().child_distribution(0), Some(Distribution::SinglePartition) ), "CrossJoinExec must declare SinglePartition on its left input" From 1aa4065feb413c7c64ba12544d5b67a893b275ac Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Thu, 6 Aug 2026 07:22:16 -0600 Subject: [PATCH 06/11] test: refresh AQE snapshot for DataFusion's null_aware join display --- .../src/state/aqe/optimizer_rule/distributed_exchange.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ballista/scheduler/src/state/aqe/optimizer_rule/distributed_exchange.rs b/ballista/scheduler/src/state/aqe/optimizer_rule/distributed_exchange.rs index 80e325566f..cc97c510c6 100644 --- a/ballista/scheduler/src/state/aqe/optimizer_rule/distributed_exchange.rs +++ b/ballista/scheduler/src/state/aqe/optimizer_rule/distributed_exchange.rs @@ -374,7 +374,7 @@ mod tests { assert_plan!(result.as_ref(), @ r" AdaptiveDatafusionExec: is_final=false, plan_id=1, stage_id=pending, stage_resolved=false - HashJoinExec: mode=CollectLeft, join_type=LeftAnti, on=[(a@0, a@0)] + HashJoinExec: mode=CollectLeft, join_type=LeftAnti, on=[(a@0, a@0)], null_aware StatisticsExec: col_count=1, row_count=Absent CoalescePartitionsExec ExchangeExec: partitioning=None, plan_id=0, stage_id=pending, stage_resolved=false From c91233f953ed5fc4f0cee73d84e000c1a5394bd3 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Thu, 6 Aug 2026 07:32:37 -0600 Subject: [PATCH 07/11] chore(python): regenerate Cargo.lock for the bumped DataFusion rev --- python/Cargo.lock | 539 ++++++++++++++++++++++++---------------------- 1 file changed, 279 insertions(+), 260 deletions(-) diff --git a/python/Cargo.lock b/python/Cargo.lock index 18951e3e05..9f8e3776b4 100644 --- a/python/Cargo.lock +++ b/python/Cargo.lock @@ -267,7 +267,7 @@ dependencies = [ "arrow-schema 58.4.0", "arrow-select 58.4.0", "atoi", - "base64", + "base64 0.22.1", "chrono", "comfy-table", "half", @@ -289,7 +289,7 @@ dependencies = [ "arrow-schema 59.1.0", "arrow-select 59.1.0", "atoi", - "base64", + "base64 0.22.1", "chrono", "comfy-table", "half", @@ -371,7 +371,7 @@ dependencies = [ "arrow-schema 59.1.0", "arrow-select 59.1.0", "arrow-string 59.1.0", - "base64", + "base64 0.22.1", "bytes", "futures", "once_cell", @@ -721,7 +721,7 @@ dependencies = [ "ballista-core", "ballista-executor", "ballista-scheduler", - "datafusion 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "datafusion 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", "log", "tokio", "url", @@ -734,10 +734,10 @@ dependencies = [ "arrow-flight", "async-trait", "chrono", - "datafusion 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", - "datafusion-functions-aggregate-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", - "datafusion-proto 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", - "datafusion-proto-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "datafusion 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", + "datafusion-functions-aggregate-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", + "datafusion-proto 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", + "datafusion-proto-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", "futures", "itertools 0.15.0", "log", @@ -768,8 +768,8 @@ dependencies = [ "ballista-core", "bytesize", "dashmap", - "datafusion 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", - "datafusion-proto 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "datafusion 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", + "datafusion-proto 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", "futures", "libc", "log", @@ -795,8 +795,8 @@ dependencies = [ "axum", "ballista-core", "dashmap", - "datafusion 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", - "datafusion-proto 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "datafusion 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", + "datafusion-proto 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", "futures", "http", "insta", @@ -812,6 +812,7 @@ dependencies = [ "tokio-stream", "tonic", "tower-http 0.7.0", + "url-escape", "uuid", ] @@ -821,6 +822,12 @@ version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" +[[package]] +name = "base64" +version = "0.23.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac07cdecf99051d9a5238b80f35af32cdeba5b336e55d957b318b50137e18da5" + [[package]] name = "bigdecimal" version = "0.4.10" @@ -1248,38 +1255,38 @@ dependencies = [ [[package]] name = "datafusion" version = "54.1.0" -source = "git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a#efa84e8309ced6614e62d1607899c47ed0b6b08a" +source = "git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11#70c26a06716eb552f0fc1d2115957a4089504f11" dependencies = [ "arrow 59.1.0", "arrow-schema 59.1.0", "async-trait", "bzip2", "chrono", - "datafusion-catalog 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", - "datafusion-catalog-listing 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", - "datafusion-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", - "datafusion-common-runtime 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", - "datafusion-datasource 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", - "datafusion-datasource-arrow 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", - "datafusion-datasource-csv 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", - "datafusion-datasource-json 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", - "datafusion-datasource-parquet 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", - "datafusion-execution 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", - "datafusion-expr 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", - "datafusion-expr-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", - "datafusion-functions 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", - "datafusion-functions-aggregate 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", - "datafusion-functions-nested 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", - "datafusion-functions-table 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", - "datafusion-functions-window 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", - "datafusion-optimizer 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", - "datafusion-physical-expr 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", - "datafusion-physical-expr-adapter 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", - "datafusion-physical-expr-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", - "datafusion-physical-optimizer 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", - "datafusion-physical-plan 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", - "datafusion-session 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", - "datafusion-sql 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "datafusion-catalog 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", + "datafusion-catalog-listing 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", + "datafusion-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", + "datafusion-common-runtime 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", + "datafusion-datasource 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", + "datafusion-datasource-arrow 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", + "datafusion-datasource-csv 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", + "datafusion-datasource-json 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", + "datafusion-datasource-parquet 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", + "datafusion-execution 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", + "datafusion-expr 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", + "datafusion-expr-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", + "datafusion-functions 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", + "datafusion-functions-aggregate 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", + "datafusion-functions-nested 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", + "datafusion-functions-table 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", + "datafusion-functions-window 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", + "datafusion-optimizer 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", + "datafusion-physical-expr 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", + "datafusion-physical-expr-adapter 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", + "datafusion-physical-expr-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", + "datafusion-physical-optimizer 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", + "datafusion-physical-plan 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", + "datafusion-session 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", + "datafusion-sql 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", "flate2", "futures", "indexmap", @@ -1325,19 +1332,19 @@ dependencies = [ [[package]] name = "datafusion-catalog" version = "54.1.0" -source = "git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a#efa84e8309ced6614e62d1607899c47ed0b6b08a" +source = "git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11#70c26a06716eb552f0fc1d2115957a4089504f11" dependencies = [ "arrow 59.1.0", "async-trait", "dashmap", - "datafusion-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", - "datafusion-common-runtime 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", - "datafusion-datasource 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", - "datafusion-execution 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", - "datafusion-expr 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", - "datafusion-physical-expr 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", - "datafusion-physical-plan 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", - "datafusion-session 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "datafusion-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", + "datafusion-common-runtime 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", + "datafusion-datasource 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", + "datafusion-execution 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", + "datafusion-expr 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", + "datafusion-physical-expr 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", + "datafusion-physical-plan 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", + "datafusion-session 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", "futures", "itertools 0.15.0", "log", @@ -1372,19 +1379,19 @@ dependencies = [ [[package]] name = "datafusion-catalog-listing" version = "54.1.0" -source = "git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a#efa84e8309ced6614e62d1607899c47ed0b6b08a" +source = "git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11#70c26a06716eb552f0fc1d2115957a4089504f11" dependencies = [ "arrow 59.1.0", "async-trait", - "datafusion-catalog 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", - "datafusion-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", - "datafusion-datasource 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", - "datafusion-execution 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", - "datafusion-expr 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", - "datafusion-physical-expr 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", - "datafusion-physical-expr-adapter 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", - "datafusion-physical-expr-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", - "datafusion-physical-plan 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "datafusion-catalog 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", + "datafusion-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", + "datafusion-datasource 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", + "datafusion-execution 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", + "datafusion-expr 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", + "datafusion-physical-expr 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", + "datafusion-physical-expr-adapter 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", + "datafusion-physical-expr-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", + "datafusion-physical-plan 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", "futures", "itertools 0.15.0", "log", @@ -1421,7 +1428,7 @@ dependencies = [ [[package]] name = "datafusion-common" version = "54.1.0" -source = "git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a#efa84e8309ced6614e62d1607899c47ed0b6b08a" +source = "git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11#70c26a06716eb552f0fc1d2115957a4089504f11" dependencies = [ "arrow 59.1.0", "arrow-ipc 59.1.0", @@ -1458,7 +1465,7 @@ dependencies = [ [[package]] name = "datafusion-common-runtime" version = "54.1.0" -source = "git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a#efa84e8309ced6614e62d1607899c47ed0b6b08a" +source = "git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11#70c26a06716eb552f0fc1d2115957a4089504f11" dependencies = [ "futures", "log", @@ -1504,7 +1511,7 @@ dependencies = [ [[package]] name = "datafusion-datasource" version = "54.1.0" -source = "git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a#efa84e8309ced6614e62d1607899c47ed0b6b08a" +source = "git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11#70c26a06716eb552f0fc1d2115957a4089504f11" dependencies = [ "arrow 59.1.0", "async-compression", @@ -1512,15 +1519,16 @@ dependencies = [ "bytes", "bzip2", "chrono", - "datafusion-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", - "datafusion-common-runtime 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", - "datafusion-execution 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", - "datafusion-expr 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", - "datafusion-physical-expr 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", - "datafusion-physical-expr-adapter 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", - "datafusion-physical-expr-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", - "datafusion-physical-plan 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", - "datafusion-session 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "datafusion-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", + "datafusion-common-runtime 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", + "datafusion-execution 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", + "datafusion-expr 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", + "datafusion-physical-expr 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", + "datafusion-physical-expr-adapter 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", + "datafusion-physical-expr-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", + "datafusion-physical-plan 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", + "datafusion-proto-models", + "datafusion-session 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", "flate2", "futures", "glob", @@ -1563,20 +1571,20 @@ dependencies = [ [[package]] name = "datafusion-datasource-arrow" version = "54.1.0" -source = "git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a#efa84e8309ced6614e62d1607899c47ed0b6b08a" +source = "git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11#70c26a06716eb552f0fc1d2115957a4089504f11" dependencies = [ "arrow 59.1.0", "arrow-ipc 59.1.0", "async-trait", "bytes", - "datafusion-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", - "datafusion-common-runtime 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", - "datafusion-datasource 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", - "datafusion-execution 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", - "datafusion-expr 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", - "datafusion-physical-expr-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", - "datafusion-physical-plan 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", - "datafusion-session 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "datafusion-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", + "datafusion-common-runtime 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", + "datafusion-datasource 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", + "datafusion-execution 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", + "datafusion-expr 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", + "datafusion-physical-expr-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", + "datafusion-physical-plan 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", + "datafusion-session 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", "futures", "itertools 0.15.0", "object_store", @@ -1628,19 +1636,19 @@ dependencies = [ [[package]] name = "datafusion-datasource-csv" version = "54.1.0" -source = "git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a#efa84e8309ced6614e62d1607899c47ed0b6b08a" +source = "git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11#70c26a06716eb552f0fc1d2115957a4089504f11" dependencies = [ "arrow 59.1.0", "async-trait", "bytes", - "datafusion-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", - "datafusion-common-runtime 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", - "datafusion-datasource 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", - "datafusion-execution 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", - "datafusion-expr 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", - "datafusion-physical-expr-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", - "datafusion-physical-plan 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", - "datafusion-session 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "datafusion-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", + "datafusion-common-runtime 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", + "datafusion-datasource 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", + "datafusion-execution 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", + "datafusion-expr 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", + "datafusion-physical-expr-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", + "datafusion-physical-plan 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", + "datafusion-session 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", "futures", "object_store", "regex", @@ -1673,19 +1681,19 @@ dependencies = [ [[package]] name = "datafusion-datasource-json" version = "54.1.0" -source = "git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a#efa84e8309ced6614e62d1607899c47ed0b6b08a" +source = "git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11#70c26a06716eb552f0fc1d2115957a4089504f11" dependencies = [ "arrow 59.1.0", "async-trait", "bytes", - "datafusion-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", - "datafusion-common-runtime 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", - "datafusion-datasource 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", - "datafusion-execution 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", - "datafusion-expr 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", - "datafusion-physical-expr-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", - "datafusion-physical-plan 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", - "datafusion-session 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "datafusion-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", + "datafusion-common-runtime 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", + "datafusion-datasource 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", + "datafusion-execution 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", + "datafusion-expr 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", + "datafusion-physical-expr-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", + "datafusion-physical-plan 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", + "datafusion-session 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", "futures", "object_store", "tokio", @@ -1726,25 +1734,25 @@ dependencies = [ [[package]] name = "datafusion-datasource-parquet" version = "54.1.0" -source = "git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a#efa84e8309ced6614e62d1607899c47ed0b6b08a" +source = "git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11#70c26a06716eb552f0fc1d2115957a4089504f11" dependencies = [ "arrow 59.1.0", "arrow-schema 59.1.0", "async-trait", "bytes", - "datafusion-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", - "datafusion-common-runtime 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", - "datafusion-datasource 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", - "datafusion-execution 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", - "datafusion-expr 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", - "datafusion-functions 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", - "datafusion-functions-aggregate-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", - "datafusion-physical-expr 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", - "datafusion-physical-expr-adapter 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", - "datafusion-physical-expr-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", - "datafusion-physical-plan 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", - "datafusion-pruning 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", - "datafusion-session 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "datafusion-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", + "datafusion-common-runtime 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", + "datafusion-datasource 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", + "datafusion-execution 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", + "datafusion-expr 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", + "datafusion-functions 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", + "datafusion-functions-aggregate-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", + "datafusion-physical-expr 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", + "datafusion-physical-expr-adapter 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", + "datafusion-physical-expr-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", + "datafusion-physical-plan 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", + "datafusion-pruning 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", + "datafusion-session 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", "futures", "itertools 0.15.0", "log", @@ -1763,7 +1771,7 @@ checksum = "d69bb69d8769e34f76839c960dbde24c1ac0c885a79b6c3c2287bdc56ec67891" [[package]] name = "datafusion-doc" version = "54.1.0" -source = "git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a#efa84e8309ced6614e62d1607899c47ed0b6b08a" +source = "git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11#70c26a06716eb552f0fc1d2115957a4089504f11" [[package]] name = "datafusion-execution" @@ -1790,16 +1798,16 @@ dependencies = [ [[package]] name = "datafusion-execution" version = "54.1.0" -source = "git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a#efa84e8309ced6614e62d1607899c47ed0b6b08a" +source = "git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11#70c26a06716eb552f0fc1d2115957a4089504f11" dependencies = [ "arrow 59.1.0", "arrow-buffer 59.1.0", "async-trait", "bytes", "dashmap", - "datafusion-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", - "datafusion-expr 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", - "datafusion-physical-expr-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "datafusion-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", + "datafusion-expr 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", + "datafusion-physical-expr-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", "futures", "log", "object_store", @@ -1838,18 +1846,18 @@ dependencies = [ [[package]] name = "datafusion-expr" version = "54.1.0" -source = "git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a#efa84e8309ced6614e62d1607899c47ed0b6b08a" +source = "git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11#70c26a06716eb552f0fc1d2115957a4089504f11" dependencies = [ "arrow 59.1.0", "arrow-schema 59.1.0", "async-trait", "chrono", - "datafusion-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", - "datafusion-doc 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", - "datafusion-expr-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", - "datafusion-functions-aggregate-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", - "datafusion-functions-window-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", - "datafusion-physical-expr-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "datafusion-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", + "datafusion-doc 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", + "datafusion-expr-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", + "datafusion-functions-aggregate-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", + "datafusion-functions-window-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", + "datafusion-physical-expr-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", "indexmap", "itertools 0.15.0", "recursive", @@ -1872,10 +1880,10 @@ dependencies = [ [[package]] name = "datafusion-expr-common" version = "54.1.0" -source = "git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a#efa84e8309ced6614e62d1607899c47ed0b6b08a" +source = "git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11#70c26a06716eb552f0fc1d2115957a4089504f11" dependencies = [ "arrow 59.1.0", - "datafusion-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "datafusion-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", "indexmap", "itertools 0.15.0", ] @@ -1921,7 +1929,7 @@ checksum = "5f64c983bbbdcb729d921a2b2ac3375598719b5cc0c30345ad664936f3176fc7" dependencies = [ "arrow 58.4.0", "arrow-buffer 58.4.0", - "base64", + "base64 0.22.1", "blake2", "blake3", "chrono", @@ -1948,22 +1956,22 @@ dependencies = [ [[package]] name = "datafusion-functions" version = "54.1.0" -source = "git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a#efa84e8309ced6614e62d1607899c47ed0b6b08a" +source = "git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11#70c26a06716eb552f0fc1d2115957a4089504f11" dependencies = [ "arrow 59.1.0", "arrow-buffer 59.1.0", - "base64", + "base64 0.23.1", "blake2", "blake3", "chrono", "chrono-tz", - "datafusion-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", - "datafusion-doc 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", - "datafusion-execution 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", - "datafusion-expr 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", - "datafusion-expr-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", - "datafusion-macros 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", - "datafusion-physical-expr-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "datafusion-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", + "datafusion-doc 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", + "datafusion-execution 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", + "datafusion-expr 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", + "datafusion-expr-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", + "datafusion-macros 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", + "datafusion-physical-expr-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", "hex", "itertools 0.15.0", "log", @@ -2000,18 +2008,19 @@ dependencies = [ [[package]] name = "datafusion-functions-aggregate" version = "54.1.0" -source = "git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a#efa84e8309ced6614e62d1607899c47ed0b6b08a" +source = "git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11#70c26a06716eb552f0fc1d2115957a4089504f11" dependencies = [ "arrow 59.1.0", - "datafusion-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", - "datafusion-doc 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", - "datafusion-execution 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", - "datafusion-expr 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", - "datafusion-functions-aggregate-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", - "datafusion-macros 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", - "datafusion-physical-expr 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", - "datafusion-physical-expr-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "datafusion-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", + "datafusion-doc 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", + "datafusion-execution 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", + "datafusion-expr 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", + "datafusion-functions-aggregate-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", + "datafusion-macros 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", + "datafusion-physical-expr 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", + "datafusion-physical-expr-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", "half", + "hashbrown 0.17.1", "log", "num-traits", ] @@ -2031,12 +2040,12 @@ dependencies = [ [[package]] name = "datafusion-functions-aggregate-common" version = "54.1.0" -source = "git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a#efa84e8309ced6614e62d1607899c47ed0b6b08a" +source = "git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11#70c26a06716eb552f0fc1d2115957a4089504f11" dependencies = [ "arrow 59.1.0", - "datafusion-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", - "datafusion-expr-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", - "datafusion-physical-expr-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "datafusion-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", + "datafusion-expr-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", + "datafusion-physical-expr-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", ] [[package]] @@ -2067,20 +2076,20 @@ dependencies = [ [[package]] name = "datafusion-functions-nested" version = "54.1.0" -source = "git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a#efa84e8309ced6614e62d1607899c47ed0b6b08a" +source = "git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11#70c26a06716eb552f0fc1d2115957a4089504f11" dependencies = [ "arrow 59.1.0", "arrow-ord 59.1.0", - "datafusion-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", - "datafusion-doc 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", - "datafusion-execution 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", - "datafusion-expr 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", - "datafusion-expr-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", - "datafusion-functions 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", - "datafusion-functions-aggregate 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", - "datafusion-functions-aggregate-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", - "datafusion-macros 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", - "datafusion-physical-expr-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "datafusion-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", + "datafusion-doc 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", + "datafusion-execution 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", + "datafusion-expr 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", + "datafusion-expr-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", + "datafusion-functions 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", + "datafusion-functions-aggregate 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", + "datafusion-functions-aggregate-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", + "datafusion-macros 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", + "datafusion-physical-expr-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", "hashbrown 0.17.1", "itertools 0.15.0", "itoa", @@ -2107,15 +2116,15 @@ dependencies = [ [[package]] name = "datafusion-functions-table" version = "54.1.0" -source = "git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a#efa84e8309ced6614e62d1607899c47ed0b6b08a" +source = "git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11#70c26a06716eb552f0fc1d2115957a4089504f11" dependencies = [ "arrow 59.1.0", "async-trait", - "datafusion-catalog 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", - "datafusion-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", - "datafusion-expr 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", - "datafusion-physical-expr 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", - "datafusion-physical-plan 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "datafusion-catalog 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", + "datafusion-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", + "datafusion-expr 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", + "datafusion-physical-expr 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", + "datafusion-physical-plan 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", "parking_lot", ] @@ -2139,16 +2148,16 @@ dependencies = [ [[package]] name = "datafusion-functions-window" version = "54.1.0" -source = "git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a#efa84e8309ced6614e62d1607899c47ed0b6b08a" +source = "git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11#70c26a06716eb552f0fc1d2115957a4089504f11" dependencies = [ "arrow 59.1.0", - "datafusion-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", - "datafusion-doc 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", - "datafusion-expr 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", - "datafusion-functions-window-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", - "datafusion-macros 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", - "datafusion-physical-expr 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", - "datafusion-physical-expr-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "datafusion-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", + "datafusion-doc 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", + "datafusion-expr 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", + "datafusion-functions-window-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", + "datafusion-macros 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", + "datafusion-physical-expr 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", + "datafusion-physical-expr-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", "log", ] @@ -2165,10 +2174,10 @@ dependencies = [ [[package]] name = "datafusion-functions-window-common" version = "54.1.0" -source = "git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a#efa84e8309ced6614e62d1607899c47ed0b6b08a" +source = "git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11#70c26a06716eb552f0fc1d2115957a4089504f11" dependencies = [ - "datafusion-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", - "datafusion-physical-expr-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "datafusion-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", + "datafusion-physical-expr-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", ] [[package]] @@ -2185,11 +2194,11 @@ dependencies = [ [[package]] name = "datafusion-macros" version = "54.1.0" -source = "git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a#efa84e8309ced6614e62d1607899c47ed0b6b08a" +source = "git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11#70c26a06716eb552f0fc1d2115957a4089504f11" dependencies = [ - "datafusion-doc 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "datafusion-doc 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", "quote", - "syn 2.0.119", + "syn 3.0.3", ] [[package]] @@ -2215,14 +2224,14 @@ dependencies = [ [[package]] name = "datafusion-optimizer" version = "54.1.0" -source = "git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a#efa84e8309ced6614e62d1607899c47ed0b6b08a" +source = "git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11#70c26a06716eb552f0fc1d2115957a4089504f11" dependencies = [ "arrow 59.1.0", "chrono", - "datafusion-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", - "datafusion-expr 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", - "datafusion-expr-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", - "datafusion-physical-expr 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "datafusion-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", + "datafusion-expr 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", + "datafusion-expr-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", + "datafusion-physical-expr 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", "indexmap", "itertools 0.15.0", "log", @@ -2256,14 +2265,14 @@ dependencies = [ [[package]] name = "datafusion-physical-expr" version = "54.1.0" -source = "git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a#efa84e8309ced6614e62d1607899c47ed0b6b08a" +source = "git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11#70c26a06716eb552f0fc1d2115957a4089504f11" dependencies = [ "arrow 59.1.0", - "datafusion-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", - "datafusion-expr 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", - "datafusion-expr-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", - "datafusion-functions-aggregate-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", - "datafusion-physical-expr-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "datafusion-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", + "datafusion-expr 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", + "datafusion-expr-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", + "datafusion-functions-aggregate-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", + "datafusion-physical-expr-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", "datafusion-proto-models", "half", "hashbrown 0.17.1", @@ -2293,14 +2302,14 @@ dependencies = [ [[package]] name = "datafusion-physical-expr-adapter" version = "54.1.0" -source = "git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a#efa84e8309ced6614e62d1607899c47ed0b6b08a" +source = "git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11#70c26a06716eb552f0fc1d2115957a4089504f11" dependencies = [ "arrow 59.1.0", - "datafusion-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", - "datafusion-expr 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", - "datafusion-functions 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", - "datafusion-physical-expr 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", - "datafusion-physical-expr-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "datafusion-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", + "datafusion-expr 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", + "datafusion-functions 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", + "datafusion-physical-expr 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", + "datafusion-physical-expr-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", "itertools 0.15.0", ] @@ -2324,12 +2333,12 @@ dependencies = [ [[package]] name = "datafusion-physical-expr-common" version = "54.1.0" -source = "git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a#efa84e8309ced6614e62d1607899c47ed0b6b08a" +source = "git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11#70c26a06716eb552f0fc1d2115957a4089504f11" dependencies = [ "arrow 59.1.0", "chrono", - "datafusion-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", - "datafusion-expr-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "datafusion-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", + "datafusion-expr-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", "datafusion-proto-models", "hashbrown 0.17.1", "indexmap", @@ -2360,17 +2369,18 @@ dependencies = [ [[package]] name = "datafusion-physical-optimizer" version = "54.1.0" -source = "git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a#efa84e8309ced6614e62d1607899c47ed0b6b08a" +source = "git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11#70c26a06716eb552f0fc1d2115957a4089504f11" dependencies = [ "arrow 59.1.0", - "datafusion-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", - "datafusion-execution 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", - "datafusion-expr 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", - "datafusion-expr-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", - "datafusion-physical-expr 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", - "datafusion-physical-expr-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", - "datafusion-physical-plan 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", - "datafusion-pruning 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "datafusion-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", + "datafusion-execution 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", + "datafusion-expr 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", + "datafusion-expr-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", + "datafusion-physical-expr 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", + "datafusion-physical-expr-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", + "datafusion-physical-plan 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", + "datafusion-pruning 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", + "datafusion-session 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", "itertools 0.15.0", "recursive", ] @@ -2411,7 +2421,7 @@ dependencies = [ [[package]] name = "datafusion-physical-plan" version = "54.1.0" -source = "git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a#efa84e8309ced6614e62d1607899c47ed0b6b08a" +source = "git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11#70c26a06716eb552f0fc1d2115957a4089504f11" dependencies = [ "arrow 59.1.0", "arrow-data 59.1.0", @@ -2420,16 +2430,16 @@ dependencies = [ "arrow-schema 59.1.0", "async-trait", "bytes", - "datafusion-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", - "datafusion-common-runtime 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", - "datafusion-execution 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", - "datafusion-expr 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", - "datafusion-functions 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", - "datafusion-functions-aggregate-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", - "datafusion-functions-window-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", - "datafusion-physical-expr 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", - "datafusion-physical-expr-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", - "datafusion-proto-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "datafusion-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", + "datafusion-common-runtime 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", + "datafusion-execution 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", + "datafusion-expr 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", + "datafusion-functions 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", + "datafusion-functions-aggregate-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", + "datafusion-functions-window-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", + "datafusion-physical-expr 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", + "datafusion-physical-expr-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", + "datafusion-proto-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", "datafusion-proto-models", "futures", "half", @@ -2474,25 +2484,25 @@ dependencies = [ [[package]] name = "datafusion-proto" version = "54.1.0" -source = "git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a#efa84e8309ced6614e62d1607899c47ed0b6b08a" +source = "git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11#70c26a06716eb552f0fc1d2115957a4089504f11" dependencies = [ "arrow 59.1.0", "chrono", - "datafusion-catalog 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", - "datafusion-catalog-listing 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", - "datafusion-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", - "datafusion-datasource 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", - "datafusion-datasource-arrow 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", - "datafusion-datasource-csv 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", - "datafusion-datasource-json 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", - "datafusion-datasource-parquet 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", - "datafusion-execution 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", - "datafusion-expr 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", - "datafusion-functions-table 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", - "datafusion-physical-expr 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", - "datafusion-physical-expr-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", - "datafusion-physical-plan 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", - "datafusion-proto-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "datafusion-catalog 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", + "datafusion-catalog-listing 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", + "datafusion-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", + "datafusion-datasource 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", + "datafusion-datasource-arrow 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", + "datafusion-datasource-csv 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", + "datafusion-datasource-json 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", + "datafusion-datasource-parquet 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", + "datafusion-execution 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", + "datafusion-expr 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", + "datafusion-functions-table 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", + "datafusion-physical-expr 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", + "datafusion-physical-expr-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", + "datafusion-physical-plan 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", + "datafusion-proto-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", "datafusion-proto-models", "object_store", "prost", @@ -2512,19 +2522,19 @@ dependencies = [ [[package]] name = "datafusion-proto-common" version = "54.1.0" -source = "git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a#efa84e8309ced6614e62d1607899c47ed0b6b08a" +source = "git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11#70c26a06716eb552f0fc1d2115957a4089504f11" dependencies = [ "arrow 59.1.0", - "datafusion-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "datafusion-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", "prost", ] [[package]] name = "datafusion-proto-models" version = "54.1.0" -source = "git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a#efa84e8309ced6614e62d1607899c47ed0b6b08a" +source = "git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11#70c26a06716eb552f0fc1d2115957a4089504f11" dependencies = [ - "datafusion-proto-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "datafusion-proto-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", "prost", ] @@ -2547,15 +2557,15 @@ dependencies = [ [[package]] name = "datafusion-pruning" version = "54.1.0" -source = "git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a#efa84e8309ced6614e62d1607899c47ed0b6b08a" +source = "git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11#70c26a06716eb552f0fc1d2115957a4089504f11" dependencies = [ "arrow 59.1.0", - "datafusion-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", - "datafusion-datasource 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", - "datafusion-expr-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", - "datafusion-physical-expr 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", - "datafusion-physical-expr-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", - "datafusion-physical-plan 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "datafusion-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", + "datafusion-datasource 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", + "datafusion-expr-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", + "datafusion-physical-expr 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", + "datafusion-physical-expr-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", + "datafusion-physical-plan 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", "log", ] @@ -2623,14 +2633,14 @@ dependencies = [ [[package]] name = "datafusion-session" version = "54.1.0" -source = "git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a#efa84e8309ced6614e62d1607899c47ed0b6b08a" +source = "git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11#70c26a06716eb552f0fc1d2115957a4089504f11" dependencies = [ "arrow-schema 59.1.0", "async-trait", - "datafusion-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", - "datafusion-execution 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", - "datafusion-expr 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", - "datafusion-physical-plan 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "datafusion-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", + "datafusion-execution 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", + "datafusion-expr 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", + "datafusion-physical-plan 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", "parking_lot", ] @@ -2685,14 +2695,14 @@ dependencies = [ [[package]] name = "datafusion-sql" version = "54.1.0" -source = "git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a#efa84e8309ced6614e62d1607899c47ed0b6b08a" +source = "git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11#70c26a06716eb552f0fc1d2115957a4089504f11" dependencies = [ "arrow 59.1.0", "bigdecimal", "chrono", - "datafusion-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", - "datafusion-expr 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", - "datafusion-functions-nested 54.1.0 (git+https://github.com/apache/datafusion.git?rev=efa84e8309ced6614e62d1607899c47ed0b6b08a)", + "datafusion-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", + "datafusion-expr 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", + "datafusion-functions-nested 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", "indexmap", "log", "recursive", @@ -3157,7 +3167,7 @@ version = "0.1.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0" dependencies = [ - "base64", + "base64 0.22.1", "bytes", "futures-channel", "futures-util", @@ -3693,7 +3703,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "622acbc9100d3c10e2ee15804b0caa40e55c933d5aa53814cd520805b7958a49" dependencies = [ "async-trait", - "base64", + "base64 0.22.1", "bytes", "chrono", "form_urlencoded", @@ -3783,7 +3793,7 @@ dependencies = [ "arrow-ipc 58.4.0", "arrow-schema 58.4.0", "arrow-select 58.4.0", - "base64", + "base64 0.22.1", "brotli", "bytes", "chrono", @@ -3819,7 +3829,7 @@ dependencies = [ "arrow-ipc 59.1.0", "arrow-schema 59.1.0", "arrow-select 59.1.0", - "base64", + "base64 0.22.1", "brotli", "bytes", "chrono", @@ -4378,7 +4388,7 @@ version = "0.12.28" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147" dependencies = [ - "base64", + "base64 0.22.1", "bytes", "futures-core", "futures-util", @@ -5072,7 +5082,7 @@ checksum = "ac2a5518c70fa84342385732db33fb3f44bc4cc748936eb5833d2df34d6445ef" dependencies = [ "async-trait", "axum", - "base64", + "base64 0.22.1", "bytes", "h2", "http", @@ -5291,6 +5301,15 @@ dependencies = [ "serde", ] +[[package]] +name = "url-escape" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd4105a4fc511f6b1c9616a372a1313ae7c15afcaac839617c1c7cf25ce0a965" +dependencies = [ + "percent-encoding", +] + [[package]] name = "utf8_iter" version = "1.0.4" From f2bcf3c6a9fd07da830222588501064099c328ee Mon Sep 17 00:00:00 2001 From: Tim Saucer Date: Fri, 7 Aug 2026 12:27:08 -0400 Subject: [PATCH 08/11] feat(python): integrate Ballista FFI query planner AI Disclosure: This code was written in part by an AI agent. --- Cargo.lock | 185 +- Cargo.toml | 19 +- ballista/core/proto/ballista.proto | 8 + .../src/execution_plans/distributed_query.rs | 30 +- ballista/core/src/serde/generated/ballista.rs | 15 +- ballista/core/src/serde/mod.rs | 321 ++- python/Cargo.lock | 2034 +++++------------ python/Cargo.toml | 25 +- python/pyproject.toml | 3 +- python/python/ballista/extension.py | 41 +- python/python/tests/test_context.py | 96 +- python/src/lib.rs | 10 + python/src/query_planner.rs | 352 +++ python/uv.lock | 17 +- 14 files changed, 1478 insertions(+), 1678 deletions(-) create mode 100644 python/src/query_planner.rs diff --git a/Cargo.lock b/Cargo.lock index 1c76fe0c90..64838e9bfd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -179,9 +179,9 @@ checksum = "d3fb67a6e08acf24fdeccbac2cb6ac4305825bd1f117462e0e6f2f193345ad56" [[package]] name = "arrow" -version = "59.1.0" +version = "59.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b952ca5a8046ad741b60f142d6eca4aeebcad615694202bc64c5341f23e32c5b" +checksum = "61d285d16bce7d0be61912f7928342b673067b6b7d7ef6cc179258ba7de1fecf" dependencies = [ "arrow-arith", "arrow-array", @@ -200,9 +200,9 @@ dependencies = [ [[package]] name = "arrow-arith" -version = "59.1.0" +version = "59.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64a13b8d3008c4e9063c597a08f46446fe3fd5789277127672d6c0bdbb43b1ff" +checksum = "757ef1836251e88222542a7da2623bc1c9cb9e20afefa6db2c41e79991cd91d4" dependencies = [ "arrow-array", "arrow-buffer", @@ -214,9 +214,9 @@ dependencies = [ [[package]] name = "arrow-array" -version = "59.1.0" +version = "59.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9486151b2f0785bafc6fa04fc5c99fcb4495455662e58787ea32eaaed33c4192" +checksum = "bc9a4a4b2b5ecd0e04df03471661cb61f28bed3c7fd50994715129b01b2edb97" dependencies = [ "ahash", "arrow-buffer", @@ -233,9 +233,9 @@ dependencies = [ [[package]] name = "arrow-avro" -version = "59.1.0" +version = "59.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e4f9b23a0d7b613acb59fa20bdbe0f80ffdae6411498378340b3915e45f5b84" +checksum = "9fb45cd6bd2b25c0965793b83200eaca82214273a8030fbbc2d783e4c7c65a61" dependencies = [ "arrow-array", "arrow-buffer", @@ -257,21 +257,21 @@ dependencies = [ [[package]] name = "arrow-buffer" -version = "59.1.0" +version = "59.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4776577a87794bfdf0b4e90e2ea12454fa7738ea2823c4be5b9d1851da7b434" +checksum = "c12b576ef18c1deb80925a248b25ad84f419198d791b8e293fc6aaa60441fe90" dependencies = [ "bytes", "half", - "num-bigint", + "num-bigint 0.5.1", "num-traits", ] [[package]] name = "arrow-cast" -version = "59.1.0" +version = "59.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9ad451ce4f98710828a455b96991b8f031deb2e67f5fcad6773f017e4a69c3a" +checksum = "68338a9096a5dc9bc11927c58c43a8526d96bf6abd2012ef6c0c9f505991cc79" dependencies = [ "arrow-array", "arrow-buffer", @@ -280,7 +280,7 @@ dependencies = [ "arrow-schema", "arrow-select", "atoi", - "base64 0.22.1", + "base64 0.23.1", "chrono", "comfy-table", "half", @@ -291,9 +291,9 @@ dependencies = [ [[package]] name = "arrow-csv" -version = "59.1.0" +version = "59.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8aa7bf96d6141a7bcca2eed57c7c9767d2a2175281857b8a7b68308992864784" +checksum = "25011b52b346407d497ef0030e12b45e4f2d0cc279efc09c4f3d09106db30e36" dependencies = [ "arrow-array", "arrow-cast", @@ -306,9 +306,9 @@ dependencies = [ [[package]] name = "arrow-data" -version = "59.1.0" +version = "59.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b38fe43e2e8704360f1464e6e8cc4fc381ef02cc4fb0192afa8df1aaa0115c66" +checksum = "723fe4aeed7604e00b9883a465af4ff0a0e6c44c03e41a68c3d1cbc403e0e44d" dependencies = [ "arrow-buffer", "arrow-schema", @@ -319,9 +319,9 @@ dependencies = [ [[package]] name = "arrow-flight" -version = "59.1.0" +version = "59.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42115e09dbb694b5955da998912121451c6910b338228cb80a5701370dba43ff" +checksum = "2bebfacc9d71f0728f6774164e4d4254b5e504d2b46812d0512d8290ec119a64" dependencies = [ "arrow-arith", "arrow-array", @@ -334,11 +334,10 @@ dependencies = [ "arrow-schema", "arrow-select", "arrow-string", - "base64 0.22.1", + "base64 0.23.1", "bytes", "futures", "once_cell", - "paste", "prost", "prost-types", "tonic", @@ -347,9 +346,9 @@ dependencies = [ [[package]] name = "arrow-ipc" -version = "59.1.0" +version = "59.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29dac499fcbc6ba74ee0324057821d381929a48526a3966bd9dffb44aa06d98c" +checksum = "149437b14371f5b9ec60f5ddc751483ae99d7a7072653c0075e5e469156eea7b" dependencies = [ "arrow-array", "arrow-buffer", @@ -363,9 +362,9 @@ dependencies = [ [[package]] name = "arrow-json" -version = "59.1.0" +version = "59.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fe05e916ddc50f4c7a363cd69c0ef5894fcee063517e9a0b8582f0c56746af6" +checksum = "f18b9123ccfec418a663f821c9a034af339711678c11ffe00d3ec07da5ff9f7e" dependencies = [ "arrow-array", "arrow-buffer", @@ -388,9 +387,9 @@ dependencies = [ [[package]] name = "arrow-ord" -version = "59.1.0" +version = "59.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e13dbdc2a9c053c10c7baa6e30faee04a180aa7ce88e471835850ce37abd20b" +checksum = "e6c08dff0686cf23ca4f562803f191ccbeb726dbae6309cd4b4aaf65e0f2c979" dependencies = [ "arrow-array", "arrow-buffer", @@ -401,9 +400,9 @@ dependencies = [ [[package]] name = "arrow-row" -version = "59.1.0" +version = "59.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d5a1f8c733d15260b305683472ee8ad89c62cbd706703ca873b90d051b41592" +checksum = "bbec439386df71ad570e6758a946111322b9e9dc8db83b5527321f0b4c9119c2" dependencies = [ "arrow-array", "arrow-buffer", @@ -414,9 +413,9 @@ dependencies = [ [[package]] name = "arrow-schema" -version = "59.1.0" +version = "59.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9e4969dc350d571766247143ab36a5187d095d3d3690970408bc630d47c69e5" +checksum = "e6fed2ca0d1eade57e811cbe73b98ad50cc08a1183e13b2d2aa43a7df593f40e" dependencies = [ "serde_core", "serde_json", @@ -424,9 +423,9 @@ dependencies = [ [[package]] name = "arrow-select" -version = "59.1.0" +version = "59.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "402770dba90865359d98d1ef92ef16e23d75c0cca9c2c880c8a05468b7743bf9" +checksum = "466b19cf75130b891dc1b23a84b343c714c62c64c9c62e365c76aa0ff90a53fb" dependencies = [ "ahash", "arrow-array", @@ -438,9 +437,9 @@ dependencies = [ [[package]] name = "arrow-string" -version = "59.1.0" +version = "59.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2b0afbb8b9016700938291123df30838b89decc3213dba00852021988b170d3" +checksum = "c838a25bb3691e919e0f617616ac51a4ff8517a952e29ca133cf0c22b2ce65b1" dependencies = [ "arrow-array", "arrow-buffer", @@ -1331,7 +1330,7 @@ checksum = "4d6867f1565b3aad85681f1015055b087fcfd840d6aeee6eee7f2da317603695" dependencies = [ "autocfg", "libm", - "num-bigint", + "num-bigint 0.4.8", "num-integer", "num-traits", ] @@ -2197,7 +2196,7 @@ dependencies = [ [[package]] name = "datafusion" version = "54.1.0" -source = "git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11#70c26a06716eb552f0fc1d2115957a4089504f11" +source = "git+https://github.com/apache/datafusion.git?rev=e08aed1e5de41dcf81d529140dae07723b942a5e#e08aed1e5de41dcf81d529140dae07723b942a5e" dependencies = [ "arrow", "arrow-schema", @@ -2250,7 +2249,7 @@ dependencies = [ [[package]] name = "datafusion-catalog" version = "54.1.0" -source = "git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11#70c26a06716eb552f0fc1d2115957a4089504f11" +source = "git+https://github.com/apache/datafusion.git?rev=e08aed1e5de41dcf81d529140dae07723b942a5e#e08aed1e5de41dcf81d529140dae07723b942a5e" dependencies = [ "arrow", "async-trait", @@ -2274,7 +2273,7 @@ dependencies = [ [[package]] name = "datafusion-catalog-listing" version = "54.1.0" -source = "git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11#70c26a06716eb552f0fc1d2115957a4089504f11" +source = "git+https://github.com/apache/datafusion.git?rev=e08aed1e5de41dcf81d529140dae07723b942a5e#e08aed1e5de41dcf81d529140dae07723b942a5e" dependencies = [ "arrow", "async-trait", @@ -2297,7 +2296,7 @@ dependencies = [ [[package]] name = "datafusion-cli" version = "54.1.0" -source = "git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11#70c26a06716eb552f0fc1d2115957a4089504f11" +source = "git+https://github.com/apache/datafusion.git?rev=e08aed1e5de41dcf81d529140dae07723b942a5e#e08aed1e5de41dcf81d529140dae07723b942a5e" dependencies = [ "arrow", "async-trait", @@ -2324,7 +2323,7 @@ dependencies = [ [[package]] name = "datafusion-common" version = "54.1.0" -source = "git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11#70c26a06716eb552f0fc1d2115957a4089504f11" +source = "git+https://github.com/apache/datafusion.git?rev=e08aed1e5de41dcf81d529140dae07723b942a5e#e08aed1e5de41dcf81d529140dae07723b942a5e" dependencies = [ "arrow", "arrow-ipc", @@ -2351,7 +2350,7 @@ dependencies = [ [[package]] name = "datafusion-common-runtime" version = "54.1.0" -source = "git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11#70c26a06716eb552f0fc1d2115957a4089504f11" +source = "git+https://github.com/apache/datafusion.git?rev=e08aed1e5de41dcf81d529140dae07723b942a5e#e08aed1e5de41dcf81d529140dae07723b942a5e" dependencies = [ "futures", "log", @@ -2361,7 +2360,7 @@ dependencies = [ [[package]] name = "datafusion-datasource" version = "54.1.0" -source = "git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11#70c26a06716eb552f0fc1d2115957a4089504f11" +source = "git+https://github.com/apache/datafusion.git?rev=e08aed1e5de41dcf81d529140dae07723b942a5e#e08aed1e5de41dcf81d529140dae07723b942a5e" dependencies = [ "arrow", "async-compression", @@ -2397,7 +2396,7 @@ dependencies = [ [[package]] name = "datafusion-datasource-arrow" version = "54.1.0" -source = "git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11#70c26a06716eb552f0fc1d2115957a4089504f11" +source = "git+https://github.com/apache/datafusion.git?rev=e08aed1e5de41dcf81d529140dae07723b942a5e#e08aed1e5de41dcf81d529140dae07723b942a5e" dependencies = [ "arrow", "arrow-ipc", @@ -2420,7 +2419,7 @@ dependencies = [ [[package]] name = "datafusion-datasource-avro" version = "54.1.0" -source = "git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11#70c26a06716eb552f0fc1d2115957a4089504f11" +source = "git+https://github.com/apache/datafusion.git?rev=e08aed1e5de41dcf81d529140dae07723b942a5e#e08aed1e5de41dcf81d529140dae07723b942a5e" dependencies = [ "arrow", "arrow-avro", @@ -2438,7 +2437,7 @@ dependencies = [ [[package]] name = "datafusion-datasource-csv" version = "54.1.0" -source = "git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11#70c26a06716eb552f0fc1d2115957a4089504f11" +source = "git+https://github.com/apache/datafusion.git?rev=e08aed1e5de41dcf81d529140dae07723b942a5e#e08aed1e5de41dcf81d529140dae07723b942a5e" dependencies = [ "arrow", "async-trait", @@ -2450,6 +2449,7 @@ dependencies = [ "datafusion-expr", "datafusion-physical-expr-common", "datafusion-physical-plan", + "datafusion-proto-models", "datafusion-session", "futures", "object_store", @@ -2460,7 +2460,7 @@ dependencies = [ [[package]] name = "datafusion-datasource-json" version = "54.1.0" -source = "git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11#70c26a06716eb552f0fc1d2115957a4089504f11" +source = "git+https://github.com/apache/datafusion.git?rev=e08aed1e5de41dcf81d529140dae07723b942a5e#e08aed1e5de41dcf81d529140dae07723b942a5e" dependencies = [ "arrow", "async-trait", @@ -2472,6 +2472,7 @@ dependencies = [ "datafusion-expr", "datafusion-physical-expr-common", "datafusion-physical-plan", + "datafusion-proto-models", "datafusion-session", "futures", "object_store", @@ -2482,7 +2483,7 @@ dependencies = [ [[package]] name = "datafusion-datasource-parquet" version = "54.1.0" -source = "git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11#70c26a06716eb552f0fc1d2115957a4089504f11" +source = "git+https://github.com/apache/datafusion.git?rev=e08aed1e5de41dcf81d529140dae07723b942a5e#e08aed1e5de41dcf81d529140dae07723b942a5e" dependencies = [ "arrow", "arrow-schema", @@ -2499,6 +2500,7 @@ dependencies = [ "datafusion-physical-expr-adapter", "datafusion-physical-expr-common", "datafusion-physical-plan", + "datafusion-proto-models", "datafusion-pruning", "datafusion-session", "futures", @@ -2513,12 +2515,12 @@ dependencies = [ [[package]] name = "datafusion-doc" version = "54.1.0" -source = "git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11#70c26a06716eb552f0fc1d2115957a4089504f11" +source = "git+https://github.com/apache/datafusion.git?rev=e08aed1e5de41dcf81d529140dae07723b942a5e#e08aed1e5de41dcf81d529140dae07723b942a5e" [[package]] name = "datafusion-execution" version = "54.1.0" -source = "git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11#70c26a06716eb552f0fc1d2115957a4089504f11" +source = "git+https://github.com/apache/datafusion.git?rev=e08aed1e5de41dcf81d529140dae07723b942a5e#e08aed1e5de41dcf81d529140dae07723b942a5e" dependencies = [ "arrow", "arrow-buffer", @@ -2544,7 +2546,7 @@ dependencies = [ [[package]] name = "datafusion-expr" version = "54.1.0" -source = "git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11#70c26a06716eb552f0fc1d2115957a4089504f11" +source = "git+https://github.com/apache/datafusion.git?rev=e08aed1e5de41dcf81d529140dae07723b942a5e#e08aed1e5de41dcf81d529140dae07723b942a5e" dependencies = [ "arrow", "arrow-schema", @@ -2566,7 +2568,7 @@ dependencies = [ [[package]] name = "datafusion-expr-common" version = "54.1.0" -source = "git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11#70c26a06716eb552f0fc1d2115957a4089504f11" +source = "git+https://github.com/apache/datafusion.git?rev=e08aed1e5de41dcf81d529140dae07723b942a5e#e08aed1e5de41dcf81d529140dae07723b942a5e" dependencies = [ "arrow", "datafusion-common", @@ -2577,7 +2579,7 @@ dependencies = [ [[package]] name = "datafusion-functions" version = "54.1.0" -source = "git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11#70c26a06716eb552f0fc1d2115957a4089504f11" +source = "git+https://github.com/apache/datafusion.git?rev=e08aed1e5de41dcf81d529140dae07723b942a5e#e08aed1e5de41dcf81d529140dae07723b942a5e" dependencies = [ "arrow", "arrow-buffer", @@ -2608,7 +2610,7 @@ dependencies = [ [[package]] name = "datafusion-functions-aggregate" version = "54.1.0" -source = "git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11#70c26a06716eb552f0fc1d2115957a4089504f11" +source = "git+https://github.com/apache/datafusion.git?rev=e08aed1e5de41dcf81d529140dae07723b942a5e#e08aed1e5de41dcf81d529140dae07723b942a5e" dependencies = [ "arrow", "datafusion-common", @@ -2628,7 +2630,7 @@ dependencies = [ [[package]] name = "datafusion-functions-aggregate-common" version = "54.1.0" -source = "git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11#70c26a06716eb552f0fc1d2115957a4089504f11" +source = "git+https://github.com/apache/datafusion.git?rev=e08aed1e5de41dcf81d529140dae07723b942a5e#e08aed1e5de41dcf81d529140dae07723b942a5e" dependencies = [ "arrow", "datafusion-common", @@ -2639,7 +2641,7 @@ dependencies = [ [[package]] name = "datafusion-functions-nested" version = "54.1.0" -source = "git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11#70c26a06716eb552f0fc1d2115957a4089504f11" +source = "git+https://github.com/apache/datafusion.git?rev=e08aed1e5de41dcf81d529140dae07723b942a5e#e08aed1e5de41dcf81d529140dae07723b942a5e" dependencies = [ "arrow", "arrow-ord", @@ -2663,7 +2665,7 @@ dependencies = [ [[package]] name = "datafusion-functions-table" version = "54.1.0" -source = "git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11#70c26a06716eb552f0fc1d2115957a4089504f11" +source = "git+https://github.com/apache/datafusion.git?rev=e08aed1e5de41dcf81d529140dae07723b942a5e#e08aed1e5de41dcf81d529140dae07723b942a5e" dependencies = [ "arrow", "async-trait", @@ -2678,7 +2680,7 @@ dependencies = [ [[package]] name = "datafusion-functions-window" version = "54.1.0" -source = "git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11#70c26a06716eb552f0fc1d2115957a4089504f11" +source = "git+https://github.com/apache/datafusion.git?rev=e08aed1e5de41dcf81d529140dae07723b942a5e#e08aed1e5de41dcf81d529140dae07723b942a5e" dependencies = [ "arrow", "datafusion-common", @@ -2694,7 +2696,7 @@ dependencies = [ [[package]] name = "datafusion-functions-window-common" version = "54.1.0" -source = "git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11#70c26a06716eb552f0fc1d2115957a4089504f11" +source = "git+https://github.com/apache/datafusion.git?rev=e08aed1e5de41dcf81d529140dae07723b942a5e#e08aed1e5de41dcf81d529140dae07723b942a5e" dependencies = [ "datafusion-common", "datafusion-physical-expr-common", @@ -2703,7 +2705,7 @@ dependencies = [ [[package]] name = "datafusion-macros" version = "54.1.0" -source = "git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11#70c26a06716eb552f0fc1d2115957a4089504f11" +source = "git+https://github.com/apache/datafusion.git?rev=e08aed1e5de41dcf81d529140dae07723b942a5e#e08aed1e5de41dcf81d529140dae07723b942a5e" dependencies = [ "datafusion-doc", "quote", @@ -2713,7 +2715,7 @@ dependencies = [ [[package]] name = "datafusion-optimizer" version = "54.1.0" -source = "git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11#70c26a06716eb552f0fc1d2115957a4089504f11" +source = "git+https://github.com/apache/datafusion.git?rev=e08aed1e5de41dcf81d529140dae07723b942a5e#e08aed1e5de41dcf81d529140dae07723b942a5e" dependencies = [ "arrow", "chrono", @@ -2732,7 +2734,7 @@ dependencies = [ [[package]] name = "datafusion-physical-expr" version = "54.1.0" -source = "git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11#70c26a06716eb552f0fc1d2115957a4089504f11" +source = "git+https://github.com/apache/datafusion.git?rev=e08aed1e5de41dcf81d529140dae07723b942a5e#e08aed1e5de41dcf81d529140dae07723b942a5e" dependencies = [ "arrow", "datafusion-common", @@ -2754,7 +2756,7 @@ dependencies = [ [[package]] name = "datafusion-physical-expr-adapter" version = "54.1.0" -source = "git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11#70c26a06716eb552f0fc1d2115957a4089504f11" +source = "git+https://github.com/apache/datafusion.git?rev=e08aed1e5de41dcf81d529140dae07723b942a5e#e08aed1e5de41dcf81d529140dae07723b942a5e" dependencies = [ "arrow", "datafusion-common", @@ -2768,7 +2770,7 @@ dependencies = [ [[package]] name = "datafusion-physical-expr-common" version = "54.1.0" -source = "git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11#70c26a06716eb552f0fc1d2115957a4089504f11" +source = "git+https://github.com/apache/datafusion.git?rev=e08aed1e5de41dcf81d529140dae07723b942a5e#e08aed1e5de41dcf81d529140dae07723b942a5e" dependencies = [ "arrow", "chrono", @@ -2785,7 +2787,7 @@ dependencies = [ [[package]] name = "datafusion-physical-optimizer" version = "54.1.0" -source = "git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11#70c26a06716eb552f0fc1d2115957a4089504f11" +source = "git+https://github.com/apache/datafusion.git?rev=e08aed1e5de41dcf81d529140dae07723b942a5e#e08aed1e5de41dcf81d529140dae07723b942a5e" dependencies = [ "arrow", "datafusion-common", @@ -2804,7 +2806,7 @@ dependencies = [ [[package]] name = "datafusion-physical-plan" version = "54.1.0" -source = "git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11#70c26a06716eb552f0fc1d2115957a4089504f11" +source = "git+https://github.com/apache/datafusion.git?rev=e08aed1e5de41dcf81d529140dae07723b942a5e#e08aed1e5de41dcf81d529140dae07723b942a5e" dependencies = [ "arrow", "arrow-data", @@ -2840,7 +2842,7 @@ dependencies = [ [[package]] name = "datafusion-proto" version = "54.1.0" -source = "git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11#70c26a06716eb552f0fc1d2115957a4089504f11" +source = "git+https://github.com/apache/datafusion.git?rev=e08aed1e5de41dcf81d529140dae07723b942a5e#e08aed1e5de41dcf81d529140dae07723b942a5e" dependencies = [ "arrow", "chrono", @@ -2867,7 +2869,7 @@ dependencies = [ [[package]] name = "datafusion-proto-common" version = "54.1.0" -source = "git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11#70c26a06716eb552f0fc1d2115957a4089504f11" +source = "git+https://github.com/apache/datafusion.git?rev=e08aed1e5de41dcf81d529140dae07723b942a5e#e08aed1e5de41dcf81d529140dae07723b942a5e" dependencies = [ "arrow", "datafusion-common", @@ -2877,7 +2879,7 @@ dependencies = [ [[package]] name = "datafusion-proto-models" version = "54.1.0" -source = "git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11#70c26a06716eb552f0fc1d2115957a4089504f11" +source = "git+https://github.com/apache/datafusion.git?rev=e08aed1e5de41dcf81d529140dae07723b942a5e#e08aed1e5de41dcf81d529140dae07723b942a5e" dependencies = [ "datafusion-proto-common", "prost", @@ -2886,7 +2888,7 @@ dependencies = [ [[package]] name = "datafusion-pruning" version = "54.1.0" -source = "git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11#70c26a06716eb552f0fc1d2115957a4089504f11" +source = "git+https://github.com/apache/datafusion.git?rev=e08aed1e5de41dcf81d529140dae07723b942a5e#e08aed1e5de41dcf81d529140dae07723b942a5e" dependencies = [ "arrow", "datafusion-common", @@ -2901,7 +2903,7 @@ dependencies = [ [[package]] name = "datafusion-session" version = "54.1.0" -source = "git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11#70c26a06716eb552f0fc1d2115957a4089504f11" +source = "git+https://github.com/apache/datafusion.git?rev=e08aed1e5de41dcf81d529140dae07723b942a5e#e08aed1e5de41dcf81d529140dae07723b942a5e" dependencies = [ "arrow-schema", "async-trait", @@ -2915,7 +2917,7 @@ dependencies = [ [[package]] name = "datafusion-spark" version = "54.1.0" -source = "git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11#70c26a06716eb552f0fc1d2115957a4089504f11" +source = "git+https://github.com/apache/datafusion.git?rev=e08aed1e5de41dcf81d529140dae07723b942a5e#e08aed1e5de41dcf81d529140dae07723b942a5e" dependencies = [ "arrow", "bigdecimal", @@ -2944,7 +2946,7 @@ dependencies = [ [[package]] name = "datafusion-sql" version = "54.1.0" -source = "git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11#70c26a06716eb552f0fc1d2115957a4089504f11" +source = "git+https://github.com/apache/datafusion.git?rev=e08aed1e5de41dcf81d529140dae07723b942a5e#e08aed1e5de41dcf81d529140dae07723b942a5e" dependencies = [ "arrow", "bigdecimal", @@ -2963,7 +2965,7 @@ dependencies = [ [[package]] name = "datafusion-substrait" version = "54.1.0" -source = "git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11#70c26a06716eb552f0fc1d2115957a4089504f11" +source = "git+https://github.com/apache/datafusion.git?rev=e08aed1e5de41dcf81d529140dae07723b942a5e#e08aed1e5de41dcf81d529140dae07723b942a5e" dependencies = [ "async-recursion", "async-trait", @@ -4471,9 +4473,9 @@ checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" [[package]] name = "lz4_flex" -version = "0.13.1" +version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ef0d4ed8669f8f8826eb00dc878084aa8f253506c4fd5e8f58f5bce72ddb97e" +checksum = "ecbdfe44b1bd960b68170b417450a628c43f7cf56bb3c5317e61cb230ee7f226" dependencies = [ "twox-hash", ] @@ -4680,7 +4682,7 @@ version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "35bd024e8b2ff75562e5f34e7f4905839deb4b22955ef5e73d2fea1b9813cb23" dependencies = [ - "num-bigint", + "num-bigint 0.4.8", "num-complex", "num-integer", "num-iter", @@ -4698,6 +4700,16 @@ dependencies = [ "num-traits", ] +[[package]] +name = "num-bigint" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93e7820bc0a80a0238e650327316f929ba18d5be054b647490a3a6a339f3e7c0" +dependencies = [ + "num-integer", + "num-traits", +] + [[package]] name = "num-complex" version = "0.4.6" @@ -4749,7 +4761,7 @@ version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f83d14da390562dca69fc84082e73e548e1ad308d24accdedd2720017cb37824" dependencies = [ - "num-bigint", + "num-bigint 0.4.8", "num-integer", "num-traits", ] @@ -4950,9 +4962,9 @@ dependencies = [ [[package]] name = "parquet" -version = "59.1.0" +version = "59.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5302d4da74d6596a1f11f9928767995b53bca657cbeea1e4e8c5074f8a1157dd" +checksum = "7065842956a20c2a536924ce8e4d9955f7422451511b9eb7500d7bfe5077e59c" dependencies = [ "ahash", "arrow-array", @@ -4961,7 +4973,7 @@ dependencies = [ "arrow-ipc", "arrow-schema", "arrow-select", - "base64 0.22.1", + "base64 0.23.1", "brotli", "bytes", "chrono", @@ -4970,11 +4982,10 @@ dependencies = [ "half", "hashbrown 0.17.1", "lz4_flex", - "num-bigint", + "num-bigint 0.5.1", "num-integer", "num-traits", "object_store", - "paste", "ring", "seq-macro", "simdutf8", @@ -5009,12 +5020,6 @@ dependencies = [ "syn 2.0.119", ] -[[package]] -name = "paste" -version = "1.0.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" - [[package]] name = "pathdiff" version = "0.2.3" diff --git a/Cargo.toml b/Cargo.toml index 71e7c374d4..b8ab443a5e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -35,19 +35,20 @@ edition = "2024" rust-version = "1.88.0" [workspace.dependencies] -arrow = { version = "59.1.0", features = ["ipc_compression"] } -arrow-flight = { version = "59.1.0", features = ["flight-sql-experimental"] } +arrow = { version = "59.2.0", features = ["ipc_compression"] } +arrow-flight = { version = "59.2.0", features = ["flight-sql-experimental"] } clap = { version = "4.5", features = ["derive", "cargo"] } # DataFusion crates are pinned to a git rev of apache/datafusion main rather than # a crates.io release. Bump this rev to advance to newer DataFusion commits. -datafusion = { git = "https://github.com/apache/datafusion.git", rev = "70c26a06716eb552f0fc1d2115957a4089504f11" } -datafusion-cli = { git = "https://github.com/apache/datafusion.git", rev = "70c26a06716eb552f0fc1d2115957a4089504f11" } -datafusion-functions-aggregate-common = { git = "https://github.com/apache/datafusion.git", rev = "70c26a06716eb552f0fc1d2115957a4089504f11" } -datafusion-proto = { git = "https://github.com/apache/datafusion.git", rev = "70c26a06716eb552f0fc1d2115957a4089504f11" } -datafusion-proto-common = { git = "https://github.com/apache/datafusion.git", rev = "70c26a06716eb552f0fc1d2115957a4089504f11" } -datafusion-spark = { git = "https://github.com/apache/datafusion.git", rev = "70c26a06716eb552f0fc1d2115957a4089504f11" } -datafusion-substrait = { git = "https://github.com/apache/datafusion.git", rev = "70c26a06716eb552f0fc1d2115957a4089504f11" } +datafusion = { git = "https://github.com/apache/datafusion.git", rev = "e08aed1e5de41dcf81d529140dae07723b942a5e" } +datafusion-cli = { git = "https://github.com/apache/datafusion.git", rev = "e08aed1e5de41dcf81d529140dae07723b942a5e" } +datafusion-ffi = { git = "https://github.com/apache/datafusion.git", rev = "e08aed1e5de41dcf81d529140dae07723b942a5e" } +datafusion-functions-aggregate-common = { git = "https://github.com/apache/datafusion.git", rev = "e08aed1e5de41dcf81d529140dae07723b942a5e" } +datafusion-proto = { git = "https://github.com/apache/datafusion.git", rev = "e08aed1e5de41dcf81d529140dae07723b942a5e" } +datafusion-proto-common = { git = "https://github.com/apache/datafusion.git", rev = "e08aed1e5de41dcf81d529140dae07723b942a5e" } +datafusion-spark = { git = "https://github.com/apache/datafusion.git", rev = "e08aed1e5de41dcf81d529140dae07723b942a5e" } +datafusion-substrait = { git = "https://github.com/apache/datafusion.git", rev = "e08aed1e5de41dcf81d529140dae07723b942a5e" } ctor = { version = "1.0" } insta = "1.47" diff --git a/ballista/core/proto/ballista.proto b/ballista/core/proto/ballista.proto index 99fceb91ea..08f540e21e 100644 --- a/ballista/core/proto/ballista.proto +++ b/ballista/core/proto/ballista.proto @@ -56,9 +56,17 @@ message BallistaPhysicalPlanNode { UnorderedRangeRepartitionExecNode unordered_range_repartition = 8; OrderedRangeRepartitionExecNode ordered_range_repartition = 9; PerPartitionFilterExecNode per_partition_filter = 10; + DistributedQueryExecNode distributed_query = 11; } } +message DistributedQueryExecNode { + string scheduler_url = 1; + repeated KeyValuePair settings = 2; + bytes logical_plan = 3; + string session_id = 4; +} + // Value-range router over N locally-sorted overlapping input partitions. // Redistributes them into K range-disjoint output partitions where each // output is fully sorted on the routing expression. Uses N × K scatter diff --git a/ballista/core/src/execution_plans/distributed_query.rs b/ballista/core/src/execution_plans/distributed_query.rs index 20f3234cfe..62673c398f 100644 --- a/ballista/core/src/execution_plans/distributed_query.rs +++ b/ballista/core/src/execution_plans/distributed_query.rs @@ -136,6 +136,26 @@ impl DistributedQueryExec { } } + /// Returns the scheduler URL used by this query. + pub fn scheduler_url(&self) -> &str { + &self.scheduler_url + } + + /// Returns the Ballista configuration used by this query. + pub fn config(&self) -> &BallistaConfig { + &self.config + } + + /// Returns the logical plan submitted to the scheduler. + pub fn logical_plan(&self) -> &LogicalPlan { + &self.plan + } + + /// Returns the client session identifier. + pub fn session_id(&self) -> &str { + &self.session_id + } + /// Returns the scheduler job id after the query has been accepted. pub fn job_id(&self) -> Option { self.job_id.lock().clone() @@ -227,8 +247,14 @@ impl ExecutionPlan for DistributedQueryExec { DataFusionError::Execution(format!("failed to encode logical plan: {e:?}")) })?; - let settings = context + // The FFI planner carries Ballista options in this node because the + // destination datafusion-python TaskContext does not know Ballista's + // ConfigExtension. Restore them before executing the query. + let session_config = context .session_config() + .clone() + .with_option_extension(self.config.clone()); + let settings = session_config .options() .entries() .iter() @@ -255,8 +281,6 @@ impl ExecutionPlan for DistributedQueryExec { let metric_total_bytes = MetricBuilder::new(&self.metrics).counter("transferred_bytes", partition); - let session_config = context.session_config().clone(); - if session_config.ballista_config().client_pull() { let stream = futures::stream::once( execute_query_pull( diff --git a/ballista/core/src/serde/generated/ballista.rs b/ballista/core/src/serde/generated/ballista.rs index 5bf6964eb2..326f950524 100644 --- a/ballista/core/src/serde/generated/ballista.rs +++ b/ballista/core/src/serde/generated/ballista.rs @@ -31,7 +31,7 @@ pub struct LogicalPlanCacheNode { pub struct BallistaPhysicalPlanNode { #[prost( oneof = "ballista_physical_plan_node::PhysicalPlanType", - tags = "1, 2, 3, 4, 5, 6, 7, 8, 9, 10" + tags = "1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11" )] pub physical_plan_type: ::core::option::Option< ballista_physical_plan_node::PhysicalPlanType, @@ -61,8 +61,21 @@ pub mod ballista_physical_plan_node { OrderedRangeRepartition(super::OrderedRangeRepartitionExecNode), #[prost(message, tag = "10")] PerPartitionFilter(super::PerPartitionFilterExecNode), + #[prost(message, tag = "11")] + DistributedQuery(super::DistributedQueryExecNode), } } +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct DistributedQueryExecNode { + #[prost(string, tag = "1")] + pub scheduler_url: ::prost::alloc::string::String, + #[prost(message, repeated, tag = "2")] + pub settings: ::prost::alloc::vec::Vec, + #[prost(bytes = "vec", tag = "3")] + pub logical_plan: ::prost::alloc::vec::Vec, + #[prost(string, tag = "4")] + pub session_id: ::prost::alloc::string::String, +} /// Value-range router over N locally-sorted overlapping input partitions. /// Redistributes them into K range-disjoint output partitions where each /// output is fully sorted on the routing expression. Uses N × K scatter diff --git a/ballista/core/src/serde/mod.rs b/ballista/core/src/serde/mod.rs index eb2ca41d15..5f1623c062 100644 --- a/ballista/core/src/serde/mod.rs +++ b/ballista/core/src/serde/mod.rs @@ -18,15 +18,20 @@ //! This crate contains code generated from the Ballista Protocol Buffer Definition as well //! as convenience code for interacting with the generated code. -use crate::extension::BallistaCacheNode; +use crate::extension::{BallistaCacheNode, SessionConfigExt, SessionConfigHelperExt}; use crate::{error::BallistaError, serde::scheduler::Action as BallistaAction}; use arrow_flight::sql::ProstMessageExt; use datafusion::arrow::datatypes::SchemaRef; use datafusion::common::{DataFusionError, Result}; use datafusion::execution::TaskContext; +use datafusion::execution::config::SessionConfig; use datafusion::logical_expr::Extension; use datafusion::physical_plan::{ExecutionPlan, Partitioning}; +use datafusion_proto::bytes::{ + logical_plan_from_bytes_with_extension_codec, + logical_plan_to_bytes_with_extension_codec, +}; use datafusion_proto::logical_plan::file_formats::{ ArrowLogicalExtensionCodec, AvroLogicalExtensionCodec, CsvLogicalExtensionCodec, JsonLogicalExtensionCodec, ParquetLogicalExtensionCodec, @@ -56,10 +61,10 @@ use std::{convert::TryInto, io::Cursor}; use crate::execution_plans::sort_shuffle::SortShuffleConfig; use crate::execution_plans::{ - BufferExec, BufferMode, ChaosExec, CoalescePlan, OrderedRangeRepartitionExec, - PartitionGroup, PerPartitionFilterExec, RuntimeStatsExec, ShuffleReaderExec, - ShuffleWriterExec, SortShuffleWriterExec, UnorderedRangeRepartitionExec, - UnresolvedShuffleExec, + BufferExec, BufferMode, ChaosExec, CoalescePlan, DistributedQueryExec, + OrderedRangeRepartitionExec, PartitionGroup, PerPartitionFilterExec, + RuntimeStatsExec, ShuffleReaderExec, ShuffleWriterExec, SortShuffleWriterExec, + UnorderedRangeRepartitionExec, UnresolvedShuffleExec, }; use crate::serde::protobuf::{ ballista_logical_plan_node::LogicalPlanType, @@ -192,6 +197,21 @@ pub struct BallistaLogicalExtensionCodec { } impl BallistaLogicalExtensionCodec { + /// Creates a Ballista logical codec that delegates non-Ballista objects to + /// `default_codec` while retaining Ballista's file-format codecs. + pub fn new(default_codec: Arc) -> Self { + Self { + default_codec, + file_format_codecs: vec![ + Arc::new(ParquetLogicalExtensionCodec {}), + Arc::new(CsvLogicalExtensionCodec {}), + Arc::new(JsonLogicalExtensionCodec {}), + Arc::new(ArrowLogicalExtensionCodec {}), + Arc::new(AvroLogicalExtensionCodec {}), + ], + } + } + /// looks for a codec which can operate on this node /// returns a position of codec in the list and result. /// @@ -220,18 +240,7 @@ impl BallistaLogicalExtensionCodec { impl Default for BallistaLogicalExtensionCodec { fn default() -> Self { - Self { - default_codec: Arc::new(DefaultLogicalExtensionCodec {}), - // Position in this list is important as it will be used for decoding. - // If new codec is added it should go to last position. - file_format_codecs: vec![ - Arc::new(ParquetLogicalExtensionCodec {}), - Arc::new(CsvLogicalExtensionCodec {}), - Arc::new(JsonLogicalExtensionCodec {}), - Arc::new(ArrowLogicalExtensionCodec {}), - Arc::new(AvroLogicalExtensionCodec {}), - ], - } + Self::new(Arc::new(DefaultLogicalExtensionCodec {})) } } @@ -349,22 +358,98 @@ impl LogicalExtensionCodec for BallistaLogicalExtensionCodec { .encode(buf) .map_err(|e| DataFusionError::Internal(e.to_string())) } + + fn try_decode_udf( + &self, + name: &str, + buf: &[u8], + ) -> Result> { + self.default_codec.try_decode_udf(name, buf) + } + + fn try_encode_udf( + &self, + node: &datafusion::logical_expr::ScalarUDF, + buf: &mut Vec, + ) -> Result<()> { + self.default_codec.try_encode_udf(node, buf) + } + + fn try_decode_higher_order_function( + &self, + name: &str, + buf: &[u8], + ) -> Result> { + self.default_codec + .try_decode_higher_order_function(name, buf) + } + + fn try_encode_higher_order_function( + &self, + node: &datafusion::logical_expr::HigherOrderUDF, + buf: &mut Vec, + ) -> Result<()> { + self.default_codec + .try_encode_higher_order_function(node, buf) + } + + fn try_decode_udaf( + &self, + name: &str, + buf: &[u8], + ) -> Result> { + self.default_codec.try_decode_udaf(name, buf) + } + + fn try_encode_udaf( + &self, + node: &datafusion::logical_expr::AggregateUDF, + buf: &mut Vec, + ) -> Result<()> { + self.default_codec.try_encode_udaf(node, buf) + } + + fn try_decode_udwf( + &self, + name: &str, + buf: &[u8], + ) -> Result> { + self.default_codec.try_decode_udwf(name, buf) + } + + fn try_encode_udwf( + &self, + node: &datafusion::logical_expr::WindowUDF, + buf: &mut Vec, + ) -> Result<()> { + self.default_codec.try_encode_udwf(node, buf) + } } /// Physical extension codec for Ballista-specific execution plan nodes. #[derive(Debug)] pub struct BallistaPhysicalExtensionCodec { default_codec: Arc, + logical_codec: Arc, } -impl Default for BallistaPhysicalExtensionCodec { - fn default() -> Self { +impl BallistaPhysicalExtensionCodec { + /// Creates a physical codec that uses `logical_codec` for logical plans + /// embedded in client-side Ballista execution nodes. + pub fn new(logical_codec: Arc) -> Self { Self { default_codec: Arc::new(DefaultPhysicalExtensionCodec {}), + logical_codec, } } } +impl Default for BallistaPhysicalExtensionCodec { + fn default() -> Self { + Self::new(Arc::new(BallistaLogicalExtensionCodec::default())) + } +} + impl PhysicalExtensionCodec for BallistaPhysicalExtensionCodec { fn try_decode( &self, @@ -663,6 +748,31 @@ impl PhysicalExtensionCodec for BallistaPhysicalExtensionCodec { predicates, )?)) } + PhysicalPlanType::DistributedQuery(distributed_query) => { + if !inputs.is_empty() { + return Err(DataFusionError::Internal(format!( + "DistributedQueryExec expects no inputs, got {}", + inputs.len() + ))); + } + let plan = logical_plan_from_bytes_with_extension_codec( + &distributed_query.logical_plan, + ctx, + self.logical_codec.as_ref(), + )?; + let config = SessionConfig::new_with_ballista() + .update_from_key_value_pair(&distributed_query.settings) + .ballista_config(); + Ok(Arc::new( + DistributedQueryExec::::with_extension( + distributed_query.scheduler_url.clone(), + config, + plan, + Arc::clone(&self.logical_codec), + distributed_query.session_id.clone(), + ), + )) + } } } @@ -924,6 +1034,38 @@ impl PhysicalExtensionCodec for BallistaPhysicalExtensionCodec { )) })?; Ok(()) + } else if let Some(exec) = + node.downcast_ref::>() + { + let logical_plan = logical_plan_to_bytes_with_extension_codec( + exec.logical_plan(), + self.logical_codec.as_ref(), + )?; + let settings = exec + .config() + .settings() + .iter() + .map(|(key, value)| protobuf::KeyValuePair { + key: key.clone(), + value: Some(value.clone()), + }) + .collect(); + let proto = protobuf::BallistaPhysicalPlanNode { + physical_plan_type: Some(PhysicalPlanType::DistributedQuery( + protobuf::DistributedQueryExecNode { + scheduler_url: exec.scheduler_url().to_string(), + settings, + logical_plan: logical_plan.to_vec(), + session_id: exec.session_id().to_string(), + }, + )), + }; + proto.encode(buf).map_err(|e| { + DataFusionError::Internal(format!( + "failed to encode distributed query execution plan: {e:?}" + )) + })?; + Ok(()) } else { Err(DataFusionError::Internal(format!( "Unsupported plan node, name: [{}] ", @@ -968,6 +1110,109 @@ mod test { use datafusion_proto::{logical_plan::AsLogicalPlan, protobuf::LogicalPlanNode}; use std::sync::Arc; + #[derive(Debug)] + struct MarkerLogicalCodec; + + impl LogicalExtensionCodec for MarkerLogicalCodec { + fn try_decode( + &self, + _buf: &[u8], + _inputs: &[LogicalPlan], + _ctx: &TaskContext, + ) -> Result { + Err(DataFusionError::Plan("marker-plan-decode".to_string())) + } + + fn try_encode(&self, _node: &Extension, _buf: &mut Vec) -> Result<()> { + Err(DataFusionError::Plan("marker-plan-encode".to_string())) + } + + fn try_decode_table_provider( + &self, + _buf: &[u8], + _table_ref: &datafusion::common::TableReference, + _schema: SchemaRef, + _ctx: &TaskContext, + ) -> Result> { + Err(DataFusionError::Plan("marker-table-decode".to_string())) + } + + fn try_encode_table_provider( + &self, + _table_ref: &datafusion::common::TableReference, + _node: Arc, + _buf: &mut Vec, + ) -> Result<()> { + Err(DataFusionError::Plan("marker-table-encode".to_string())) + } + + fn try_decode_udf( + &self, + _name: &str, + _buf: &[u8], + ) -> Result> { + Err(DataFusionError::Plan("marker-scalar".to_string())) + } + + fn try_decode_higher_order_function( + &self, + _name: &str, + _buf: &[u8], + ) -> Result> { + Err(DataFusionError::Plan("marker-higher-order".to_string())) + } + + fn try_decode_udaf( + &self, + _name: &str, + _buf: &[u8], + ) -> Result> { + Err(DataFusionError::Plan("marker-aggregate".to_string())) + } + + fn try_decode_udwf( + &self, + _name: &str, + _buf: &[u8], + ) -> Result> { + Err(DataFusionError::Plan("marker-window".to_string())) + } + } + + #[test] + fn logical_codec_delegates_function_decoding() { + let codec = BallistaLogicalExtensionCodec::new(Arc::new(MarkerLogicalCodec)); + + assert!( + codec + .try_decode_udf("udf", &[]) + .unwrap_err() + .to_string() + .contains("marker-scalar") + ); + assert!( + codec + .try_decode_udaf("udaf", &[]) + .unwrap_err() + .to_string() + .contains("marker-aggregate") + ); + assert!( + codec + .try_decode_udwf("udwf", &[]) + .unwrap_err() + .to_string() + .contains("marker-window") + ); + assert!( + codec + .try_decode_higher_order_function("higher", &[]) + .unwrap_err() + .to_string() + .contains("marker-higher-order") + ); + } + #[tokio::test] async fn file_format_serialization_roundtrip() { let ctx = SessionContext::new().task_ctx(); @@ -1093,6 +1338,44 @@ mod test { ); } + #[tokio::test] + async fn test_distributed_query_exec_roundtrip() { + let logical_plan = LogicalPlan::EmptyRelation(EmptyRelation { + produce_one_row: true, + schema: Arc::new(DFSchema::empty()), + }); + let config = SessionConfig::new_with_ballista() + .set_bool(crate::config::BALLISTA_CLIENT_PULL, true) + .ballista_config(); + let original_exec = DistributedQueryExec::::new( + "http://localhost:50050".to_string(), + config, + logical_plan.clone(), + "test-session".to_string(), + ); + + let codec = BallistaPhysicalExtensionCodec::default(); + let converter = DefaultPhysicalProtoConverter {}; + let mut buf = Vec::new(); + codec + .try_encode(Arc::new(original_exec), &mut buf, &converter) + .unwrap(); + + let ctx = SessionContext::new().task_ctx(); + let decoded_plan = codec.try_decode(&buf, &[], &ctx, &converter).unwrap(); + let decoded_exec = decoded_plan + .downcast_ref::>() + .expect("Expected DistributedQueryExec"); + + assert_eq!(decoded_exec.scheduler_url(), "http://localhost:50050"); + assert_eq!(decoded_exec.session_id(), "test-session"); + assert!(decoded_exec.config().client_pull()); + assert_eq!( + decoded_exec.logical_plan().display_indent().to_string(), + logical_plan.display_indent().to_string() + ); + } + #[tokio::test] async fn test_unresolved_shuffle_exec_roundtrip() { let schema = create_test_schema(); diff --git a/python/Cargo.lock b/python/Cargo.lock index 9f8e3776b4..1a21218128 100644 --- a/python/Cargo.lock +++ b/python/Cargo.lock @@ -99,108 +99,55 @@ checksum = "d3fb67a6e08acf24fdeccbac2cb6ac4305825bd1f117462e0e6f2f193345ad56" [[package]] name = "arrow" -version = "58.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6cfdd0833e32a9874d2b55089333ad310c0be208aafa277385ce2461dec90be3" -dependencies = [ - "arrow-arith 58.4.0", - "arrow-array 58.4.0", - "arrow-buffer 58.4.0", - "arrow-cast 58.4.0", - "arrow-csv 58.4.0", - "arrow-data 58.4.0", - "arrow-ipc 58.4.0", - "arrow-json 58.4.0", - "arrow-ord 58.4.0", +version = "59.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61d285d16bce7d0be61912f7928342b673067b6b7d7ef6cc179258ba7de1fecf" +dependencies = [ + "arrow-arith", + "arrow-array", + "arrow-buffer", + "arrow-cast", + "arrow-csv", + "arrow-data", + "arrow-ipc", + "arrow-json", + "arrow-ord", "arrow-pyarrow", - "arrow-row 58.4.0", - "arrow-schema 58.4.0", - "arrow-select 58.4.0", - "arrow-string 58.4.0", -] - -[[package]] -name = "arrow" -version = "59.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b952ca5a8046ad741b60f142d6eca4aeebcad615694202bc64c5341f23e32c5b" -dependencies = [ - "arrow-arith 59.1.0", - "arrow-array 59.1.0", - "arrow-buffer 59.1.0", - "arrow-cast 59.1.0", - "arrow-csv 59.1.0", - "arrow-data 59.1.0", - "arrow-ipc 59.1.0", - "arrow-json 59.1.0", - "arrow-ord 59.1.0", - "arrow-row 59.1.0", - "arrow-schema 59.1.0", - "arrow-select 59.1.0", - "arrow-string 59.1.0", + "arrow-row", + "arrow-schema", + "arrow-select", + "arrow-string", ] [[package]] name = "arrow-arith" -version = "58.4.0" +version = "59.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0a41203398f0eaa6f7ec8e62c0da742a21abf282c148fc157f6c35c90e29981a" +checksum = "757ef1836251e88222542a7da2623bc1c9cb9e20afefa6db2c41e79991cd91d4" dependencies = [ - "arrow-array 58.4.0", - "arrow-buffer 58.4.0", - "arrow-data 58.4.0", - "arrow-schema 58.4.0", - "chrono", - "num-traits", -] - -[[package]] -name = "arrow-arith" -version = "59.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64a13b8d3008c4e9063c597a08f46446fe3fd5789277127672d6c0bdbb43b1ff" -dependencies = [ - "arrow-array 59.1.0", - "arrow-buffer 59.1.0", - "arrow-data 59.1.0", - "arrow-schema 59.1.0", + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", "chrono", "num-traits", ] [[package]] name = "arrow-array" -version = "58.4.0" +version = "59.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae33dad492b7df00a217563a7b0ef2874df68a0deea1b1a3acf628152f7f7a69" +checksum = "bc9a4a4b2b5ecd0e04df03471661cb61f28bed3c7fd50994715129b01b2edb97" dependencies = [ "ahash", - "arrow-buffer 58.4.0", - "arrow-data 58.4.0", - "arrow-schema 58.4.0", - "chrono", - "chrono-tz", - "half", - "hashbrown 0.17.1", - "num-complex", - "num-integer", - "num-traits", -] - -[[package]] -name = "arrow-array" -version = "59.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9486151b2f0785bafc6fa04fc5c99fcb4495455662e58787ea32eaaed33c4192" -dependencies = [ - "ahash", - "arrow-buffer 59.1.0", - "arrow-data 59.1.0", - "arrow-schema 59.1.0", + "arrow-buffer", + "arrow-data", + "arrow-schema", "chrono", "chrono-tz", "half", "hashbrown 0.17.1", + "libc", "num-complex", "num-integer", "num-traits", @@ -208,13 +155,13 @@ dependencies = [ [[package]] name = "arrow-avro" -version = "58.4.0" +version = "59.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8108cbcae32141577bc55ab490449944c63cfc93d47384906b475d13d89d43d" +checksum = "9fb45cd6bd2b25c0965793b83200eaca82214273a8030fbbc2d783e4c7c65a61" dependencies = [ - "arrow-array 58.4.0", - "arrow-buffer 58.4.0", - "arrow-schema 58.4.0", + "arrow-array", + "arrow-buffer", + "arrow-schema", "bytes", "bzip2", "crc", @@ -232,64 +179,30 @@ dependencies = [ [[package]] name = "arrow-buffer" -version = "58.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9552f96391c005e6ab449fa941420935e7e062489b12b8b1b08879b2163f5b5" -dependencies = [ - "bytes", - "half", - "num-bigint", - "num-traits", -] - -[[package]] -name = "arrow-buffer" -version = "59.1.0" +version = "59.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4776577a87794bfdf0b4e90e2ea12454fa7738ea2823c4be5b9d1851da7b434" +checksum = "c12b576ef18c1deb80925a248b25ad84f419198d791b8e293fc6aaa60441fe90" dependencies = [ "bytes", "half", - "num-bigint", - "num-traits", -] - -[[package]] -name = "arrow-cast" -version = "58.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a8a327c9649f30d8406995f27642b68df354713cca3baaaf100f076f18d5f34" -dependencies = [ - "arrow-array 58.4.0", - "arrow-buffer 58.4.0", - "arrow-data 58.4.0", - "arrow-ord 58.4.0", - "arrow-schema 58.4.0", - "arrow-select 58.4.0", - "atoi", - "base64 0.22.1", - "chrono", - "comfy-table", - "half", - "lexical-core", + "num-bigint 0.5.1", "num-traits", - "ryu", ] [[package]] name = "arrow-cast" -version = "59.1.0" +version = "59.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9ad451ce4f98710828a455b96991b8f031deb2e67f5fcad6773f017e4a69c3a" +checksum = "68338a9096a5dc9bc11927c58c43a8526d96bf6abd2012ef6c0c9f505991cc79" dependencies = [ - "arrow-array 59.1.0", - "arrow-buffer 59.1.0", - "arrow-data 59.1.0", - "arrow-ord 59.1.0", - "arrow-schema 59.1.0", - "arrow-select 59.1.0", + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-ord", + "arrow-schema", + "arrow-select", "atoi", - "base64 0.22.1", + "base64 0.23.1", "chrono", "comfy-table", "half", @@ -300,55 +213,27 @@ dependencies = [ [[package]] name = "arrow-csv" -version = "58.4.0" +version = "59.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af0dd6d90d1955e9f9a014c1e563ee8aeffc21909085d25623e1da44d96eca26" +checksum = "25011b52b346407d497ef0030e12b45e4f2d0cc279efc09c4f3d09106db30e36" dependencies = [ - "arrow-array 58.4.0", - "arrow-cast 58.4.0", - "arrow-schema 58.4.0", + "arrow-array", + "arrow-cast", + "arrow-schema", "chrono", "csv", "csv-core", "regex", ] -[[package]] -name = "arrow-csv" -version = "59.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8aa7bf96d6141a7bcca2eed57c7c9767d2a2175281857b8a7b68308992864784" -dependencies = [ - "arrow-array 59.1.0", - "arrow-cast 59.1.0", - "arrow-schema 59.1.0", - "chrono", - "csv", - "csv-core", - "regex", -] - -[[package]] -name = "arrow-data" -version = "58.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b24852db04738907e06c04ea61e42fe7fda962a34513022dc0d0e754fb7976b" -dependencies = [ - "arrow-buffer 58.4.0", - "arrow-schema 58.4.0", - "half", - "num-integer", - "num-traits", -] - [[package]] name = "arrow-data" -version = "59.1.0" +version = "59.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b38fe43e2e8704360f1464e6e8cc4fc381ef02cc4fb0192afa8df1aaa0115c66" +checksum = "723fe4aeed7604e00b9883a465af4ff0a0e6c44c03e41a68c3d1cbc403e0e44d" dependencies = [ - "arrow-buffer 59.1.0", - "arrow-schema 59.1.0", + "arrow-buffer", + "arrow-schema", "half", "num-integer", "num-traits", @@ -356,26 +241,25 @@ dependencies = [ [[package]] name = "arrow-flight" -version = "59.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42115e09dbb694b5955da998912121451c6910b338228cb80a5701370dba43ff" -dependencies = [ - "arrow-arith 59.1.0", - "arrow-array 59.1.0", - "arrow-buffer 59.1.0", - "arrow-cast 59.1.0", - "arrow-data 59.1.0", - "arrow-ipc 59.1.0", - "arrow-ord 59.1.0", - "arrow-row 59.1.0", - "arrow-schema 59.1.0", - "arrow-select 59.1.0", - "arrow-string 59.1.0", - "base64 0.22.1", +version = "59.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2bebfacc9d71f0728f6774164e4d4254b5e504d2b46812d0512d8290ec119a64" +dependencies = [ + "arrow-arith", + "arrow-array", + "arrow-buffer", + "arrow-cast", + "arrow-data", + "arrow-ipc", + "arrow-ord", + "arrow-row", + "arrow-schema", + "arrow-select", + "arrow-string", + "base64 0.23.1", "bytes", "futures", "once_cell", - "paste", "prost", "prost-types", "tonic", @@ -384,31 +268,15 @@ dependencies = [ [[package]] name = "arrow-ipc" -version = "58.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29a908a11fcfb3fb2f6730f4ac15e367bc644e419155e96238f68cf3adde572b" -dependencies = [ - "arrow-array 58.4.0", - "arrow-buffer 58.4.0", - "arrow-data 58.4.0", - "arrow-schema 58.4.0", - "arrow-select 58.4.0", - "flatbuffers", - "lz4_flex", - "zstd", -] - -[[package]] -name = "arrow-ipc" -version = "59.1.0" +version = "59.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29dac499fcbc6ba74ee0324057821d381929a48526a3966bd9dffb44aa06d98c" +checksum = "149437b14371f5b9ec60f5ddc751483ae99d7a7072653c0075e5e469156eea7b" dependencies = [ - "arrow-array 59.1.0", - "arrow-buffer 59.1.0", - "arrow-data 59.1.0", - "arrow-schema 59.1.0", - "arrow-select 59.1.0", + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "arrow-select", "flatbuffers", "lz4_flex", "zstd", @@ -416,41 +284,16 @@ dependencies = [ [[package]] name = "arrow-json" -version = "58.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8a96aed3931c076adee39ec2a40d8219fc7f09e79bcdaca1df16272993e1e14" -dependencies = [ - "arrow-array 58.4.0", - "arrow-buffer 58.4.0", - "arrow-cast 58.4.0", - "arrow-ord 58.4.0", - "arrow-schema 58.4.0", - "arrow-select 58.4.0", - "chrono", - "half", - "indexmap", - "itoa", - "lexical-core", - "memchr", - "num-traits", - "ryu", - "serde_core", - "serde_json", - "simdutf8", -] - -[[package]] -name = "arrow-json" -version = "59.1.0" +version = "59.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fe05e916ddc50f4c7a363cd69c0ef5894fcee063517e9a0b8582f0c56746af6" +checksum = "f18b9123ccfec418a663f821c9a034af339711678c11ffe00d3ec07da5ff9f7e" dependencies = [ - "arrow-array 59.1.0", - "arrow-buffer 59.1.0", - "arrow-cast 59.1.0", - "arrow-ord 59.1.0", - "arrow-schema 59.1.0", - "arrow-select 59.1.0", + "arrow-array", + "arrow-buffer", + "arrow-cast", + "arrow-ord", + "arrow-schema", + "arrow-select", "chrono", "half", "indexmap", @@ -466,145 +309,78 @@ dependencies = [ [[package]] name = "arrow-ord" -version = "58.4.0" +version = "59.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "63a083ec750f5c043f02946b4baf05fcdbb55f4560a3277055caca5cc99f3eb0" +checksum = "e6c08dff0686cf23ca4f562803f191ccbeb726dbae6309cd4b4aaf65e0f2c979" dependencies = [ - "arrow-array 58.4.0", - "arrow-buffer 58.4.0", - "arrow-data 58.4.0", - "arrow-schema 58.4.0", - "arrow-select 58.4.0", -] - -[[package]] -name = "arrow-ord" -version = "59.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e13dbdc2a9c053c10c7baa6e30faee04a180aa7ce88e471835850ce37abd20b" -dependencies = [ - "arrow-array 59.1.0", - "arrow-buffer 59.1.0", - "arrow-data 59.1.0", - "arrow-schema 59.1.0", - "arrow-select 59.1.0", + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "arrow-select", ] [[package]] name = "arrow-pyarrow" -version = "58.4.0" +version = "59.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3ffb9be5a873590f825aef50df20e0f8dff5fd42a77058e28bbe7bd44bb53dec" +checksum = "c196ecc25b3a8dcbc1d842f2619cee653dcfa2fb8b56a291bc0481c3cf5c3821" dependencies = [ - "arrow-array 58.4.0", - "arrow-data 58.4.0", - "arrow-schema 58.4.0", + "arrow-array", + "arrow-data", + "arrow-schema", "pyo3", ] [[package]] name = "arrow-row" -version = "58.4.0" +version = "59.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "514ba0ef0d4c5896202dae736251ce415abb43a950bed570fb7981b8716c0e4c" +checksum = "bbec439386df71ad570e6758a946111322b9e9dc8db83b5527321f0b4c9119c2" dependencies = [ - "arrow-array 58.4.0", - "arrow-buffer 58.4.0", - "arrow-data 58.4.0", - "arrow-schema 58.4.0", - "half", -] - -[[package]] -name = "arrow-row" -version = "59.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d5a1f8c733d15260b305683472ee8ad89c62cbd706703ca873b90d051b41592" -dependencies = [ - "arrow-array 59.1.0", - "arrow-buffer 59.1.0", - "arrow-data 59.1.0", - "arrow-schema 59.1.0", + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", "half", ] [[package]] name = "arrow-schema" -version = "58.4.0" +version = "59.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "21ca356ad6425cecb6eb7b28e4f659f1ee7880fbb1a16127de7dd62901efee9e" +checksum = "e6fed2ca0d1eade57e811cbe73b98ad50cc08a1183e13b2d2aa43a7df593f40e" dependencies = [ "bitflags", "serde_core", "serde_json", ] -[[package]] -name = "arrow-schema" -version = "59.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9e4969dc350d571766247143ab36a5187d095d3d3690970408bc630d47c69e5" -dependencies = [ - "serde_core", - "serde_json", -] - [[package]] name = "arrow-select" -version = "58.4.0" +version = "59.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c58da39eb3d8350ad4a549e5c2bc49284dac554016c69829310350f1731b0aad" +checksum = "466b19cf75130b891dc1b23a84b343c714c62c64c9c62e365c76aa0ff90a53fb" dependencies = [ "ahash", - "arrow-array 58.4.0", - "arrow-buffer 58.4.0", - "arrow-data 58.4.0", - "arrow-schema 58.4.0", + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", "num-traits", ] -[[package]] -name = "arrow-select" -version = "59.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "402770dba90865359d98d1ef92ef16e23d75c0cca9c2c880c8a05468b7743bf9" -dependencies = [ - "ahash", - "arrow-array 59.1.0", - "arrow-buffer 59.1.0", - "arrow-data 59.1.0", - "arrow-schema 59.1.0", - "num-traits", -] - -[[package]] -name = "arrow-string" -version = "58.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6789b388467525e3271326b6b4915666ecfdf5142aef09779445c954b67543c" -dependencies = [ - "arrow-array 58.4.0", - "arrow-buffer 58.4.0", - "arrow-data 58.4.0", - "arrow-schema 58.4.0", - "arrow-select 58.4.0", - "memchr", - "num-traits", - "regex", - "regex-syntax", -] - [[package]] name = "arrow-string" -version = "59.1.0" +version = "59.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2b0afbb8b9016700938291123df30838b89decc3213dba00852021988b170d3" +checksum = "c838a25bb3691e919e0f617616ac51a4ff8517a952e29ca133cf0c22b2ce65b1" dependencies = [ - "arrow-array 59.1.0", - "arrow-buffer 59.1.0", - "arrow-data 59.1.0", - "arrow-schema 59.1.0", - "arrow-select 59.1.0", + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "arrow-select", "memchr", "num-traits", "regex", @@ -721,7 +497,7 @@ dependencies = [ "ballista-core", "ballista-executor", "ballista-scheduler", - "datafusion 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", + "datafusion", "log", "tokio", "url", @@ -734,10 +510,10 @@ dependencies = [ "arrow-flight", "async-trait", "chrono", - "datafusion 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", - "datafusion-functions-aggregate-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", - "datafusion-proto 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", - "datafusion-proto-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", + "datafusion", + "datafusion-functions-aggregate-common", + "datafusion-proto", + "datafusion-proto-common", "futures", "itertools 0.15.0", "log", @@ -762,14 +538,14 @@ dependencies = [ name = "ballista-executor" version = "54.0.0" dependencies = [ - "arrow 59.1.0", + "arrow", "arrow-flight", "async-trait", "ballista-core", "bytesize", "dashmap", - "datafusion 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", - "datafusion-proto 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", + "datafusion", + "datafusion-proto", "futures", "libc", "log", @@ -795,8 +571,8 @@ dependencies = [ "axum", "ballista-core", "dashmap", - "datafusion 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", - "datafusion-proto 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", + "datafusion", + "datafusion-proto", "futures", "http", "insta", @@ -836,7 +612,7 @@ checksum = "4d6867f1565b3aad85681f1015055b087fcfd840d6aeee6eee7f2da317603695" dependencies = [ "autocfg", "libm", - "num-bigint", + "num-bigint 0.4.8", "num-integer", "num-traits", ] @@ -915,12 +691,6 @@ version = "3.20.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" -[[package]] -name = "byteorder" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" - [[package]] name = "bytes" version = "1.12.1" @@ -1201,92 +971,39 @@ dependencies = [ [[package]] name = "datafusion" version = "54.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "754ef4e8f073922a26f5b23133b9db4829342362b09be0bc94309cf261c2f098" +source = "git+https://github.com/apache/datafusion.git?rev=e08aed1e5de41dcf81d529140dae07723b942a5e#e08aed1e5de41dcf81d529140dae07723b942a5e" dependencies = [ - "arrow 58.4.0", - "arrow-schema 58.4.0", + "arrow", + "arrow-schema", "async-trait", "bzip2", "chrono", - "datafusion-catalog 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "datafusion-catalog-listing 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "datafusion-common 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "datafusion-common-runtime 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "datafusion-datasource 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "datafusion-datasource-arrow 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-catalog", + "datafusion-catalog-listing", + "datafusion-common", + "datafusion-common-runtime", + "datafusion-datasource", + "datafusion-datasource-arrow", "datafusion-datasource-avro", - "datafusion-datasource-csv 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "datafusion-datasource-json 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "datafusion-datasource-parquet 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "datafusion-execution 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "datafusion-expr 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "datafusion-expr-common 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "datafusion-functions 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "datafusion-functions-aggregate 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "datafusion-functions-nested 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "datafusion-functions-table 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "datafusion-functions-window 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "datafusion-optimizer 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "datafusion-physical-expr 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "datafusion-physical-expr-adapter 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "datafusion-physical-expr-common 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "datafusion-physical-optimizer 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "datafusion-physical-plan 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "datafusion-session 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "datafusion-sql 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "flate2", - "futures", - "indexmap", - "itertools 0.14.0", - "liblzma", - "log", - "object_store", - "parking_lot", - "parquet 58.4.0", - "sqlparser", - "tempfile", - "tokio", - "url", - "uuid", - "zstd", -] - -[[package]] -name = "datafusion" -version = "54.1.0" -source = "git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11#70c26a06716eb552f0fc1d2115957a4089504f11" -dependencies = [ - "arrow 59.1.0", - "arrow-schema 59.1.0", - "async-trait", - "bzip2", - "chrono", - "datafusion-catalog 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", - "datafusion-catalog-listing 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", - "datafusion-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", - "datafusion-common-runtime 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", - "datafusion-datasource 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", - "datafusion-datasource-arrow 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", - "datafusion-datasource-csv 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", - "datafusion-datasource-json 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", - "datafusion-datasource-parquet 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", - "datafusion-execution 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", - "datafusion-expr 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", - "datafusion-expr-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", - "datafusion-functions 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", - "datafusion-functions-aggregate 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", - "datafusion-functions-nested 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", - "datafusion-functions-table 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", - "datafusion-functions-window 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", - "datafusion-optimizer 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", - "datafusion-physical-expr 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", - "datafusion-physical-expr-adapter 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", - "datafusion-physical-expr-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", - "datafusion-physical-optimizer 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", - "datafusion-physical-plan 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", - "datafusion-session 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", - "datafusion-sql 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", + "datafusion-datasource-csv", + "datafusion-datasource-json", + "datafusion-datasource-parquet", + "datafusion-execution", + "datafusion-expr", + "datafusion-expr-common", + "datafusion-functions", + "datafusion-functions-aggregate", + "datafusion-functions-nested", + "datafusion-functions-table", + "datafusion-functions-window", + "datafusion-optimizer", + "datafusion-physical-expr", + "datafusion-physical-expr-adapter", + "datafusion-physical-expr-common", + "datafusion-physical-optimizer", + "datafusion-physical-plan", + "datafusion-session", + "datafusion-sql", "flate2", "futures", "indexmap", @@ -1295,7 +1012,7 @@ dependencies = [ "log", "object_store", "parking_lot", - "parquet 59.1.0", + "parquet", "sqlparser", "tempfile", "tokio", @@ -1307,44 +1024,19 @@ dependencies = [ [[package]] name = "datafusion-catalog" version = "54.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "06afd1e38dd27bbb1258685a1fc6524df6aff4e07b25b393a47de59635178d99" +source = "git+https://github.com/apache/datafusion.git?rev=e08aed1e5de41dcf81d529140dae07723b942a5e#e08aed1e5de41dcf81d529140dae07723b942a5e" dependencies = [ - "arrow 58.4.0", + "arrow", "async-trait", "dashmap", - "datafusion-common 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "datafusion-common-runtime 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "datafusion-datasource 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "datafusion-execution 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "datafusion-expr 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "datafusion-physical-expr 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "datafusion-physical-plan 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "datafusion-session 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "futures", - "itertools 0.14.0", - "log", - "object_store", - "parking_lot", - "tokio", -] - -[[package]] -name = "datafusion-catalog" -version = "54.1.0" -source = "git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11#70c26a06716eb552f0fc1d2115957a4089504f11" -dependencies = [ - "arrow 59.1.0", - "async-trait", - "dashmap", - "datafusion-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", - "datafusion-common-runtime 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", - "datafusion-datasource 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", - "datafusion-execution 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", - "datafusion-expr 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", - "datafusion-physical-expr 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", - "datafusion-physical-plan 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", - "datafusion-session 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", + "datafusion-common", + "datafusion-common-runtime", + "datafusion-datasource", + "datafusion-execution", + "datafusion-expr", + "datafusion-physical-expr", + "datafusion-physical-plan", + "datafusion-session", "futures", "itertools 0.15.0", "log", @@ -1356,42 +1048,19 @@ dependencies = [ [[package]] name = "datafusion-catalog-listing" version = "54.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0668fb32c12065ec242be0e5b4bc62bd7a06a0be3ecd83791ef877e4be67e02" -dependencies = [ - "arrow 58.4.0", - "async-trait", - "datafusion-catalog 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "datafusion-common 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "datafusion-datasource 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "datafusion-execution 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "datafusion-expr 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "datafusion-physical-expr 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "datafusion-physical-expr-adapter 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "datafusion-physical-expr-common 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "datafusion-physical-plan 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "futures", - "itertools 0.14.0", - "log", - "object_store", -] - -[[package]] -name = "datafusion-catalog-listing" -version = "54.1.0" -source = "git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11#70c26a06716eb552f0fc1d2115957a4089504f11" +source = "git+https://github.com/apache/datafusion.git?rev=e08aed1e5de41dcf81d529140dae07723b942a5e#e08aed1e5de41dcf81d529140dae07723b942a5e" dependencies = [ - "arrow 59.1.0", + "arrow", "async-trait", - "datafusion-catalog 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", - "datafusion-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", - "datafusion-datasource 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", - "datafusion-execution 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", - "datafusion-expr 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", - "datafusion-physical-expr 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", - "datafusion-physical-expr-adapter 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", - "datafusion-physical-expr-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", - "datafusion-physical-plan 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", + "datafusion-catalog", + "datafusion-common", + "datafusion-datasource", + "datafusion-execution", + "datafusion-expr", + "datafusion-physical-expr", + "datafusion-physical-expr-adapter", + "datafusion-physical-expr-common", + "datafusion-physical-plan", "futures", "itertools 0.15.0", "log", @@ -1402,37 +1071,11 @@ dependencies = [ [[package]] name = "datafusion-common" version = "54.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca43b263cdff57042cfa8fb817fb3469f4878933380dccff25f5e793580abbf9" +source = "git+https://github.com/apache/datafusion.git?rev=e08aed1e5de41dcf81d529140dae07723b942a5e#e08aed1e5de41dcf81d529140dae07723b942a5e" dependencies = [ - "arrow 58.4.0", - "arrow-ipc 58.4.0", - "arrow-schema 58.4.0", - "chrono", - "foldhash 0.2.0", - "half", - "hashbrown 0.17.1", - "indexmap", - "itertools 0.14.0", - "libc", - "log", - "object_store", - "parquet 58.4.0", - "recursive", - "sqlparser", - "tokio", - "uuid", - "web-time", -] - -[[package]] -name = "datafusion-common" -version = "54.1.0" -source = "git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11#70c26a06716eb552f0fc1d2115957a4089504f11" -dependencies = [ - "arrow 59.1.0", - "arrow-ipc 59.1.0", - "arrow-schema 59.1.0", + "arrow", + "arrow-ipc", + "arrow-schema", "chrono", "foldhash 0.2.0", "half", @@ -1443,7 +1086,7 @@ dependencies = [ "log", "num-traits", "object_store", - "parquet 59.1.0", + "parquet", "recursive", "sqlparser", "tokio", @@ -1454,81 +1097,34 @@ dependencies = [ [[package]] name = "datafusion-common-runtime" version = "54.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05f0ba2b864792bdca4d76c59a1de0ab6e1b61946596b9936888dbd6360035f2" +source = "git+https://github.com/apache/datafusion.git?rev=e08aed1e5de41dcf81d529140dae07723b942a5e#e08aed1e5de41dcf81d529140dae07723b942a5e" dependencies = [ "futures", "log", "tokio", ] -[[package]] -name = "datafusion-common-runtime" -version = "54.1.0" -source = "git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11#70c26a06716eb552f0fc1d2115957a4089504f11" -dependencies = [ - "futures", - "log", - "tokio", -] - -[[package]] -name = "datafusion-datasource" -version = "54.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b840a8bce0bcbf5afad02946d438591e7c373f7afccaf3d874c04485772514dd" -dependencies = [ - "arrow 58.4.0", - "async-compression", - "async-trait", - "bytes", - "bzip2", - "chrono", - "datafusion-common 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "datafusion-common-runtime 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "datafusion-execution 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "datafusion-expr 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "datafusion-physical-expr 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "datafusion-physical-expr-adapter 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "datafusion-physical-expr-common 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "datafusion-physical-plan 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "datafusion-session 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "flate2", - "futures", - "glob", - "itertools 0.14.0", - "liblzma", - "log", - "object_store", - "parking_lot", - "rand 0.9.5", - "tokio", - "tokio-util", - "url", - "zstd", -] - [[package]] name = "datafusion-datasource" version = "54.1.0" -source = "git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11#70c26a06716eb552f0fc1d2115957a4089504f11" +source = "git+https://github.com/apache/datafusion.git?rev=e08aed1e5de41dcf81d529140dae07723b942a5e#e08aed1e5de41dcf81d529140dae07723b942a5e" dependencies = [ - "arrow 59.1.0", + "arrow", "async-compression", "async-trait", "bytes", "bzip2", "chrono", - "datafusion-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", - "datafusion-common-runtime 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", - "datafusion-execution 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", - "datafusion-expr 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", - "datafusion-physical-expr 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", - "datafusion-physical-expr-adapter 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", - "datafusion-physical-expr-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", - "datafusion-physical-plan 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", + "datafusion-common", + "datafusion-common-runtime", + "datafusion-execution", + "datafusion-expr", + "datafusion-physical-expr", + "datafusion-physical-expr-adapter", + "datafusion-physical-expr-common", + "datafusion-physical-plan", "datafusion-proto-models", - "datafusion-session 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", + "datafusion-session", "flate2", "futures", "glob", @@ -1547,44 +1143,20 @@ dependencies = [ [[package]] name = "datafusion-datasource-arrow" version = "54.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a24cc0b9cf6e367f27f27406eff13abf48a11b72446aaa40b3105c0ded5c17d9" +source = "git+https://github.com/apache/datafusion.git?rev=e08aed1e5de41dcf81d529140dae07723b942a5e#e08aed1e5de41dcf81d529140dae07723b942a5e" dependencies = [ - "arrow 58.4.0", - "arrow-ipc 58.4.0", + "arrow", + "arrow-ipc", "async-trait", "bytes", - "datafusion-common 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "datafusion-common-runtime 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "datafusion-datasource 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "datafusion-execution 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "datafusion-expr 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "datafusion-physical-expr-common 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "datafusion-physical-plan 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "datafusion-session 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "futures", - "itertools 0.14.0", - "object_store", - "tokio", -] - -[[package]] -name = "datafusion-datasource-arrow" -version = "54.1.0" -source = "git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11#70c26a06716eb552f0fc1d2115957a4089504f11" -dependencies = [ - "arrow 59.1.0", - "arrow-ipc 59.1.0", - "async-trait", - "bytes", - "datafusion-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", - "datafusion-common-runtime 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", - "datafusion-datasource 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", - "datafusion-execution 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", - "datafusion-expr 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", - "datafusion-physical-expr-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", - "datafusion-physical-plan 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", - "datafusion-session 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", + "datafusion-common", + "datafusion-common-runtime", + "datafusion-datasource", + "datafusion-execution", + "datafusion-expr", + "datafusion-physical-expr-common", + "datafusion-physical-plan", + "datafusion-session", "futures", "itertools 0.15.0", "object_store", @@ -1594,18 +1166,17 @@ dependencies = [ [[package]] name = "datafusion-datasource-avro" version = "54.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f26a312bb06528c17b4bb86b798b0c4321e7656d85e0ef65c2e9adf07b0a518d" +source = "git+https://github.com/apache/datafusion.git?rev=e08aed1e5de41dcf81d529140dae07723b942a5e#e08aed1e5de41dcf81d529140dae07723b942a5e" dependencies = [ - "arrow 58.4.0", + "arrow", "arrow-avro", "async-trait", "bytes", - "datafusion-common 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "datafusion-datasource 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "datafusion-physical-expr-adapter 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "datafusion-physical-plan 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "datafusion-session 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-common", + "datafusion-datasource", + "datafusion-physical-expr-adapter", + "datafusion-physical-plan", + "datafusion-session", "futures", "object_store", ] @@ -1613,42 +1184,20 @@ dependencies = [ [[package]] name = "datafusion-datasource-csv" version = "54.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e1abe56b2a7a2d1d6de5117dd1a203181e267f28529faa5da546947621b697d7" +source = "git+https://github.com/apache/datafusion.git?rev=e08aed1e5de41dcf81d529140dae07723b942a5e#e08aed1e5de41dcf81d529140dae07723b942a5e" dependencies = [ - "arrow 58.4.0", + "arrow", "async-trait", "bytes", - "datafusion-common 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "datafusion-common-runtime 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "datafusion-datasource 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "datafusion-execution 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "datafusion-expr 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "datafusion-physical-expr-common 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "datafusion-physical-plan 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "datafusion-session 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "futures", - "object_store", - "regex", - "tokio", -] - -[[package]] -name = "datafusion-datasource-csv" -version = "54.1.0" -source = "git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11#70c26a06716eb552f0fc1d2115957a4089504f11" -dependencies = [ - "arrow 59.1.0", - "async-trait", - "bytes", - "datafusion-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", - "datafusion-common-runtime 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", - "datafusion-datasource 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", - "datafusion-execution 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", - "datafusion-expr 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", - "datafusion-physical-expr-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", - "datafusion-physical-plan 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", - "datafusion-session 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", + "datafusion-common", + "datafusion-common-runtime", + "datafusion-datasource", + "datafusion-execution", + "datafusion-expr", + "datafusion-physical-expr-common", + "datafusion-physical-plan", + "datafusion-proto-models", + "datafusion-session", "futures", "object_store", "regex", @@ -1658,42 +1207,20 @@ dependencies = [ [[package]] name = "datafusion-datasource-json" version = "54.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c3e467f0611ad7bdd5aad17c63c9bb6182d04e5282e5496d897ea2b49c024ba" -dependencies = [ - "arrow 58.4.0", - "async-trait", - "bytes", - "datafusion-common 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "datafusion-common-runtime 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "datafusion-datasource 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "datafusion-execution 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "datafusion-expr 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "datafusion-physical-expr-common 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "datafusion-physical-plan 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "datafusion-session 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "futures", - "object_store", - "tokio", - "tokio-stream", -] - -[[package]] -name = "datafusion-datasource-json" -version = "54.1.0" -source = "git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11#70c26a06716eb552f0fc1d2115957a4089504f11" +source = "git+https://github.com/apache/datafusion.git?rev=e08aed1e5de41dcf81d529140dae07723b942a5e#e08aed1e5de41dcf81d529140dae07723b942a5e" dependencies = [ - "arrow 59.1.0", + "arrow", "async-trait", "bytes", - "datafusion-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", - "datafusion-common-runtime 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", - "datafusion-datasource 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", - "datafusion-execution 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", - "datafusion-expr 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", - "datafusion-physical-expr-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", - "datafusion-physical-plan 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", - "datafusion-session 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", + "datafusion-common", + "datafusion-common-runtime", + "datafusion-datasource", + "datafusion-execution", + "datafusion-expr", + "datafusion-physical-expr-common", + "datafusion-physical-plan", + "datafusion-proto-models", + "datafusion-session", "futures", "object_store", "tokio", @@ -1703,111 +1230,53 @@ dependencies = [ [[package]] name = "datafusion-datasource-parquet" version = "54.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4cc35b92cd560082155e80d9c826929c852d3c51543f4affd3a51c464a0aab3a" +source = "git+https://github.com/apache/datafusion.git?rev=e08aed1e5de41dcf81d529140dae07723b942a5e#e08aed1e5de41dcf81d529140dae07723b942a5e" dependencies = [ - "arrow 58.4.0", + "arrow", + "arrow-schema", "async-trait", "bytes", - "datafusion-common 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "datafusion-common-runtime 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "datafusion-datasource 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "datafusion-execution 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "datafusion-expr 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "datafusion-functions 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "datafusion-functions-aggregate-common 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "datafusion-physical-expr 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "datafusion-physical-expr-adapter 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "datafusion-physical-expr-common 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "datafusion-physical-plan 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "datafusion-pruning 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "datafusion-session 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "futures", - "itertools 0.14.0", - "log", - "object_store", - "parking_lot", - "parquet 58.4.0", - "tokio", -] - -[[package]] -name = "datafusion-datasource-parquet" -version = "54.1.0" -source = "git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11#70c26a06716eb552f0fc1d2115957a4089504f11" -dependencies = [ - "arrow 59.1.0", - "arrow-schema 59.1.0", - "async-trait", - "bytes", - "datafusion-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", - "datafusion-common-runtime 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", - "datafusion-datasource 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", - "datafusion-execution 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", - "datafusion-expr 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", - "datafusion-functions 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", - "datafusion-functions-aggregate-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", - "datafusion-physical-expr 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", - "datafusion-physical-expr-adapter 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", - "datafusion-physical-expr-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", - "datafusion-physical-plan 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", - "datafusion-pruning 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", - "datafusion-session 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", + "datafusion-common", + "datafusion-common-runtime", + "datafusion-datasource", + "datafusion-execution", + "datafusion-expr", + "datafusion-functions", + "datafusion-functions-aggregate-common", + "datafusion-physical-expr", + "datafusion-physical-expr-adapter", + "datafusion-physical-expr-common", + "datafusion-physical-plan", + "datafusion-proto-models", + "datafusion-pruning", + "datafusion-session", "futures", "itertools 0.15.0", "log", "object_store", "parking_lot", - "parquet 59.1.0", + "parquet", "tokio", ] [[package]] name = "datafusion-doc" version = "54.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d69bb69d8769e34f76839c960dbde24c1ac0c885a79b6c3c2287bdc56ec67891" - -[[package]] -name = "datafusion-doc" -version = "54.1.0" -source = "git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11#70c26a06716eb552f0fc1d2115957a4089504f11" +source = "git+https://github.com/apache/datafusion.git?rev=e08aed1e5de41dcf81d529140dae07723b942a5e#e08aed1e5de41dcf81d529140dae07723b942a5e" [[package]] name = "datafusion-execution" version = "54.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d8eac0a09bc8d263f52025cad9e001da4d8138d633fa288edda4d06b1772eae6" +source = "git+https://github.com/apache/datafusion.git?rev=e08aed1e5de41dcf81d529140dae07723b942a5e#e08aed1e5de41dcf81d529140dae07723b942a5e" dependencies = [ - "arrow 58.4.0", - "arrow-buffer 58.4.0", - "async-trait", - "dashmap", - "datafusion-common 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "datafusion-expr 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "datafusion-physical-expr-common 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "futures", - "log", - "object_store", - "parking_lot", - "rand 0.9.5", - "tempfile", - "url", -] - -[[package]] -name = "datafusion-execution" -version = "54.1.0" -source = "git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11#70c26a06716eb552f0fc1d2115957a4089504f11" -dependencies = [ - "arrow 59.1.0", - "arrow-buffer 59.1.0", + "arrow", + "arrow-buffer", "async-trait", "bytes", "dashmap", - "datafusion-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", - "datafusion-expr 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", - "datafusion-physical-expr-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", + "datafusion-common", + "datafusion-expr", + "datafusion-physical-expr-common", "futures", "log", "object_store", @@ -1823,41 +1292,18 @@ dependencies = [ [[package]] name = "datafusion-expr" version = "54.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eeb14d374767ee0fc62dc79a5ba8bcf8a63c14e993c7d992d0e63adfa23d77d3" +source = "git+https://github.com/apache/datafusion.git?rev=e08aed1e5de41dcf81d529140dae07723b942a5e#e08aed1e5de41dcf81d529140dae07723b942a5e" dependencies = [ - "arrow 58.4.0", - "arrow-schema 58.4.0", + "arrow", + "arrow-schema", "async-trait", "chrono", - "datafusion-common 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "datafusion-doc 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "datafusion-expr-common 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "datafusion-functions-aggregate-common 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "datafusion-functions-window-common 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "datafusion-physical-expr-common 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "indexmap", - "itertools 0.14.0", - "recursive", - "serde_json", - "sqlparser", -] - -[[package]] -name = "datafusion-expr" -version = "54.1.0" -source = "git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11#70c26a06716eb552f0fc1d2115957a4089504f11" -dependencies = [ - "arrow 59.1.0", - "arrow-schema 59.1.0", - "async-trait", - "chrono", - "datafusion-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", - "datafusion-doc 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", - "datafusion-expr-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", - "datafusion-functions-aggregate-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", - "datafusion-functions-window-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", - "datafusion-physical-expr-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", + "datafusion-common", + "datafusion-doc", + "datafusion-expr-common", + "datafusion-functions-aggregate-common", + "datafusion-functions-window-common", + "datafusion-physical-expr-common", "indexmap", "itertools 0.15.0", "recursive", @@ -1868,22 +1314,10 @@ dependencies = [ [[package]] name = "datafusion-expr-common" version = "54.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b7b19a8c95522bee8cbb313d74263b85e355d2b52f42e67ef5694bf5de9e9356" -dependencies = [ - "arrow 58.4.0", - "datafusion-common 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "indexmap", - "itertools 0.14.0", -] - -[[package]] -name = "datafusion-expr-common" -version = "54.1.0" -source = "git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11#70c26a06716eb552f0fc1d2115957a4089504f11" +source = "git+https://github.com/apache/datafusion.git?rev=e08aed1e5de41dcf81d529140dae07723b942a5e#e08aed1e5de41dcf81d529140dae07723b942a5e" dependencies = [ - "arrow 59.1.0", - "datafusion-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", + "arrow", + "datafusion-common", "indexmap", "itertools 0.15.0", ] @@ -1891,27 +1325,26 @@ dependencies = [ [[package]] name = "datafusion-ffi" version = "54.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9a4a09f24f1387408810acca070207a9cf6c7fc2368122982a8e7671f643d1e" +source = "git+https://github.com/apache/datafusion.git?rev=e08aed1e5de41dcf81d529140dae07723b942a5e#e08aed1e5de41dcf81d529140dae07723b942a5e" dependencies = [ - "arrow 58.4.0", - "arrow-schema 58.4.0", + "arrow", + "arrow-schema", "async-ffi", "async-trait", "chrono", - "datafusion-catalog 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "datafusion-common 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "datafusion-datasource 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "datafusion-execution 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "datafusion-expr 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "datafusion-functions-aggregate-common 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "datafusion-physical-expr 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "datafusion-physical-expr-common 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "datafusion-physical-optimizer 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "datafusion-physical-plan 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "datafusion-proto 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "datafusion-proto-common 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "datafusion-session 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-catalog", + "datafusion-common", + "datafusion-datasource", + "datafusion-execution", + "datafusion-expr", + "datafusion-functions-aggregate-common", + "datafusion-physical-expr", + "datafusion-physical-expr-common", + "datafusion-physical-optimizer", + "datafusion-physical-plan", + "datafusion-proto", + "datafusion-proto-common", + "datafusion-session", "futures", "libloading", "log", @@ -1924,101 +1357,48 @@ dependencies = [ [[package]] name = "datafusion-functions" version = "54.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f64c983bbbdcb729d921a2b2ac3375598719b5cc0c30345ad664936f3176fc7" -dependencies = [ - "arrow 58.4.0", - "arrow-buffer 58.4.0", - "base64 0.22.1", - "blake2", - "blake3", - "chrono", - "chrono-tz", - "datafusion-common 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "datafusion-doc 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "datafusion-execution 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "datafusion-expr 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "datafusion-expr-common 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "datafusion-macros 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "datafusion-physical-expr-common 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "hex", - "itertools 0.14.0", - "log", - "md-5 0.11.0", - "memchr", - "num-traits", - "rand 0.9.5", - "regex", - "sha2", - "uuid", -] - -[[package]] -name = "datafusion-functions" -version = "54.1.0" -source = "git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11#70c26a06716eb552f0fc1d2115957a4089504f11" +source = "git+https://github.com/apache/datafusion.git?rev=e08aed1e5de41dcf81d529140dae07723b942a5e#e08aed1e5de41dcf81d529140dae07723b942a5e" dependencies = [ - "arrow 59.1.0", - "arrow-buffer 59.1.0", + "arrow", + "arrow-buffer", "base64 0.23.1", "blake2", "blake3", "chrono", "chrono-tz", - "datafusion-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", - "datafusion-doc 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", - "datafusion-execution 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", - "datafusion-expr 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", - "datafusion-expr-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", - "datafusion-macros 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", - "datafusion-physical-expr-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", + "datafusion-common", + "datafusion-doc", + "datafusion-execution", + "datafusion-expr", + "datafusion-expr-common", + "datafusion-macros", + "datafusion-physical-expr-common", "hex", "itertools 0.15.0", "log", "md-5 0.11.0", - "memchr", - "num-traits", - "rand 0.9.5", - "regex", - "sha2", - "uuid", -] - -[[package]] -name = "datafusion-functions-aggregate" -version = "54.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89bc17041e424a47ed062f43df24d84aab8b57c4c3221e5c1a5eef46d6c5718b" -dependencies = [ - "arrow 58.4.0", - "datafusion-common 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "datafusion-doc 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "datafusion-execution 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "datafusion-expr 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "datafusion-functions-aggregate-common 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "datafusion-macros 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "datafusion-physical-expr 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "datafusion-physical-expr-common 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "foldhash 0.2.0", - "half", - "log", + "memchr", "num-traits", + "rand 0.9.5", + "regex", + "sha2", + "uuid", ] [[package]] name = "datafusion-functions-aggregate" version = "54.1.0" -source = "git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11#70c26a06716eb552f0fc1d2115957a4089504f11" -dependencies = [ - "arrow 59.1.0", - "datafusion-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", - "datafusion-doc 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", - "datafusion-execution 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", - "datafusion-expr 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", - "datafusion-functions-aggregate-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", - "datafusion-macros 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", - "datafusion-physical-expr 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", - "datafusion-physical-expr-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", +source = "git+https://github.com/apache/datafusion.git?rev=e08aed1e5de41dcf81d529140dae07723b942a5e#e08aed1e5de41dcf81d529140dae07723b942a5e" +dependencies = [ + "arrow", + "datafusion-common", + "datafusion-doc", + "datafusion-execution", + "datafusion-expr", + "datafusion-functions-aggregate-common", + "datafusion-macros", + "datafusion-physical-expr", + "datafusion-physical-expr-common", "half", "hashbrown 0.17.1", "log", @@ -2028,68 +1408,31 @@ dependencies = [ [[package]] name = "datafusion-functions-aggregate-common" version = "54.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97dd2a9e865c6108059f5b37b77934f84b50bfb108f837bd0e5c9536e03f0545" -dependencies = [ - "arrow 58.4.0", - "datafusion-common 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "datafusion-expr-common 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "datafusion-physical-expr-common 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "datafusion-functions-aggregate-common" -version = "54.1.0" -source = "git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11#70c26a06716eb552f0fc1d2115957a4089504f11" +source = "git+https://github.com/apache/datafusion.git?rev=e08aed1e5de41dcf81d529140dae07723b942a5e#e08aed1e5de41dcf81d529140dae07723b942a5e" dependencies = [ - "arrow 59.1.0", - "datafusion-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", - "datafusion-expr-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", - "datafusion-physical-expr-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", -] - -[[package]] -name = "datafusion-functions-nested" -version = "54.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75f0bdfeef16d96417b9632ef855645376b242e9006a126dfd0bedfc54a93f5f" -dependencies = [ - "arrow 58.4.0", - "arrow-ord 58.4.0", - "datafusion-common 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "datafusion-doc 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "datafusion-execution 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "datafusion-expr 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "datafusion-expr-common 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "datafusion-functions 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "datafusion-functions-aggregate 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "datafusion-functions-aggregate-common 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "datafusion-macros 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "datafusion-physical-expr-common 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "hashbrown 0.17.1", - "itertools 0.14.0", - "itoa", - "log", - "memchr", + "arrow", + "datafusion-common", + "datafusion-expr-common", + "datafusion-physical-expr-common", ] [[package]] name = "datafusion-functions-nested" version = "54.1.0" -source = "git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11#70c26a06716eb552f0fc1d2115957a4089504f11" -dependencies = [ - "arrow 59.1.0", - "arrow-ord 59.1.0", - "datafusion-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", - "datafusion-doc 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", - "datafusion-execution 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", - "datafusion-expr 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", - "datafusion-expr-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", - "datafusion-functions 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", - "datafusion-functions-aggregate 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", - "datafusion-functions-aggregate-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", - "datafusion-macros 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", - "datafusion-physical-expr-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", +source = "git+https://github.com/apache/datafusion.git?rev=e08aed1e5de41dcf81d529140dae07723b942a5e#e08aed1e5de41dcf81d529140dae07723b942a5e" +dependencies = [ + "arrow", + "arrow-ord", + "datafusion-common", + "datafusion-doc", + "datafusion-execution", + "datafusion-expr", + "datafusion-expr-common", + "datafusion-functions", + "datafusion-functions-aggregate", + "datafusion-functions-aggregate-common", + "datafusion-macros", + "datafusion-physical-expr-common", "hashbrown 0.17.1", "itertools 0.15.0", "itoa", @@ -2100,103 +1443,49 @@ dependencies = [ [[package]] name = "datafusion-functions-table" version = "54.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f4e4941673c917819616877e9993da4503e4f4739812be0bc32c5356184c6383" -dependencies = [ - "arrow 58.4.0", - "async-trait", - "datafusion-catalog 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "datafusion-common 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "datafusion-expr 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "datafusion-physical-expr 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "datafusion-physical-plan 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "parking_lot", -] - -[[package]] -name = "datafusion-functions-table" -version = "54.1.0" -source = "git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11#70c26a06716eb552f0fc1d2115957a4089504f11" +source = "git+https://github.com/apache/datafusion.git?rev=e08aed1e5de41dcf81d529140dae07723b942a5e#e08aed1e5de41dcf81d529140dae07723b942a5e" dependencies = [ - "arrow 59.1.0", + "arrow", "async-trait", - "datafusion-catalog 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", - "datafusion-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", - "datafusion-expr 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", - "datafusion-physical-expr 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", - "datafusion-physical-plan 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", + "datafusion-catalog", + "datafusion-common", + "datafusion-expr", + "datafusion-physical-expr", + "datafusion-physical-plan", "parking_lot", ] [[package]] name = "datafusion-functions-window" version = "54.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12dd2e16c12b84b6f6b41b19f55b366dd1c46876bb35b86896c6349067379e8d" -dependencies = [ - "arrow 58.4.0", - "datafusion-common 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "datafusion-doc 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "datafusion-expr 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "datafusion-functions-window-common 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "datafusion-macros 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "datafusion-physical-expr 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "datafusion-physical-expr-common 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "log", -] - -[[package]] -name = "datafusion-functions-window" -version = "54.1.0" -source = "git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11#70c26a06716eb552f0fc1d2115957a4089504f11" -dependencies = [ - "arrow 59.1.0", - "datafusion-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", - "datafusion-doc 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", - "datafusion-expr 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", - "datafusion-functions-window-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", - "datafusion-macros 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", - "datafusion-physical-expr 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", - "datafusion-physical-expr-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", +source = "git+https://github.com/apache/datafusion.git?rev=e08aed1e5de41dcf81d529140dae07723b942a5e#e08aed1e5de41dcf81d529140dae07723b942a5e" +dependencies = [ + "arrow", + "datafusion-common", + "datafusion-doc", + "datafusion-expr", + "datafusion-functions-window-common", + "datafusion-macros", + "datafusion-physical-expr", + "datafusion-physical-expr-common", "log", ] [[package]] name = "datafusion-functions-window-common" version = "54.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4cdc5e4b6f8b6ef823cc1c761f85088ad4c884fe8df64df3cbcc6b2b84698441" -dependencies = [ - "datafusion-common 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "datafusion-physical-expr-common 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "datafusion-functions-window-common" -version = "54.1.0" -source = "git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11#70c26a06716eb552f0fc1d2115957a4089504f11" -dependencies = [ - "datafusion-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", - "datafusion-physical-expr-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", -] - -[[package]] -name = "datafusion-macros" -version = "54.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a3614234dd93578c92428cb4f408e020874f0d2b7e6c90c928d9d28b5df2ceb" +source = "git+https://github.com/apache/datafusion.git?rev=e08aed1e5de41dcf81d529140dae07723b942a5e#e08aed1e5de41dcf81d529140dae07723b942a5e" dependencies = [ - "datafusion-doc 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "quote", - "syn 2.0.119", + "datafusion-common", + "datafusion-physical-expr-common", ] [[package]] name = "datafusion-macros" version = "54.1.0" -source = "git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11#70c26a06716eb552f0fc1d2115957a4089504f11" +source = "git+https://github.com/apache/datafusion.git?rev=e08aed1e5de41dcf81d529140dae07723b942a5e#e08aed1e5de41dcf81d529140dae07723b942a5e" dependencies = [ - "datafusion-doc 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", + "datafusion-doc", "quote", "syn 3.0.3", ] @@ -2204,34 +1493,14 @@ dependencies = [ [[package]] name = "datafusion-optimizer" version = "54.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a0635620b050b81bb92764e99250868f654e2cd5ad1bece413283b3f73c83179" -dependencies = [ - "arrow 58.4.0", - "chrono", - "datafusion-common 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "datafusion-expr 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "datafusion-expr-common 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "datafusion-physical-expr 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "indexmap", - "itertools 0.14.0", - "log", - "recursive", - "regex", - "regex-syntax", -] - -[[package]] -name = "datafusion-optimizer" -version = "54.1.0" -source = "git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11#70c26a06716eb552f0fc1d2115957a4089504f11" +source = "git+https://github.com/apache/datafusion.git?rev=e08aed1e5de41dcf81d529140dae07723b942a5e#e08aed1e5de41dcf81d529140dae07723b942a5e" dependencies = [ - "arrow 59.1.0", + "arrow", "chrono", - "datafusion-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", - "datafusion-expr 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", - "datafusion-expr-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", - "datafusion-physical-expr 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", + "datafusion-common", + "datafusion-expr", + "datafusion-expr-common", + "datafusion-physical-expr", "indexmap", "itertools 0.15.0", "log", @@ -2243,36 +1512,14 @@ dependencies = [ [[package]] name = "datafusion-physical-expr" version = "54.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8cabf7a86eb70b816729e33c81bf7767c936ee1226f607a114f5dac2decac8d0" -dependencies = [ - "arrow 58.4.0", - "datafusion-common 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "datafusion-expr 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "datafusion-expr-common 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "datafusion-functions-aggregate-common 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "datafusion-physical-expr-common 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "half", - "hashbrown 0.17.1", - "indexmap", - "itertools 0.14.0", - "parking_lot", - "petgraph", - "recursive", - "tokio", -] - -[[package]] -name = "datafusion-physical-expr" -version = "54.1.0" -source = "git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11#70c26a06716eb552f0fc1d2115957a4089504f11" -dependencies = [ - "arrow 59.1.0", - "datafusion-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", - "datafusion-expr 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", - "datafusion-expr-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", - "datafusion-functions-aggregate-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", - "datafusion-physical-expr-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", +source = "git+https://github.com/apache/datafusion.git?rev=e08aed1e5de41dcf81d529140dae07723b942a5e#e08aed1e5de41dcf81d529140dae07723b942a5e" +dependencies = [ + "arrow", + "datafusion-common", + "datafusion-expr", + "datafusion-expr-common", + "datafusion-functions-aggregate-common", + "datafusion-physical-expr-common", "datafusion-proto-models", "half", "hashbrown 0.17.1", @@ -2287,58 +1534,26 @@ dependencies = [ [[package]] name = "datafusion-physical-expr-adapter" version = "54.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "de222e04f7e6744501555a54ab0abe26bfdfebee380af79a9bdc175704246859" -dependencies = [ - "arrow 58.4.0", - "datafusion-common 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "datafusion-expr 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "datafusion-functions 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "datafusion-physical-expr 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "datafusion-physical-expr-common 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "itertools 0.14.0", -] - -[[package]] -name = "datafusion-physical-expr-adapter" -version = "54.1.0" -source = "git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11#70c26a06716eb552f0fc1d2115957a4089504f11" -dependencies = [ - "arrow 59.1.0", - "datafusion-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", - "datafusion-expr 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", - "datafusion-functions 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", - "datafusion-physical-expr 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", - "datafusion-physical-expr-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", +source = "git+https://github.com/apache/datafusion.git?rev=e08aed1e5de41dcf81d529140dae07723b942a5e#e08aed1e5de41dcf81d529140dae07723b942a5e" +dependencies = [ + "arrow", + "datafusion-common", + "datafusion-expr", + "datafusion-functions", + "datafusion-physical-expr", + "datafusion-physical-expr-common", "itertools 0.15.0", ] [[package]] name = "datafusion-physical-expr-common" version = "54.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72d0d0057fc5a502d45c870cb6d47c66eb7bdd5edb1bd71ad6f3f724975ac2a8" -dependencies = [ - "arrow 58.4.0", - "chrono", - "datafusion-common 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "datafusion-expr-common 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "hashbrown 0.17.1", - "indexmap", - "itertools 0.14.0", - "parking_lot", - "pin-project", -] - -[[package]] -name = "datafusion-physical-expr-common" -version = "54.1.0" -source = "git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11#70c26a06716eb552f0fc1d2115957a4089504f11" +source = "git+https://github.com/apache/datafusion.git?rev=e08aed1e5de41dcf81d529140dae07723b942a5e#e08aed1e5de41dcf81d529140dae07723b942a5e" dependencies = [ - "arrow 59.1.0", + "arrow", "chrono", - "datafusion-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", - "datafusion-expr-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", + "datafusion-common", + "datafusion-expr-common", "datafusion-proto-models", "hashbrown 0.17.1", "indexmap", @@ -2350,37 +1565,18 @@ dependencies = [ [[package]] name = "datafusion-physical-optimizer" version = "54.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86046eed10950c5f9aaed9acfd148e9bd2e1dfdfe4f9aef607d1447b271e4183" -dependencies = [ - "arrow 58.4.0", - "datafusion-common 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "datafusion-execution 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "datafusion-expr 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "datafusion-expr-common 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "datafusion-physical-expr 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "datafusion-physical-expr-common 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "datafusion-physical-plan 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "datafusion-pruning 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "itertools 0.14.0", - "recursive", -] - -[[package]] -name = "datafusion-physical-optimizer" -version = "54.1.0" -source = "git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11#70c26a06716eb552f0fc1d2115957a4089504f11" -dependencies = [ - "arrow 59.1.0", - "datafusion-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", - "datafusion-execution 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", - "datafusion-expr 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", - "datafusion-expr-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", - "datafusion-physical-expr 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", - "datafusion-physical-expr-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", - "datafusion-physical-plan 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", - "datafusion-pruning 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", - "datafusion-session 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", +source = "git+https://github.com/apache/datafusion.git?rev=e08aed1e5de41dcf81d529140dae07723b942a5e#e08aed1e5de41dcf81d529140dae07723b942a5e" +dependencies = [ + "arrow", + "datafusion-common", + "datafusion-execution", + "datafusion-expr", + "datafusion-expr-common", + "datafusion-physical-expr", + "datafusion-physical-expr-common", + "datafusion-physical-plan", + "datafusion-pruning", + "datafusion-session", "itertools 0.15.0", "recursive", ] @@ -2388,58 +1584,25 @@ dependencies = [ [[package]] name = "datafusion-physical-plan" version = "54.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9bc84da934c903407ba297971ebcc020c4c1a38aafd765d6c144c76eff3fa6a1" -dependencies = [ - "arrow 58.4.0", - "arrow-data 58.4.0", - "arrow-ipc 58.4.0", - "arrow-ord 58.4.0", - "arrow-schema 58.4.0", - "async-trait", - "datafusion-common 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "datafusion-common-runtime 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "datafusion-execution 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "datafusion-expr 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "datafusion-functions 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "datafusion-functions-aggregate-common 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "datafusion-functions-window-common 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "datafusion-physical-expr 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "datafusion-physical-expr-common 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "futures", - "half", - "hashbrown 0.17.1", - "indexmap", - "itertools 0.14.0", - "log", - "num-traits", - "parking_lot", - "pin-project-lite", - "tokio", -] - -[[package]] -name = "datafusion-physical-plan" -version = "54.1.0" -source = "git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11#70c26a06716eb552f0fc1d2115957a4089504f11" +source = "git+https://github.com/apache/datafusion.git?rev=e08aed1e5de41dcf81d529140dae07723b942a5e#e08aed1e5de41dcf81d529140dae07723b942a5e" dependencies = [ - "arrow 59.1.0", - "arrow-data 59.1.0", - "arrow-ipc 59.1.0", - "arrow-ord 59.1.0", - "arrow-schema 59.1.0", + "arrow", + "arrow-data", + "arrow-ipc", + "arrow-ord", + "arrow-schema", "async-trait", "bytes", - "datafusion-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", - "datafusion-common-runtime 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", - "datafusion-execution 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", - "datafusion-expr 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", - "datafusion-functions 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", - "datafusion-functions-aggregate-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", - "datafusion-functions-window-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", - "datafusion-physical-expr 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", - "datafusion-physical-expr-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", - "datafusion-proto-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", + "datafusion-common", + "datafusion-common-runtime", + "datafusion-execution", + "datafusion-expr", + "datafusion-functions", + "datafusion-functions-aggregate-common", + "datafusion-functions-window-common", + "datafusion-physical-expr", + "datafusion-physical-expr-common", + "datafusion-proto-common", "datafusion-proto-models", "futures", "half", @@ -2457,52 +1620,25 @@ dependencies = [ [[package]] name = "datafusion-proto" version = "54.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67791dcfacd142a9d95f8f73c2a161094cc936d231d80c235f5017dacf24e84e" -dependencies = [ - "arrow 58.4.0", - "chrono", - "datafusion-catalog 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "datafusion-catalog-listing 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "datafusion-common 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "datafusion-datasource 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "datafusion-datasource-arrow 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "datafusion-datasource-csv 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "datafusion-datasource-json 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "datafusion-datasource-parquet 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "datafusion-execution 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "datafusion-expr 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "datafusion-functions-table 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "datafusion-physical-expr 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "datafusion-physical-expr-common 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "datafusion-physical-plan 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "datafusion-proto-common 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "object_store", - "prost", -] - -[[package]] -name = "datafusion-proto" -version = "54.1.0" -source = "git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11#70c26a06716eb552f0fc1d2115957a4089504f11" +source = "git+https://github.com/apache/datafusion.git?rev=e08aed1e5de41dcf81d529140dae07723b942a5e#e08aed1e5de41dcf81d529140dae07723b942a5e" dependencies = [ - "arrow 59.1.0", + "arrow", "chrono", - "datafusion-catalog 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", - "datafusion-catalog-listing 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", - "datafusion-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", - "datafusion-datasource 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", - "datafusion-datasource-arrow 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", - "datafusion-datasource-csv 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", - "datafusion-datasource-json 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", - "datafusion-datasource-parquet 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", - "datafusion-execution 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", - "datafusion-expr 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", - "datafusion-functions-table 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", - "datafusion-physical-expr 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", - "datafusion-physical-expr-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", - "datafusion-physical-plan 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", - "datafusion-proto-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", + "datafusion-catalog", + "datafusion-catalog-listing", + "datafusion-common", + "datafusion-datasource", + "datafusion-datasource-arrow", + "datafusion-datasource-csv", + "datafusion-datasource-json", + "datafusion-datasource-parquet", + "datafusion-execution", + "datafusion-expr", + "datafusion-functions-table", + "datafusion-physical-expr", + "datafusion-physical-expr-common", + "datafusion-physical-plan", + "datafusion-proto-common", "datafusion-proto-models", "object_store", "prost", @@ -2511,79 +1647,52 @@ dependencies = [ [[package]] name = "datafusion-proto-common" version = "54.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8cd9e80d637891645d074db0f6c650b591117367247deb313fdfb78dff559cb" -dependencies = [ - "arrow 58.4.0", - "datafusion-common 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "prost", -] - -[[package]] -name = "datafusion-proto-common" -version = "54.1.0" -source = "git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11#70c26a06716eb552f0fc1d2115957a4089504f11" +source = "git+https://github.com/apache/datafusion.git?rev=e08aed1e5de41dcf81d529140dae07723b942a5e#e08aed1e5de41dcf81d529140dae07723b942a5e" dependencies = [ - "arrow 59.1.0", - "datafusion-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", + "arrow", + "datafusion-common", "prost", ] [[package]] name = "datafusion-proto-models" version = "54.1.0" -source = "git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11#70c26a06716eb552f0fc1d2115957a4089504f11" +source = "git+https://github.com/apache/datafusion.git?rev=e08aed1e5de41dcf81d529140dae07723b942a5e#e08aed1e5de41dcf81d529140dae07723b942a5e" dependencies = [ - "datafusion-proto-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", + "datafusion-proto-common", "prost", ] [[package]] name = "datafusion-pruning" version = "54.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb63eeac6de19be40f487b65dd84e546195f783c5a9928618e0c4f2a3569b0d7" -dependencies = [ - "arrow 58.4.0", - "datafusion-common 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "datafusion-datasource 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "datafusion-expr-common 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "datafusion-physical-expr 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "datafusion-physical-expr-common 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "datafusion-physical-plan 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "log", -] - -[[package]] -name = "datafusion-pruning" -version = "54.1.0" -source = "git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11#70c26a06716eb552f0fc1d2115957a4089504f11" -dependencies = [ - "arrow 59.1.0", - "datafusion-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", - "datafusion-datasource 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", - "datafusion-expr-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", - "datafusion-physical-expr 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", - "datafusion-physical-expr-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", - "datafusion-physical-plan 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", +source = "git+https://github.com/apache/datafusion.git?rev=e08aed1e5de41dcf81d529140dae07723b942a5e#e08aed1e5de41dcf81d529140dae07723b942a5e" +dependencies = [ + "arrow", + "datafusion-common", + "datafusion-datasource", + "datafusion-expr-common", + "datafusion-physical-expr", + "datafusion-physical-expr-common", + "datafusion-physical-plan", "log", ] [[package]] name = "datafusion-python" version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b5da6aab44da83d488e6fb05855f5cd4480c7ab6b44f164166b495cba9fbbb8" +source = "git+https://github.com/apache/datafusion-python.git?rev=55f43b5e231da5591e7bbd632dac85f304a22cca#55f43b5e231da5591e7bbd632dac85f304a22cca" dependencies = [ - "arrow 58.4.0", - "arrow-select 58.4.0", + "arrow", + "arrow-select", "async-trait", "chrono", "cstr", - "datafusion 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion", "datafusion-ffi", - "datafusion-proto 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-proto", "datafusion-python-util", + "datafusion-session", "datafusion-spark", "futures", "log", @@ -2593,7 +1702,7 @@ dependencies = [ "prost-types", "pyo3", "pyo3-async-runtimes", - "pyo3-build-config 0.28.3", + "pyo3-build-config", "pyo3-log", "serde_json", "tokio", @@ -2604,13 +1713,12 @@ dependencies = [ [[package]] name = "datafusion-python-util" version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba491c15adfebc9ae53aa11f65231814b5199627e9815e2fb62d2f81f4a17fb2" +source = "git+https://github.com/apache/datafusion-python.git?rev=55f43b5e231da5591e7bbd632dac85f304a22cca#55f43b5e231da5591e7bbd632dac85f304a22cca" dependencies = [ - "arrow 58.4.0", - "datafusion 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "arrow", + "datafusion", "datafusion-ffi", - "datafusion-proto 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-proto", "prost", "pyo3", "tokio", @@ -2619,49 +1727,35 @@ dependencies = [ [[package]] name = "datafusion-session" version = "54.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f961d209177f91bd014db5cbb2c33b7d28a2597b9003e77f17aeb712964315a" -dependencies = [ - "async-trait", - "datafusion-common 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "datafusion-execution 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "datafusion-expr 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "datafusion-physical-plan 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "parking_lot", -] - -[[package]] -name = "datafusion-session" -version = "54.1.0" -source = "git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11#70c26a06716eb552f0fc1d2115957a4089504f11" +source = "git+https://github.com/apache/datafusion.git?rev=e08aed1e5de41dcf81d529140dae07723b942a5e#e08aed1e5de41dcf81d529140dae07723b942a5e" dependencies = [ - "arrow-schema 59.1.0", + "arrow-schema", "async-trait", - "datafusion-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", - "datafusion-execution 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", - "datafusion-expr 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", - "datafusion-physical-plan 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", + "datafusion-common", + "datafusion-execution", + "datafusion-expr", + "datafusion-physical-plan", "parking_lot", ] [[package]] name = "datafusion-spark" version = "54.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1ccd16a6949503e56c084df1b90c8889db826ec9347d2f0f51a7837d6fa011e" +source = "git+https://github.com/apache/datafusion.git?rev=e08aed1e5de41dcf81d529140dae07723b942a5e#e08aed1e5de41dcf81d529140dae07723b942a5e" dependencies = [ - "arrow 58.4.0", + "arrow", "bigdecimal", "chrono", "crc32fast", - "datafusion-catalog 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "datafusion-common 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "datafusion-execution 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "datafusion-expr 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "datafusion-functions 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "datafusion-functions-aggregate 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "datafusion-functions-aggregate-common 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "datafusion-functions-nested 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion", + "datafusion-catalog", + "datafusion-common", + "datafusion-execution", + "datafusion-expr", + "datafusion-functions", + "datafusion-functions-aggregate", + "datafusion-functions-aggregate-common", + "datafusion-functions-nested", "log", "num-traits", "percent-encoding", @@ -2676,33 +1770,14 @@ dependencies = [ [[package]] name = "datafusion-sql" version = "54.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1d71cb454da682b2af7488e1fc1ddd72ee1b28f19297b8ccad73f0a21ee9a69" -dependencies = [ - "arrow 58.4.0", - "bigdecimal", - "chrono", - "datafusion-common 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "datafusion-expr 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "datafusion-functions-nested 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "indexmap", - "log", - "recursive", - "regex", - "sqlparser", -] - -[[package]] -name = "datafusion-sql" -version = "54.1.0" -source = "git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11#70c26a06716eb552f0fc1d2115957a4089504f11" +source = "git+https://github.com/apache/datafusion.git?rev=e08aed1e5de41dcf81d529140dae07723b942a5e#e08aed1e5de41dcf81d529140dae07723b942a5e" dependencies = [ - "arrow 59.1.0", + "arrow", "bigdecimal", "chrono", - "datafusion-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", - "datafusion-expr 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", - "datafusion-functions-nested 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)", + "datafusion-common", + "datafusion-expr", + "datafusion-functions-nested", "indexmap", "log", "recursive", @@ -3333,12 +2408,6 @@ dependencies = [ "tempfile", ] -[[package]] -name = "integer-encoding" -version = "3.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8bb03732005da905c88227371639bf1ad885cc712789c011c31c5fb3ab3ccf02" - [[package]] name = "ipnet" version = "2.12.0" @@ -3539,9 +2608,9 @@ checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" [[package]] name = "lz4_flex" -version = "0.13.1" +version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ef0d4ed8669f8f8826eb00dc878084aa8f253506c4fd5e8f58f5bce72ddb97e" +checksum = "ecbdfe44b1bd960b68170b417450a628c43f7cf56bb3c5317e61cb230ee7f226" dependencies = [ "twox-hash", ] @@ -3640,6 +2709,16 @@ dependencies = [ "num-traits", ] +[[package]] +name = "num-bigint" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93e7820bc0a80a0238e650327316f929ba18d5be054b647490a3a6a339f3e7c0" +dependencies = [ + "num-integer", + "num-traits", +] + [[package]] name = "num-complex" version = "0.4.6" @@ -3748,15 +2827,6 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe" -[[package]] -name = "ordered-float" -version = "2.10.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68f19d67e5a2795c94e73e0bb1cc1a7edeb2e28efd39e2e1c9b7a40c1108b11c" -dependencies = [ - "num-traits", -] - [[package]] name = "parking_lot" version = "0.12.5" @@ -3782,54 +2852,18 @@ dependencies = [ [[package]] name = "parquet" -version = "58.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d298093b2dec60289dce0684c986d0f7679e9dd15771c2c65406e1aaf604a704" -dependencies = [ - "ahash", - "arrow-array 58.4.0", - "arrow-buffer 58.4.0", - "arrow-data 58.4.0", - "arrow-ipc 58.4.0", - "arrow-schema 58.4.0", - "arrow-select 58.4.0", - "base64 0.22.1", - "brotli", - "bytes", - "chrono", - "flate2", - "futures", - "half", - "hashbrown 0.17.1", - "lz4_flex", - "num-bigint", - "num-integer", - "num-traits", - "object_store", - "paste", - "seq-macro", - "simdutf8", - "snap", - "thrift", - "tokio", - "twox-hash", - "zstd", -] - -[[package]] -name = "parquet" -version = "59.1.0" +version = "59.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5302d4da74d6596a1f11f9928767995b53bca657cbeea1e4e8c5074f8a1157dd" +checksum = "7065842956a20c2a536924ce8e4d9955f7422451511b9eb7500d7bfe5077e59c" dependencies = [ "ahash", - "arrow-array 59.1.0", - "arrow-buffer 59.1.0", - "arrow-data 59.1.0", - "arrow-ipc 59.1.0", - "arrow-schema 59.1.0", - "arrow-select 59.1.0", - "base64 0.22.1", + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-ipc", + "arrow-schema", + "arrow-select", + "base64 0.23.1", "brotli", "bytes", "chrono", @@ -3838,11 +2872,10 @@ dependencies = [ "half", "hashbrown 0.17.1", "lz4_flex", - "num-bigint", + "num-bigint 0.5.1", "num-integer", "num-traits", "object_store", - "paste", "seq-macro", "simdutf8", "snap", @@ -3851,12 +2884,6 @@ dependencies = [ "zstd", ] -[[package]] -name = "paste" -version = "1.0.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" - [[package]] name = "percent-encoding" version = "2.3.2" @@ -4069,36 +3096,38 @@ dependencies = [ "ballista-core", "ballista-executor", "ballista-scheduler", - "datafusion 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "datafusion-proto 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion", + "datafusion-ffi", + "datafusion-proto", "datafusion-python", "log", "pyo3", - "pyo3-build-config 0.29.0", + "pyo3-build-config", "pyo3-log", "tokio", "tonic", + "url", ] [[package]] name = "pyo3" -version = "0.28.3" +version = "0.29.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91fd8e38a3b50ed1167fb981cd6fd60147e091784c427b8f7183a7ee32c31c12" +checksum = "4688ddedf473e32662b9b067670129a8afb8c18e351482c70d62ba4a88171e8b" dependencies = [ "libc", "once_cell", "portable-atomic", - "pyo3-build-config 0.28.3", + "pyo3-build-config", "pyo3-ffi", "pyo3-macros", ] [[package]] name = "pyo3-async-runtimes" -version = "0.28.0" +version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e7364a95bf00e8377bbf9b0f09d7ff9715a29d8fcf93b47d1a967363b973178" +checksum = "b3ef68daa7316a3fac65e5e18b2203f010346de1c1c53456811a2624673ab046" dependencies = [ "futures-channel", "futures-util", @@ -4110,31 +3139,21 @@ dependencies = [ [[package]] name = "pyo3-build-config" -version = "0.28.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e368e7ddfdeb98c9bca7f8383be1648fd84ab466bf2bc015e94008db6d35611e" -dependencies = [ - "python3-dll-a", - "target-lexicon", -] - -[[package]] -name = "pyo3-build-config" -version = "0.29.0" +version = "0.29.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c5e2a7d2f0d013342f295c048ad19237add5154a55b1c5a254c0ec93d4109078" +checksum = "f41027e41b4bd03f6e60f9f417fe24a6341a6bb744edd62b6f709f2a52ea30e9" dependencies = [ "target-lexicon", ] [[package]] name = "pyo3-ffi" -version = "0.28.3" +version = "0.29.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f29e10af80b1f7ccaf7f69eace800a03ecd13e883acfacc1e5d0988605f651e" +checksum = "e591a95526fead067432c3b3a33fc74770b87b1e04e73671090d9c2055a2b327" dependencies = [ "libc", - "pyo3-build-config 0.28.3", + "pyo3-build-config", ] [[package]] @@ -4150,9 +3169,9 @@ dependencies = [ [[package]] name = "pyo3-macros" -version = "0.28.3" +version = "0.29.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df6e520eff47c45997d2fc7dd8214b25dd1310918bbb2642156ef66a67f29813" +checksum = "73225868fc1cd84eef2c3c230ddb91273bf1de46aeb8a4248da76d32a0924a1c" dependencies = [ "proc-macro2", "pyo3-macros-backend", @@ -4162,26 +3181,16 @@ dependencies = [ [[package]] name = "pyo3-macros-backend" -version = "0.28.3" +version = "0.29.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4cdc218d835738f81c2338f822078af45b4afdf8b2e33cbb5916f108b813acb" +checksum = "571575aa3749fa6216757dd47d2a3e7ef360f329a40f0666a9fbd14889024952" dependencies = [ "heck", "proc-macro2", - "pyo3-build-config 0.28.3", "quote", "syn 2.0.119", ] -[[package]] -name = "python3-dll-a" -version = "0.2.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d80ba7540edb18890d444c5aa8e1f1f99b1bdf26fb26ae383135325f4a36042b" -dependencies = [ - "cc", -] - [[package]] name = "quick-xml" version = "0.39.4" @@ -4935,17 +3944,6 @@ dependencies = [ "syn 3.0.3", ] -[[package]] -name = "thrift" -version = "0.17.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e54bc85fc7faa8bc175c4bab5b92ba8d9a3ce893d0e9f42cc455c8ab16a9e09" -dependencies = [ - "byteorder", - "integer-encoding", - "ordered-float", -] - [[package]] name = "tiny-keccak" version = "2.0.2" diff --git a/python/Cargo.toml b/python/Cargo.toml index 0712605831..10527a71b0 100644 --- a/python/Cargo.toml +++ b/python/Cargo.toml @@ -35,14 +35,16 @@ ballista = { path = "../ballista/client", version = "54.0.0" } ballista-core = { path = "../ballista/core", version = "54.0.0" } ballista-executor = { path = "../ballista/executor", version = "54.0.0", default-features = false } ballista-scheduler = { path = "../ballista/scheduler", version = "54.0.0", default-features = false } -datafusion = { version = "54", features = ["avro"] } -datafusion-proto = { version = "54" } -datafusion-python = { version = "54", default-features = false } +datafusion = { git = "https://github.com/apache/datafusion.git", rev = "e08aed1e5de41dcf81d529140dae07723b942a5e", features = ["avro"] } +datafusion-ffi = { git = "https://github.com/apache/datafusion.git", rev = "e08aed1e5de41dcf81d529140dae07723b942a5e" } +datafusion-proto = { git = "https://github.com/apache/datafusion.git", rev = "e08aed1e5de41dcf81d529140dae07723b942a5e" } +datafusion-python = { git = "https://github.com/apache/datafusion-python.git", rev = "55f43b5e231da5591e7bbd632dac85f304a22cca", default-features = false } log = { version = "0.4" } -pyo3 = { version = "0.28", features = ["extension-module", "abi3", "abi3-py310"] } +pyo3 = { version = "0.29", features = ["extension-module", "abi3", "abi3-py310"] } pyo3-log = "0.13" tokio = { version = "1.48", features = ["macros", "rt", "rt-multi-thread", "sync"] } tonic = { version = "0.14" } +url = { version = "2.5" } [lib] crate-type = ["cdylib"] @@ -50,3 +52,18 @@ name = "ballista" [build-dependencies] pyo3-build-config = "0.29" + +# datafusion-python's workspace patch does not apply when it is consumed as a +# dependency. Repeat its exact DataFusion patch here so the standalone Python +# crate uses one ABI-compatible DataFusion source family. +[patch.crates-io] +datafusion = { git = "https://github.com/apache/datafusion.git", rev = "e08aed1e5de41dcf81d529140dae07723b942a5e" } +datafusion-catalog = { git = "https://github.com/apache/datafusion.git", rev = "e08aed1e5de41dcf81d529140dae07723b942a5e" } +datafusion-common = { git = "https://github.com/apache/datafusion.git", rev = "e08aed1e5de41dcf81d529140dae07723b942a5e" } +datafusion-expr = { git = "https://github.com/apache/datafusion.git", rev = "e08aed1e5de41dcf81d529140dae07723b942a5e" } +datafusion-ffi = { git = "https://github.com/apache/datafusion.git", rev = "e08aed1e5de41dcf81d529140dae07723b942a5e" } +datafusion-functions-aggregate = { git = "https://github.com/apache/datafusion.git", rev = "e08aed1e5de41dcf81d529140dae07723b942a5e" } +datafusion-functions-window = { git = "https://github.com/apache/datafusion.git", rev = "e08aed1e5de41dcf81d529140dae07723b942a5e" } +datafusion-proto = { git = "https://github.com/apache/datafusion.git", rev = "e08aed1e5de41dcf81d529140dae07723b942a5e" } +datafusion-session = { git = "https://github.com/apache/datafusion.git", rev = "e08aed1e5de41dcf81d529140dae07723b942a5e" } +datafusion-spark = { git = "https://github.com/apache/datafusion.git", rev = "e08aed1e5de41dcf81d529140dae07723b942a5e" } diff --git a/python/pyproject.toml b/python/pyproject.toml index 081c504b4d..4e85fbb075 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -41,7 +41,8 @@ classifiers = [ ] dependencies = [ "pyarrow>=23.0.1", - "datafusion==54", + # Temporary PR #1672 pin. Use its released version before publishing. + "datafusion @ git+https://github.com/apache/datafusion-python.git@55f43b5e231da5591e7bbd632dac85f304a22cca", "typing-extensions;python_version<'3.13'", ] dynamic = ["version"] diff --git a/python/python/ballista/extension.py b/python/python/ballista/extension.py index 645c99518a..3455da013a 100644 --- a/python/python/ballista/extension.py +++ b/python/python/ballista/extension.py @@ -26,7 +26,9 @@ from typing import Dict, List, Union, Optional, Callable import warnings +from ._internal_ballista import ballista_datafusion_config_defaults from ._internal_ballista import create_ballista_data_frame +from ._internal_ballista import with_ballista_query_planner from ._internal_ballista import ParquetColumnOptions as ParquetColumnOptionsInternal from ._internal_ballista import ParquetWriterOptions as ParquetWriterOptionsInternal @@ -53,9 +55,9 @@ def __new__(cls, name, bases, attrs): def __wrap_dataframe_execution(func): def method_wrapper(*args, **kwargs): slf, *argz = args - df = slf._to_internal_df() - - return getattr(df, func)(*argz, **kwargs) + # The DataFrame already carries the FFI-installed Ballista + # planner, so normal terminal operations execute it directly. + return getattr(DataFrame, func)(slf, *argz, **kwargs) return method_wrapper @@ -66,7 +68,12 @@ def method_wrapper(*args, **kwargs): address = args[0].address session_id = args[0].session_id df = func(*args, **kwargs) - return DistributedDataFrame(df, session_id, address) + return DistributedDataFrame( + df, + session_id, + address, + args[0].cluster_config, + ) return method_wrapper @@ -531,6 +538,9 @@ def __repr__(self) -> str: return f"ExecutionPlanVisualization(analyze={self.analyze})\n{self.plan_str}" +# Keep the compatibility wrapper for write_* methods because DataFusion's +# logical codec FFI does not yet transport the file-format factories used by COPY. +# Normal terminal operations execute through the installed FFI planner. class BallistaSessionContext(SessionContext, metaclass=RedefiningSessionContextMeta): """ A session context for connecting to and querying a Ballista cluster. @@ -571,20 +581,17 @@ def __init__( if cluster_config is not None else None ) - # Apply overrides to the local SessionContext too: some settings - # (e.g. datafusion.execution.listing_table_factory_infer_partitions) - # are consulted during local table registration / planning, before - # the plan is shipped to the scheduler. Ballista-namespaced keys - # are not understood by the local SessionConfig and are forwarded - # to the scheduler only. - if self.cluster_config: - if config is None: - config = SessionConfig() - for key, value in self.cluster_config.items(): - if key.startswith("ballista."): - continue + if config is None: + config = SessionConfig() + for key, value in ballista_datafusion_config_defaults().items(): config = config.set(key, value) - super().__init__(config, runtime) + source_ctx = SessionContext(config, runtime) + configured_ctx = with_ballista_query_planner( + source_ctx, + address, + self.cluster_config, + ) + self.ctx = configured_ctx.ctx self.address = address self.session_id_internal = super().session_id() diff --git a/python/python/tests/test_context.py b/python/python/tests/test_context.py index 6e51803784..e823a5ed89 100644 --- a/python/python/tests/test_context.py +++ b/python/python/tests/test_context.py @@ -16,8 +16,19 @@ # under the License. from ballista import BallistaSessionContext, setup_test_cluster -from ballista.extension import DataFrame, DistributedDataFrame, SessionContext +from ballista._internal_ballista import ( + BallistaQueryPlanner, + ballista_datafusion_config_defaults, + with_ballista_query_planner, +) +from ballista.extension import ( + DataFrame, + DistributedDataFrame, + SessionConfig, + SessionContext, +) from datafusion import col, lit +import ctypes import pytest import pyarrow as pa @@ -28,14 +39,82 @@ def ctx(): return BallistaSessionContext(address=f"df://{address}:{port}") +def assert_uses_ballista(df): + assert "DistributedQueryExec" in str(df.execution_plan()) + + +def test_query_planner_capsules_use_current_datafusion_abi(): + source_ctx = SessionContext() + planner = BallistaQueryPlanner("df://localhost:50050", source_ctx) + is_valid = ctypes.pythonapi.PyCapsule_IsValid + is_valid.argtypes = [ctypes.py_object, ctypes.c_char_p] + is_valid.restype = ctypes.c_int + + capsules = { + b"datafusion_query_planner": planner.__datafusion_query_planner__(), + b"datafusion_logical_extension_codec": ( + planner.__datafusion_logical_extension_codec__() + ), + b"datafusion_physical_extension_codec": ( + planner.__datafusion_physical_extension_codec__() + ), + } + for name, capsule in capsules.items(): + assert is_valid(capsule, name) == 1 + + +def test_low_level_helper_returns_standard_distributed_dataframe(): + address, port = setup_test_cluster() + source_ctx = SessionContext() + source_ctx.register_csv("registered", "testdata/test.csv", has_header=True) + configured_ctx = with_ballista_query_planner( + source_ctx, + f"df://{address}:{port}", + {"datafusion.execution.target_partitions": "7"}, + ) + + assert type(configured_ctx) is SessionContext + assert "datafusion.execution.target_partitions = 7" in str(source_ctx) + assert "datafusion.execution.target_partitions = 7" in str(configured_ctx) + df = configured_ctx.sql("SELECT COUNT(*) FROM registered") + assert type(df) is DataFrame + assert_uses_ballista(df) + assert df.collect()[0].column(0).to_pylist() == [5] + + +def test_ballista_datafusion_defaults_are_applied(ctx): + config_text = str(ctx) + + for key, value in ballista_datafusion_config_defaults().items(): + assert f"{key} = {value}" in config_text + + +def test_caller_session_config_is_not_overwritten(): + key = "datafusion.execution.target_partitions" + config = SessionConfig().set(key, "7") + ctx = BallistaSessionContext("df://localhost:50050", config=config) + + assert f"{key} = 7" in str(ctx) + + override_config = SessionConfig().set(key, "7") + override_ctx = BallistaSessionContext( + "df://localhost:50050", + config=override_config, + cluster_config={key: "8"}, + ) + assert f"{key} = 8" in str(override_ctx) + + def test_select_one(ctx): df = ctx.sql("SELECT 1") + assert_uses_ballista(df) batches = df.collect() assert len(batches) == 1 def test_read_csv(ctx): df = ctx.read_csv("testdata/test.csv", has_header=True) + assert_uses_ballista(df) batches = df.collect() assert len(batches) == 1 assert len(batches[0]) == 5 @@ -44,6 +123,7 @@ def test_read_csv(ctx): def test_register_csv(ctx): ctx.register_csv("test", "testdata/test.csv", has_header=True) df = ctx.sql("SELECT * FROM test") + assert_uses_ballista(df) batches = df.collect() assert len(batches) == 1 assert len(batches[0]) == 5 @@ -51,6 +131,7 @@ def test_register_csv(ctx): def test_read_parquet(ctx): df = ctx.read_parquet("testdata/test.parquet") + assert_uses_ballista(df) batches = df.collect() assert len(batches) == 1 assert len(batches[0]) == 8 @@ -59,6 +140,7 @@ def test_read_parquet(ctx): def test_register_parquet(ctx): ctx.register_parquet("test", "testdata/test.parquet") df = ctx.sql("SELECT * FROM test") + assert_uses_ballista(df) batches = df.collect() assert len(batches) == 1 assert len(batches[0]) == 8 @@ -70,6 +152,7 @@ def test_read_dataframe_api(ctx): .select("a", "b") .filter(col("a") > lit(2)) ) + assert_uses_ballista(df) result = df.collect()[0] assert result.column(0) == pa.array([3, 4, 5]) @@ -92,6 +175,8 @@ def test_cluster_config_propagates_to_distributed_dataframe(): df = ctx.sql("SELECT 1") assert df.cluster_config == overrides + assert_uses_ballista(df) + assert len(df.collect()) == 1 def test_cluster_config_accepts_ballista_namespaced_keys(): @@ -114,9 +199,18 @@ def test_cluster_config_accepts_ballista_namespaced_keys(): df = ctx.sql("SELECT 1") assert df.cluster_config == overrides + assert_uses_ballista(df) assert len(df.collect()) == 1 +def test_malformed_scheduler_url_fails_lazily(): + ctx = BallistaSessionContext(address="not a scheduler URL") + ctx.register_csv("test", "testdata/test.csv", has_header=True) + + with pytest.raises(Exception, match="relative URL without a base"): + ctx.sql("SELECT * FROM test").collect() + + def test_write_csv(ctx, tmp_path): df = ctx.read_csv("testdata/test.csv", has_header=True) out_dir = str(tmp_path / "out") diff --git a/python/src/lib.rs b/python/src/lib.rs index d927311bec..ba58070d30 100644 --- a/python/src/lib.rs +++ b/python/src/lib.rs @@ -28,6 +28,7 @@ use pyo3::prelude::*; use std::collections::HashMap; mod cluster; +mod query_planner; mod utils; pub(crate) struct TokioRuntime(tokio::runtime::Runtime); @@ -38,12 +39,21 @@ fn _internal_ballista(_py: Python, m: Bound<'_, PyModule>) -> PyResult<()> { m.add_class::()?; m.add_class::()?; + m.add_class::()?; m.add_class::()?; m.add_class::()?; m.add_class::()?; m.add_function(wrap_pyfunction!(create_ballista_data_frame, m.clone())?)?; + m.add_function(wrap_pyfunction!( + query_planner::ballista_datafusion_config_defaults, + m.clone() + )?)?; + m.add_function(wrap_pyfunction!( + query_planner::with_ballista_query_planner, + m.clone() + )?)?; m.add_function(wrap_pyfunction!( crate::cluster::setup_test_cluster, m.clone() diff --git a/python/src/query_planner.rs b/python/src/query_planner.rs new file mode 100644 index 0000000000..403ca99ae6 --- /dev/null +++ b/python/src/query_planner.rs @@ -0,0 +1,352 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use ballista_core::extension::SessionConfigExt; +use ballista_core::planner::BallistaQueryPlanner; +use ballista_core::serde::{ + BallistaLogicalExtensionCodec, BallistaPhysicalExtensionCodec, +}; +use datafusion::catalog::Session; +use datafusion::common::{DataFusionError, Result}; +use datafusion::execution::config::SessionConfig; +use datafusion::execution::context::QueryPlanner; +use datafusion::logical_expr::LogicalPlan; +use datafusion::physical_plan::ExecutionPlan; +use datafusion_ffi::execution::FFI_TaskContextProvider; +use datafusion_ffi::proto::logical_extension_codec::FFI_LogicalExtensionCodec; +use datafusion_ffi::proto::physical_extension_codec::FFI_PhysicalExtensionCodec; +use datafusion_ffi::query_planner::FFI_QueryPlanner; +use datafusion_proto::logical_plan::{ + DefaultLogicalExtensionCodec, LogicalExtensionCodec, +}; +use datafusion_proto::physical_plan::PhysicalExtensionCodec; +use datafusion_proto::protobuf::LogicalPlanNode; +use pyo3::exceptions::PyValueError; +use pyo3::prelude::*; +use pyo3::types::PyCapsule; +use std::collections::HashMap; +use std::ptr::NonNull; +use std::sync::{Arc, OnceLock}; +use tokio::runtime::{Handle, Runtime}; +use url::Url; + +const DEFAULT_SCHEDULER_PORT: u16 = 50050; + +#[pyclass( + name = "BallistaQueryPlanner", + module = "ballista._internal_ballista", + skip_from_py_object +)] +pub(crate) struct PyBallistaQueryPlanner { + address: String, + config: ballista_core::config::BallistaConfig, + logical_codec: Arc, + ffi_logical_codec: FFI_LogicalExtensionCodec, + ffi_physical_codec: FFI_PhysicalExtensionCodec, + // The FFI codecs hold a weak task-context provider. Retain its source + // context for as long as Python retains this planner. + _session_ctx: Py, +} + +#[pymethods] +impl PyBallistaQueryPlanner { + #[new] + #[pyo3(signature = (address, session_ctx, config_overrides=None))] + fn new( + address: String, + session_ctx: Bound<'_, PyAny>, + config_overrides: Option>, + ) -> PyResult { + let user_logical_codec: Arc = + (&ffi_logical_codec_from_python(session_ctx.clone())?).into(); + // The planner-side codec must retain the source context's Python-aware + // codecs because DistributedQueryExec embeds a serialized logical plan. + let logical_codec: Arc = + Arc::new(BallistaLogicalExtensionCodec::new(user_logical_codec)); + let physical_codec: Arc = Arc::new( + BallistaPhysicalExtensionCodec::new(Arc::clone(&logical_codec)), + ); + + // The receiving SessionContext composes an installed codec with its own + // codecs. Export only the Ballista layer here; wrapping the source codec + // again would make the receiver traverse the same codec stack twice. + let installed_logical_codec: Arc = Arc::new( + BallistaLogicalExtensionCodec::new(Arc::new( + DefaultLogicalExtensionCodec {}, + )), + ); + let task_ctx_provider = ffi_task_ctx_provider_from_python(session_ctx.clone())?; + let runtime = tokio_runtime_handle(); + let ffi_logical_codec = FFI_LogicalExtensionCodec::new( + installed_logical_codec, + Some(runtime.clone()), + task_ctx_provider.clone(), + ); + let ffi_physical_codec = FFI_PhysicalExtensionCodec::new( + physical_codec, + Some(runtime), + task_ctx_provider.clone(), + ); + + let mut session_config = SessionConfig::new_with_ballista(); + if let Some(overrides) = config_overrides { + for (key, value) in overrides { + if !key.starts_with("ballista.") { + return Err(PyValueError::new_err(format!( + "unsupported Ballista configuration key: {key}" + ))); + } + session_config + .options_mut() + .set(&key, &value) + .map_err(|error| PyValueError::new_err(error.to_string()))?; + } + } + + Ok(Self { + address, + config: session_config.ballista_config(), + logical_codec, + ffi_logical_codec, + ffi_physical_codec, + _session_ctx: session_ctx.unbind(), + }) + } + + fn __datafusion_logical_extension_codec__<'py>( + &self, + py: Python<'py>, + ) -> PyResult> { + PyCapsule::new_with_value( + py, + self.ffi_logical_codec.clone(), + cr"datafusion_logical_extension_codec", + ) + } + + fn __datafusion_physical_extension_codec__<'py>( + &self, + py: Python<'py>, + ) -> PyResult> { + PyCapsule::new_with_value( + py, + self.ffi_physical_codec.clone(), + cr"datafusion_physical_extension_codec", + ) + } + + fn __datafusion_query_planner__<'py>( + &self, + py: Python<'py>, + ) -> PyResult> { + let planner: Arc = + Arc::new(LazyBallistaQueryPlanner { + address: self.address.clone(), + config: self.config.clone(), + logical_codec: Arc::clone(&self.logical_codec), + _session_ctx: self._session_ctx.clone_ref(py), + }); + let ffi = FFI_QueryPlanner::new_with_ffi_codecs( + planner, + self.ffi_logical_codec.clone(), + self.ffi_physical_codec.clone(), + ); + + // This exact name is the ABI consumed by datafusion-python 55f43b5e. + PyCapsule::new_with_value(py, ffi, cr"datafusion_query_planner") + } +} + +struct LazyBallistaQueryPlanner { + address: String, + config: ballista_core::config::BallistaConfig, + logical_codec: Arc, + _session_ctx: Py, +} + +impl std::fmt::Debug for LazyBallistaQueryPlanner { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("LazyBallistaQueryPlanner") + .field("address", &self.address) + .field("config", &self.config) + .finish_non_exhaustive() + } +} + +#[async_trait::async_trait] +impl QueryPlanner for LazyBallistaQueryPlanner { + async fn create_physical_plan( + &self, + logical_plan: &LogicalPlan, + session: &dyn Session, + ) -> Result> { + if matches!(logical_plan, LogicalPlan::Analyze(_)) { + return Err(DataFusionError::NotImplemented( + "EXPLAIN ANALYZE is not yet supported by the Ballista Python FFI planner" + .to_string(), + )); + } + + // Parse only when DataFusion asks for a physical plan so malformed + // addresses fail lazily, not during context construction. + let scheduler_url = parse_scheduler_url(&self.address)?; + let planner = BallistaQueryPlanner::::with_extension( + scheduler_url, + self.config.clone(), + Arc::clone(&self.logical_codec), + ); + planner.create_physical_plan(logical_plan, session).await + } +} + +#[pyfunction] +pub(crate) fn ballista_datafusion_config_defaults() -> HashMap { + let defaults = SessionConfig::new(); + let ballista = SessionConfig::new_with_ballista(); + let default_values = defaults + .options() + .entries() + .into_iter() + .map(|entry| (entry.key, entry.value)) + .collect::>(); + + ballista + .options() + .entries() + .into_iter() + .filter_map(|entry| { + if !entry.key.starts_with("datafusion.") { + return None; + } + let value = entry.value?; + (default_values.get(&entry.key) != Some(&Some(value.clone()))) + .then_some((entry.key, value)) + }) + .collect() +} + +#[pyfunction] +#[pyo3(signature = (session_ctx, address, config_overrides=None))] +pub(crate) fn with_ballista_query_planner( + py: Python<'_>, + session_ctx: Bound<'_, PyAny>, + address: String, + config_overrides: Option>, +) -> PyResult> { + let mut ballista_overrides = HashMap::new(); + if let Some(overrides) = config_overrides { + for (key, value) in overrides { + if key.starts_with("datafusion.") { + apply_datafusion_override(&session_ctx, &key, &value)?; + } else if key.starts_with("ballista.") { + ballista_overrides.insert(key, value); + } else { + return Err(PyValueError::new_err(format!( + "configuration key must start with 'datafusion.' or 'ballista.': {key}" + ))); + } + } + } + + let planner = PyBallistaQueryPlanner::new( + address, + session_ctx.clone(), + Some(ballista_overrides), + )?; + let planner = Py::new(py, planner)?; + + // datafusion-python deliberately rebinds imported planners to the codecs + // installed on the receiving context. Install Ballista's codecs first so + // that rebind keeps DistributedQueryExec serialization active. + let ctx = session_ctx.call_method1( + "with_logical_extension_codec", + (planner.bind(py),), + )?; + let ctx = ctx.call_method1( + "with_physical_extension_codec", + (planner.bind(py),), + )?; + Ok(ctx + .call_method1("with_query_planner", (planner.bind(py),))? + .unbind()) +} + +fn apply_datafusion_override( + session_ctx: &Bound<'_, PyAny>, + key: &str, + value: &str, +) -> PyResult<()> { + if !key + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'.' | b'_')) + { + return Err(PyValueError::new_err(format!( + "invalid DataFusion configuration key: {key}" + ))); + } + + // SET mutates the shared session state. The contexts derived below retain + // that state, including registered tables and functions. + let escaped_value = value.replace('\'', "''"); + let statement = format!("SET {key} = '{escaped_value}'"); + session_ctx.call_method1("sql", (statement,))?; + Ok(()) +} + +fn parse_scheduler_url(address: &str) -> Result { + let url = + Url::parse(address).map_err(|e| DataFusionError::Configuration(e.to_string()))?; + let host = url.host().ok_or_else(|| { + DataFusionError::Configuration("hostname should be provided".to_string()) + })?; + let port = url.port().unwrap_or(DEFAULT_SCHEDULER_PORT); + Ok(format!("http://{host}:{port}")) +} + +fn tokio_runtime_handle() -> Handle { + static RUNTIME: OnceLock = OnceLock::new(); + RUNTIME + .get_or_init(|| Runtime::new().expect("tokio runtime for Ballista Python FFI")) + .handle() + .clone() +} + +fn ffi_logical_codec_from_python( + obj: Bound<'_, PyAny>, +) -> PyResult { + let capsule = obj + .getattr("__datafusion_logical_extension_codec__")? + .call0()?; + let capsule = capsule.cast::()?; + let data: NonNull = capsule + .pointer_checked(Some(c"datafusion_logical_extension_codec"))? + .cast(); + Ok(unsafe { data.as_ref().clone() }) +} + +fn ffi_task_ctx_provider_from_python( + obj: Bound<'_, PyAny>, +) -> PyResult { + let capsule = obj + .getattr("__datafusion_task_context_provider__")? + .call0()?; + let capsule = capsule.cast::()?; + let data: NonNull = capsule + .pointer_checked(Some(c"datafusion_task_context_provider"))? + .cast(); + Ok(unsafe { data.as_ref().clone() }) +} diff --git a/python/uv.lock b/python/uv.lock index 81ee8f45bb..f467ad3174 100644 --- a/python/uv.lock +++ b/python/uv.lock @@ -52,7 +52,7 @@ dev = [ [package.metadata] requires-dist = [ - { name = "datafusion", specifier = "==54" }, + { name = "datafusion", git = "https://github.com/apache/datafusion-python.git?rev=55f43b5e231da5591e7bbd632dac85f304a22cca" }, { name = "ipython", marker = "extra == 'jupyter'", specifier = ">=8.0.0" }, { name = "pyarrow", specifier = ">=23.0.1" }, { name = "typing-extensions", marker = "python_full_version < '3.13'" }, @@ -328,25 +328,12 @@ wheels = [ [[package]] name = "datafusion" version = "54.0.0" -source = { registry = "https://pypi.org/simple" } +source = { git = "https://github.com/apache/datafusion-python.git?rev=55f43b5e231da5591e7bbd632dac85f304a22cca#55f43b5e231da5591e7bbd632dac85f304a22cca" } dependencies = [ { name = "cloudpickle" }, { name = "pyarrow" }, { name = "typing-extensions", marker = "python_full_version < '3.13'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/60/90/886f7e9cf827f07ebd60bd293e54e0a028a50dd49bbaef0ee42aae1981ea/datafusion-54.0.0.tar.gz", hash = "sha256:cfe7e8dfc026efc05824f49b53ad6a72caf5c2d6820759b6212a09e245a427ed", size = 276448, upload-time = "2026-06-29T11:19:34.816Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/46/58/4c5b981e3d9ade32a906c15a4941eef50c9b862781cdc14bf4dff48d026a/datafusion-54.0.0-cp310-abi3-macosx_10_12_x86_64.whl", hash = "sha256:946f55e48b8d523d7b4ac106bdf588b4493c2c66f81877d6952aafeaf7c3ec73", size = 39810553, upload-time = "2026-06-29T11:19:02.1Z" }, - { url = "https://files.pythonhosted.org/packages/66/e5/5e4dbd42ce9a2affb3be90d9ab17cebde1a6f28b0d9fb4b83d612d5c8e42/datafusion-54.0.0-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:2a3bf43185c7e43e25242e5fb17b6a11b86bf976434c0bc493fdedbd9a080969", size = 37145255, upload-time = "2026-06-29T11:19:05.491Z" }, - { url = "https://files.pythonhosted.org/packages/c6/5e/dbb9e6e3e5006d34f295d7ac73f1302c8f2df140666402a06e6c55028edb/datafusion-54.0.0-cp310-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:9432bf162381e9282cbc74915b8b773895de18be836f7e3f6d0de4d981f24630", size = 38853856, upload-time = "2026-06-29T11:19:08.732Z" }, - { url = "https://files.pythonhosted.org/packages/a8/81/e69008e3479f4d0134875bc4ae39503bedcd55ca2597e71392c963c651b4/datafusion-54.0.0-cp310-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:3bcd4d213fa74710e75e6e182cc468c2bdbc5ffc74a08c8155d414fbbfa1b3f6", size = 41050149, upload-time = "2026-06-29T11:19:12.108Z" }, - { url = "https://files.pythonhosted.org/packages/61/d4/8ba6e3fe3291c9ccc94b5ca3ec3c1fbcbfbe5ece5ffb965e4550844e2c56/datafusion-54.0.0-cp310-abi3-win_amd64.whl", hash = "sha256:b934e097e1bdca7d5768a81ac1bc4a1812cb459269f8b1a5d892a5d930f18376", size = 43444869, upload-time = "2026-06-29T11:19:15.963Z" }, - { url = "https://files.pythonhosted.org/packages/9d/41/5608323226f21a0fa180823c531dbc0ed270e9b694f299b7647505cb6a06/datafusion-54.0.0-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:c4e79048da82ad89b768bd0be7df39254cd2a0afe2b719d1f129e8a7229af683", size = 39796248, upload-time = "2026-06-29T11:19:19.208Z" }, - { url = "https://files.pythonhosted.org/packages/18/81/392ee323104ab14ca689384723b69e137064a828233c165574f97a74c0e9/datafusion-54.0.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:fe57038003b18e28b90752c1e32b44af74ec4f552a1904aee725e1129a00c447", size = 37153577, upload-time = "2026-06-29T11:19:22.397Z" }, - { url = "https://files.pythonhosted.org/packages/40/c4/ebd5ef5349ecbea7f5f9da76c213581c13e7bbe1b5735c9925b279eeb4eb/datafusion-54.0.0-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:574f642832a106456cfc4f32aa82484c504fc32f4be2b510202bcb579de8e6d1", size = 38849839, upload-time = "2026-06-29T11:19:25.783Z" }, - { url = "https://files.pythonhosted.org/packages/5a/b9/2383d30d317bb913cab97dbf2e6e1d5f37f594860d5c5bc176e025cf7d4a/datafusion-54.0.0-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:796fd5683927443c5bc61999d00b9007ef9b5ce107725ea8d241df718860985d", size = 41074623, upload-time = "2026-06-29T11:19:29.119Z" }, - { url = "https://files.pythonhosted.org/packages/35/5c/553fd1107dede0a56727fda7216a7198d41394f2d19697f4fb104cc695ea/datafusion-54.0.0-cp314-cp314t-win_amd64.whl", hash = "sha256:64973c63874ec31670dd97b32b18af7b07fad679cb20d58ed154038e3a5c204e", size = 43438801, upload-time = "2026-06-29T11:19:32.799Z" }, -] [[package]] name = "decorator" From 9e813516a9cec047a7497e73e902e5ec6ef0b957 Mon Sep 17 00:00:00 2001 From: Tim Saucer Date: Fri, 7 Aug 2026 13:27:44 -0400 Subject: [PATCH 09/11] feat(python): expose Ballista extension codecs AI Disclosure: This code was written in part by an AI agent. --- python/python/ballista/__init__.py | 8 +- python/python/tests/test_context.py | 45 ++++- python/src/lib.rs | 2 + python/src/query_planner.rs | 251 ++++++++++++++++++++-------- 4 files changed, 223 insertions(+), 83 deletions(-) diff --git a/python/python/ballista/__init__.py b/python/python/ballista/__init__.py index 5a5ef7bb02..e892ff8ea5 100644 --- a/python/python/ballista/__init__.py +++ b/python/python/ballista/__init__.py @@ -21,8 +21,11 @@ import importlib_metadata from ._internal_ballista import ( - BallistaScheduler, BallistaExecutor, + BallistaLogicalExtensionCodec, + BallistaPhysicalExtensionCodec, + BallistaQueryPlanner, + BallistaScheduler, setup_test_cluster, ) from .extension import ( @@ -37,6 +40,9 @@ "setup_test_cluster", "BallistaScheduler", "BallistaExecutor", + "BallistaLogicalExtensionCodec", + "BallistaPhysicalExtensionCodec", + "BallistaQueryPlanner", "BallistaSessionContext", "DistributedDataFrame", "ExecutionPlanVisualization", diff --git a/python/python/tests/test_context.py b/python/python/tests/test_context.py index e823a5ed89..7fc0168062 100644 --- a/python/python/tests/test_context.py +++ b/python/python/tests/test_context.py @@ -15,9 +15,14 @@ # specific language governing permissions and limitations # under the License. -from ballista import BallistaSessionContext, setup_test_cluster -from ballista._internal_ballista import ( +from ballista import ( + BallistaLogicalExtensionCodec, + BallistaPhysicalExtensionCodec, BallistaQueryPlanner, + BallistaSessionContext, + setup_test_cluster, +) +from ballista._internal_ballista import ( ballista_datafusion_config_defaults, with_ballista_query_planner, ) @@ -25,9 +30,8 @@ DataFrame, DistributedDataFrame, SessionConfig, - SessionContext, ) -from datafusion import col, lit +from datafusion import SessionContext, col, lit import ctypes import pytest import pyarrow as pa @@ -43,25 +47,50 @@ def assert_uses_ballista(df): assert "DistributedQueryExec" in str(df.execution_plan()) -def test_query_planner_capsules_use_current_datafusion_abi(): +def test_ballista_ffi_capsules_use_current_datafusion_abi(): source_ctx = SessionContext() + logical_codec = BallistaLogicalExtensionCodec(source_ctx) + physical_codec = BallistaPhysicalExtensionCodec(source_ctx, logical_codec) planner = BallistaQueryPlanner("df://localhost:50050", source_ctx) is_valid = ctypes.pythonapi.PyCapsule_IsValid is_valid.argtypes = [ctypes.py_object, ctypes.c_char_p] is_valid.restype = ctypes.c_int capsules = { - b"datafusion_query_planner": planner.__datafusion_query_planner__(), b"datafusion_logical_extension_codec": ( - planner.__datafusion_logical_extension_codec__() + logical_codec.__datafusion_logical_extension_codec__() ), b"datafusion_physical_extension_codec": ( - planner.__datafusion_physical_extension_codec__() + physical_codec.__datafusion_physical_extension_codec__() ), + b"datafusion_query_planner": planner.__datafusion_query_planner__(), } for name, capsule in capsules.items(): assert is_valid(capsule, name) == 1 + assert not hasattr(logical_codec, "__datafusion_physical_extension_codec__") + assert not hasattr(physical_codec, "__datafusion_logical_extension_codec__") + assert not hasattr(planner, "__datafusion_logical_extension_codec__") + assert not hasattr(planner, "__datafusion_physical_extension_codec__") + + +def test_manual_query_planner_and_codec_composition(): + address, port = setup_test_cluster() + source_ctx = SessionContext() + source_ctx.register_csv("registered", "testdata/test.csv", has_header=True) + logical_codec = BallistaLogicalExtensionCodec(source_ctx) + physical_codec = BallistaPhysicalExtensionCodec(source_ctx, logical_codec) + planner = BallistaQueryPlanner(f"df://{address}:{port}", source_ctx) + + configured_ctx = source_ctx.with_logical_extension_codec(logical_codec) + configured_ctx = configured_ctx.with_physical_extension_codec(physical_codec) + configured_ctx = configured_ctx.with_query_planner(planner) + + df = configured_ctx.sql("SELECT COUNT(*) FROM registered") + assert type(df) is DataFrame + assert_uses_ballista(df) + assert df.collect()[0].column(0).to_pylist() == [5] + def test_low_level_helper_returns_standard_distributed_dataframe(): address, port = setup_test_cluster() diff --git a/python/src/lib.rs b/python/src/lib.rs index ba58070d30..7e44e2f204 100644 --- a/python/src/lib.rs +++ b/python/src/lib.rs @@ -39,6 +39,8 @@ fn _internal_ballista(_py: Python, m: Bound<'_, PyModule>) -> PyResult<()> { m.add_class::()?; m.add_class::()?; + m.add_class::()?; + m.add_class::()?; m.add_class::()?; m.add_class::()?; diff --git a/python/src/query_planner.rs b/python/src/query_planner.rs index 403ca99ae6..c934bb9ce0 100644 --- a/python/src/query_planner.rs +++ b/python/src/query_planner.rs @@ -18,7 +18,8 @@ use ballista_core::extension::SessionConfigExt; use ballista_core::planner::BallistaQueryPlanner; use ballista_core::serde::{ - BallistaLogicalExtensionCodec, BallistaPhysicalExtensionCodec, + BallistaLogicalExtensionCodec as NativeBallistaLogicalExtensionCodec, + BallistaPhysicalExtensionCodec as NativeBallistaPhysicalExtensionCodec, }; use datafusion::catalog::Session; use datafusion::common::{DataFusionError, Result}; @@ -30,9 +31,7 @@ use datafusion_ffi::execution::FFI_TaskContextProvider; use datafusion_ffi::proto::logical_extension_codec::FFI_LogicalExtensionCodec; use datafusion_ffi::proto::physical_extension_codec::FFI_PhysicalExtensionCodec; use datafusion_ffi::query_planner::FFI_QueryPlanner; -use datafusion_proto::logical_plan::{ - DefaultLogicalExtensionCodec, LogicalExtensionCodec, -}; +use datafusion_proto::logical_plan::LogicalExtensionCodec; use datafusion_proto::physical_plan::PhysicalExtensionCodec; use datafusion_proto::protobuf::LogicalPlanNode; use pyo3::exceptions::PyValueError; @@ -47,82 +46,35 @@ use url::Url; const DEFAULT_SCHEDULER_PORT: u16 = 50050; #[pyclass( - name = "BallistaQueryPlanner", + name = "BallistaLogicalExtensionCodec", module = "ballista._internal_ballista", skip_from_py_object )] -pub(crate) struct PyBallistaQueryPlanner { - address: String, - config: ballista_core::config::BallistaConfig, +pub(crate) struct PyBallistaLogicalExtensionCodec { logical_codec: Arc, ffi_logical_codec: FFI_LogicalExtensionCodec, - ffi_physical_codec: FFI_PhysicalExtensionCodec, - // The FFI codecs hold a weak task-context provider. Retain its source - // context for as long as Python retains this planner. + // The FFI codec holds a weak task-context provider. _session_ctx: Py, } #[pymethods] -impl PyBallistaQueryPlanner { +impl PyBallistaLogicalExtensionCodec { #[new] - #[pyo3(signature = (address, session_ctx, config_overrides=None))] - fn new( - address: String, - session_ctx: Bound<'_, PyAny>, - config_overrides: Option>, - ) -> PyResult { + fn new(session_ctx: Bound<'_, PyAny>) -> PyResult { let user_logical_codec: Arc = (&ffi_logical_codec_from_python(session_ctx.clone())?).into(); - // The planner-side codec must retain the source context's Python-aware - // codecs because DistributedQueryExec embeds a serialized logical plan. let logical_codec: Arc = - Arc::new(BallistaLogicalExtensionCodec::new(user_logical_codec)); - let physical_codec: Arc = Arc::new( - BallistaPhysicalExtensionCodec::new(Arc::clone(&logical_codec)), - ); - - // The receiving SessionContext composes an installed codec with its own - // codecs. Export only the Ballista layer here; wrapping the source codec - // again would make the receiver traverse the same codec stack twice. - let installed_logical_codec: Arc = Arc::new( - BallistaLogicalExtensionCodec::new(Arc::new( - DefaultLogicalExtensionCodec {}, - )), - ); + Arc::new(NativeBallistaLogicalExtensionCodec::new(user_logical_codec)); let task_ctx_provider = ffi_task_ctx_provider_from_python(session_ctx.clone())?; - let runtime = tokio_runtime_handle(); let ffi_logical_codec = FFI_LogicalExtensionCodec::new( - installed_logical_codec, - Some(runtime.clone()), - task_ctx_provider.clone(), - ); - let ffi_physical_codec = FFI_PhysicalExtensionCodec::new( - physical_codec, - Some(runtime), - task_ctx_provider.clone(), + Arc::clone(&logical_codec), + Some(tokio_runtime_handle()), + task_ctx_provider, ); - let mut session_config = SessionConfig::new_with_ballista(); - if let Some(overrides) = config_overrides { - for (key, value) in overrides { - if !key.starts_with("ballista.") { - return Err(PyValueError::new_err(format!( - "unsupported Ballista configuration key: {key}" - ))); - } - session_config - .options_mut() - .set(&key, &value) - .map_err(|error| PyValueError::new_err(error.to_string()))?; - } - } - Ok(Self { - address, - config: session_config.ballista_config(), logical_codec, ffi_logical_codec, - ffi_physical_codec, _session_ctx: session_ctx.unbind(), }) } @@ -137,6 +89,56 @@ impl PyBallistaQueryPlanner { cr"datafusion_logical_extension_codec", ) } +} + +#[pyclass( + name = "BallistaPhysicalExtensionCodec", + module = "ballista._internal_ballista", + skip_from_py_object +)] +pub(crate) struct PyBallistaPhysicalExtensionCodec { + ffi_physical_codec: FFI_PhysicalExtensionCodec, + // The FFI codec holds a weak task-context provider. + _session_ctx: Py, + // Retain the Python logical codec as well as its native Arc. + _logical_codec: Py, +} + +impl PyBallistaPhysicalExtensionCodec { + fn from_logical_codec( + session_ctx: Bound<'_, PyAny>, + logical_codec: Arc, + logical_codec_object: Py, + ) -> PyResult { + let ffi_physical_codec = + ballista_physical_ffi_codec(session_ctx.clone(), logical_codec)?; + + Ok(Self { + ffi_physical_codec, + _session_ctx: session_ctx.unbind(), + _logical_codec: logical_codec_object, + }) + } +} + +#[pymethods] +impl PyBallistaPhysicalExtensionCodec { + #[new] + fn new( + session_ctx: Bound<'_, PyAny>, + logical_codec: Bound<'_, PyAny>, + ) -> PyResult { + let native_logical_codec = { + let logical_codec = + logical_codec.extract::>()?; + Arc::clone(&logical_codec.logical_codec) + }; + Self::from_logical_codec( + session_ctx, + native_logical_codec, + logical_codec.unbind(), + ) + } fn __datafusion_physical_extension_codec__<'py>( &self, @@ -148,6 +150,47 @@ impl PyBallistaQueryPlanner { cr"datafusion_physical_extension_codec", ) } +} + +#[pyclass( + name = "BallistaQueryPlanner", + module = "ballista._internal_ballista", + skip_from_py_object +)] +pub(crate) struct PyBallistaQueryPlanner { + address: String, + config: ballista_core::config::BallistaConfig, + logical_codec: Arc, + fallback_logical_codec: FFI_LogicalExtensionCodec, + fallback_physical_codec: FFI_PhysicalExtensionCodec, + // The fallback FFI codecs hold a weak task-context provider. Retain its + // source context for as long as Python retains this planner. + _session_ctx: Py, +} + +#[pymethods] +impl PyBallistaQueryPlanner { + #[new] + #[pyo3(signature = (address, session_ctx, config_overrides=None))] + fn new( + address: String, + session_ctx: Bound<'_, PyAny>, + config_overrides: Option>, + ) -> PyResult { + let logical_codec = PyBallistaLogicalExtensionCodec::new(session_ctx.clone())?; + let fallback_physical_codec = ballista_physical_ffi_codec( + session_ctx.clone(), + Arc::clone(&logical_codec.logical_codec), + )?; + Self::from_codecs( + address, + session_ctx, + config_overrides, + logical_codec.logical_codec, + logical_codec.ffi_logical_codec, + fallback_physical_codec, + ) + } fn __datafusion_query_planner__<'py>( &self, @@ -162,8 +205,8 @@ impl PyBallistaQueryPlanner { }); let ffi = FFI_QueryPlanner::new_with_ffi_codecs( planner, - self.ffi_logical_codec.clone(), - self.ffi_physical_codec.clone(), + self.fallback_logical_codec.clone(), + self.fallback_physical_codec.clone(), ); // This exact name is the ABI consumed by datafusion-python 55f43b5e. @@ -171,6 +214,41 @@ impl PyBallistaQueryPlanner { } } +impl PyBallistaQueryPlanner { + fn from_codecs( + address: String, + session_ctx: Bound<'_, PyAny>, + config_overrides: Option>, + logical_codec: Arc, + fallback_logical_codec: FFI_LogicalExtensionCodec, + fallback_physical_codec: FFI_PhysicalExtensionCodec, + ) -> PyResult { + let mut session_config = SessionConfig::new_with_ballista(); + if let Some(overrides) = config_overrides { + for (key, value) in overrides { + if !key.starts_with("ballista.") { + return Err(PyValueError::new_err(format!( + "unsupported Ballista configuration key: {key}" + ))); + } + session_config + .options_mut() + .set(&key, &value) + .map_err(|error| PyValueError::new_err(error.to_string()))?; + } + } + + Ok(Self { + address, + config: session_config.ballista_config(), + logical_codec, + fallback_logical_codec, + fallback_physical_codec, + _session_ctx: session_ctx.unbind(), + }) + } +} + struct LazyBallistaQueryPlanner { address: String, config: ballista_core::config::BallistaConfig, @@ -262,29 +340,54 @@ pub(crate) fn with_ballista_query_planner( } } - let planner = PyBallistaQueryPlanner::new( + let logical_codec = PyBallistaLogicalExtensionCodec::new(session_ctx.clone())?; + let logical_native = Arc::clone(&logical_codec.logical_codec); + let fallback_logical_codec = logical_codec.ffi_logical_codec.clone(); + let logical_codec = Py::new(py, logical_codec)?; + + let physical_codec = PyBallistaPhysicalExtensionCodec::from_logical_codec( + session_ctx.clone(), + Arc::clone(&logical_native), + logical_codec.clone_ref(py).into_any(), + )?; + let fallback_physical_codec = physical_codec.ffi_physical_codec.clone(); + let physical_codec = Py::new(py, physical_codec)?; + + let planner = PyBallistaQueryPlanner::from_codecs( address, session_ctx.clone(), Some(ballista_overrides), + logical_native, + fallback_logical_codec, + fallback_physical_codec, )?; let planner = Py::new(py, planner)?; - // datafusion-python deliberately rebinds imported planners to the codecs - // installed on the receiving context. Install Ballista's codecs first so - // that rebind keeps DistributedQueryExec serialization active. - let ctx = session_ctx.call_method1( - "with_logical_extension_codec", - (planner.bind(py),), - )?; - let ctx = ctx.call_method1( - "with_physical_extension_codec", - (planner.bind(py),), - )?; + // Install each Ballista codec once before the planner. datafusion-python + // then rebinds the planner to the receiving context's codec chains. + let ctx = session_ctx + .call_method1("with_logical_extension_codec", (logical_codec.bind(py),))?; + let ctx = + ctx.call_method1("with_physical_extension_codec", (physical_codec.bind(py),))?; Ok(ctx .call_method1("with_query_planner", (planner.bind(py),))? .unbind()) } +fn ballista_physical_ffi_codec( + session_ctx: Bound<'_, PyAny>, + logical_codec: Arc, +) -> PyResult { + let physical_codec: Arc = + Arc::new(NativeBallistaPhysicalExtensionCodec::new(logical_codec)); + let task_ctx_provider = ffi_task_ctx_provider_from_python(session_ctx)?; + Ok(FFI_PhysicalExtensionCodec::new( + physical_codec, + Some(tokio_runtime_handle()), + task_ctx_provider, + )) +} + fn apply_datafusion_override( session_ctx: &Bound<'_, PyAny>, key: &str, From 44169f1d0859b9f9da136e52ad2397d29df35790 Mon Sep 17 00:00:00 2001 From: Tim Saucer Date: Fri, 7 Aug 2026 13:29:37 -0400 Subject: [PATCH 10/11] Update example to use new way to export codecs and query planner to datafusion-python --- python/examples/getting_started.ipynb | 38 +++++++++++++++++++++++---- 1 file changed, 33 insertions(+), 5 deletions(-) diff --git a/python/examples/getting_started.ipynb b/python/examples/getting_started.ipynb index b2f725cbab..b2b938ba59 100644 --- a/python/examples/getting_started.ipynb +++ b/python/examples/getting_started.ipynb @@ -58,7 +58,14 @@ "outputs": [], "source": [ "import ballista\n", - "from ballista import BallistaSessionContext, setup_test_cluster\n", + "from ballista._internal_ballista import ballista_datafusion_config_defaults\n", + "from ballista import (\n", + " BallistaQueryPlanner,\n", + " BallistaLogicalExtensionCodec,\n", + " BallistaPhysicalExtensionCodec,\n", + " setup_test_cluster,\n", + ")\n", + "from datafusion import SessionConfig, SessionContext\n", "%load_ext autoreload\n", "%autoreload 2\n", "\n", @@ -77,10 +84,31 @@ "# ctx = BallistaSessionContext(\"df://your-scheduler:50050\")\n", "\n", "host, port = setup_test_cluster()\n", - "ctx = BallistaSessionContext(f\"df://{host}:{port}\")\n", + "address = f\"df://{host}:{port}\"\n", + "\n", + "ballista_config = ballista_datafusion_config_defaults()\n", + "ctx = SessionContext(SessionConfig(ballista_config))\n", + "logical_codec = BallistaLogicalExtensionCodec(ctx)\n", + "physical_codec = BallistaPhysicalExtensionCodec(ctx, logical_codec)\n", + "\n", + "ctx = (\n", + " ctx.with_logical_extension_codec(logical_codec)\n", + " .with_physical_extension_codec(physical_codec)\n", + " .with_query_planner(BallistaQueryPlanner(address, ctx))\n", + ")\n", "\n", "print(f\"Connected to Ballista at {host}:{port}\")\n", - "print(f\"Session ID: {ctx.session_id}\")" + "print(f\"Session ID: {ctx.session_id()}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "ctx.deregister_table(\"public.test_data_v1\")\n", + "ctx.deregister_table(\"public.test_data_v2\")" ] }, { @@ -94,7 +122,7 @@ "ctx.register_parquet(\"public.test_data_v2\", \"../testdata/test.parquet\")\n", "\n", "# List registered tables\n", - "print(\"Registered schemas and tables:\\n\", ctx.get_tables())" + "print(\"Registered schemas and tables:\\n\", ctx.catalog().schema().table_names())" ] }, { @@ -321,7 +349,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.12.3" + "version": "3.12.9" } }, "nbformat": 4, From 0a07776f6b2c3194d743d6c6b3e9e8ed725b1138 Mon Sep 17 00:00:00 2001 From: Tim Saucer Date: Fri, 7 Aug 2026 16:07:37 -0400 Subject: [PATCH 11/11] feat(python): use atomic Ballista session extensions AI Disclosure: This code was written in part by an AI agent.: --- python/Cargo.lock | 4 +- python/Cargo.toml | 2 +- python/examples/getting_started.ipynb | 16 +-- python/pyproject.toml | 4 +- python/python/ballista/__init__.py | 4 + python/python/tests/test_context.py | 32 ++++++ python/requirements.txt | 2 +- python/src/lib.rs | 1 + python/src/query_planner.rs | 145 ++++++++++++++++++-------- python/uv.lock | 4 +- 10 files changed, 150 insertions(+), 64 deletions(-) diff --git a/python/Cargo.lock b/python/Cargo.lock index 1a21218128..49a856bd5e 100644 --- a/python/Cargo.lock +++ b/python/Cargo.lock @@ -1681,7 +1681,7 @@ dependencies = [ [[package]] name = "datafusion-python" version = "54.0.0" -source = "git+https://github.com/apache/datafusion-python.git?rev=55f43b5e231da5591e7bbd632dac85f304a22cca#55f43b5e231da5591e7bbd632dac85f304a22cca" +source = "git+https://github.com/apache/datafusion-python.git?rev=f1526210338d75ef105246e5c948e0e4a4a0f5ce#f1526210338d75ef105246e5c948e0e4a4a0f5ce" dependencies = [ "arrow", "arrow-select", @@ -1713,7 +1713,7 @@ dependencies = [ [[package]] name = "datafusion-python-util" version = "54.0.0" -source = "git+https://github.com/apache/datafusion-python.git?rev=55f43b5e231da5591e7bbd632dac85f304a22cca#55f43b5e231da5591e7bbd632dac85f304a22cca" +source = "git+https://github.com/apache/datafusion-python.git?rev=f1526210338d75ef105246e5c948e0e4a4a0f5ce#f1526210338d75ef105246e5c948e0e4a4a0f5ce" dependencies = [ "arrow", "datafusion", diff --git a/python/Cargo.toml b/python/Cargo.toml index 10527a71b0..3f45c53648 100644 --- a/python/Cargo.toml +++ b/python/Cargo.toml @@ -38,7 +38,7 @@ ballista-scheduler = { path = "../ballista/scheduler", version = "54.0.0", defau datafusion = { git = "https://github.com/apache/datafusion.git", rev = "e08aed1e5de41dcf81d529140dae07723b942a5e", features = ["avro"] } datafusion-ffi = { git = "https://github.com/apache/datafusion.git", rev = "e08aed1e5de41dcf81d529140dae07723b942a5e" } datafusion-proto = { git = "https://github.com/apache/datafusion.git", rev = "e08aed1e5de41dcf81d529140dae07723b942a5e" } -datafusion-python = { git = "https://github.com/apache/datafusion-python.git", rev = "55f43b5e231da5591e7bbd632dac85f304a22cca", default-features = false } +datafusion-python = { git = "https://github.com/apache/datafusion-python.git", rev = "f1526210338d75ef105246e5c948e0e4a4a0f5ce", default-features = false } log = { version = "0.4" } pyo3 = { version = "0.29", features = ["extension-module", "abi3", "abi3-py310"] } pyo3-log = "0.13" diff --git a/python/examples/getting_started.ipynb b/python/examples/getting_started.ipynb index b2b938ba59..60eb5c3dd1 100644 --- a/python/examples/getting_started.ipynb +++ b/python/examples/getting_started.ipynb @@ -58,11 +58,9 @@ "outputs": [], "source": [ "import ballista\n", - "from ballista._internal_ballista import ballista_datafusion_config_defaults\n", "from ballista import (\n", - " BallistaQueryPlanner,\n", - " BallistaLogicalExtensionCodec,\n", - " BallistaPhysicalExtensionCodec,\n", + " BallistaExtension,\n", + " ballista_datafusion_config_defaults,\n", " setup_test_cluster,\n", ")\n", "from datafusion import SessionConfig, SessionContext\n", @@ -87,14 +85,8 @@ "address = f\"df://{host}:{port}\"\n", "\n", "ballista_config = ballista_datafusion_config_defaults()\n", - "ctx = SessionContext(SessionConfig(ballista_config))\n", - "logical_codec = BallistaLogicalExtensionCodec(ctx)\n", - "physical_codec = BallistaPhysicalExtensionCodec(ctx, logical_codec)\n", - "\n", - "ctx = (\n", - " ctx.with_logical_extension_codec(logical_codec)\n", - " .with_physical_extension_codec(physical_codec)\n", - " .with_query_planner(BallistaQueryPlanner(address, ctx))\n", + "ctx = SessionContext(SessionConfig(ballista_config)).with_extensions(\n", + " BallistaExtension(address)\n", ")\n", "\n", "print(f\"Connected to Ballista at {host}:{port}\")\n", diff --git a/python/pyproject.toml b/python/pyproject.toml index 4e85fbb075..dbc82006cf 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -41,8 +41,8 @@ classifiers = [ ] dependencies = [ "pyarrow>=23.0.1", - # Temporary PR #1672 pin. Use its released version before publishing. - "datafusion @ git+https://github.com/apache/datafusion-python.git@55f43b5e231da5591e7bbd632dac85f304a22cca", + # Temporary PR #1679 pin. Use its released version before publishing. + "datafusion @ git+https://github.com/apache/datafusion-python.git@f1526210338d75ef105246e5c948e0e4a4a0f5ce", "typing-extensions;python_version<'3.13'", ] dynamic = ["version"] diff --git a/python/python/ballista/__init__.py b/python/python/ballista/__init__.py index e892ff8ea5..97ba8ef657 100644 --- a/python/python/ballista/__init__.py +++ b/python/python/ballista/__init__.py @@ -22,10 +22,12 @@ from ._internal_ballista import ( BallistaExecutor, + BallistaExtension, BallistaLogicalExtensionCodec, BallistaPhysicalExtensionCodec, BallistaQueryPlanner, BallistaScheduler, + ballista_datafusion_config_defaults, setup_test_cluster, ) from .extension import ( @@ -38,8 +40,10 @@ __all__ = [ "setup_test_cluster", + "ballista_datafusion_config_defaults", "BallistaScheduler", "BallistaExecutor", + "BallistaExtension", "BallistaLogicalExtensionCodec", "BallistaPhysicalExtensionCodec", "BallistaQueryPlanner", diff --git a/python/python/tests/test_context.py b/python/python/tests/test_context.py index 7fc0168062..ab8a2a7598 100644 --- a/python/python/tests/test_context.py +++ b/python/python/tests/test_context.py @@ -15,7 +15,10 @@ # specific language governing permissions and limitations # under the License. +import gc + from ballista import ( + BallistaExtension, BallistaLogicalExtensionCodec, BallistaPhysicalExtensionCodec, BallistaQueryPlanner, @@ -92,6 +95,35 @@ def test_manual_query_planner_and_codec_composition(): assert df.collect()[0].column(0).to_pylist() == [5] +def test_ballista_extension_uses_atomic_context_installation(): + address, port = setup_test_cluster() + source_ctx = SessionContext() + source_ctx.register_csv("registered", "testdata/test.csv", has_header=True) + + configured_ctx = source_ctx.with_extensions( + BallistaExtension(f"df://{address}:{port}") + ) + + df = configured_ctx.sql("SELECT COUNT(*) FROM registered") + assert type(df) is DataFrame + assert_uses_ballista(df) + assert df.collect()[0].column(0).to_pylist() == [5] + + +def test_ballista_extension_context_must_outlive_dataframe(): + address, port = setup_test_cluster() + configured_ctx = SessionContext().with_extensions( + BallistaExtension(f"df://{address}:{port}") + ) + df = configured_ctx.sql("SELECT 1") + + del configured_ctx + gc.collect() + + with pytest.raises(Exception, match="went out of scope"): + df.collect() + + def test_low_level_helper_returns_standard_distributed_dataframe(): address, port = setup_test_cluster() source_ctx = SessionContext() diff --git a/python/requirements.txt b/python/requirements.txt index bc9cfb34d9..ae3a415c68 100644 --- a/python/requirements.txt +++ b/python/requirements.txt @@ -1,4 +1,4 @@ -datafusion==52.0.0 +datafusion @ git+https://github.com/apache/datafusion-python.git@f1526210338d75ef105246e5c948e0e4a4a0f5ce pyarrow pytest maturin>=1.8.0,<2.0.0 diff --git a/python/src/lib.rs b/python/src/lib.rs index 7e44e2f204..810c2dae92 100644 --- a/python/src/lib.rs +++ b/python/src/lib.rs @@ -39,6 +39,7 @@ fn _internal_ballista(_py: Python, m: Bound<'_, PyModule>) -> PyResult<()> { m.add_class::()?; m.add_class::()?; + m.add_class::()?; m.add_class::()?; m.add_class::()?; m.add_class::()?; diff --git a/python/src/query_planner.rs b/python/src/query_planner.rs index c934bb9ce0..0d10423cc0 100644 --- a/python/src/query_planner.rs +++ b/python/src/query_planner.rs @@ -36,7 +36,7 @@ use datafusion_proto::physical_plan::PhysicalExtensionCodec; use datafusion_proto::protobuf::LogicalPlanNode; use pyo3::exceptions::PyValueError; use pyo3::prelude::*; -use pyo3::types::PyCapsule; +use pyo3::types::{PyCapsule, PyDict}; use std::collections::HashMap; use std::ptr::NonNull; use std::sync::{Arc, OnceLock}; @@ -45,6 +45,41 @@ use url::Url; const DEFAULT_SCHEDULER_PORT: u16 = 50050; +#[pyclass( + name = "BallistaExtension", + module = "ballista._internal_ballista", + skip_from_py_object +)] +pub(crate) struct PyBallistaExtension { + address: String, + config_overrides: Option>, +} + +#[pymethods] +impl PyBallistaExtension { + #[new] + #[pyo3(signature = (address, config_overrides=None))] + fn new(address: String, config_overrides: Option>) -> Self { + Self { + address, + config_overrides, + } + } + + fn __datafusion_session_extension__<'py>( + &self, + py: Python<'py>, + session_ctx: Bound<'py, PyAny>, + ) -> PyResult> { + ballista_extension_components( + py, + session_ctx, + self.address.clone(), + self.config_overrides.clone(), + ) + } +} + #[pyclass( name = "BallistaLogicalExtensionCodec", module = "ballista._internal_ballista", @@ -53,8 +88,9 @@ const DEFAULT_SCHEDULER_PORT: u16 = 50050; pub(crate) struct PyBallistaLogicalExtensionCodec { logical_codec: Arc, ffi_logical_codec: FFI_LogicalExtensionCodec, - // The FFI codec holds a weak task-context provider. - _session_ctx: Py, + // Retain the source context for the legacy low-level API. Atomic extension + // installation leaves this empty so the returned context owns the provider. + _session_ctx: Option>, } #[pymethods] @@ -75,7 +111,7 @@ impl PyBallistaLogicalExtensionCodec { Ok(Self { logical_codec, ffi_logical_codec, - _session_ctx: session_ctx.unbind(), + _session_ctx: Some(session_ctx.unbind()), }) } @@ -98,8 +134,9 @@ impl PyBallistaLogicalExtensionCodec { )] pub(crate) struct PyBallistaPhysicalExtensionCodec { ffi_physical_codec: FFI_PhysicalExtensionCodec, - // The FFI codec holds a weak task-context provider. - _session_ctx: Py, + // Retain the source context for the legacy low-level API. Atomic extension + // installation leaves this empty so the returned context owns the provider. + _session_ctx: Option>, // Retain the Python logical codec as well as its native Arc. _logical_codec: Py, } @@ -115,7 +152,7 @@ impl PyBallistaPhysicalExtensionCodec { Ok(Self { ffi_physical_codec, - _session_ctx: session_ctx.unbind(), + _session_ctx: Some(session_ctx.unbind()), _logical_codec: logical_codec_object, }) } @@ -163,9 +200,9 @@ pub(crate) struct PyBallistaQueryPlanner { logical_codec: Arc, fallback_logical_codec: FFI_LogicalExtensionCodec, fallback_physical_codec: FFI_PhysicalExtensionCodec, - // The fallback FFI codecs hold a weak task-context provider. Retain its - // source context for as long as Python retains this planner. - _session_ctx: Py, + // Retain the source context for the legacy low-level API. Atomic extension + // installation leaves this empty so the returned context owns the provider. + _session_ctx: Option>, } #[pymethods] @@ -201,7 +238,10 @@ impl PyBallistaQueryPlanner { address: self.address.clone(), config: self.config.clone(), logical_codec: Arc::clone(&self.logical_codec), - _session_ctx: self._session_ctx.clone_ref(py), + _session_ctx: self + ._session_ctx + .as_ref() + .map(|session_ctx| session_ctx.clone_ref(py)), }); let ffi = FFI_QueryPlanner::new_with_ffi_codecs( planner, @@ -209,7 +249,7 @@ impl PyBallistaQueryPlanner { self.fallback_physical_codec.clone(), ); - // This exact name is the ABI consumed by datafusion-python 55f43b5e. + // This exact name is the ABI consumed by datafusion-python. PyCapsule::new_with_value(py, ffi, cr"datafusion_query_planner") } } @@ -244,7 +284,7 @@ impl PyBallistaQueryPlanner { logical_codec, fallback_logical_codec, fallback_physical_codec, - _session_ctx: session_ctx.unbind(), + _session_ctx: Some(session_ctx.unbind()), }) } } @@ -253,7 +293,7 @@ struct LazyBallistaQueryPlanner { address: String, config: ballista_core::config::BallistaConfig, logical_codec: Arc, - _session_ctx: Py, + _session_ctx: Option>, } impl std::fmt::Debug for LazyBallistaQueryPlanner { @@ -317,6 +357,48 @@ pub(crate) fn ballista_datafusion_config_defaults() -> HashMap { .collect() } +fn ballista_extension_components<'py>( + py: Python<'py>, + session_ctx: Bound<'py, PyAny>, + address: String, + config_overrides: Option>, +) -> PyResult> { + let mut logical_codec = PyBallistaLogicalExtensionCodec::new(session_ctx.clone())?; + let logical_native = Arc::clone(&logical_codec.logical_codec); + let fallback_logical_codec = logical_codec.ffi_logical_codec.clone(); + logical_codec._session_ctx = None; + let logical_codec = Py::new(py, logical_codec)?; + + let mut physical_codec = PyBallistaPhysicalExtensionCodec::from_logical_codec( + session_ctx.clone(), + Arc::clone(&logical_native), + logical_codec.clone_ref(py).into_any(), + )?; + let fallback_physical_codec = physical_codec.ffi_physical_codec.clone(); + physical_codec._session_ctx = None; + let physical_codec = Py::new(py, physical_codec)?; + + let mut planner = PyBallistaQueryPlanner::from_codecs( + address, + session_ctx, + config_overrides, + logical_native, + fallback_logical_codec, + fallback_physical_codec, + )?; + planner._session_ctx = None; + let planner = Py::new(py, planner)?; + + let components = py + .import("datafusion")? + .getattr("SessionExtensionComponents")?; + let kwargs = PyDict::new(py); + kwargs.set_item("logical_extension_codecs", (logical_codec,))?; + kwargs.set_item("physical_extension_codecs", (physical_codec,))?; + kwargs.set_item("query_planner", planner)?; + components.call((), Some(&kwargs)) +} + #[pyfunction] #[pyo3(signature = (session_ctx, address, config_overrides=None))] pub(crate) fn with_ballista_query_planner( @@ -340,37 +422,12 @@ pub(crate) fn with_ballista_query_planner( } } - let logical_codec = PyBallistaLogicalExtensionCodec::new(session_ctx.clone())?; - let logical_native = Arc::clone(&logical_codec.logical_codec); - let fallback_logical_codec = logical_codec.ffi_logical_codec.clone(); - let logical_codec = Py::new(py, logical_codec)?; - - let physical_codec = PyBallistaPhysicalExtensionCodec::from_logical_codec( - session_ctx.clone(), - Arc::clone(&logical_native), - logical_codec.clone_ref(py).into_any(), + let extension = Py::new( + py, + PyBallistaExtension::new(address, Some(ballista_overrides)), )?; - let fallback_physical_codec = physical_codec.ffi_physical_codec.clone(); - let physical_codec = Py::new(py, physical_codec)?; - - let planner = PyBallistaQueryPlanner::from_codecs( - address, - session_ctx.clone(), - Some(ballista_overrides), - logical_native, - fallback_logical_codec, - fallback_physical_codec, - )?; - let planner = Py::new(py, planner)?; - - // Install each Ballista codec once before the planner. datafusion-python - // then rebinds the planner to the receiving context's codec chains. - let ctx = session_ctx - .call_method1("with_logical_extension_codec", (logical_codec.bind(py),))?; - let ctx = - ctx.call_method1("with_physical_extension_codec", (physical_codec.bind(py),))?; - Ok(ctx - .call_method1("with_query_planner", (planner.bind(py),))? + Ok(session_ctx + .call_method1("with_extensions", (extension.bind(py),))? .unbind()) } diff --git a/python/uv.lock b/python/uv.lock index f467ad3174..4966fc4bed 100644 --- a/python/uv.lock +++ b/python/uv.lock @@ -52,7 +52,7 @@ dev = [ [package.metadata] requires-dist = [ - { name = "datafusion", git = "https://github.com/apache/datafusion-python.git?rev=55f43b5e231da5591e7bbd632dac85f304a22cca" }, + { name = "datafusion", git = "https://github.com/apache/datafusion-python.git?rev=f1526210338d75ef105246e5c948e0e4a4a0f5ce" }, { name = "ipython", marker = "extra == 'jupyter'", specifier = ">=8.0.0" }, { name = "pyarrow", specifier = ">=23.0.1" }, { name = "typing-extensions", marker = "python_full_version < '3.13'" }, @@ -328,7 +328,7 @@ wheels = [ [[package]] name = "datafusion" version = "54.0.0" -source = { git = "https://github.com/apache/datafusion-python.git?rev=55f43b5e231da5591e7bbd632dac85f304a22cca#55f43b5e231da5591e7bbd632dac85f304a22cca" } +source = { git = "https://github.com/apache/datafusion-python.git?rev=f1526210338d75ef105246e5c948e0e4a4a0f5ce#f1526210338d75ef105246e5c948e0e4a4a0f5ce" } dependencies = [ { name = "cloudpickle" }, { name = "pyarrow" },