From 7b94a4a9fab6c12fbfdef6fab96d1c0d530b0900 Mon Sep 17 00:00:00 2001 From: Leonardo Yvens Date: Thu, 20 Nov 2025 18:07:07 -0300 Subject: [PATCH 1/3] upgrade arrow and datafusion --- Cargo.lock | 437 ++++++++++---------- Cargo.toml | 10 +- crates/arrow-to-postgres/Cargo.toml | 4 +- crates/bin/ampsync/Cargo.toml | 4 +- crates/core/common/src/catalog/physical.rs | 2 +- crates/core/common/src/catalog/reader.rs | 13 +- crates/core/common/src/evm/udfs/eth_call.rs | 6 + crates/core/common/src/evm/udfs/mod.rs | 8 +- crates/core/common/src/js_udf.rs | 1 + crates/core/js-runtime/src/convert/mod.rs | 2 + 10 files changed, 258 insertions(+), 229 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f8fa27c18..edee72db2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -163,7 +163,7 @@ checksum = "6068f356948cd84b5ad9ac30c50478e433847f14a50714d2b68f15d052724049" dependencies = [ "alloy-primitives", "num_enum", - "strum 0.27.2", + "strum", ] [[package]] @@ -1238,9 +1238,9 @@ dependencies = [ [[package]] name = "arrow" -version = "56.2.0" +version = "57.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e833808ff2d94ed40d9379848a950d995043c7fb3e81a30b383f4c6033821cc" +checksum = "4df8bb5b0bd64c0b9bc61317fcc480bad0f00e56d3bc32c69a4c8dada4786bae" dependencies = [ "arrow-arith", "arrow-array", @@ -1259,23 +1259,23 @@ dependencies = [ [[package]] name = "arrow-arith" -version = "56.2.0" +version = "57.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad08897b81588f60ba983e3ca39bda2b179bdd84dced378e7df81a5313802ef8" +checksum = "a1a640186d3bd30a24cb42264c2dafb30e236a6f50d510e56d40b708c9582491" dependencies = [ "arrow-array", "arrow-buffer", "arrow-data", "arrow-schema", "chrono", - "num", + "num-traits", ] [[package]] name = "arrow-array" -version = "56.2.0" +version = "57.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8548ca7c070d8db9ce7aa43f37393e4bfcf3f2d3681df278490772fd1673d08d" +checksum = "219fe420e6800979744c8393b687afb0252b3f8a89b91027d27887b72aa36d31" dependencies = [ "ahash", "arrow-buffer", @@ -1285,25 +1285,28 @@ dependencies = [ "chrono-tz", "half", "hashbrown 0.16.0", - "num", + "num-complex", + "num-integer", + "num-traits", ] [[package]] name = "arrow-buffer" -version = "56.2.0" +version = "57.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e003216336f70446457e280807a73899dd822feaf02087d31febca1363e2fccc" +checksum = "76885a2697a7edf6b59577f568b456afc94ce0e2edc15b784ce3685b6c3c5c27" dependencies = [ "bytes", "half", - "num", + "num-bigint", + "num-traits", ] [[package]] name = "arrow-cast" -version = "56.2.0" +version = "57.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "919418a0681298d3a77d1a315f625916cb5678ad0d74b9c60108eb15fd083023" +checksum = "9c9ebb4c987e6b3b236fb4a14b20b34835abfdd80acead3ccf1f9bf399e1f168" dependencies = [ "arrow-array", "arrow-buffer", @@ -1316,15 +1319,15 @@ dependencies = [ "comfy-table", "half", "lexical-core", - "num", + "num-traits", "ryu", ] [[package]] name = "arrow-csv" -version = "56.2.0" +version = "57.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfa9bf02705b5cf762b6f764c65f04ae9082c7cfc4e96e0c33548ee3f67012eb" +checksum = "92386159c8d4bce96f8bd396b0642a0d544d471bdc2ef34d631aec80db40a09c" dependencies = [ "arrow-array", "arrow-cast", @@ -1337,21 +1340,22 @@ dependencies = [ [[package]] name = "arrow-data" -version = "56.2.0" +version = "57.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a5c64fff1d142f833d78897a772f2e5b55b36cb3e6320376f0961ab0db7bd6d0" +checksum = "727681b95de313b600eddc2a37e736dcb21980a40f640314dcf360e2f36bc89b" dependencies = [ "arrow-buffer", "arrow-schema", "half", - "num", + "num-integer", + "num-traits", ] [[package]] name = "arrow-flight" -version = "56.2.0" +version = "57.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c8b0ba0784d56bc6266b79f5de7a24b47024e7b3a0045d2ad4df3d9b686099f" +checksum = "f70bb56412a007b0cfc116d15f24dda6adeed9611a213852a004cda20085a3b9" dependencies = [ "arrow-arith", "arrow-array", @@ -1369,16 +1373,17 @@ dependencies = [ "futures", "once_cell", "paste", - "prost 0.13.5", - "prost-types", - "tonic 0.13.1", + "prost 0.14.1", + "prost-types 0.14.1", + "tonic 0.14.2", + "tonic-prost", ] [[package]] name = "arrow-ipc" -version = "56.2.0" +version = "57.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d3594dcddccc7f20fd069bc8e9828ce37220372680ff638c5e00dea427d88f5" +checksum = "da9ba92e3de170295c98a84e5af22e2b037f0c7b32449445e6c493b5fca27f27" dependencies = [ "arrow-array", "arrow-buffer", @@ -1392,9 +1397,9 @@ dependencies = [ [[package]] name = "arrow-json" -version = "56.2.0" +version = "57.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "88cf36502b64a127dc659e3b305f1d993a544eab0d48cce704424e62074dc04b" +checksum = "b969b4a421ae83828591c6bf5450bd52e6d489584142845ad6a861f42fe35df8" dependencies = [ "arrow-array", "arrow-buffer", @@ -1404,19 +1409,21 @@ dependencies = [ "chrono", "half", "indexmap 2.12.0", + "itoa", "lexical-core", "memchr", - "num", - "serde", + "num-traits", + "ryu", + "serde_core", "serde_json", "simdutf8", ] [[package]] name = "arrow-ord" -version = "56.2.0" +version = "57.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c8f82583eb4f8d84d4ee55fd1cb306720cddead7596edce95b50ee418edf66f" +checksum = "141c05298b21d03e88062317a1f1a73f5ba7b6eb041b350015b1cd6aabc0519b" dependencies = [ "arrow-array", "arrow-buffer", @@ -1427,9 +1434,9 @@ dependencies = [ [[package]] name = "arrow-row" -version = "56.2.0" +version = "57.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d07ba24522229d9085031df6b94605e0f4b26e099fb7cdeec37abd941a73753" +checksum = "c5f3c06a6abad6164508ed283c7a02151515cef3de4b4ff2cebbcaeb85533db2" dependencies = [ "arrow-array", "arrow-buffer", @@ -1440,33 +1447,34 @@ dependencies = [ [[package]] name = "arrow-schema" -version = "56.2.0" +version = "57.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b3aa9e59c611ebc291c28582077ef25c97f1975383f1479b12f3b9ffee2ffabe" +checksum = "9cfa7a03d1eee2a4d061476e1840ad5c9867a544ca6c4c59256496af5d0a8be5" dependencies = [ "serde", + "serde_core", "serde_json", ] [[package]] name = "arrow-select" -version = "56.2.0" +version = "57.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c41dbbd1e97bfcaee4fcb30e29105fb2c75e4d82ae4de70b792a5d3f66b2e7a" +checksum = "bafa595babaad59f2455f4957d0f26448fb472722c186739f4fac0823a1bdb47" dependencies = [ "ahash", "arrow-array", "arrow-buffer", "arrow-data", "arrow-schema", - "num", + "num-traits", ] [[package]] name = "arrow-string" -version = "56.2.0" +version = "57.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53f5183c150fbc619eede22b861ea7c0eebed8eaac0333eaa7f6da5205fd504d" +checksum = "32f46457dbbb99f2650ff3ac23e46a929e0ab81db809b02aa5511c258348bef2" dependencies = [ "arrow-array", "arrow-buffer", @@ -1474,7 +1482,7 @@ dependencies = [ "arrow-schema", "arrow-select", "memchr", - "num", + "num-traits", "regex", "regex-syntax", ] @@ -2353,14 +2361,12 @@ checksum = "b05b61dc5112cbb17e4b6cd61790d9845d13888356391624cbe7e41efeac1e75" [[package]] name = "comfy-table" -version = "7.1.2" +version = "7.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0d05af1e006a2407bedef5af410552494ce5be9090444dbbcb57258c1af3d56" +checksum = "b03b7db8e0b4b2fdad6c551e634134e99ec000e5c8c3b6856c65e8bbaded7a3b" dependencies = [ - "crossterm 0.27.0", - "crossterm 0.28.1", - "strum 0.26.3", - "strum_macros 0.26.4", + "crossterm", + "unicode-segmentation", "unicode-width", ] @@ -2446,7 +2452,7 @@ checksum = "8030735ecb0d128428b64cd379809817e620a40e5001c54465b99ec5feec2857" dependencies = [ "futures-core", "prost 0.13.5", - "prost-types", + "prost-types 0.13.5", "tonic 0.12.3", "tracing-core", ] @@ -2465,7 +2471,7 @@ dependencies = [ "humantime", "hyper-util", "prost 0.13.5", - "prost-types", + "prost-types 0.13.5", "serde", "serde_json", "thread_local", @@ -2647,28 +2653,18 @@ checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" [[package]] name = "crossterm" -version = "0.27.0" +version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f476fe445d41c9e991fd07515a6f463074b782242ccf4a5b7b1d1012e70824df" +checksum = "d8b9f2e4c67f833b660cdb0a3523065869fb35570177239812ed4c905aeff87b" dependencies = [ "bitflags", "crossterm_winapi", - "libc", + "document-features", "parking_lot", + "rustix 1.1.2", "winapi", ] -[[package]] -name = "crossterm" -version = "0.28.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "829d955a0bb380ef178a640b91779e3987da38c9aea133b20614cfed8cdea9c6" -dependencies = [ - "bitflags", - "parking_lot", - "rustix 0.38.44", -] - [[package]] name = "crossterm_winapi" version = "0.9.1" @@ -2906,12 +2902,11 @@ checksum = "2a2330da5de22e8a3cb63252ce2abb30116bf5265e89c0e01bc17015ce30a476" [[package]] name = "datafusion" -version = "50.3.0" +version = "51.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2af15bb3c6ffa33011ef579f6b0bcbe7c26584688bd6c994f548e44df67f011a" +checksum = "8ba7cb113e9c0bedf9e9765926031e132fa05a1b09ba6e93a6d1a4d7044457b8" dependencies = [ "arrow", - "arrow-ipc", "arrow-schema", "async-trait", "bytes", @@ -2922,6 +2917,7 @@ dependencies = [ "datafusion-common", "datafusion-common-runtime", "datafusion-datasource", + "datafusion-datasource-arrow", "datafusion-datasource-csv", "datafusion-datasource-json", "datafusion-datasource-parquet", @@ -2950,6 +2946,7 @@ dependencies = [ "parquet", "rand 0.9.2", "regex", + "rstest", "serde", "sqlparser", "tempfile", @@ -2962,9 +2959,9 @@ dependencies = [ [[package]] name = "datafusion-catalog" -version = "50.3.0" +version = "51.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "187622262ad8f7d16d3be9202b4c1e0116f1c9aa387e5074245538b755261621" +checksum = "66a3a799f914a59b1ea343906a0486f17061f39509af74e874a866428951130d" dependencies = [ "arrow", "async-trait", @@ -2977,7 +2974,6 @@ dependencies = [ "datafusion-physical-expr", "datafusion-physical-plan", "datafusion-session", - "datafusion-sql", "futures", "itertools 0.14.0", "log", @@ -2988,9 +2984,9 @@ dependencies = [ [[package]] name = "datafusion-catalog-listing" -version = "50.3.0" +version = "51.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9657314f0a32efd0382b9a46fdeb2d233273ece64baa68a7c45f5a192daf0f83" +checksum = "6db1b113c80d7a0febcd901476a57aef378e717c54517a163ed51417d87621b0" dependencies = [ "arrow", "async-trait", @@ -3000,10 +2996,11 @@ dependencies = [ "datafusion-execution", "datafusion-expr", "datafusion-physical-expr", + "datafusion-physical-expr-adapter", "datafusion-physical-expr-common", "datafusion-physical-plan", - "datafusion-session", "futures", + "itertools 0.14.0", "log", "object_store", "tokio", @@ -3011,14 +3008,13 @@ dependencies = [ [[package]] name = "datafusion-common" -version = "50.3.0" +version = "51.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a83760d9a13122d025fbdb1d5d5aaf93dd9ada5e90ea229add92aa30898b2d1" +checksum = "7c10f7659e96127d25e8366be7c8be4109595d6a2c3eac70421f380a7006a1b0" dependencies = [ "ahash", "arrow", "arrow-ipc", - "base64 0.22.1", "chrono", "half", "hashbrown 0.14.5", @@ -3036,9 +3032,9 @@ dependencies = [ [[package]] name = "datafusion-common-runtime" -version = "50.3.0" +version = "51.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b6234a6c7173fe5db1c6c35c01a12b2aa0f803a3007feee53483218817f8b1e" +checksum = "b92065bbc6532c6651e2f7dd30b55cba0c7a14f860c7e1d15f165c41a1868d95" dependencies = [ "futures", "log", @@ -3047,9 +3043,9 @@ dependencies = [ [[package]] name = "datafusion-datasource" -version = "50.3.0" +version = "51.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7256c9cb27a78709dd42d0c80f0178494637209cac6e29d5c93edd09b6721b86" +checksum = "fde13794244bc7581cd82f6fff217068ed79cdc344cafe4ab2c3a1c3510b38d6" dependencies = [ "arrow", "async-compression", @@ -3072,9 +3068,7 @@ dependencies = [ "itertools 0.14.0", "log", "object_store", - "parquet", "rand 0.9.2", - "tempfile", "tokio", "tokio-util", "url", @@ -3082,22 +3076,44 @@ dependencies = [ "zstd", ] +[[package]] +name = "datafusion-datasource-arrow" +version = "51.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "804fa9b4ecf3157982021770617200ef7c1b2979d57bec9044748314775a9aea" +dependencies = [ + "arrow", + "arrow-ipc", + "async-trait", + "bytes", + "datafusion-common", + "datafusion-common-runtime", + "datafusion-datasource", + "datafusion-execution", + "datafusion-expr", + "datafusion-physical-expr-common", + "datafusion-physical-plan", + "datafusion-session", + "futures", + "itertools 0.14.0", + "object_store", + "tokio", +] + [[package]] name = "datafusion-datasource-csv" -version = "50.3.0" +version = "51.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64533a90f78e1684bfb113d200b540f18f268134622d7c96bbebc91354d04825" +checksum = "61a1641a40b259bab38131c5e6f48fac0717bedb7dc93690e604142a849e0568" dependencies = [ "arrow", "async-trait", "bytes", - "datafusion-catalog", "datafusion-common", "datafusion-common-runtime", "datafusion-datasource", "datafusion-execution", "datafusion-expr", - "datafusion-physical-expr", "datafusion-physical-expr-common", "datafusion-physical-plan", "datafusion-session", @@ -3109,49 +3125,44 @@ dependencies = [ [[package]] name = "datafusion-datasource-json" -version = "50.3.0" +version = "51.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d7ebeb12c77df0aacad26f21b0d033aeede423a64b2b352f53048a75bf1d6e6" +checksum = "adeacdb00c1d37271176f8fb6a1d8ce096baba16ea7a4b2671840c5c9c64fe85" dependencies = [ "arrow", "async-trait", "bytes", - "datafusion-catalog", "datafusion-common", "datafusion-common-runtime", "datafusion-datasource", "datafusion-execution", "datafusion-expr", - "datafusion-physical-expr", "datafusion-physical-expr-common", "datafusion-physical-plan", "datafusion-session", "futures", "object_store", - "serde_json", "tokio", ] [[package]] name = "datafusion-datasource-parquet" -version = "50.3.0" +version = "51.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09e783c4c7d7faa1199af2df4761c68530634521b176a8d1331ddbc5a5c75133" +checksum = "43d0b60ffd66f28bfb026565d62b0a6cbc416da09814766a3797bba7d85a3cd9" dependencies = [ "arrow", "async-trait", "bytes", - "datafusion-catalog", "datafusion-common", "datafusion-common-runtime", "datafusion-datasource", "datafusion-execution", "datafusion-expr", - "datafusion-functions-aggregate", + "datafusion-functions-aggregate-common", "datafusion-physical-expr", "datafusion-physical-expr-adapter", "datafusion-physical-expr-common", - "datafusion-physical-optimizer", "datafusion-physical-plan", "datafusion-pruning", "datafusion-session", @@ -3161,21 +3172,20 @@ dependencies = [ "object_store", "parking_lot", "parquet", - "rand 0.9.2", "tokio", ] [[package]] name = "datafusion-doc" -version = "50.3.0" +version = "51.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "99ee6b1d9a80d13f9deb2291f45c07044b8e62fb540dbde2453a18be17a36429" +checksum = "2b99e13947667b36ad713549237362afb054b2d8f8cc447751e23ec61202db07" [[package]] name = "datafusion-execution" -version = "50.3.0" +version = "51.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4cec0a57653bec7b933fb248d3ffa3fa3ab3bd33bd140dc917f714ac036f531" +checksum = "63695643190679037bc946ad46a263b62016931547bf119859c511f7ff2f5178" dependencies = [ "arrow", "async-trait", @@ -3193,9 +3203,9 @@ dependencies = [ [[package]] name = "datafusion-expr" -version = "50.3.0" +version = "51.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef76910bdca909722586389156d0aa4da4020e1631994d50fadd8ad4b1aa05fe" +checksum = "f9a4787cbf5feb1ab351f789063398f67654a6df75c4d37d7f637dc96f951a91" dependencies = [ "arrow", "async-trait", @@ -3207,6 +3217,7 @@ dependencies = [ "datafusion-functions-window-common", "datafusion-physical-expr-common", "indexmap 2.12.0", + "itertools 0.14.0", "paste", "recursive", "serde_json", @@ -3215,9 +3226,9 @@ dependencies = [ [[package]] name = "datafusion-expr-common" -version = "50.3.0" +version = "51.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d155ccbda29591ca71a1344dd6bed26c65a4438072b400df9db59447f590bb6" +checksum = "5ce2fb1b8c15c9ac45b0863c30b268c69dc9ee7a1ee13ecf5d067738338173dc" dependencies = [ "arrow", "datafusion-common", @@ -3228,9 +3239,9 @@ dependencies = [ [[package]] name = "datafusion-functions" -version = "50.3.0" +version = "51.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7de2782136bd6014670fd84fe3b0ca3b3e4106c96403c3ae05c0598577139977" +checksum = "794a9db7f7b96b3346fc007ff25e994f09b8f0511b4cf7dff651fadfe3ebb28f" dependencies = [ "arrow", "arrow-buffer", @@ -3248,6 +3259,7 @@ dependencies = [ "itertools 0.14.0", "log", "md-5", + "num-traits", "rand 0.9.2", "regex", "sha2", @@ -3257,9 +3269,9 @@ dependencies = [ [[package]] name = "datafusion-functions-aggregate" -version = "50.3.0" +version = "51.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07331fc13603a9da97b74fd8a273f4238222943dffdbbed1c4c6f862a30105bf" +checksum = "1c25210520a9dcf9c2b2cbbce31ebd4131ef5af7fc60ee92b266dc7d159cb305" dependencies = [ "ahash", "arrow", @@ -3278,9 +3290,9 @@ dependencies = [ [[package]] name = "datafusion-functions-aggregate-common" -version = "50.3.0" +version = "51.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5951e572a8610b89968a09b5420515a121fbc305c0258651f318dc07c97ab17" +checksum = "62f4a66f3b87300bb70f4124b55434d2ae3fe80455f3574701d0348da040b55d" dependencies = [ "ahash", "arrow", @@ -3291,9 +3303,9 @@ dependencies = [ [[package]] name = "datafusion-functions-nested" -version = "50.3.0" +version = "51.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fdacca9302c3d8fc03f3e94f338767e786a88a33f5ebad6ffc0e7b50364b9ea3" +checksum = "ae5c06eed03918dc7fe7a9f082a284050f0e9ecf95d72f57712d1496da03b8c4" dependencies = [ "arrow", "arrow-ord", @@ -3301,6 +3313,7 @@ dependencies = [ "datafusion-doc", "datafusion-execution", "datafusion-expr", + "datafusion-expr-common", "datafusion-functions", "datafusion-functions-aggregate", "datafusion-functions-aggregate-common", @@ -3313,9 +3326,9 @@ dependencies = [ [[package]] name = "datafusion-functions-table" -version = "50.3.0" +version = "51.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c37ff8a99434fbbad604a7e0669717c58c7c4f14c472d45067c4b016621d981" +checksum = "db4fed1d71738fbe22e2712d71396db04c25de4111f1ec252b8f4c6d3b25d7f5" dependencies = [ "arrow", "async-trait", @@ -3329,9 +3342,9 @@ dependencies = [ [[package]] name = "datafusion-functions-window" -version = "50.3.0" +version = "51.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48e2aea7c79c926cffabb13dc27309d4eaeb130f4a21c8ba91cdd241c813652b" +checksum = "1d92206aa5ae21892f1552b4d61758a862a70956e6fd7a95cb85db1de74bc6d1" dependencies = [ "arrow", "datafusion-common", @@ -3347,9 +3360,9 @@ dependencies = [ [[package]] name = "datafusion-functions-window-common" -version = "50.3.0" +version = "51.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fead257ab5fd2ffc3b40fda64da307e20de0040fe43d49197241d9de82a487f" +checksum = "53ae9bcc39800820d53a22d758b3b8726ff84a5a3e24cecef04ef4e5fdf1c7cc" dependencies = [ "datafusion-common", "datafusion-physical-expr-common", @@ -3357,20 +3370,20 @@ dependencies = [ [[package]] name = "datafusion-macros" -version = "50.3.0" +version = "51.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec6f637bce95efac05cdfb9b6c19579ed4aa5f6b94d951cfa5bb054b7bb4f730" +checksum = "1063ad4c9e094b3f798acee16d9a47bd7372d9699be2de21b05c3bd3f34ab848" dependencies = [ - "datafusion-expr", + "datafusion-doc", "quote", "syn 2.0.110", ] [[package]] name = "datafusion-optimizer" -version = "50.3.0" +version = "51.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c6583ef666ae000a613a837e69e456681a9faa96347bf3877661e9e89e141d8a" +checksum = "9f35f9ec5d08b87fd1893a30c2929f2559c2f9806ca072d8fefca5009dc0f06a" dependencies = [ "arrow", "chrono", @@ -3388,9 +3401,9 @@ dependencies = [ [[package]] name = "datafusion-physical-expr" -version = "50.3.0" +version = "51.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8668103361a272cbbe3a61f72eca60c9b7c706e87cc3565bcf21e2b277b84f6" +checksum = "c30cc8012e9eedcb48bbe112c6eff4ae5ed19cf3003cb0f505662e88b7014c5d" dependencies = [ "ahash", "arrow", @@ -3403,7 +3416,6 @@ dependencies = [ "hashbrown 0.14.5", "indexmap 2.12.0", "itertools 0.14.0", - "log", "parking_lot", "paste", "petgraph 0.8.3", @@ -3411,9 +3423,9 @@ dependencies = [ [[package]] name = "datafusion-physical-expr-adapter" -version = "50.3.0" +version = "51.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "815acced725d30601b397e39958e0e55630e0a10d66ef7769c14ae6597298bb0" +checksum = "7f9ff2dbd476221b1f67337699eff432781c4e6e1713d2aefdaa517dfbf79768" dependencies = [ "arrow", "datafusion-common", @@ -3426,9 +3438,9 @@ dependencies = [ [[package]] name = "datafusion-physical-expr-common" -version = "50.3.0" +version = "51.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6652fe7b5bf87e85ed175f571745305565da2c0b599d98e697bcbedc7baa47c3" +checksum = "90da43e1ec550b172f34c87ec68161986ced70fd05c8d2a2add66eef9c276f03" dependencies = [ "ahash", "arrow", @@ -3440,9 +3452,9 @@ dependencies = [ [[package]] name = "datafusion-physical-optimizer" -version = "50.3.0" +version = "51.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49b7d623eb6162a3332b564a0907ba00895c505d101b99af78345f1acf929b5c" +checksum = "ce9804f799acd7daef3be7aaffe77c0033768ed8fdbf5fb82fc4c5f2e6bc14e6" dependencies = [ "arrow", "datafusion-common", @@ -3454,15 +3466,14 @@ dependencies = [ "datafusion-physical-plan", "datafusion-pruning", "itertools 0.14.0", - "log", "recursive", ] [[package]] name = "datafusion-physical-plan" -version = "50.3.0" +version = "51.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2f7f778a1a838dec124efb96eae6144237d546945587557c9e6936b3414558c" +checksum = "0acf0ad6b6924c6b1aa7d213b181e012e2d3ec0a64ff5b10ee6282ab0f8532ac" dependencies = [ "ahash", "arrow", @@ -3491,12 +3502,11 @@ dependencies = [ [[package]] name = "datafusion-pruning" -version = "50.3.0" +version = "51.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd1e59e2ca14fe3c30f141600b10ad8815e2856caa59ebbd0e3e07cd3d127a65" +checksum = "ac2c2498a1f134a9e11a9f5ed202a2a7d7e9774bd9249295593053ea3be999db" dependencies = [ "arrow", - "arrow-schema", "datafusion-common", "datafusion-datasource", "datafusion-expr-common", @@ -3509,36 +3519,27 @@ dependencies = [ [[package]] name = "datafusion-session" -version = "50.3.0" +version = "51.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "21ef8e2745583619bd7a49474e8f45fbe98ebb31a133f27802217125a7b3d58d" +checksum = "8f96eebd17555386f459037c65ab73aae8df09f464524c709d6a3134ad4f4776" dependencies = [ - "arrow", "async-trait", - "dashmap", "datafusion-common", - "datafusion-common-runtime", "datafusion-execution", "datafusion-expr", - "datafusion-physical-expr", "datafusion-physical-plan", - "datafusion-sql", - "futures", - "itertools 0.14.0", - "log", - "object_store", "parking_lot", - "tokio", ] [[package]] name = "datafusion-sql" -version = "50.3.0" +version = "51.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89abd9868770386fede29e5a4b14f49c0bf48d652c3b9d7a8a0332329b87d50b" +checksum = "3fc195fe60634b2c6ccfd131b487de46dc30eccae8a3c35a13f136e7f440414f" dependencies = [ "arrow", "bigdecimal", + "chrono", "datafusion-common", "datafusion-expr", "indexmap 2.12.0", @@ -3550,9 +3551,9 @@ dependencies = [ [[package]] name = "datafusion-tracing" -version = "50.0.2" +version = "51.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "255d381dd658f367650c18454e5393fe25b897a58026949828f56b2155335276" +checksum = "6c3343373b8c20944edd104e07e7cc6734cdcaefa796979d19ad5e58b1da6dca" dependencies = [ "comfy-table", "datafusion", @@ -3854,6 +3855,15 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "aac81fa3e28d21450aa4d2ac065992ba96a1d7303efbce51a95f4fd175b67562" +[[package]] +name = "document-features" +version = "0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4b8a88685455ed29a21542a33abd9cb6510b6b129abadabdcef0f4c55bc8f61" +dependencies = [ + "litrs", +] + [[package]] name = "dotenvy" version = "0.15.7" @@ -5553,6 +5563,12 @@ version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6373607a59f0be73a39b6fe456b8192fcc3585f602af20751600e974dd455e77" +[[package]] +name = "litrs" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11d3d7f243d5c5a8b9bb5d6dd2b1602c0cb0b9db1621bafc7ed66e35ff9fe092" + [[package]] name = "lmdb-master-sys" version = "0.2.5" @@ -5903,20 +5919,6 @@ dependencies = [ "windows-sys 0.61.2", ] -[[package]] -name = "num" -version = "0.4.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "35bd024e8b2ff75562e5f34e7f4905839deb4b22955ef5e73d2fea1b9813cb23" -dependencies = [ - "num-bigint", - "num-complex", - "num-integer", - "num-iter", - "num-rational", - "num-traits", -] - [[package]] name = "num-bigint" version = "0.4.6" @@ -5978,17 +5980,6 @@ dependencies = [ "num-traits", ] -[[package]] -name = "num-rational" -version = "0.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f83d14da390562dca69fc84082e73e548e1ad308d24accdedd2720017cb37824" -dependencies = [ - "num-bigint", - "num-integer", - "num-traits", -] - [[package]] name = "num-traits" version = "0.2.19" @@ -6383,9 +6374,9 @@ dependencies = [ [[package]] name = "parquet" -version = "56.2.0" +version = "57.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0dbd48ad52d7dccf8ea1b90a3ddbfaea4f69878dd7683e51c507d4bc52b5b27" +checksum = "7a0f31027ef1af7549f7cec603a9a21dce706d3f8d7c2060a68f43c1773be95a" dependencies = [ "ahash", "arrow-array", @@ -6404,11 +6395,11 @@ dependencies = [ "half", "hashbrown 0.16.0", "lz4_flex", - "num", "num-bigint", + "num-integer", + "num-traits", "object_store", "paste", - "ring", "seq-macro", "simdutf8", "snap", @@ -6963,7 +6954,7 @@ dependencies = [ "petgraph 0.7.1", "prettyplease", "prost 0.13.5", - "prost-types", + "prost-types 0.13.5", "regex", "syn 2.0.110", "tempfile", @@ -7004,6 +6995,15 @@ dependencies = [ "prost 0.13.5", ] +[[package]] +name = "prost-types" +version = "0.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9b4db3d6da204ed77bb26ba83b6122a73aeb2e87e25fbf7ad2e84c4ccbf8f72" +dependencies = [ + "prost 0.14.1", +] + [[package]] name = "psm" version = "0.1.28" @@ -7304,6 +7304,12 @@ dependencies = [ "memchr", ] +[[package]] +name = "relative-path" +version = "1.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba39f3699c378cd8970968dcbff9c43159ea4cfbd88d43c00b22f2ef10a435d2" + [[package]] name = "reqwest" version = "0.12.24" @@ -7407,6 +7413,35 @@ dependencies = [ "zeroize", ] +[[package]] +name = "rstest" +version = "0.26.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f5a3193c063baaa2a95a33f03035c8a72b83d97a54916055ba22d35ed3839d49" +dependencies = [ + "futures-timer", + "futures-util", + "rstest_macros", +] + +[[package]] +name = "rstest_macros" +version = "0.26.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c845311f0ff7951c5506121a9ad75aec44d083c31583b2ea5a30bcb0b0abba0" +dependencies = [ + "cfg-if", + "glob", + "proc-macro-crate", + "proc-macro2", + "quote", + "regex", + "relative-path", + "rustc_version 0.4.1", + "syn 2.0.110", + "unicode-ident", +] + [[package]] name = "ruint" version = "1.17.0" @@ -8172,9 +8207,9 @@ dependencies = [ [[package]] name = "sqlparser" -version = "0.58.0" +version = "0.59.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec4b661c54b1e4b603b37873a18c59920e4c51ea8ea2cf527d925424dbd4437c" +checksum = "4591acadbcf52f0af60eafbb2c003232b2b4cd8de5f0e9437cb8b1b59046cc0f" dependencies = [ "log", "recursive", @@ -8432,32 +8467,13 @@ version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" -[[package]] -name = "strum" -version = "0.26.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fec0f0aef304996cf250b31b5a10dee7980c85da9d759361292b8bca5a18f06" - [[package]] name = "strum" version = "0.27.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "af23d6f6c1a224baef9d3f61e287d2761385a5b88fdab4eb4c6f11aeb54c4bcf" dependencies = [ - "strum_macros 0.27.2", -] - -[[package]] -name = "strum_macros" -version = "0.26.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c6bee85a5a24955dc440386795aa378cd9cf82acd5f764469152d2270e581be" -dependencies = [ - "heck", - "proc-macro2", - "quote", - "rustversion", - "syn 2.0.110", + "strum_macros", ] [[package]] @@ -9025,8 +9041,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eb7613188ce9f7df5bfe185db26c5814347d110db17920415cf2fbcad85e7203" dependencies = [ "async-trait", + "axum 0.8.6", "base64 0.22.1", "bytes", + "h2", "http", "http-body", "http-body-util", @@ -9035,6 +9053,7 @@ dependencies = [ "hyper-util", "percent-encoding", "pin-project", + "socket2 0.6.1", "sync_wrapper", "tokio", "tokio-stream", @@ -9053,7 +9072,7 @@ dependencies = [ "prettyplease", "proc-macro2", "prost-build", - "prost-types", + "prost-types 0.13.5", "quote", "syn 2.0.110", ] diff --git a/Cargo.toml b/Cargo.toml index 0873e8b99..466de8f4b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -102,7 +102,7 @@ semver = { version = "1.0.18", features = ["serde"] } serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" serde_yaml = "0.9.34" -sqlparser = "0.58" +sqlparser = "0.59" tempfile = "3.13.0" thiserror = "2.0" tokio = { version = "1.36.0", features = [ @@ -124,10 +124,10 @@ url = { version = "2.5.0", features = ["serde"] } uuid = { version = "1.11.0", features = ["v7"] } # Datafusion and Arrow crates -arrow-flight = { version = "56", features = ["flight-sql-experimental"] } -datafusion = { version = "50", features = ["serde"] } -datafusion-tracing = { version = "50" } -datafusion-datasource = { version = "50" } +arrow-flight = { version = "57", features = ["flight-sql-experimental"] } +datafusion = { version = "51", features = ["serde"] } +datafusion-tracing = { version = "51" } +datafusion-datasource = { version = "51" } object_store = { version = "0.12", features = ["aws", "gcp", "azure"] } # Crates that should follow the version used by DataFusion and Arrow diff --git a/crates/arrow-to-postgres/Cargo.toml b/crates/arrow-to-postgres/Cargo.toml index a08d3f4b9..94cc099e6 100644 --- a/crates/arrow-to-postgres/Cargo.toml +++ b/crates/arrow-to-postgres/Cargo.toml @@ -6,8 +6,8 @@ version.workspace = true license-file.workspace = true [dependencies] -arrow-array = { version = "56", default-features = false } -arrow-schema = { version = "56" } +arrow-array = { version = "57", default-features = false } +arrow-schema = { version = "57" } bytes = "1.10.1" enum_dispatch = "0.3.13" serde_json = "1.0" diff --git a/crates/bin/ampsync/Cargo.toml b/crates/bin/ampsync/Cargo.toml index 4f6fad375..1f1753942 100644 --- a/crates/bin/ampsync/Cargo.toml +++ b/crates/bin/ampsync/Cargo.toml @@ -21,8 +21,8 @@ default = [] amp-client = { path = "../../client", features = ["postgres"] } ampctl = { path = "../ampctl" } anyhow.workspace = true -arrow-array = { version = "56", default-features = false } -arrow-schema = { version = "56" } +arrow-array = { version = "57", default-features = false } +arrow-schema = { version = "57" } arrow-to-postgres = { path = "../../arrow-to-postgres" } backon.workspace = true bytes = "1.10.1" diff --git a/crates/core/common/src/catalog/physical.rs b/crates/core/common/src/catalog/physical.rs index 97ac04b0c..d7439647e 100644 --- a/crates/core/common/src/catalog/physical.rs +++ b/crates/core/common/src/catalog/physical.rs @@ -786,7 +786,7 @@ impl TableProvider for TableSnapshot { .with_file_groups(file_groups) .with_limit(limit) .with_output_ordering(output_ordering) - .with_projection(projection.cloned()) + .with_projection_indices(projection.cloned()) .with_statistics(statistics) .build(), ); diff --git a/crates/core/common/src/catalog/reader.rs b/crates/core/common/src/catalog/reader.rs index aaff579aa..48ca15d2f 100644 --- a/crates/core/common/src/catalog/reader.rs +++ b/crates/core/common/src/catalog/reader.rs @@ -2,7 +2,7 @@ use std::{ops::Range, sync::Arc}; use bytes::Bytes; use datafusion::{ - datasource::physical_plan::{FileMeta, ParquetFileMetrics, ParquetFileReaderFactory}, + datasource::physical_plan::{ParquetFileMetrics, ParquetFileReaderFactory}, error::{DataFusionError, Result as DataFusionResult}, parquet::{ arrow::{ @@ -14,6 +14,7 @@ use datafusion::{ }, physical_plan::metrics::ExecutionPlanMetricsSet, }; +use datafusion_datasource::PartitionedFile; use foyer::Cache; use futures::{TryFutureExt as _, future::BoxFuture}; use metadata_db::{FileId, LocationId, MetadataDb}; @@ -50,18 +51,18 @@ impl ParquetFileReaderFactory for AmpReaderFactory { fn create_reader( &self, partition_index: usize, - file_meta: FileMeta, + partitioned_file: PartitionedFile, _metadata_size_hint: Option, metrics: &ExecutionPlanMetricsSet, ) -> DataFusionResult> { - let path = file_meta.location(); + let file_meta = &partitioned_file.object_meta; + let path = &file_meta.location; let file_metrics = ParquetFileMetrics::new(partition_index, path.as_ref(), metrics); let metadata_db = self.metadata_db.clone(); let store = Arc::clone(&self.object_store); - let inner = ParquetObjectReader::new(store, path.clone()) - .with_file_size(file_meta.object_meta.size); + let inner = ParquetObjectReader::new(store, path.clone()).with_file_size(file_meta.size); let location_id = self.location_id; - let file_id = file_meta + let file_id = partitioned_file .extensions .ok_or(DataFusionError::Execution(format!( "FileMeta missing extensions for location_id: {}", diff --git a/crates/core/common/src/evm/udfs/eth_call.rs b/crates/core/common/src/evm/udfs/eth_call.rs index 03820717e..2a74687fc 100644 --- a/crates/core/common/src/evm/udfs/eth_call.rs +++ b/crates/core/common/src/evm/udfs/eth_call.rs @@ -70,6 +70,12 @@ impl EthCall { DataType::Utf8, ]), volatility: Volatility::Volatile, + parameter_names: Some(vec![ + "from".to_string(), + "to".to_string(), + "input_data".to_string(), + "block".to_string(), + ]), }, fields: Fields::from_iter([ Field::new("data", DataType::Binary, true), diff --git a/crates/core/common/src/evm/udfs/mod.rs b/crates/core/common/src/evm/udfs/mod.rs index 4325ffe52..502ac755c 100644 --- a/crates/core/common/src/evm/udfs/mod.rs +++ b/crates/core/common/src/evm/udfs/mod.rs @@ -611,7 +611,7 @@ fn append_sol_value_to_builder( .append_value(i64::try_from(s)?), n if n <= DEC_128_MAX_BINARY_PREC => { let val = i128::try_from(s)?; - validate_decimal_precision(val, DEC128_PREC)?; + validate_decimal_precision(val, DEC128_PREC, 0)?; let builder = builder .as_any_mut() .downcast_mut::() @@ -622,7 +622,7 @@ fn append_sol_value_to_builder( } n if n <= DEC_256_MAX_BINARY_PREC => { let val = i256::from_le_bytes(s.to_le_bytes()); - validate_decimal256_precision(val, DEC256_PREC)?; + validate_decimal256_precision(val, DEC256_PREC, 0)?; let builder = builder .as_any_mut() .downcast_mut::() @@ -671,7 +671,7 @@ fn append_sol_value_to_builder( .append_value(u64::try_from(u)?), n if n <= DEC_128_MAX_BINARY_PREC => { let val = i128::try_from(u)?; - validate_decimal_precision(val, DEC128_PREC)?; + validate_decimal_precision(val, DEC128_PREC, 0)?; let builder = builder .as_any_mut() .downcast_mut::() @@ -682,7 +682,7 @@ fn append_sol_value_to_builder( } n if n <= DEC_256_MAX_BINARY_PREC => { let val = i256::from_le_bytes(u.to_le_bytes()); - validate_decimal256_precision(val, DEC256_PREC)?; + validate_decimal256_precision(val, DEC256_PREC, 0)?; let builder = builder .as_any_mut() .downcast_mut::() diff --git a/crates/core/common/src/js_udf.rs b/crates/core/common/src/js_udf.rs index e86df903d..b692ae5ba 100644 --- a/crates/core/common/src/js_udf.rs +++ b/crates/core/common/src/js_udf.rs @@ -61,6 +61,7 @@ impl JsUdf { let signature = Signature { type_signature: TypeSignature::Exact(input_types), volatility: Volatility::Immutable, + parameter_names: None, }; // Create UDF name based on whether schema is provided diff --git a/crates/core/js-runtime/src/convert/mod.rs b/crates/core/js-runtime/src/convert/mod.rs index cf52022fe..d35ac0aa5 100644 --- a/crates/core/js-runtime/src/convert/mod.rs +++ b/crates/core/js-runtime/src/convert/mod.rs @@ -327,6 +327,8 @@ impl ToV8 for ScalarValue { | ScalarValue::DurationMicrosecond(_) | ScalarValue::DurationNanosecond(_) | ScalarValue::Union(_, _, _) + | ScalarValue::Decimal32(_, _, _) + | ScalarValue::Decimal64(_, _, _) | ScalarValue::Dictionary(_, _) => Err(BoxError::from(format!( "{} not yet supported in functions", self.data_type() From 4ef36a2e4ffb96af019a008ab81b90ae16335f84 Mon Sep 17 00:00:00 2001 From: Leonardo Yvens Date: Thu, 20 Nov 2025 18:08:17 -0300 Subject: [PATCH 2/3] upgrde prost and tonic --- Cargo.lock | 346 ++++++------------ Cargo.toml | 9 +- crates/bin/ampd/Cargo.toml | 2 +- crates/core/dump/src/compaction/plan.rs | 4 +- crates/core/dump/src/parquet_writer.rs | 4 +- crates/extractors/firehose/Cargo.toml | 3 +- crates/extractors/firehose/build.rs | 4 +- crates/extractors/firehose/src/client.rs | 1 - crates/extractors/firehose/src/lib.rs | 1 - .../firehose/src/proto/google.protobuf.rs | 17 +- .../src/proto/sf.ethereum.r#type.v2.rs | 120 +++--- .../firehose/src/proto/sf.firehose.v2.rs | 128 +++---- crates/services/server/src/flight.rs | 22 +- tests/src/testlib/fixtures/flight_client.rs | 5 +- 14 files changed, 269 insertions(+), 397 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index edee72db2..d27459e01 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -19,7 +19,7 @@ name = "admin-api" version = "0.1.0" dependencies = [ "async-trait", - "axum 0.8.6", + "axum", "common", "datafusion", "dataset-store", @@ -507,7 +507,7 @@ dependencies = [ "serde_json", "tokio", "tokio-stream", - "tower 0.5.2", + "tower", "tracing", "wasmtimer", ] @@ -553,7 +553,7 @@ dependencies = [ "serde_json", "tokio", "tokio-stream", - "tower 0.5.2", + "tower", "tracing", "url", "wasmtimer", @@ -741,7 +741,7 @@ dependencies = [ "serde_json", "thiserror 2.0.17", "tokio", - "tower 0.5.2", + "tower", "tracing", "url", "wasmtimer", @@ -756,7 +756,7 @@ dependencies = [ "alloy-transport", "reqwest", "serde_json", - "tower 0.5.2", + "tower", "tracing", "url", ] @@ -844,7 +844,7 @@ dependencies = [ "tempfile", "thiserror 2.0.17", "tokio", - "tonic 0.13.1", + "tonic", "tracing", "uuid", ] @@ -1373,9 +1373,9 @@ dependencies = [ "futures", "once_cell", "paste", - "prost 0.14.1", - "prost-types 0.14.1", - "tonic 0.14.2", + "prost", + "prost-types", + "tonic", "tonic-prost", ] @@ -1607,7 +1607,7 @@ name = "auth-http" version = "0.1.0" dependencies = [ "anyhow", - "axum 0.8.6", + "axum", "axum-extra", "chrono", "headers", @@ -1620,7 +1620,7 @@ dependencies = [ "serde_json", "thiserror 2.0.17", "tokio", - "tower 0.5.2", + "tower", "tracing", ] @@ -1677,40 +1677,13 @@ dependencies = [ "fs_extra", ] -[[package]] -name = "axum" -version = "0.7.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "edca88bc138befd0323b20752846e6587272d3b03b0343c8ea28a6f819e6e71f" -dependencies = [ - "async-trait", - "axum-core 0.4.5", - "bytes", - "futures-util", - "http", - "http-body", - "http-body-util", - "itoa", - "matchit 0.7.3", - "memchr", - "mime", - "percent-encoding", - "pin-project-lite", - "rustversion", - "serde", - "sync_wrapper", - "tower 0.5.2", - "tower-layer", - "tower-service", -] - [[package]] name = "axum" version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a18ed336352031311f4e0b4dd2ff392d4fbb370777c9d18d7fc9d7359f73871" dependencies = [ - "axum-core 0.5.5", + "axum-core", "axum-macros", "bytes", "form_urlencoded", @@ -1721,7 +1694,7 @@ dependencies = [ "hyper", "hyper-util", "itoa", - "matchit 0.8.4", + "matchit", "memchr", "mime", "percent-encoding", @@ -1732,32 +1705,12 @@ dependencies = [ "serde_urlencoded", "sync_wrapper", "tokio", - "tower 0.5.2", + "tower", "tower-layer", "tower-service", "tracing", ] -[[package]] -name = "axum-core" -version = "0.4.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09f2bd6146b97ae3359fa0cc6d6b376d9539582c7b4220f041a33ec24c226199" -dependencies = [ - "async-trait", - "bytes", - "futures-util", - "http", - "http-body", - "http-body-util", - "mime", - "pin-project-lite", - "rustversion", - "sync_wrapper", - "tower-layer", - "tower-service", -] - [[package]] name = "axum-core" version = "0.5.5" @@ -1783,8 +1736,8 @@ version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5136e6c5e7e7978fe23e9876fb924af2c0f84c72127ac6ac17e7c46f457d362c" dependencies = [ - "axum 0.8.6", - "axum-core 0.5.5", + "axum", + "axum-core", "bytes", "futures-core", "futures-util", @@ -2377,7 +2330,7 @@ dependencies = [ "alloy", "async-stream", "async-trait", - "axum 0.8.6", + "axum", "backon", "bytes", "chrono", @@ -2403,7 +2356,7 @@ dependencies = [ "tempfile", "thiserror 2.0.17", "tokio", - "tower 0.5.2", + "tower", "tracing", "url", "uuid", @@ -2446,22 +2399,23 @@ dependencies = [ [[package]] name = "console-api" -version = "0.8.1" +version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8030735ecb0d128428b64cd379809817e620a40e5001c54465b99ec5feec2857" +checksum = "e8599749b6667e2f0c910c1d0dff6901163ff698a52d5a39720f61b5be4b20d3" dependencies = [ "futures-core", - "prost 0.13.5", - "prost-types 0.13.5", - "tonic 0.12.3", + "prost", + "prost-types", + "tonic", + "tonic-prost", "tracing-core", ] [[package]] name = "console-subscriber" -version = "0.4.1" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6539aa9c6a4cd31f4b1c040f860a1eac9aa80e7df6b05d506a6e7179936d6a01" +checksum = "fb4915b7d8dd960457a1b6c380114c2944f728e7c65294ab247ae6b6f1f37592" dependencies = [ "console-api", "crossbeam-channel", @@ -2470,14 +2424,14 @@ dependencies = [ "hdrhistogram", "humantime", "hyper-util", - "prost 0.13.5", - "prost-types 0.13.5", + "prost", + "prost-types", "serde", "serde_json", "thread_local", "tokio", "tokio-stream", - "tonic 0.12.3", + "tonic", "tracing", "tracing-core", "tracing-subscriber", @@ -2553,7 +2507,7 @@ version = "0.1.0" dependencies = [ "admin-api", "async-trait", - "axum 0.8.6", + "axum", "common", "dataset-store", "dump", @@ -4057,7 +4011,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -4206,15 +4160,16 @@ dependencies = [ "futures", "hex", "monitoring", - "prost 0.13.5", + "prost", "prost-build", "schemars 1.1.0", "serde", "serde_json", "thiserror 2.0.17", "tokio", - "tonic 0.13.1", - "tonic-build", + "tonic", + "tonic-prost", + "tonic-prost-build", "tracing", ] @@ -5042,7 +4997,7 @@ dependencies = [ "libc", "percent-encoding", "pin-project-lite", - "socket2 0.6.1", + "socket2", "system-configuration", "tokio", "tower-service", @@ -5733,12 +5688,6 @@ dependencies = [ "regex-automata", ] -[[package]] -name = "matchit" -version = "0.7.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e7465ac9959cc2b1404e8e2367b43684a6d13790fe23056cc8c6c5a6b7bcb94" - [[package]] name = "matchit" version = "0.8.4" @@ -5916,7 +5865,7 @@ version = "0.50.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -6203,7 +6152,7 @@ version = "0.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5fa342b7ded339a38c6df9a3aef833298bca24a7aeec098397b4b3ec741b0e52" dependencies = [ - "axum 0.8.6", + "axum", "futures-util", "http", "http-body", @@ -6225,11 +6174,11 @@ dependencies = [ "opentelemetry-http", "opentelemetry-proto", "opentelemetry_sdk", - "prost 0.14.1", + "prost", "reqwest", "thiserror 2.0.17", "tokio", - "tonic 0.14.2", + "tonic", "tracing", ] @@ -6241,8 +6190,8 @@ checksum = "a7175df06de5eaee9909d4805a3d07e28bb752c34cab57fa9cff549da596b30f" dependencies = [ "opentelemetry", "opentelemetry_sdk", - "prost 0.14.1", - "tonic 0.14.2", + "prost", + "tonic", "tonic-prost", ] @@ -6920,16 +6869,6 @@ dependencies = [ "unarray", ] -[[package]] -name = "prost" -version = "0.13.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2796faa41db3ec313a31f7624d9286acf277b52de526150b7e69f3debf891ee5" -dependencies = [ - "bytes", - "prost-derive 0.13.5", -] - [[package]] name = "prost" version = "0.14.1" @@ -6937,14 +6876,14 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7231bd9b3d3d33c86b58adbac74b5ec0ad9f496b19d22801d773636feaa95f3d" dependencies = [ "bytes", - "prost-derive 0.14.1", + "prost-derive", ] [[package]] name = "prost-build" -version = "0.13.5" +version = "0.14.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be769465445e8c1474e9c5dac2018218498557af32d9ed057325ec9a41ae81bf" +checksum = "ac6c3320f9abac597dcbc668774ef006702672474aad53c6d596b62e487b40b1" dependencies = [ "heck", "itertools 0.14.0", @@ -6953,8 +6892,10 @@ dependencies = [ "once_cell", "petgraph 0.7.1", "prettyplease", - "prost 0.13.5", - "prost-types 0.13.5", + "prost", + "prost-types", + "pulldown-cmark", + "pulldown-cmark-to-cmark", "regex", "syn 2.0.110", "tempfile", @@ -6962,9 +6903,9 @@ dependencies = [ [[package]] name = "prost-derive" -version = "0.13.5" +version = "0.14.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a56d757972c98b346a9b766e3f02746cde6dd1cd1d1d563472929fdd74bec4d" +checksum = "9120690fafc389a67ba3803df527d0ec9cbbc9cc45e4cc20b332996dfb672425" dependencies = [ "anyhow", "itertools 0.14.0", @@ -6974,44 +6915,42 @@ dependencies = [ ] [[package]] -name = "prost-derive" +name = "prost-types" version = "0.14.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9120690fafc389a67ba3803df527d0ec9cbbc9cc45e4cc20b332996dfb672425" +checksum = "b9b4db3d6da204ed77bb26ba83b6122a73aeb2e87e25fbf7ad2e84c4ccbf8f72" dependencies = [ - "anyhow", - "itertools 0.14.0", - "proc-macro2", - "quote", - "syn 2.0.110", + "prost", ] [[package]] -name = "prost-types" -version = "0.13.5" +name = "psm" +version = "0.1.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52c2c1bf36ddb1a1c396b3601a3cec27c2462e45f07c386894ec3ccf5332bd16" +checksum = "d11f2fedc3b7dafdc2851bc52f277377c5473d378859be234bc7ebb593144d01" dependencies = [ - "prost 0.13.5", + "ar_archive_writer", + "cc", ] [[package]] -name = "prost-types" -version = "0.14.1" +name = "pulldown-cmark" +version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9b4db3d6da204ed77bb26ba83b6122a73aeb2e87e25fbf7ad2e84c4ccbf8f72" +checksum = "1e8bbe1a966bd2f362681a44f6edce3c2310ac21e4d5067a6e7ec396297a6ea0" dependencies = [ - "prost 0.14.1", + "bitflags", + "memchr", + "unicase", ] [[package]] -name = "psm" -version = "0.1.28" +name = "pulldown-cmark-to-cmark" +version = "21.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d11f2fedc3b7dafdc2851bc52f277377c5473d378859be234bc7ebb593144d01" +checksum = "8246feae3db61428fd0bb94285c690b460e4517d83152377543ca802357785f1" dependencies = [ - "ar_archive_writer", - "cc", + "pulldown-cmark", ] [[package]] @@ -7058,7 +6997,7 @@ dependencies = [ "quinn-udp", "rustc-hash", "rustls", - "socket2 0.6.1", + "socket2", "thiserror 2.0.17", "tokio", "tracing", @@ -7095,7 +7034,7 @@ dependencies = [ "cfg_aliases", "libc", "once_cell", - "socket2 0.6.1", + "socket2", "tracing", "windows-sys 0.60.2", ] @@ -7348,7 +7287,7 @@ dependencies = [ "tokio-native-tls", "tokio-rustls", "tokio-util", - "tower 0.5.2", + "tower", "tower-http", "tower-service", "url", @@ -7529,7 +7468,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys 0.11.0", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -7997,7 +7936,7 @@ dependencies = [ "arrow-flight", "async-stream", "async-trait", - "axum 0.8.6", + "axum", "bincode 2.0.1", "bytes", "common", @@ -8008,12 +7947,12 @@ dependencies = [ "futures", "metadata-db", "monitoring", - "prost 0.13.5", + "prost", "rand 0.9.2", "serde_json", "thiserror 2.0.17", "tokio", - "tonic 0.13.1", + "tonic", "tower-http", "tracing", ] @@ -8157,16 +8096,6 @@ dependencies = [ "cmake", ] -[[package]] -name = "socket2" -version = "0.5.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e22376abed350d73dd1cd119b57ffccad95b4e585a7cda43e286245ce23c0678" -dependencies = [ - "libc", - "windows-sys 0.52.0", -] - [[package]] name = "socket2" version = "0.6.1" @@ -8594,7 +8523,7 @@ dependencies = [ "getrandom 0.3.4", "once_cell", "rustix 1.1.2", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -8646,7 +8575,7 @@ dependencies = [ "thiserror 2.0.17", "tokio", "toml 0.8.23", - "tonic 0.13.1", + "tonic", "tracing", "url", "worker", @@ -8800,7 +8729,7 @@ dependencies = [ "parking_lot", "pin-project-lite", "signal-hook-registry", - "socket2 0.6.1", + "socket2", "tokio-macros", "tracing", "windows-sys 0.61.2", @@ -8972,68 +8901,6 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "df8b2b54733674ad286d16267dcfc7a71ed5c776e4ac7aa3c3e2561f7c637bf2" -[[package]] -name = "tonic" -version = "0.12.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "877c5b330756d856ffcc4553ab34a5684481ade925ecc54bcd1bf02b1d0d4d52" -dependencies = [ - "async-stream", - "async-trait", - "axum 0.7.9", - "base64 0.22.1", - "bytes", - "h2", - "http", - "http-body", - "http-body-util", - "hyper", - "hyper-timeout", - "hyper-util", - "percent-encoding", - "pin-project", - "prost 0.13.5", - "socket2 0.5.10", - "tokio", - "tokio-stream", - "tower 0.4.13", - "tower-layer", - "tower-service", - "tracing", -] - -[[package]] -name = "tonic" -version = "0.13.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e581ba15a835f4d9ea06c55ab1bd4dce26fc53752c69a04aac00703bfb49ba9" -dependencies = [ - "async-trait", - "axum 0.8.6", - "base64 0.22.1", - "bytes", - "flate2", - "h2", - "http", - "http-body", - "http-body-util", - "hyper", - "hyper-timeout", - "hyper-util", - "percent-encoding", - "pin-project", - "prost 0.13.5", - "rustls-native-certs", - "socket2 0.5.10", - "tokio", - "tokio-rustls", - "tokio-stream", - "tower 0.5.2", - "tower-layer", - "tower-service", - "tracing", -] - [[package]] name = "tonic" version = "0.14.2" @@ -9041,9 +8908,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eb7613188ce9f7df5bfe185db26c5814347d110db17920415cf2fbcad85e7203" dependencies = [ "async-trait", - "axum 0.8.6", + "axum", "base64 0.22.1", "bytes", + "flate2", "h2", "http", "http-body", @@ -9053,11 +8921,13 @@ dependencies = [ "hyper-util", "percent-encoding", "pin-project", - "socket2 0.6.1", + "rustls-native-certs", + "socket2", "sync_wrapper", "tokio", + "tokio-rustls", "tokio-stream", - "tower 0.5.2", + "tower", "tower-layer", "tower-service", "tracing", @@ -9065,14 +8935,12 @@ dependencies = [ [[package]] name = "tonic-build" -version = "0.13.1" +version = "0.14.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eac6f67be712d12f0b41328db3137e0d0757645d8904b4cb7d51cd9c2279e847" +checksum = "4c40aaccc9f9eccf2cd82ebc111adc13030d23e887244bc9cfa5d1d636049de3" dependencies = [ "prettyplease", "proc-macro2", - "prost-build", - "prost-types 0.13.5", "quote", "syn 2.0.110", ] @@ -9084,28 +8952,24 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "66bd50ad6ce1252d87ef024b3d64fe4c3cf54a86fb9ef4c631fdd0ded7aeaa67" dependencies = [ "bytes", - "prost 0.14.1", - "tonic 0.14.2", + "prost", + "tonic", ] [[package]] -name = "tower" -version = "0.4.13" +name = "tonic-prost-build" +version = "0.14.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8fa9be0de6cf49e536ce1851f987bd21a43b771b09473c3549a6c853db37c1c" +checksum = "b4a16cba4043dc3ff43fcb3f96b4c5c154c64cbd18ca8dce2ab2c6a451d058a2" dependencies = [ - "futures-core", - "futures-util", - "indexmap 1.9.3", - "pin-project", - "pin-project-lite", - "rand 0.8.5", - "slab", - "tokio", - "tokio-util", - "tower-layer", - "tower-service", - "tracing", + "prettyplease", + "proc-macro2", + "prost-build", + "prost-types", + "quote", + "syn 2.0.110", + "tempfile", + "tonic-build", ] [[package]] @@ -9144,7 +9008,7 @@ dependencies = [ "pin-project-lite", "tokio", "tokio-util", - "tower 0.5.2", + "tower", "tower-layer", "tower-service", ] @@ -9374,6 +9238,12 @@ dependencies = [ "version_check", ] +[[package]] +name = "unicase" +version = "2.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75b844d17643ee918803943289730bec8aac480150456169e647ed0b576ba539" + [[package]] name = "unicode-bidi" version = "0.3.18" @@ -9825,7 +9695,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.48.0", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 466de8f4b..ad6898d38 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -131,11 +131,12 @@ datafusion-datasource = { version = "51" } object_store = { version = "0.12", features = ["aws", "gcp", "azure"] } # Crates that should follow the version used by DataFusion and Arrow -prost = "0.13.3" -prost-build = "0.13.3" -tonic = { version = "0.13", features = [ +prost = "0.14.1" +prost-build = "0.14.1" +tonic = { version = "0.14", features = [ "transport", "gzip", "tls-native-roots", ] } -tonic-build = "0.13" +tonic-prost = "0.14" +tonic-prost-build = "0.14" diff --git a/crates/bin/ampd/Cargo.toml b/crates/bin/ampd/Cargo.toml index 82656ee6d..c8bc74d82 100644 --- a/crates/bin/ampd/Cargo.toml +++ b/crates/bin/ampd/Cargo.toml @@ -16,7 +16,7 @@ snmalloc = ["dep:snmalloc-rs"] [dependencies] clap.workspace = true common = { path = "../../core/common" } -console-subscriber = { version = "0.4.1", default-features = false, optional = true } +console-subscriber = { version = "0.5.0", default-features = false, optional = true } controller = { path = "../../services/controller" } metadata-db = { path = "../../core/metadata-db" } monitoring = { path = "../../core/monitoring" } diff --git a/crates/core/dump/src/compaction/plan.rs b/crates/core/dump/src/compaction/plan.rs index 4c65490c5..eb84bc7a9 100644 --- a/crates/core/dump/src/compaction/plan.rs +++ b/crates/core/dump/src/compaction/plan.rs @@ -22,7 +22,7 @@ use common::{ }, }; use datafusion::{ - datasource::physical_plan::{FileMeta, ParquetFileReaderFactory}, + datasource::{listing::PartitionedFile, physical_plan::ParquetFileReaderFactory}, error::DataFusionError, execution::SendableRecordBatchStream, physical_plan::{metrics::ExecutionPlanMetricsSet, stream::RecordBatchStreamAdapter}, @@ -57,7 +57,7 @@ impl CompactionFile { let file_id = segment.id; let range = segment.range.clone(); - let mut file_meta = FileMeta::from(segment.object.clone()); + let mut file_meta = PartitionedFile::from(segment.object.clone()); file_meta.extensions = Some(Arc::new(file_id)); diff --git a/crates/core/dump/src/parquet_writer.rs b/crates/core/dump/src/parquet_writer.rs index 4335d9f22..421afc675 100644 --- a/crates/core/dump/src/parquet_writer.rs +++ b/crates/core/dump/src/parquet_writer.rs @@ -11,7 +11,7 @@ use common::{ }, segments::BlockRange, }, - parquet::{arrow::AsyncArrowWriter, errors::ParquetError, format::KeyValue}, + parquet::{arrow::AsyncArrowWriter, errors::ParquetError, file::metadata::KeyValue}, }; use metadata_db::{FileId, FooterBytes, LocationId, MetadataDb}; use object_store::{ObjectMeta, buffered::BufWriter, path::Path}; @@ -146,7 +146,7 @@ impl ParquetFileWriter { self.filename, range.start(), range.end(), - meta.num_rows, + meta.file_metadata().num_rows(), ); let location = Path::from_url_path(self.file_url.path())?; diff --git a/crates/extractors/firehose/Cargo.toml b/crates/extractors/firehose/Cargo.toml index c6314acc5..3ed594c5a 100644 --- a/crates/extractors/firehose/Cargo.toml +++ b/crates/extractors/firehose/Cargo.toml @@ -21,6 +21,7 @@ serde_json = { workspace = true, optional = true } thiserror.workspace = true tokio.workspace = true tonic.workspace = true +tonic-prost.workspace = true tracing.workspace = true [dev-dependencies] @@ -30,7 +31,7 @@ serde_json.workspace = true # These dependencies are only included when the gen_proto cfg flag is enabled [target.'cfg(gen_proto)'.build-dependencies] prost-build = { workspace = true } -tonic-build = { workspace = true } +tonic-prost-build = { workspace = true } [lints.rust] # Allow the gen_proto cfg flag used for conditional protobuf code generation diff --git a/crates/extractors/firehose/build.rs b/crates/extractors/firehose/build.rs index e39f12481..b0993659d 100644 --- a/crates/extractors/firehose/build.rs +++ b/crates/extractors/firehose/build.rs @@ -8,7 +8,7 @@ fn main() -> Result<(), Box> { // These comments break doc tests, so we disable them. prost_config.disable_comments(&["google.protobuf.Timestamp", "google.protobuf.Any"]); - let config = tonic_build::configure() + let config = tonic_prost_build::configure() .build_server(false) .out_dir("src/proto"); @@ -22,7 +22,7 @@ fn main() -> Result<(), Box> { .type_attribute(".", "#[derive(serde::Serialize, serde::Deserialize)]") .emit_rerun_if_changed(false); // See https://github.com/hyperium/tonic/issues/1070 - config.compile_protos_with_config( + config.compile_with_config( prost_config, &["proto/firehose.proto", "proto/ethereum.proto"], &[""], diff --git a/crates/extractors/firehose/src/client.rs b/crates/extractors/firehose/src/client.rs index 76650d4dc..b4933f44a 100644 --- a/crates/extractors/firehose/src/client.rs +++ b/crates/extractors/firehose/src/client.rs @@ -134,7 +134,6 @@ pub struct AuthInterceptor { } impl AuthInterceptor { - #[expect(clippy::result_large_err)] pub fn new(token: Option) -> Result { Ok(AuthInterceptor { token: token diff --git a/crates/extractors/firehose/src/lib.rs b/crates/extractors/firehose/src/lib.rs index 505f48b02..e2bb1d337 100644 --- a/crates/extractors/firehose/src/lib.rs +++ b/crates/extractors/firehose/src/lib.rs @@ -10,7 +10,6 @@ pub mod dataset; mod dataset_kind; pub mod evm; pub mod metrics; -#[expect(clippy::doc_overindented_list_items)] #[expect(clippy::enum_variant_names)] mod proto; diff --git a/crates/extractors/firehose/src/proto/google.protobuf.rs b/crates/extractors/firehose/src/proto/google.protobuf.rs index 6d4927a35..7f186fcd7 100644 --- a/crates/extractors/firehose/src/proto/google.protobuf.rs +++ b/crates/extractors/firehose/src/proto/google.protobuf.rs @@ -1,6 +1,6 @@ // This file is @generated by prost-build. #[derive(serde::Serialize, serde::Deserialize)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct Any { /// A URL/resource name that uniquely identifies the type of the serialized /// protocol buffer message. This string must contain at least @@ -15,13 +15,13 @@ pub struct Any { /// server that maps type URLs to message definitions as follows: /// /// * If no scheme is provided, `https` is assumed. - /// * An HTTP GET on the URL must yield a [google.protobuf.Type][] - /// value in binary format, or produce an error. + /// * An HTTP GET on the URL must yield a \[google.protobuf.Type\]\[\] + /// value in binary format, or produce an error. /// * Applications are allowed to cache lookup results based on the - /// URL, or have them precompiled into a binary to avoid any - /// lookup. Therefore, binary compatibility needs to be preserved - /// on changes to types. (Use versioned type names to manage - /// breaking changes.) + /// URL, or have them precompiled into a binary to avoid any + /// lookup. Therefore, binary compatibility needs to be preserved + /// on changes to types. (Use versioned type names to manage + /// breaking changes.) /// /// Note: this functionality is not currently available in the official /// protobuf release, and it is not used for type URLs beginning with @@ -30,7 +30,6 @@ pub struct Any { /// /// Schemes other than `http`, `https` (or the empty scheme) might be /// used with implementation specific semantics. - /// #[prost(string, tag = "1")] pub type_url: ::prost::alloc::string::String, /// Must be a valid serialized protocol buffer of the above specified type. @@ -38,7 +37,7 @@ pub struct Any { pub value: ::prost::alloc::vec::Vec, } #[derive(serde::Serialize, serde::Deserialize)] -#[derive(Clone, Copy, PartialEq, ::prost::Message)] +#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] pub struct Timestamp { /// Represents seconds of UTC time since Unix epoch /// 1970-01-01T00:00:00Z. Must be from 0001-01-01T00:00:00Z to diff --git a/crates/extractors/firehose/src/proto/sf.ethereum.r#type.v2.rs b/crates/extractors/firehose/src/proto/sf.ethereum.r#type.v2.rs index 001a7ba30..cb596a2d2 100644 --- a/crates/extractors/firehose/src/proto/sf.ethereum.r#type.v2.rs +++ b/crates/extractors/firehose/src/proto/sf.ethereum.r#type.v2.rs @@ -169,8 +169,7 @@ pub mod block { pub struct BlockHeader { #[prost(bytes = "vec", tag = "1")] pub parent_hash: ::prost::alloc::vec::Vec, - /// Uncle hash of the block, some reference it as `sha3Uncles`, but `sha3`` is badly worded, so we prefer `uncle_hash`, also - /// referred as `ommers` in EIP specification. + /// Uncle hash of the block, some reference it as `sha3Uncles`, but ```sha3`` is badly worded, so we prefer ```uncle_hash`, also referred as `ommers\` in EIP specification. /// /// If the Block containing this `BlockHeader` has been produced using the Proof of Stake /// consensus algorithm, this field will actually be constant and set to `0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347`. @@ -215,7 +214,7 @@ pub struct BlockHeader { /// forks are using bigger values to carry special consensus data. /// /// If the Block containing this `BlockHeader` has been produced using the Proof of Stake - /// consensus algorithm, this field is strictly enforced to be <= 32 bytes. + /// consensus algorithm, this field is strictly enforced to be \<= 32 bytes. #[prost(bytes = "vec", tag = "13")] pub extra_data: ::prost::alloc::vec::Vec, /// MixHash is used to prove, when combined with the `nonce` that sufficient amount of computation has been @@ -231,29 +230,28 @@ pub struct BlockHeader { pub nonce: u64, /// Hash is the hash of the block which is actually the computation: /// - /// Keccak256(rlp([ - /// parent_hash, - /// uncle_hash, - /// coinbase, - /// state_root, - /// transactions_root, - /// receipt_root, - /// logs_bloom, - /// difficulty, - /// number, - /// gas_limit, - /// gas_used, - /// timestamp, - /// extra_data, - /// mix_hash, - /// nonce, - /// base_fee_per_gas (to be included only if London fork is active) - /// withdrawals_root (to be included only if Shangai fork is active) - /// blob_gas_used (to be included only if Cancun fork is active) - /// excess_blob_gas (to be included only if Cancun fork is active) - /// parent_beacon_root (to be included only if Cancun fork is active) - /// ])) - /// + /// Keccak256(rlp(\[ + /// parent_hash, + /// uncle_hash, + /// coinbase, + /// state_root, + /// transactions_root, + /// receipt_root, + /// logs_bloom, + /// difficulty, + /// number, + /// gas_limit, + /// gas_used, + /// timestamp, + /// extra_data, + /// mix_hash, + /// nonce, + /// base_fee_per_gas (to be included only if London fork is active) + /// withdrawals_root (to be included only if Shangai fork is active) + /// blob_gas_used (to be included only if Cancun fork is active) + /// excess_blob_gas (to be included only if Cancun fork is active) + /// parent_beacon_root (to be included only if Cancun fork is active) + /// \])) #[prost(bytes = "vec", tag = "16")] pub hash: ::prost::alloc::vec::Vec, /// Base fee per gas according to EIP-1559 (e.g. London Fork) rules, only set if London is present/active on the chain. @@ -291,13 +289,13 @@ pub struct Uint64NestedArray { pub val: ::prost::alloc::vec::Vec, } #[derive(serde::Serialize, serde::Deserialize)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct Uint64Array { #[prost(uint64, repeated, tag = "1")] pub val: ::prost::alloc::vec::Vec, } #[derive(serde::Serialize, serde::Deserialize)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct BigInt { #[prost(bytes = "vec", tag = "1")] pub bytes: ::prost::alloc::vec::Vec, @@ -423,7 +421,7 @@ pub struct TransactionTrace { /// ## Explanation relevant only for blocks with `DetailLevel: EXTENDED` /// /// A successful transaction has been recorded to the blockchain's state for calls in it that were successful. - /// This means it's possible only a subset of the calls were properly recorded, refer to \[calls[\].state_reverted] field + /// This means it's possible only a subset of the calls were properly recorded, refer to \[calls\[\].state_reverted\] field /// to determine which calls were reverted. /// /// A quirks of the Ethereum protocol is that a transaction `FAILED` or `REVERTED` still affects the blockchain's @@ -573,7 +571,7 @@ pub mod transaction_trace { /// AccessTuple represents a list of storage keys for a given contract's address and is used /// for AccessList construction. #[derive(serde::Serialize, serde::Deserialize)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct AccessTuple { #[prost(bytes = "vec", tag = "1")] pub address: ::prost::alloc::vec::Vec, @@ -585,12 +583,12 @@ pub struct AccessTuple { pub struct TransactionReceipt { /// State root is an intermediate state_root hash, computed in-between transactions to make /// **sure** you could build a proof and point to state in the middle of a block. Geth client - /// uses `PostState + root + PostStateOrStatus`` while Parity used `status_code, root...`` this piles + /// uses ```PostState + root + PostStateOrStatus`` while Parity used ```status_code, root...\`` this piles /// hardforks, see (read the EIPs first): - /// - /// - /// Moreover, the notion of `Outcome`` in parity, which segregates the two concepts, which are - /// stored in the same field `status_code`` can be computed based on such a hack of the `state_root` + /// * + /// + /// Moreover, the notion of ```Outcome`` in parity, which segregates the two concepts, which are stored in the same field ```status_code\`` can be computed based on such a hack of the `state_root` /// field, following `EIP-658`. /// /// Before Byzantinium hard fork, this field is always empty. @@ -621,7 +619,7 @@ pub struct TransactionReceipt { pub blob_gas_price: ::core::option::Option, } #[derive(serde::Serialize, serde::Deserialize)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct Log { #[prost(bytes = "vec", tag = "1")] pub address: ::prost::alloc::vec::Vec, @@ -642,8 +640,9 @@ pub struct Log { /// that emitted the log has been reverted by the chain. /// /// Currently, there is two locations where a Log can be obtained: - /// - block.transaction_traces\[\].receipt.logs\[\] - /// - block.transaction_traces\[\].calls\[\].logs\[\] + /// + /// * block.transaction_traces\[\].receipt.logs\[\] + /// * block.transaction_traces\[\].calls\[\].logs\[\] /// /// In the `receipt` case, the logs will be populated only when the call /// that emitted them has not been reverted by the chain and when in this @@ -706,9 +705,10 @@ pub struct Call { #[prost(message, repeated, tag = "28")] pub gas_changes: ::prost::alloc::vec::Vec, /// In Ethereum, a call can be either: - /// - Successfull, execution passes without any problem encountered - /// - Failed, execution failed, and remaining gas should be consumed - /// - Reverted, execution failed, but only gas consumed so far is billed, remaining gas is refunded + /// + /// * Successfull, execution passes without any problem encountered + /// * Failed, execution failed, and remaining gas should be consumed + /// * Reverted, execution failed, but only gas consumed so far is billed, remaining gas is refunded /// /// When a call is either `failed` or `reverted`, the `status_failed` field /// below is set to `true`. If the status is `reverted`, then both `status_failed` @@ -731,13 +731,13 @@ pub struct Call { /// has a status of `SUCCESS`, the chain might have reverted all the state /// changes it performed. /// - /// ```text - /// Trx 1 - /// Call #1 - /// Call #2 - /// Call #3 - /// |--- Failure here - /// Call #4 + /// ```text,text + /// Trx 1 + /// Call #1 + /// Call #2 + /// Call #3 + /// |--- Failure here + /// Call #4 /// ``` /// /// In the transaction above, while Call #2 and Call #3 would have the @@ -764,7 +764,7 @@ pub struct Call { pub account_creations: ::prost::alloc::vec::Vec, } #[derive(serde::Serialize, serde::Deserialize)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct StorageChange { #[prost(bytes = "vec", tag = "1")] pub address: ::prost::alloc::vec::Vec, @@ -780,7 +780,7 @@ pub struct StorageChange { pub ordinal: u64, } #[derive(serde::Serialize, serde::Deserialize)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct BalanceChange { /// Address is the address of the account that has changed balance. #[prost(bytes = "vec", tag = "1")] @@ -790,7 +790,8 @@ pub struct BalanceChange { /// It is safe in those case(s) to consider the balance as being 0. /// /// If you consume this from a Substreams, you can safely use: - /// ```rust,ignore + /// + /// ```text,rust,ignore /// let old_value = old_value.unwrap_or_default(); /// ``` #[prost(message, optional, tag = "2")] @@ -801,7 +802,8 @@ pub struct BalanceChange { /// 0. /// /// If you consume this from a Substreams, you can safely use: - /// ```rust,ignore + /// + /// ```text,rust,ignore /// let new_value = new_value.unwrap_or_default(); /// ``` #[prost(message, optional, tag = "3")] @@ -819,7 +821,7 @@ pub struct BalanceChange { pub mod balance_change { /// Obtain all balanche change reasons under deep mind repository: /// - /// ```shell + /// ```text,shell /// ack -ho 'BalanceChangeReason\(".*"\)' | grep -Eo '".*"' | sort | uniq /// ``` #[derive(serde::Serialize, serde::Deserialize)] @@ -916,7 +918,7 @@ pub mod balance_change { } } #[derive(serde::Serialize, serde::Deserialize)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct NonceChange { #[prost(bytes = "vec", tag = "1")] pub address: ::prost::alloc::vec::Vec, @@ -930,7 +932,7 @@ pub struct NonceChange { pub ordinal: u64, } #[derive(serde::Serialize, serde::Deserialize)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct AccountCreation { #[prost(bytes = "vec", tag = "1")] pub account: ::prost::alloc::vec::Vec, @@ -940,7 +942,7 @@ pub struct AccountCreation { pub ordinal: u64, } #[derive(serde::Serialize, serde::Deserialize)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct CodeChange { #[prost(bytes = "vec", tag = "1")] pub address: ::prost::alloc::vec::Vec, @@ -964,7 +966,7 @@ pub struct CodeChange { /// Hence, we only index some of them, those that are costy like all the calls /// one, log events, return data, etc. #[derive(serde::Serialize, serde::Deserialize)] -#[derive(Clone, Copy, PartialEq, ::prost::Message)] +#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] pub struct GasChange { #[prost(uint64, tag = "1")] pub old_value: u64, @@ -981,7 +983,7 @@ pub struct GasChange { pub mod gas_change { /// Obtain all gas change reasons under deep mind repository: /// - /// ```shell + /// ```text,shell /// ack -ho 'GasChangeReason\(".*"\)' | grep -Eo '".*"' | sort | uniq /// ``` #[derive(serde::Serialize, serde::Deserialize)] @@ -1176,13 +1178,13 @@ pub struct TransactionTraceWithBlockRef { pub block_ref: ::core::option::Option, } #[derive(serde::Serialize, serde::Deserialize)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct TransactionRefs { #[prost(bytes = "vec", repeated, tag = "1")] pub hashes: ::prost::alloc::vec::Vec<::prost::alloc::vec::Vec>, } #[derive(serde::Serialize, serde::Deserialize)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct BlockRef { #[prost(bytes = "vec", tag = "1")] pub hash: ::prost::alloc::vec::Vec, diff --git a/crates/extractors/firehose/src/proto/sf.firehose.v2.rs b/crates/extractors/firehose/src/proto/sf.firehose.v2.rs index 22d86705e..8773a8057 100644 --- a/crates/extractors/firehose/src/proto/sf.firehose.v2.rs +++ b/crates/extractors/firehose/src/proto/sf.firehose.v2.rs @@ -1,6 +1,5 @@ // This file is @generated by prost-build. -#[derive(serde::Serialize, serde::Deserialize)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(serde::Serialize, serde::Deserialize, Clone, PartialEq, ::prost::Message)] pub struct SingleBlockRequest { #[prost(message, repeated, tag = "6")] pub transforms: ::prost::alloc::vec::Vec, @@ -10,15 +9,17 @@ pub struct SingleBlockRequest { /// Nested message and enum types in `SingleBlockRequest`. pub mod single_block_request { /// Get the current known canonical version of a block at with this number - #[derive(serde::Serialize, serde::Deserialize)] - #[derive(Clone, Copy, PartialEq, ::prost::Message)] + #[derive( + serde::Serialize, serde::Deserialize, Clone, Copy, PartialEq, Eq, Hash, ::prost::Message, + )] pub struct BlockNumber { #[prost(uint64, tag = "1")] pub num: u64, } /// Get the current block with specific hash and number - #[derive(serde::Serialize, serde::Deserialize)] - #[derive(Clone, PartialEq, ::prost::Message)] + #[derive( + serde::Serialize, serde::Deserialize, Clone, PartialEq, Eq, Hash, ::prost::Message, + )] pub struct BlockHashAndNumber { #[prost(uint64, tag = "1")] pub num: u64, @@ -26,14 +27,14 @@ pub mod single_block_request { pub hash: ::prost::alloc::string::String, } /// Get the block that generated a specific cursor - #[derive(serde::Serialize, serde::Deserialize)] - #[derive(Clone, PartialEq, ::prost::Message)] + #[derive( + serde::Serialize, serde::Deserialize, Clone, PartialEq, Eq, Hash, ::prost::Message, + )] pub struct Cursor { #[prost(string, tag = "1")] pub cursor: ::prost::alloc::string::String, } - #[derive(serde::Serialize, serde::Deserialize)] - #[derive(Clone, PartialEq, ::prost::Oneof)] + #[derive(serde::Serialize, serde::Deserialize, Clone, PartialEq, Eq, Hash, ::prost::Oneof)] pub enum Reference { #[prost(message, tag = "3")] BlockNumber(BlockNumber), @@ -43,14 +44,12 @@ pub mod single_block_request { Cursor(Cursor), } } -#[derive(serde::Serialize, serde::Deserialize)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(serde::Serialize, serde::Deserialize, Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct SingleBlockResponse { #[prost(message, optional, tag = "1")] pub block: ::core::option::Option, } -#[derive(serde::Serialize, serde::Deserialize)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(serde::Serialize, serde::Deserialize, Clone, PartialEq, ::prost::Message)] pub struct Request { /// Controls where the stream of blocks will start. /// @@ -62,7 +61,7 @@ pub struct Request { /// /// Can be negative, will be resolved relative to the chain head block, assuming /// a chain at head block #100, then using `-50` as the value will start at block - /// #50. If it resolves before first streamable block of chain, we assume start + /// \#50. If it resolves before first streamable block of chain, we assume start /// of chain. /// /// If `start_cursor` is given, this value is ignored and the stream instead starts @@ -90,13 +89,13 @@ pub struct Request { #[prost(message, repeated, tag = "10")] pub transforms: ::prost::alloc::vec::Vec, } -#[derive(serde::Serialize, serde::Deserialize)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(serde::Serialize, serde::Deserialize, Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct Response { /// Chain specific block payload, ex: - /// - sf.eosio.type.v1.Block - /// - sf.ethereum.type.v1.Block - /// - sf.near.type.v1.Block + /// + /// * sf.eosio.type.v1.Block + /// * sf.ethereum.type.v1.Block + /// * sf.near.type.v1.Block #[prost(message, optional, tag = "1")] pub block: ::core::option::Option, #[prost(enumeration = "ForkStep", tag = "6")] @@ -104,8 +103,19 @@ pub struct Response { #[prost(string, tag = "10")] pub cursor: ::prost::alloc::string::String, } -#[derive(serde::Serialize, serde::Deserialize)] -#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] +#[derive( + serde::Serialize, + serde::Deserialize, + Clone, + Copy, + Debug, + PartialEq, + Eq, + Hash, + PartialOrd, + Ord, + ::prost::Enumeration, +)] #[repr(i32)] pub enum ForkStep { StepUnset = 0, @@ -148,10 +158,10 @@ pub mod stream_client { dead_code, missing_docs, clippy::wildcard_imports, - clippy::let_unit_value, + clippy::let_unit_value )] - use tonic::codegen::*; use tonic::codegen::http::Uri; + use tonic::codegen::*; #[derive(Debug, Clone)] pub struct StreamClient { inner: tonic::client::Grpc, @@ -190,14 +200,13 @@ pub mod stream_client { F: tonic::service::Interceptor, T::ResponseBody: Default, T: tonic::codegen::Service< - http::Request, - Response = http::Response< - >::ResponseBody, + http::Request, + Response = http::Response< + >::ResponseBody, + >, >, - >, - , - >>::Error: Into + std::marker::Send + std::marker::Sync, + >>::Error: + Into + std::marker::Send + std::marker::Sync, { StreamClient::new(InterceptedService::new(inner, interceptor)) } @@ -239,18 +248,11 @@ pub mod stream_client { tonic::Response>, tonic::Status, > { - self.inner - .ready() - .await - .map_err(|e| { - tonic::Status::unknown( - format!("Service was not ready: {}", e.into()), - ) - })?; - let codec = tonic::codec::ProstCodec::default(); - let path = http::uri::PathAndQuery::from_static( - "/sf.firehose.v2.Stream/Blocks", - ); + self.inner.ready().await.map_err(|e| { + tonic::Status::unknown(format!("Service was not ready: {}", e.into())) + })?; + let codec = tonic_prost::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static("/sf.firehose.v2.Stream/Blocks"); let mut req = request.into_request(); req.extensions_mut() .insert(GrpcMethod::new("sf.firehose.v2.Stream", "Blocks")); @@ -265,10 +267,10 @@ pub mod fetch_client { dead_code, missing_docs, clippy::wildcard_imports, - clippy::let_unit_value, + clippy::let_unit_value )] - use tonic::codegen::*; use tonic::codegen::http::Uri; + use tonic::codegen::*; #[derive(Debug, Clone)] pub struct FetchClient { inner: tonic::client::Grpc, @@ -307,14 +309,13 @@ pub mod fetch_client { F: tonic::service::Interceptor, T::ResponseBody: Default, T: tonic::codegen::Service< - http::Request, - Response = http::Response< - >::ResponseBody, + http::Request, + Response = http::Response< + >::ResponseBody, + >, >, - >, - , - >>::Error: Into + std::marker::Send + std::marker::Sync, + >>::Error: + Into + std::marker::Send + std::marker::Sync, { FetchClient::new(InterceptedService::new(inner, interceptor)) } @@ -352,22 +353,13 @@ pub mod fetch_client { pub async fn block( &mut self, request: impl tonic::IntoRequest, - ) -> std::result::Result< - tonic::Response, - tonic::Status, - > { - self.inner - .ready() - .await - .map_err(|e| { - tonic::Status::unknown( - format!("Service was not ready: {}", e.into()), - ) - })?; - let codec = tonic::codec::ProstCodec::default(); - let path = http::uri::PathAndQuery::from_static( - "/sf.firehose.v2.Fetch/Block", - ); + ) -> std::result::Result, tonic::Status> + { + self.inner.ready().await.map_err(|e| { + tonic::Status::unknown(format!("Service was not ready: {}", e.into())) + })?; + let codec = tonic_prost::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static("/sf.firehose.v2.Fetch/Block"); let mut req = request.into_request(); req.extensions_mut() .insert(GrpcMethod::new("sf.firehose.v2.Fetch", "Block")); diff --git a/crates/services/server/src/flight.rs b/crates/services/server/src/flight.rs index 4bd74fb8b..26e2a30d6 100644 --- a/crates/services/server/src/flight.rs +++ b/crates/services/server/src/flight.rs @@ -24,7 +24,7 @@ use common::{ self, array::RecordBatch, datatypes::SchemaRef, - ipc::writer::{DictionaryTracker, IpcDataGenerator, IpcWriteOptions}, + ipc::writer::{CompressionContext, DictionaryTracker, IpcDataGenerator, IpcWriteOptions}, }, catalog::{ errors::{CatalogForSqlError, GetPhysicalCatalogError, PlanningCtxForSqlError}, @@ -161,7 +161,11 @@ impl Service { resume_watermark: Option, ) -> Result { let query_start_time = std::time::Instant::now(); - let schema: SchemaRef = plan.schema().as_ref().clone().into(); + let schema = { + let schema = plan.schema(); + let schema: &arrow::datatypes::Schema = schema.as_ref().as_ref(); + Arc::new(schema.clone()) + }; // If not streaming or metadata db is not available, execute once if !is_streaming { @@ -535,7 +539,7 @@ fn ipc_schema(schema: &DFSchema) -> Bytes { let ipc_opts = &Default::default(); let mut dictionary_tracker = DictionaryTracker::new(true); let encoded = IpcDataGenerator::default().schema_to_bytes_with_dictionary_tracker( - &schema.into(), + schema.as_ref(), &mut dictionary_tracker, ipc_opts, ); @@ -672,6 +676,7 @@ fn flight_data_stream(query_result_stream: QueryResultStream) -> TonicStream TonicStream { - match encode_record_batch(batch, None, &mut dictionary_tracker) { + match encode_record_batch(batch, None, &mut dictionary_tracker, &mut compression_context) { Ok(encoded) => { for message in encoded { yield Ok(message); @@ -709,6 +714,7 @@ fn flight_data_stream(query_result_stream: QueryResultStream) -> TonicStream { stream! { let mut dictionary_tracker = DictionaryTracker::new(true); + let mut compression_context = CompressionContext::default(); let mut ranges: Vec = Default::default(); let mut first_message = true; while let Some(result) = incremental_stream.next().await { @@ -734,7 +740,7 @@ fn flight_data_stream(query_result_stream: QueryResultStream) -> TonicStream { for message in encoded { yield Ok(message); @@ -754,7 +760,7 @@ fn flight_data_stream(query_result_stream: QueryResultStream) -> TonicStream { for message in encoded { yield Ok(message); @@ -776,11 +782,11 @@ fn flight_data_stream(query_result_stream: QueryResultStream) -> TonicStream, dictionary_tracker: &mut DictionaryTracker, + compression_context: &mut CompressionContext, ) -> Result, Status> { let ipc = IpcDataGenerator::default(); let options = IpcWriteOptions::default(); @@ -789,7 +795,7 @@ pub fn encode_record_batch( for batch in split_batch_for_grpc_response(batch, GRPC_TARGET_MAX_FLIGHT_SIZE_BYTES) { let (encoded_dictionaries, encoded_batch) = ipc - .encoded_batch(&batch, dictionary_tracker, &options) + .encode(&batch, dictionary_tracker, &options, compression_context) .map_err(FlightError::from)?; for encoded_dictionary in encoded_dictionaries { encoded.push(encoded_dictionary.into()); diff --git a/tests/src/testlib/fixtures/flight_client.rs b/tests/src/testlib/fixtures/flight_client.rs index 98256a080..5d1361222 100644 --- a/tests/src/testlib/fixtures/flight_client.rs +++ b/tests/src/testlib/fixtures/flight_client.rs @@ -34,7 +34,10 @@ pub struct FlightClient { impl FlightClient { /// Create a new Flight client connected to the provided Flight server URL. pub async fn new(url: impl Into) -> Result { - let flight_client = FlightServiceClient::connect(url.into()).await?; + let conn = tonic::transport::Endpoint::from_shared(url.into())? + .connect() + .await?; + let flight_client = FlightServiceClient::new(conn); let client = FlightSqlServiceClient::new_from_inner(flight_client); Ok(Self { From 42714fcf526ef052cc856e3feda7ae8fc1db7ecf Mon Sep 17 00:00:00 2001 From: Leonardo Yvens Date: Thu, 20 Nov 2025 19:48:45 -0300 Subject: [PATCH 3/3] update mds --- docs/schemas/evm-rpc.md | 140 ++++++++++++------------- docs/schemas/firehose-evm.md | 196 +++++++++++++++++------------------ 2 files changed, 168 insertions(+), 168 deletions(-) diff --git a/docs/schemas/evm-rpc.md b/docs/schemas/evm-rpc.md index 1047c6999..e462884c4 100644 --- a/docs/schemas/evm-rpc.md +++ b/docs/schemas/evm-rpc.md @@ -3,80 +3,80 @@ Auto-generated file. See `to_markdown` in `crates/core/datasets-raw/src/schema.r ## blocks ```` -+--------------------+---------------------------------------+-------------+ -| column_name | data_type | is_nullable | -+--------------------+---------------------------------------+-------------+ -| _block_num | UInt64 | NO | -| block_num | UInt64 | NO | -| timestamp | Timestamp(Nanosecond, Some("+00:00")) | NO | -| hash | FixedSizeBinary(32) | NO | -| parent_hash | FixedSizeBinary(32) | NO | -| ommers_hash | FixedSizeBinary(32) | NO | -| miner | FixedSizeBinary(20) | NO | -| state_root | FixedSizeBinary(32) | NO | -| transactions_root | FixedSizeBinary(32) | NO | -| receipt_root | FixedSizeBinary(32) | NO | -| logs_bloom | Binary | NO | -| difficulty | Decimal128(38, 0) | NO | -| total_difficulty | Decimal128(38, 0) | YES | -| gas_limit | UInt64 | NO | -| gas_used | UInt64 | NO | -| extra_data | Binary | NO | -| mix_hash | FixedSizeBinary(32) | NO | -| nonce | UInt64 | NO | -| base_fee_per_gas | Decimal128(38, 0) | YES | -| withdrawals_root | FixedSizeBinary(32) | YES | -| blob_gas_used | UInt64 | YES | -| excess_blob_gas | UInt64 | YES | -| parent_beacon_root | FixedSizeBinary(32) | YES | -+--------------------+---------------------------------------+-------------+ ++--------------------+-------------------------+-------------+ +| column_name | data_type | is_nullable | ++--------------------+-------------------------+-------------+ +| _block_num | UInt64 | NO | +| block_num | UInt64 | NO | +| timestamp | Timestamp(ns, "+00:00") | NO | +| hash | FixedSizeBinary(32) | NO | +| parent_hash | FixedSizeBinary(32) | NO | +| ommers_hash | FixedSizeBinary(32) | NO | +| miner | FixedSizeBinary(20) | NO | +| state_root | FixedSizeBinary(32) | NO | +| transactions_root | FixedSizeBinary(32) | NO | +| receipt_root | FixedSizeBinary(32) | NO | +| logs_bloom | Binary | NO | +| difficulty | Decimal128(38, 0) | NO | +| total_difficulty | Decimal128(38, 0) | YES | +| gas_limit | UInt64 | NO | +| gas_used | UInt64 | NO | +| extra_data | Binary | NO | +| mix_hash | FixedSizeBinary(32) | NO | +| nonce | UInt64 | NO | +| base_fee_per_gas | Decimal128(38, 0) | YES | +| withdrawals_root | FixedSizeBinary(32) | YES | +| blob_gas_used | UInt64 | YES | +| excess_blob_gas | UInt64 | YES | +| parent_beacon_root | FixedSizeBinary(32) | YES | ++--------------------+-------------------------+-------------+ ```` ## transactions ```` -+--------------------------+---------------------------------------+-------------+ -| column_name | data_type | is_nullable | -+--------------------------+---------------------------------------+-------------+ -| _block_num | UInt64 | NO | -| block_hash | FixedSizeBinary(32) | NO | -| block_num | UInt64 | NO | -| timestamp | Timestamp(Nanosecond, Some("+00:00")) | NO | -| tx_index | UInt32 | NO | -| tx_hash | FixedSizeBinary(32) | NO | -| to | FixedSizeBinary(20) | YES | -| nonce | UInt64 | NO | -| gas_price | Decimal128(38, 0) | YES | -| gas_limit | UInt64 | NO | -| value | Decimal128(38, 0) | NO | -| input | Binary | NO | -| v | Binary | NO | -| r | Binary | NO | -| s | Binary | NO | -| gas_used | UInt64 | NO | -| type | Int32 | NO | -| max_fee_per_gas | Decimal128(38, 0) | YES | -| max_priority_fee_per_gas | Decimal128(38, 0) | YES | -| max_fee_per_blob_gas | Decimal128(38, 0) | YES | -| from | FixedSizeBinary(20) | NO | -| status | Boolean | NO | -+--------------------------+---------------------------------------+-------------+ ++--------------------------+-------------------------+-------------+ +| column_name | data_type | is_nullable | ++--------------------------+-------------------------+-------------+ +| _block_num | UInt64 | NO | +| block_hash | FixedSizeBinary(32) | NO | +| block_num | UInt64 | NO | +| timestamp | Timestamp(ns, "+00:00") | NO | +| tx_index | UInt32 | NO | +| tx_hash | FixedSizeBinary(32) | NO | +| to | FixedSizeBinary(20) | YES | +| nonce | UInt64 | NO | +| gas_price | Decimal128(38, 0) | YES | +| gas_limit | UInt64 | NO | +| value | Decimal128(38, 0) | NO | +| input | Binary | NO | +| v | Binary | NO | +| r | Binary | NO | +| s | Binary | NO | +| gas_used | UInt64 | NO | +| type | Int32 | NO | +| max_fee_per_gas | Decimal128(38, 0) | YES | +| max_priority_fee_per_gas | Decimal128(38, 0) | YES | +| max_fee_per_blob_gas | Decimal128(38, 0) | YES | +| from | FixedSizeBinary(20) | NO | +| status | Boolean | NO | ++--------------------------+-------------------------+-------------+ ```` ## logs ```` -+-------------+---------------------------------------+-------------+ -| column_name | data_type | is_nullable | -+-------------+---------------------------------------+-------------+ -| _block_num | UInt64 | NO | -| block_hash | FixedSizeBinary(32) | NO | -| block_num | UInt64 | NO | -| timestamp | Timestamp(Nanosecond, Some("+00:00")) | NO | -| tx_hash | FixedSizeBinary(32) | NO | -| tx_index | UInt32 | NO | -| log_index | UInt32 | NO | -| address | FixedSizeBinary(20) | NO | -| topic0 | FixedSizeBinary(32) | YES | -| topic1 | FixedSizeBinary(32) | YES | -| topic2 | FixedSizeBinary(32) | YES | -| topic3 | FixedSizeBinary(32) | YES | -| data | Binary | NO | -+-------------+---------------------------------------+-------------+ ++-------------+-------------------------+-------------+ +| column_name | data_type | is_nullable | ++-------------+-------------------------+-------------+ +| _block_num | UInt64 | NO | +| block_hash | FixedSizeBinary(32) | NO | +| block_num | UInt64 | NO | +| timestamp | Timestamp(ns, "+00:00") | NO | +| tx_hash | FixedSizeBinary(32) | NO | +| tx_index | UInt32 | NO | +| log_index | UInt32 | NO | +| address | FixedSizeBinary(20) | NO | +| topic0 | FixedSizeBinary(32) | YES | +| topic1 | FixedSizeBinary(32) | YES | +| topic2 | FixedSizeBinary(32) | YES | +| topic3 | FixedSizeBinary(32) | YES | +| data | Binary | NO | ++-------------+-------------------------+-------------+ ```` diff --git a/docs/schemas/firehose-evm.md b/docs/schemas/firehose-evm.md index 85f44138f..47a070577 100644 --- a/docs/schemas/firehose-evm.md +++ b/docs/schemas/firehose-evm.md @@ -3,111 +3,111 @@ Auto-generated file. See `to_markdown` in `crates/core/datasets-raw/src/schema.r ## blocks ```` -+--------------------+---------------------------------------+-------------+ -| column_name | data_type | is_nullable | -+--------------------+---------------------------------------+-------------+ -| _block_num | UInt64 | NO | -| block_num | UInt64 | NO | -| timestamp | Timestamp(Nanosecond, Some("+00:00")) | NO | -| hash | FixedSizeBinary(32) | NO | -| parent_hash | FixedSizeBinary(32) | NO | -| ommers_hash | FixedSizeBinary(32) | NO | -| miner | FixedSizeBinary(20) | NO | -| state_root | FixedSizeBinary(32) | NO | -| transactions_root | FixedSizeBinary(32) | NO | -| receipt_root | FixedSizeBinary(32) | NO | -| logs_bloom | Binary | NO | -| difficulty | Decimal128(38, 0) | NO | -| total_difficulty | Decimal128(38, 0) | YES | -| gas_limit | UInt64 | NO | -| gas_used | UInt64 | NO | -| extra_data | Binary | NO | -| mix_hash | FixedSizeBinary(32) | NO | -| nonce | UInt64 | NO | -| base_fee_per_gas | Decimal128(38, 0) | YES | -| withdrawals_root | FixedSizeBinary(32) | YES | -| blob_gas_used | UInt64 | YES | -| excess_blob_gas | UInt64 | YES | -| parent_beacon_root | FixedSizeBinary(32) | YES | -+--------------------+---------------------------------------+-------------+ ++--------------------+-------------------------+-------------+ +| column_name | data_type | is_nullable | ++--------------------+-------------------------+-------------+ +| _block_num | UInt64 | NO | +| block_num | UInt64 | NO | +| timestamp | Timestamp(ns, "+00:00") | NO | +| hash | FixedSizeBinary(32) | NO | +| parent_hash | FixedSizeBinary(32) | NO | +| ommers_hash | FixedSizeBinary(32) | NO | +| miner | FixedSizeBinary(20) | NO | +| state_root | FixedSizeBinary(32) | NO | +| transactions_root | FixedSizeBinary(32) | NO | +| receipt_root | FixedSizeBinary(32) | NO | +| logs_bloom | Binary | NO | +| difficulty | Decimal128(38, 0) | NO | +| total_difficulty | Decimal128(38, 0) | YES | +| gas_limit | UInt64 | NO | +| gas_used | UInt64 | NO | +| extra_data | Binary | NO | +| mix_hash | FixedSizeBinary(32) | NO | +| nonce | UInt64 | NO | +| base_fee_per_gas | Decimal128(38, 0) | YES | +| withdrawals_root | FixedSizeBinary(32) | YES | +| blob_gas_used | UInt64 | YES | +| excess_blob_gas | UInt64 | YES | +| parent_beacon_root | FixedSizeBinary(32) | YES | ++--------------------+-------------------------+-------------+ ```` ## transactions ```` -+--------------------------+---------------------------------------+-------------+ -| column_name | data_type | is_nullable | -+--------------------------+---------------------------------------+-------------+ -| _block_num | UInt64 | NO | -| block_hash | FixedSizeBinary(32) | NO | -| block_num | UInt64 | NO | -| timestamp | Timestamp(Nanosecond, Some("+00:00")) | NO | -| tx_index | UInt32 | NO | -| tx_hash | FixedSizeBinary(32) | NO | -| to | FixedSizeBinary(20) | YES | -| nonce | UInt64 | NO | -| gas_price | Decimal128(38, 0) | YES | -| gas_limit | UInt64 | NO | -| value | Decimal128(38, 0) | YES | -| input | Binary | NO | -| v | Binary | NO | -| r | Binary | NO | -| s | Binary | NO | -| gas_used | UInt64 | NO | -| type | Int32 | NO | -| max_fee_per_gas | Decimal128(38, 0) | YES | -| max_priority_fee_per_gas | Decimal128(38, 0) | YES | -| from | FixedSizeBinary(20) | NO | -| status | Int32 | NO | -| return_data | Binary | NO | -| public_key | Binary | NO | -| begin_ordinal | UInt64 | NO | -| end_ordinal | UInt64 | NO | -+--------------------------+---------------------------------------+-------------+ ++--------------------------+-------------------------+-------------+ +| column_name | data_type | is_nullable | ++--------------------------+-------------------------+-------------+ +| _block_num | UInt64 | NO | +| block_hash | FixedSizeBinary(32) | NO | +| block_num | UInt64 | NO | +| timestamp | Timestamp(ns, "+00:00") | NO | +| tx_index | UInt32 | NO | +| tx_hash | FixedSizeBinary(32) | NO | +| to | FixedSizeBinary(20) | YES | +| nonce | UInt64 | NO | +| gas_price | Decimal128(38, 0) | YES | +| gas_limit | UInt64 | NO | +| value | Decimal128(38, 0) | YES | +| input | Binary | NO | +| v | Binary | NO | +| r | Binary | NO | +| s | Binary | NO | +| gas_used | UInt64 | NO | +| type | Int32 | NO | +| max_fee_per_gas | Decimal128(38, 0) | YES | +| max_priority_fee_per_gas | Decimal128(38, 0) | YES | +| from | FixedSizeBinary(20) | NO | +| status | Int32 | NO | +| return_data | Binary | NO | +| public_key | Binary | NO | +| begin_ordinal | UInt64 | NO | +| end_ordinal | UInt64 | NO | ++--------------------------+-------------------------+-------------+ ```` ## calls ```` -+---------------+---------------------------------------+-------------+ -| column_name | data_type | is_nullable | -+---------------+---------------------------------------+-------------+ -| _block_num | UInt64 | NO | -| block_hash | FixedSizeBinary(32) | NO | -| block_num | UInt64 | NO | -| timestamp | Timestamp(Nanosecond, Some("+00:00")) | NO | -| tx_index | UInt32 | NO | -| tx_hash | FixedSizeBinary(32) | NO | -| index | UInt32 | NO | -| parent_index | UInt32 | NO | -| depth | UInt32 | NO | -| call_type | Int32 | NO | -| caller | FixedSizeBinary(20) | NO | -| address | FixedSizeBinary(20) | NO | -| value | Decimal128(38, 0) | YES | -| gas_limit | UInt64 | NO | -| gas_consumed | UInt64 | NO | -| return_data | Binary | NO | -| input | Binary | NO | -| selfdestruct | Boolean | NO | -| executed_code | Boolean | NO | -| begin_ordinal | UInt64 | NO | -| end_ordinal | UInt64 | NO | -+---------------+---------------------------------------+-------------+ ++---------------+-------------------------+-------------+ +| column_name | data_type | is_nullable | ++---------------+-------------------------+-------------+ +| _block_num | UInt64 | NO | +| block_hash | FixedSizeBinary(32) | NO | +| block_num | UInt64 | NO | +| timestamp | Timestamp(ns, "+00:00") | NO | +| tx_index | UInt32 | NO | +| tx_hash | FixedSizeBinary(32) | NO | +| index | UInt32 | NO | +| parent_index | UInt32 | NO | +| depth | UInt32 | NO | +| call_type | Int32 | NO | +| caller | FixedSizeBinary(20) | NO | +| address | FixedSizeBinary(20) | NO | +| value | Decimal128(38, 0) | YES | +| gas_limit | UInt64 | NO | +| gas_consumed | UInt64 | NO | +| return_data | Binary | NO | +| input | Binary | NO | +| selfdestruct | Boolean | NO | +| executed_code | Boolean | NO | +| begin_ordinal | UInt64 | NO | +| end_ordinal | UInt64 | NO | ++---------------+-------------------------+-------------+ ```` ## logs ```` -+-------------+---------------------------------------+-------------+ -| column_name | data_type | is_nullable | -+-------------+---------------------------------------+-------------+ -| _block_num | UInt64 | NO | -| block_hash | FixedSizeBinary(32) | NO | -| block_num | UInt64 | NO | -| timestamp | Timestamp(Nanosecond, Some("+00:00")) | NO | -| tx_hash | FixedSizeBinary(32) | NO | -| tx_index | UInt32 | NO | -| log_index | UInt32 | NO | -| address | FixedSizeBinary(20) | NO | -| topic0 | FixedSizeBinary(32) | YES | -| topic1 | FixedSizeBinary(32) | YES | -| topic2 | FixedSizeBinary(32) | YES | -| topic3 | FixedSizeBinary(32) | YES | -| data | Binary | NO | -+-------------+---------------------------------------+-------------+ ++-------------+-------------------------+-------------+ +| column_name | data_type | is_nullable | ++-------------+-------------------------+-------------+ +| _block_num | UInt64 | NO | +| block_hash | FixedSizeBinary(32) | NO | +| block_num | UInt64 | NO | +| timestamp | Timestamp(ns, "+00:00") | NO | +| tx_hash | FixedSizeBinary(32) | NO | +| tx_index | UInt32 | NO | +| log_index | UInt32 | NO | +| address | FixedSizeBinary(20) | NO | +| topic0 | FixedSizeBinary(32) | YES | +| topic1 | FixedSizeBinary(32) | YES | +| topic2 | FixedSizeBinary(32) | YES | +| topic3 | FixedSizeBinary(32) | YES | +| data | Binary | NO | ++-------------+-------------------------+-------------+ ````