Update to arrow and parquet 27.0.0 (apache#4199)

* Update to arrow 27 * Update q6 expected plan * Update test for rounding * Update datafusion-cli Co-authored-by: Andrew Lamb <[email protected]>
sunchao · Nov 15, 2022 · ebb24c5 · ebb24c5
1 parent 4dcf985
commit ebb24c5
Show file tree

Hide file tree

Showing 21 changed files with 513 additions and 425 deletions.
diff --git a/benchmarks/Cargo.toml b/benchmarks/Cargo.toml
@@ -33,14 +33,14 @@ simd = ["datafusion/simd"]
 snmalloc = ["snmalloc-rs"]
 
 [dependencies]
-arrow = "26.0.0"
+arrow = "27.0.0"
 datafusion = { path = "../datafusion/core", version = "14.0.0" }
 env_logger = "0.9"
 futures = "0.3"
 mimalloc = { version = "0.1", optional = true, default-features = false }
 num_cpus = "1.13.0"
 object_store = "0.5.0"
-parquet = "26.0.0"
+parquet = "27.0.0"
 parquet-test-utils = { path = "../parquet-test-utils/", version = "0.1.0" }
 rand = "0.8.4"
 serde = { version = "1.0.136", features = ["derive"] }

diff --git a/benchmarks/expected-plans/q6.txt b/benchmarks/expected-plans/q6.txt
@@ -1,6 +1,4 @@
 Projection: SUM(lineitem.l_extendedprice * lineitem.l_discount) AS revenue
  Aggregate: groupBy=[[]], aggr=[[SUM(lineitem.l_extendedprice * lineitem.l_discount)]]
- Projection: lineitem.l_extendedprice, lineitem.l_discount
- Filter: lineitem.l_shipdate >= Date32("8766") AND lineitem.l_shipdate < Date32("9131") AND CAST(lineitem.l_discount AS Decimal128(30, 15))lineitem.l_discount >= Decimal128(Some(49999999999999),30,15) AND CAST(lineitem.l_discount AS Decimal128(30, 15))lineitem.l_discount <= Decimal128(Some(69999999999999),30,15) AND lineitem.l_quantity < Decimal128(Some(2400),15,2)
- Projection: CAST(lineitem.l_discount AS Decimal128(30, 15)) AS CAST(lineitem.l_discount AS Decimal128(30, 15))lineitem.l_discount, lineitem.l_quantity, lineitem.l_extendedprice, lineitem.l_discount, lineitem.l_shipdate
- TableScan: lineitem projection=[l_quantity, l_extendedprice, l_discount, l_shipdate]
+ Filter: lineitem.l_shipdate >= Date32("8766") AND lineitem.l_shipdate < Date32("9131") AND lineitem.l_discount >= Decimal128(Some(5),15,2) AND lineitem.l_discount <= Decimal128(Some(7),15,2) AND lineitem.l_quantity < Decimal128(Some(2400),15,2)
+ TableScan: lineitem projection=[l_quantity, l_extendedprice, l_discount, l_shipdate]
diff --git a/datafusion-cli/Cargo.lock b/datafusion-cli/Cargo.lock
diff --git a/datafusion-cli/Cargo.toml b/datafusion-cli/Cargo.toml
@@ -29,7 +29,7 @@ rust-version = "1.62"
 readme = "README.md"
 
 [dependencies]
-arrow = "26.0.0"
+arrow = "27.0.0"
 clap = { version = "3", features = ["derive", "cargo"] }
 datafusion = { path = "../datafusion/core", version = "14.0.0" }
 dirs = "4.0.0"

diff --git a/datafusion-examples/Cargo.toml b/datafusion-examples/Cargo.toml
@@ -34,8 +34,8 @@ path = "examples/avro_sql.rs"
 required-features = ["datafusion/avro"]
 
 [dev-dependencies]
-arrow = "26.0.0"
-arrow-flight = "26.0.0"
+arrow = "27.0.0"
+arrow-flight = "27.0.0"
 async-trait = "0.1.41"
 datafusion = { path = "../datafusion/core" }
 datafusion-common = { path = "../datafusion/common" }

diff --git a/datafusion/common/Cargo.toml b/datafusion/common/Cargo.toml
@@ -40,10 +40,10 @@ pyarrow = ["pyo3", "arrow/pyarrow"]
 
 [dependencies]
 apache-avro = { version = "0.14", default-features = false, features = ["snappy"], optional = true }
-arrow = { version = "26.0.0", default-features = false }
+arrow = { version = "27.0.0", default-features = false }
 chrono = { version = "0.4", default-features = false }
 cranelift-module = { version = "0.89.0", optional = true }
 object_store = { version = "0.5.0", default-features = false, optional = true }
-parquet = { version = "26.0.0", default-features = false, optional = true }
+parquet = { version = "27.0.0", default-features = false, optional = true }
 pyo3 = { version = "0.17.1", optional = true }
 sqlparser = "0.26"
diff --git a/datafusion/core/Cargo.toml b/datafusion/core/Cargo.toml
@@ -56,7 +56,7 @@ unicode_expressions = ["datafusion-physical-expr/regex_expressions", "datafusion
 [dependencies]
 ahash = { version = "0.8", default-features = false, features = ["runtime-rng"] }
 apache-avro = { version = "0.14", optional = true }
-arrow = { version = "26.0.0", features = ["prettyprint"] }
+arrow = { version = "27.0.0", features = ["prettyprint"] }
 async-compression = { version = "0.3.14", features = ["bzip2", "gzip", "futures-io", "tokio"] }
 async-trait = "0.1.41"
 bytes = "1.1"
@@ -81,7 +81,7 @@ num-traits = { version = "0.2", optional = true }
 num_cpus = "1.13.0"
 object_store = "0.5.0"
 parking_lot = "0.12"
-parquet = { version = "26.0.0", features = ["arrow", "async"] }
+parquet = { version = "27.0.0", features = ["arrow", "async"] }
 paste = "^1.0"
 percent-encoding = "2.2.0"
 pin-project-lite = "^0.2.7"
@@ -98,7 +98,7 @@ url = "2.2"
 uuid = { version = "1.0", features = ["v4"] }
 
 [dev-dependencies]
-arrow = { version = "26.0.0", features = ["prettyprint", "dyn_cmp_dict"] }
+arrow = { version = "27.0.0", features = ["prettyprint", "dyn_cmp_dict"] }
 async-trait = "0.1.53"
 criterion = "0.4"
 csv = "1.1.6"

diff --git a/datafusion/core/src/physical_plan/sorts/sort_preserving_merge.rs b/datafusion/core/src/physical_plan/sorts/sort_preserving_merge.rs
@@ -356,7 +356,7 @@ impl SortPreservingMergeStream {
  Ok(SortField::new_with_options(data_type, expr.options))
  })
  .collect::<Result<Vec<_>>>()?;
- let row_converter = RowConverter::new(sort_fields);
+ let row_converter = RowConverter::new(sort_fields)?;
 
  Ok(Self {
  schema,

diff --git a/datafusion/core/tests/sort_key_cursor.rs b/datafusion/core/tests/sort_key_cursor.rs
@@ -26,7 +26,7 @@ use datafusion::physical_plan::sorts::{RowIndex, SortKeyCursor};
 
 #[test]
 fn test_single_column() {
- let mut converter = RowConverter::new(vec![SortField::new(DataType::Int64)]);
+ let mut converter = RowConverter::new(vec![SortField::new(DataType::Int64)]).unwrap();
  let batch1 = int64_batch(vec![Some(1), Some(2), Some(5), Some(6)]);
  let batch2 = int64_batch(vec![Some(3), Some(4), Some(8), Some(9)]);
 
@@ -56,7 +56,7 @@ fn test_single_column() {
 
 #[test]
 fn test_stable_compare() {
- let mut converter = RowConverter::new(vec![SortField::new(DataType::Int64)]);
+ let mut converter = RowConverter::new(vec![SortField::new(DataType::Int64)]).unwrap();
  // Validate ties are broken by the lower stream idx to ensure stable sort
  let batch1 = int64_batch(vec![Some(3), Some(4)]);
  let batch2 = int64_batch(vec![Some(3)]);

diff --git a/datafusion/expr/Cargo.toml b/datafusion/expr/Cargo.toml
@@ -36,7 +36,7 @@ path = "src/lib.rs"
 
 [dependencies]
 ahash = { version = "0.8", default-features = false, features = ["runtime-rng"] }
-arrow = { version = "26.0.0", default-features = false }
+arrow = { version = "27.0.0", default-features = false }
 datafusion-common = { path = "../common", version = "14.0.0" }
 log = "^0.4"
 sqlparser = "0.26"
diff --git a/datafusion/jit/Cargo.toml b/datafusion/jit/Cargo.toml
@@ -36,7 +36,7 @@ path = "src/lib.rs"
 jit = []
 
 [dependencies]
-arrow = { version = "26.0.0", default-features = false }
+arrow = { version = "27.0.0", default-features = false }
 cranelift = "0.89.0"
 cranelift-jit = "0.89.0"
 cranelift-module = "0.89.0"

diff --git a/datafusion/optimizer/Cargo.toml b/datafusion/optimizer/Cargo.toml
@@ -37,7 +37,7 @@ default = ["unicode_expressions"]
 unicode_expressions = []
 
 [dependencies]
-arrow = { version = "26.0.0", features = ["prettyprint"] }
+arrow = { version = "27.0.0", features = ["prettyprint"] }
 async-trait = "0.1.41"
 chrono = { version = "0.4.23", default-features = false }
 datafusion-common = { path = "../common", version = "14.0.0" }