Structify ConfigOptions (apache#4517) (apache#4771)

* Structify ConfigOptions (apache#4517) * Simplify * Fix environment variables * Tweaks * Update datafusion-cli * Format * Misc cleanup * Further fixes * Format * Update config.md * Flatten configuration * Document macro
sunchao · Jan 3, 2023 · 21169b9 · 21169b9
1 parent 6c81c10
commit 21169b9
Show file tree

Hide file tree

Showing 22 changed files with 715 additions and 906 deletions.
diff --git a/benchmarks/src/bin/h2o.rs b/benchmarks/src/bin/h2o.rs
@@ -18,12 +18,13 @@
 //! DataFusion h2o benchmarks
 
 use datafusion::arrow::datatypes::{DataType, Field, Schema};
+use datafusion::config::ConfigOptions;
 use datafusion::datasource::file_format::csv::CsvFormat;
 use datafusion::datasource::listing::{
  ListingOptions, ListingTable, ListingTableConfig, ListingTableUrl,
 };
 use datafusion::datasource::MemTable;
-use datafusion::prelude::{CsvReadOptions, SessionConfig};
+use datafusion::prelude::CsvReadOptions;
 use datafusion::{arrow::util::pretty, error::Result, prelude::SessionContext};
 use std::path::PathBuf;
 use std::sync::Arc;
@@ -63,9 +64,10 @@ async fn main() -> Result<()> {
 
 async fn group_by(opt: &GroupBy) -> Result<()> {
  let path = opt.path.to_str().unwrap();
- let config = SessionConfig::from_env().with_batch_size(65535);
+ let mut config = ConfigOptions::from_env()?;
+ config.execution.batch_size = 65535;
 
- let ctx = SessionContext::with_config(config);
+ let ctx = SessionContext::with_config(config.into());
 
  let schema = Schema::new(vec![
  Field::new("id1", DataType::Utf8, false),

diff --git a/datafusion-cli/src/main.rs b/datafusion-cli/src/main.rs
@@ -96,7 +96,7 @@ pub async fn main() -> Result<()> {
  env::set_current_dir(p).unwrap();
  };
 
- let mut session_config = SessionConfig::from_env().with_information_schema(true);
+ let mut session_config = SessionConfig::from_env()?.with_information_schema(true);
 
  if let Some(batch_size) = args.batch_size {
  session_config = session_config.with_batch_size(batch_size);

diff --git a/datafusion/core/src/bin/print_config_docs.rs b/datafusion/core/src/bin/print_config_docs.rs
@@ -15,9 +15,9 @@
 // specific language governing permissions and limitations
 // under the License.
 
-use datafusion::config::BuiltInConfigs;
+use datafusion::config::ConfigOptions;
 
 fn main() {
- let docs = BuiltInConfigs::generate_config_markdown();
+ let docs = ConfigOptions::generate_config_markdown();
  println!("{docs}");
 }
diff --git a/datafusion/core/src/catalog/information_schema.rs b/datafusion/core/src/catalog/information_schema.rs
@@ -27,7 +27,7 @@ use arrow::{
  record_batch::RecordBatch,
 };
 
-use crate::config::ConfigOptions;
+use crate::config::{ConfigEntry, ConfigOptions};
 use crate::datasource::streaming::{PartitionStream, StreamingTable};
 use crate::datasource::TableProvider;
 use crate::execution::context::TaskContext;
@@ -162,8 +162,8 @@ impl InformationSchemaConfig {
  config_options: &ConfigOptions,
  builder: &mut InformationSchemaDfSettingsBuilder,
  ) {
- for (name, setting) in config_options.options() {
- builder.add_setting(name, setting.to_string());
+ for entry in config_options.entries() {
+ builder.add_setting(entry);
  }
  }
 }
@@ -611,7 +611,7 @@ impl InformationSchemaDfSettings {
  fn new(config: InformationSchemaConfig) -> Self {
  let schema = Arc::new(Schema::new(vec![
  Field::new("name", DataType::Utf8, false),
- Field::new("setting", DataType::Utf8, false),
+ Field::new("setting", DataType::Utf8, true),
  ]));
 
  Self { schema, config }
@@ -656,9 +656,9 @@ struct InformationSchemaDfSettingsBuilder {
 }
 
 impl InformationSchemaDfSettingsBuilder {
- fn add_setting(&mut self, name: impl AsRef<str>, setting: impl AsRef<str>) {
- self.names.append_value(name.as_ref());
- self.settings.append_value(setting.as_ref());
+ fn add_setting(&mut self, entry: ConfigEntry) {
+ self.names.append_value(entry.key);
+ self.settings.append_option(entry.value);
  }
 
  fn finish(&mut self) -> RecordBatch {