From dc4cb585dda86b3abba547a46cf803f633740b80 Mon Sep 17 00:00:00 2001 From: Maciej Date: Tue, 8 Sep 2020 19:36:15 +0200 Subject: [PATCH] Allow both str and StructType as schema in DataFrameReader and DataStreamReader csv / json (#522) --- third_party/3/pyspark/sql/readwriter.pyi | 4 ++-- third_party/3/pyspark/sql/streaming.pyi | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/third_party/3/pyspark/sql/readwriter.pyi b/third_party/3/pyspark/sql/readwriter.pyi index d9d6d09f..339880ee 100644 --- a/third_party/3/pyspark/sql/readwriter.pyi +++ b/third_party/3/pyspark/sql/readwriter.pyi @@ -31,7 +31,7 @@ class DataFrameReader(OptionUtils): def json( self, path: Union[str, List[str], RDD[str]], - schema: Optional[StructType] = ..., + schema: Optional[Union[StructType, str]] = ..., primitivesAsString: Optional[Union[bool, str]] = ..., prefersDecimal: Optional[Union[bool, str]] = ..., allowComments: Optional[Union[bool, str]] = ..., @@ -64,7 +64,7 @@ class DataFrameReader(OptionUtils): def csv( self, path: PathOrPaths, - schema: Optional[StructType] = ..., + schema: Optional[Union[StructType, str]] = ..., sep: Optional[str] = ..., encoding: Optional[str] = ..., quote: Optional[str] = ..., diff --git a/third_party/3/pyspark/sql/streaming.pyi b/third_party/3/pyspark/sql/streaming.pyi index 52f8943d..5ed0ac29 100644 --- a/third_party/3/pyspark/sql/streaming.pyi +++ b/third_party/3/pyspark/sql/streaming.pyi @@ -59,7 +59,7 @@ class DataStreamReader(OptionUtils): def json( self, path: str, - schema: Optional[str] = ..., + schema: Optional[Union[StructType, str]] = ..., primitivesAsString: Optional[Union[bool, str]] = ..., prefersDecimal: Optional[Union[bool, str]] = ..., allowComments: Optional[Union[bool, str]] = ..., @@ -101,7 +101,7 @@ class DataStreamReader(OptionUtils): def csv( self, path: str, - schema: Optional[StructType] = ..., + schema: Optional[Union[StructType, str]] = ..., sep: Optional[str] = ..., encoding: Optional[str] = ..., quote: Optional[str] = ...,