Backport DataFrameWriter argument changes (zero323#556)

requirements-tests-py3.txt

@@ -3,4 +3,4 @@ black
 flake8==3.7.8
 flake8-bugbear==19.8.0
 flake8-pyi==19.3.0
-pyspark>=3.2.0
+pyspark>=3.0.0dev0,<3.1.0

setup.py

@@ -25,15 +25,15 @@ def list_packages(src_path=src_path):
 setup(
  name="pyspark-stubs",
  package_dir={"": src_path},
- version="3.2.0",
+ version="3.0.0.post4",
  description="A collection of the Apache Spark stub files",
  long_description=(
  open("README.rst").read() if os.path.exists("README.rst") else ""
  ),
  url="https://github.com/zero323/pyspark-stubs",
  packages=list(list_packages()),
  package_data={"": ["*.pyi", "py.typed"]},
- install_requires=["pyspark>=3.2.0"],
+ install_requires=["pyspark>=3.0.0.dev0,<3.1.0"],
  classifiers=[
  "Programming Language :: Python :: 3 :: Only",
  "Development Status :: 4 - Beta",

third_party/3/pyspark/__init__.pyi

@@ -1,20 +1,5 @@
+# Stubs for pyspark (Python 3)
 #
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# https://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
 
 from typing import Callable, Optional, TypeVar
 
@@ -44,54 +29,6 @@ from pyspark.taskcontext import (
 # Compatiblity imports
 from pyspark.sql import SQLContext as SQLContext, HiveContext as HiveContext, Row as Row
 
-from typing import Callable, Optional, TypeVar
-
-from pyspark.accumulators import ( # noqa: F401
- Accumulator as Accumulator,
- AccumulatorParam as AccumulatorParam,
-)
-from pyspark.broadcast import Broadcast as Broadcast # noqa: F401
-from pyspark.conf import SparkConf as SparkConf # noqa: F401
-from pyspark.context import SparkContext as SparkContext # noqa: F401
-from pyspark.files import SparkFiles as SparkFiles # noqa: F401
-from pyspark.status import (
- StatusTracker as StatusTracker,
- SparkJobInfo as SparkJobInfo,
- SparkStageInfo as SparkStageInfo,
-) # noqa: F401
-from pyspark.profiler import ( # noqa: F401
- BasicProfiler as BasicProfiler,
- Profiler as Profiler,
-)
-from pyspark.rdd import RDD as RDD, RDDBarrier as RDDBarrier # noqa: F401
-from pyspark.serializers import ( # noqa: F401
- MarshalSerializer as MarshalSerializer,
- PickleSerializer as PickleSerializer,
-)
-from pyspark.status import ( # noqa: F401
- SparkJobInfo as SparkJobInfo,
- SparkStageInfo as SparkStageInfo,
- StatusTracker as StatusTracker,
-)
-from pyspark.storagelevel import StorageLevel as StorageLevel # noqa: F401
-from pyspark.taskcontext import ( # noqa: F401
- BarrierTaskContext as BarrierTaskContext,
- BarrierTaskInfo as BarrierTaskInfo,
- TaskContext as TaskContext,
-)
-from pyspark.util import (
- InheritableThread as InheritableThread, # noqa: F401
- inheritable_thread_target as inheritable_thread_target, # noqa: F401
-)
-from pyspark.version import __version__ as __version__
-
-# Compatibility imports
-from pyspark.sql import ( # noqa: F401
- SQLContext as SQLContext,
- HiveContext as HiveContext,
- Row as Row,
-)
-
 T = TypeVar("T")
 F = TypeVar("F", bound=Callable)
 
@@ -103,3 +40,8 @@ def copy_func(
  doc: Optional[str] = ...,
 ) -> F: ...
 def keyword_only(func: F) -> F: ...
+
+# Names in __all__ with no definition:
+# SparkJobInfo
+# SparkStageInfo
+# StatusTracker

third_party/3/pyspark/_globals.pyi

@@ -1,20 +1,6 @@
+# Stubs for pyspark._globals (Python 3.7)
 #
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# https://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
+# NOTE: This dynamically typed stub was automatically generated by stubgen.
 
 from typing import Any
 

third_party/3/pyspark/_typing.pyi

@@ -1,25 +1,6 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# https://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-from typing import Iterable, Sized, TypeVar, Union, Callable
+from typing import Any, Generic, Iterable, List, Optional, Sized, TypeVar, Union
 from typing_extensions import Protocol
 
-F = TypeVar("F", bound=Callable)
 T = TypeVar("T", covariant=True)
 
 PrimitiveType = Union[bool, float, int, str]
@@ -31,6 +12,3 @@ class SupportsOrdering(Protocol):
  def __le__(self, other: SupportsOrdering) -> bool: ...
 
 class SizedIterable(Protocol, Sized, Iterable[T]): ...
-
-
-

third_party/3/pyspark/accumulators.pyi

@@ -1,22 +1,7 @@
+# Stubs for pyspark.accumulators (Python 3.7)
 #
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# https://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
 
-from typing import Callable, Generic, Tuple, Type, TypeVar, Dict
+from typing import Callable, Generic, Tuple, Type, TypeVar
 
 import socketserver.BaseRequestHandler # type: ignore
 
@@ -26,8 +11,7 @@ T = TypeVar("T")
 U = TypeVar("U", bound=SupportsIAdd)
 
 import socketserver as SocketServer
-
-_accumulatorRegistry: Dict[int, Accumulator]
+from typing import Any
 
 class Accumulator(Generic[T]):
  aid: int
@@ -71,7 +55,3 @@ class AccumulatorServer(SocketServer.TCPServer):
  ) -> None: ...
  server_shutdown: bool
  def shutdown(self) -> None: ...
-
-class AccumulatorParam(Generic[T]):
- def zero(self, value: T) -> T: ...
- def addInPlace(self, value1: T, value2: T) -> T: ...

third_party/3/pyspark/broadcast.pyi

@@ -1,29 +1,11 @@
+# Stubs for pyspark.broadcast (Python 3.5)
 #
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# https://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
 
 import threading
-from typing import Any, Generic, Optional, TypeVar, Dict, Tuple, Callable
+from typing import Any, Generic, Optional, TypeVar
 
 T = TypeVar("T")
 
-_broadcastRegistry: Dict[int, Broadcast]
-
 class Broadcast(Generic[T]):
  def __init__(
  self,
@@ -33,14 +15,14 @@ class Broadcast(Generic[T]):
  path: Optional[Any] = ...,
  sock_file: Optional[Any] = ...,
  ) -> None: ...
- def dump(self, value: T, f: Any) -> None: ...
- def load_from_path(self, path: Any) -> T: ...
- def load(self, file: Any) -> T: ...
+ def dump(self, value: Any, f: Any) -> None: ...
+ def load_from_path(self, path: Any): ...
+ def load(self, file: Any): ...
  @property
  def value(self) -> T: ...
  def unpersist(self, blocking: bool = ...) -> None: ...
  def destroy(self, blocking: bool = ...) -> None: ...
- def __reduce__(self) -> Tuple[Callable[[int], T], Tuple[int]]: ...
+ def __reduce__(self): ...
 
 class BroadcastPickleRegistry(threading.local):
  def __init__(self) -> None: ...

third_party/3/pyspark/cloudpickle.pyi

@@ -1,21 +1,3 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# https://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
 import pickle
 from functools import partial as partial
 from pickle import _Pickler as Pickler # type: ignore[attr-defined]

third_party/3/pyspark/conf.pyi

@@ -1,21 +1,5 @@
+# Stubs for pyspark.conf (Python 3.5)
 #
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# https://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
 
 from typing import overload
 from typing import Any, List, Optional, Tuple