From 897330ae4bd8e4c29549c9a930106b7386dede47 Mon Sep 17 00:00:00 2001 From: Maciej Date: Sun, 6 Sep 2020 22:37:17 +0200 Subject: [PATCH] Disallow missing, star and unused imports (#518) - Resolves #516 - Resolves #517 --- mypy.ini | 18 ++++ third_party/3/pyspark/__init__.pyi | 50 ++++++----- third_party/3/pyspark/_typing.pyi | 2 +- third_party/3/pyspark/accumulators.pyi | 1 - third_party/3/pyspark/cloudpickle.pyi | 86 ------------------- third_party/3/pyspark/conf.pyi | 2 +- third_party/3/pyspark/context.pyi | 2 +- third_party/3/pyspark/daemon.pyi | 7 +- third_party/3/pyspark/join.pyi | 5 +- third_party/3/pyspark/ml/__init__.pyi | 9 +- third_party/3/pyspark/ml/base.pyi | 24 +++--- third_party/3/pyspark/ml/classification.pyi | 48 +++++++---- third_party/3/pyspark/ml/clustering.pyi | 26 ++++-- third_party/3/pyspark/ml/evaluation.pyi | 2 +- third_party/3/pyspark/ml/feature.pyi | 18 +++- third_party/3/pyspark/ml/fpm.pyi | 7 +- third_party/3/pyspark/ml/functions.pyi | 2 +- third_party/3/pyspark/ml/image.pyi | 2 +- third_party/3/pyspark/ml/linalg/__init__.pyi | 2 +- third_party/3/pyspark/ml/pipeline.pyi | 12 ++- third_party/3/pyspark/ml/recommendation.pyi | 20 +++-- third_party/3/pyspark/ml/regression.pyi | 42 ++++++--- third_party/3/pyspark/ml/stat.pyi | 4 +- third_party/3/pyspark/ml/tree.pyi | 23 +++-- third_party/3/pyspark/ml/tuning.pyi | 8 +- third_party/3/pyspark/ml/util.pyi | 8 +- third_party/3/pyspark/ml/wrapper.pyi | 3 +- third_party/3/pyspark/mllib/_typing.pyi | 4 +- .../3/pyspark/mllib/classification.pyi | 2 +- third_party/3/pyspark/mllib/clustering.pyi | 2 +- third_party/3/pyspark/mllib/evaluation.pyi | 2 +- third_party/3/pyspark/mllib/feature.pyi | 2 +- third_party/3/pyspark/mllib/fpm.pyi | 2 +- .../3/pyspark/mllib/linalg/distributed.pyi | 4 +- third_party/3/pyspark/mllib/random.pyi | 2 +- .../3/pyspark/mllib/recommendation.pyi | 2 +- third_party/3/pyspark/mllib/regression.pyi | 2 +- .../3/pyspark/mllib/stat/KernelDensity.pyi | 2 +- third_party/3/pyspark/mllib/stat/__init__.pyi | 23 +++-- third_party/3/pyspark/mllib/tree.pyi | 2 +- third_party/3/pyspark/mllib/util.pyi | 2 +- third_party/3/pyspark/profiler.pyi | 3 - third_party/3/pyspark/rdd.pyi | 5 +- third_party/3/pyspark/rddsampler.pyi | 2 +- third_party/3/pyspark/resource/__init__.pyi | 8 +- third_party/3/pyspark/resource/profile.pyi | 2 +- third_party/3/pyspark/resultiterable.pyi | 2 +- third_party/3/pyspark/shell.pyi | 16 ++-- third_party/3/pyspark/shuffle.pyi | 4 +- third_party/3/pyspark/sql/__init__.pyi | 27 +++--- third_party/3/pyspark/sql/_typing.pyi | 10 +-- third_party/3/pyspark/sql/avro/functions.pyi | 2 +- third_party/3/pyspark/sql/catalog.pyi | 6 +- third_party/3/pyspark/sql/column.pyi | 8 +- third_party/3/pyspark/sql/conf.pyi | 2 +- third_party/3/pyspark/sql/context.pyi | 1 - third_party/3/pyspark/sql/dataframe.pyi | 11 ++- third_party/3/pyspark/sql/functions.pyi | 15 ++-- third_party/3/pyspark/sql/group.pyi | 4 +- .../3/pyspark/sql/pandas/_typing/__init__.pyi | 3 - .../3/pyspark/sql/pandas/conversion.pyi | 24 ++++-- .../3/pyspark/sql/pandas/functions.pyi | 11 +-- .../3/pyspark/sql/pandas/group_ops.pyi | 7 +- third_party/3/pyspark/sql/pandas/map_ops.pyi | 9 +- .../3/pyspark/sql/pandas/serializers.pyi | 2 +- .../3/pyspark/sql/pandas/typehints.pyi | 2 +- third_party/3/pyspark/sql/pandas/types.pyi | 18 +++- third_party/3/pyspark/sql/readwriter.pyi | 4 +- third_party/3/pyspark/sql/session.pyi | 2 +- third_party/3/pyspark/sql/utils.pyi | 2 +- third_party/3/pyspark/sql/window.pyi | 4 - third_party/3/pyspark/status.pyi | 5 +- third_party/3/pyspark/storagelevel.pyi | 4 +- third_party/3/pyspark/streaming/__init__.pyi | 12 ++- third_party/3/pyspark/streaming/dstream.pyi | 1 - third_party/3/pyspark/taskcontext.pyi | 5 +- third_party/3/pyspark/worker.pyi | 15 +++- 77 files changed, 395 insertions(+), 349 deletions(-) delete mode 100644 third_party/3/pyspark/cloudpickle.pyi diff --git a/mypy.ini b/mypy.ini index f8b1844b..ee0bf95d 100644 --- a/mypy.ini +++ b/mypy.ini @@ -1 +1,19 @@ [mypy] + +[mypy-pyspark.cloudpickle.*] +ignore_errors = True + +[mypy-py4j.*] +ignore_missing_imports = True + +[mypy-numpy.*] +ignore_missing_imports = True + +[mypy-scipy.*] +ignore_missing_imports = True + +[mypy-pandas.*] +ignore_missing_imports = True + +[mypy-pyarrow] +ignore_missing_imports = True diff --git a/third_party/3/pyspark/__init__.pyi b/third_party/3/pyspark/__init__.pyi index 007a2799..98bd4068 100644 --- a/third_party/3/pyspark/__init__.pyi +++ b/third_party/3/pyspark/__init__.pyi @@ -16,36 +16,49 @@ # specific language governing permissions and limitations # under the License. -# Stubs for pyspark (Python 3) -# - from typing import Callable, Optional, TypeVar -from pyspark.status import * -from pyspark.accumulators import ( +from pyspark.accumulators import ( # noqa: F401 Accumulator as Accumulator, AccumulatorParam as AccumulatorParam, ) -from pyspark.broadcast import Broadcast as Broadcast -from pyspark.conf import SparkConf as SparkConf -from pyspark.context import SparkContext as SparkContext -from pyspark.files import SparkFiles as SparkFiles -from pyspark.profiler import BasicProfiler as BasicProfiler, Profiler as Profiler -from pyspark.rdd import RDD as RDD, RDDBarrier as RDDBarrier -from pyspark.serializers import ( +from pyspark.broadcast import Broadcast as Broadcast # noqa: F401 +from pyspark.conf import SparkConf as SparkConf # noqa: F401 +from pyspark.context import SparkContext as SparkContext # noqa: F401 +from pyspark.files import SparkFiles as SparkFiles # noqa: F401 +from pyspark.status import ( + StatusTracker as StatusTracker, + SparkJobInfo as SparkJobInfo, + SparkStageInfo as SparkStageInfo, +) # noqa: F401 +from pyspark.profiler import ( # noqa: F401 + BasicProfiler as BasicProfiler, + Profiler as Profiler, +) +from pyspark.rdd import RDD as RDD, RDDBarrier as RDDBarrier # noqa: F401 +from pyspark.serializers import ( # noqa: F401 MarshalSerializer as MarshalSerializer, PickleSerializer as PickleSerializer, ) -from pyspark.storagelevel import StorageLevel as StorageLevel -from pyspark.taskcontext import ( +from pyspark.status import ( # noqa: F401 + SparkJobInfo as SparkJobInfo, + SparkStageInfo as SparkStageInfo, + StatusTracker as StatusTracker, +) +from pyspark.storagelevel import StorageLevel as StorageLevel # noqa: F401 +from pyspark.taskcontext import ( # noqa: F401 BarrierTaskContext as BarrierTaskContext, BarrierTaskInfo as BarrierTaskInfo, TaskContext as TaskContext, ) -from pyspark.util import InheritableThread as InheritableThread +from pyspark.util import InheritableThread as InheritableThread # noqa: F401 # Compatiblity imports -from pyspark.sql import SQLContext as SQLContext, HiveContext as HiveContext, Row as Row +from pyspark.sql import ( # noqa: F401 + SQLContext as SQLContext, + HiveContext as HiveContext, + Row as Row, +) T = TypeVar("T") F = TypeVar("F", bound=Callable) @@ -58,8 +71,3 @@ def copy_func( doc: Optional[str] = ..., ) -> F: ... def keyword_only(func: F) -> F: ... - -# Names in __all__ with no definition: -# SparkJobInfo -# SparkStageInfo -# StatusTracker diff --git a/third_party/3/pyspark/_typing.pyi b/third_party/3/pyspark/_typing.pyi index 8aec8cd7..3f5f804b 100644 --- a/third_party/3/pyspark/_typing.pyi +++ b/third_party/3/pyspark/_typing.pyi @@ -16,7 +16,7 @@ # specific language governing permissions and limitations # under the License. -from typing import Any, Generic, Iterable, List, Optional, Sized, TypeVar, Union +from typing import Iterable, Sized, TypeVar, Union from typing_extensions import Protocol T = TypeVar("T", covariant=True) diff --git a/third_party/3/pyspark/accumulators.pyi b/third_party/3/pyspark/accumulators.pyi index 8fe35359..a44e90ab 100644 --- a/third_party/3/pyspark/accumulators.pyi +++ b/third_party/3/pyspark/accumulators.pyi @@ -29,7 +29,6 @@ T = TypeVar("T") U = TypeVar("U", bound=SupportsIAdd) import socketserver as SocketServer -from typing import Any class Accumulator(Generic[T]): aid: int diff --git a/third_party/3/pyspark/cloudpickle.pyi b/third_party/3/pyspark/cloudpickle.pyi deleted file mode 100644 index 48bbf5b4..00000000 --- a/third_party/3/pyspark/cloudpickle.pyi +++ /dev/null @@ -1,86 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -import pickle -from functools import partial as partial -from pickle import _Pickler as Pickler # type: ignore[attr-defined] -from typing import Any, Optional - -DEFAULT_PROTOCOL: Any -string_types: Any -PY3: bool -PY2: bool - -def cell_set(cell: Any, value: Any): ... - -STORE_GLOBAL: Any -DELETE_GLOBAL: Any -LOAD_GLOBAL: Any -GLOBAL_OPS: Any -HAVE_ARGUMENT: Any -EXTENDED_ARG: Any - -def islambda(func: Any): ... - -class CloudPickler(Pickler): - dispatch: Any = ... - globals_ref: Any = ... - def __init__(self, file: Any, protocol: Optional[Any] = ...) -> None: ... - def dump(self, obj: Any): ... - def save_memoryview(self, obj: Any) -> None: ... - def save_module(self, obj: Any) -> None: ... - def save_codeobject(self, obj: Any) -> None: ... - def save_function(self, obj: Any, name: Optional[Any] = ...): ... - def save_dynamic_class(self, obj: Any) -> None: ... - def save_function_tuple(self, func: Any) -> None: ... - @classmethod - def extract_code_globals(cls, co: Any): ... - def extract_func_data(self, func: Any): ... - def save_builtin_function(self, obj: Any): ... - def save_global(self, obj: Any, name: Optional[Any] = ..., pack: Any = ...): ... - def save_instancemethod(self, obj: Any) -> None: ... - def save_inst(self, obj: Any) -> None: ... - def save_property(self, obj: Any) -> None: ... - def save_classmethod(self, obj: Any) -> None: ... - def save_itemgetter(self, obj: Any): ... - attrs: Any = ... - index: Any = ... - def save_attrgetter(self, obj: Any): ... - def save_file(self, obj: Any): ... - def save_ellipsis(self, obj: Any) -> None: ... - def save_not_implemented(self, obj: Any) -> None: ... - def save_weakset(self, obj: Any) -> None: ... - def save_logger(self, obj: Any) -> None: ... - def save_root_logger(self, obj: Any) -> None: ... - def save_mappingproxy(self, obj: Any) -> None: ... - def inject_addons(self) -> None: ... - -def is_tornado_coroutine(func: Any): ... -def dump(obj: Any, file: Any, protocol: Optional[Any] = ...) -> None: ... -def dumps(obj: Any, protocol: Optional[Any] = ...): ... - -load = pickle.load -loads = pickle.loads - -def subimport(name: Any): ... -def dynamic_subimport(name: Any, vars: Any): ... -def instance(cls): ... - -class _empty_cell_value: - @classmethod - def __reduce__(cls): ... diff --git a/third_party/3/pyspark/conf.pyi b/third_party/3/pyspark/conf.pyi index 332be94d..aeaca83b 100644 --- a/third_party/3/pyspark/conf.pyi +++ b/third_party/3/pyspark/conf.pyi @@ -20,7 +20,7 @@ # from typing import overload -from typing import Any, List, Optional, Tuple +from typing import List, Optional, Tuple from py4j.java_gateway import JVMView, JavaObject # type: ignore[import] diff --git a/third_party/3/pyspark/context.pyi b/third_party/3/pyspark/context.pyi index 663d36cc..90ae5d6d 100644 --- a/third_party/3/pyspark/context.pyi +++ b/third_party/3/pyspark/context.pyi @@ -26,7 +26,7 @@ from py4j.java_gateway import JavaGateway, JavaObject # type: ignore[import] from pyspark.accumulators import Accumulator, AccumulatorParam from pyspark.broadcast import Broadcast from pyspark.conf import SparkConf -from pyspark.profiler import Profiler +from pyspark.profiler import Profiler # noqa: F401 from pyspark.resource.information import ResourceInformation from pyspark.rdd import RDD from pyspark.serializers import Serializer diff --git a/third_party/3/pyspark/daemon.pyi b/third_party/3/pyspark/daemon.pyi index 4495f6d5..dfacf30a 100644 --- a/third_party/3/pyspark/daemon.pyi +++ b/third_party/3/pyspark/daemon.pyi @@ -16,7 +16,12 @@ # specific language governing permissions and limitations # under the License. -from pyspark.serializers import UTF8Deserializer as UTF8Deserializer, read_int as read_int, write_int as write_int, write_with_length as write_with_length # type: ignore[attr-defined] +from pyspark.serializers import ( # noqa: F401 + UTF8Deserializer as UTF8Deserializer, + read_int as read_int, + write_int as write_int, + write_with_length as write_with_length, +) from typing import Any def compute_real_exit_code(exit_code: Any): ... diff --git a/third_party/3/pyspark/join.pyi b/third_party/3/pyspark/join.pyi index 0571f411..e89e0fbb 100644 --- a/third_party/3/pyspark/join.pyi +++ b/third_party/3/pyspark/join.pyi @@ -16,10 +16,7 @@ # specific language governing permissions and limitations # under the License. -# Stubs for pyspark.join (Python 3) -# - -from typing import Hashable, Iterable, List, Optional, Tuple, TypeVar +from typing import Hashable, Iterable, Optional, Tuple, TypeVar from pyspark.resultiterable import ResultIterable import pyspark.rdd diff --git a/third_party/3/pyspark/ml/__init__.pyi b/third_party/3/pyspark/ml/__init__.pyi index ab5daae3..8e3b8a5d 100644 --- a/third_party/3/pyspark/ml/__init__.pyi +++ b/third_party/3/pyspark/ml/__init__.pyi @@ -16,7 +16,7 @@ # specific language governing permissions and limitations # under the License. -from pyspark.ml import ( +from pyspark.ml import ( # noqa: F401 classification as classification, clustering as clustering, evaluation as evaluation, @@ -31,7 +31,7 @@ from pyspark.ml import ( tuning as tuning, util as util, ) -from pyspark.ml.base import ( +from pyspark.ml.base import ( # noqa: F401 Estimator as Estimator, Model as Model, PredictionModel as PredictionModel, @@ -39,4 +39,7 @@ from pyspark.ml.base import ( Transformer as Transformer, UnaryTransformer as UnaryTransformer, ) -from pyspark.ml.pipeline import Pipeline as Pipeline, PipelineModel as PipelineModel +from pyspark.ml.pipeline import ( # noqa: F401 + Pipeline as Pipeline, + PipelineModel as PipelineModel, +) diff --git a/third_party/3/pyspark/ml/base.pyi b/third_party/3/pyspark/ml/base.pyi index 108f39f5..09e76729 100644 --- a/third_party/3/pyspark/ml/base.pyi +++ b/third_party/3/pyspark/ml/base.pyi @@ -18,16 +18,13 @@ from typing import overload from typing import ( - Any, Callable, - Dict, Generic, Iterable, List, Optional, Tuple, Type, - TypeVar, ) from pyspark.ml._typing import M, P, T, ParamMap @@ -35,18 +32,25 @@ import _thread import abc from abc import abstractmethod -from pyspark.ml.common import inherit_doc as inherit_doc -from pyspark.ml.param import Params, Param -from pyspark.ml.param.shared import * -from pyspark.sql.column import Column -from pyspark.sql.dataframe import DataFrame -from pyspark.sql.functions import udf as udf -from pyspark.sql.types import ( +from pyspark import since as since # noqa: F401 +from pyspark.ml.common import inherit_doc as inherit_doc # noqa: F401 +from pyspark.ml.param.shared import ( + HasFeaturesCol as HasFeaturesCol, + HasInputCol as HasInputCol, + HasLabelCol as HasLabelCol, + HasOutputCol as HasOutputCol, + HasPredictionCol as HasPredictionCol, + Params as Params, +) +from pyspark.sql.functions import udf as udf # noqa: F401 +from pyspark.sql.types import ( # noqa: F401 DataType, StructField as StructField, StructType as StructType, ) +from pyspark.sql.dataframe import DataFrame + class _FitMultipleIterator: fitSingleModel: Callable[[int], Transformer] numModel: int diff --git a/third_party/3/pyspark/ml/classification.pyi b/third_party/3/pyspark/ml/classification.pyi index 04302204..0a61e056 100644 --- a/third_party/3/pyspark/ml/classification.pyi +++ b/third_party/3/pyspark/ml/classification.pyi @@ -16,36 +16,48 @@ # specific language governing permissions and limitations # under the License. -from typing import Any, Dict, List, Optional, Type, TypeVar +from typing import Any, List, Optional, Type from pyspark.ml._typing import JM, M, P, T, ParamMap import abc from abc import abstractmethod -from pyspark.ml.base import Estimator, Model, Transformer, PredictionModel, Predictor +from pyspark.ml import Estimator, Model, PredictionModel, Predictor, Transformer from pyspark.ml.base import _PredictorParams -from pyspark.ml.linalg import Matrix, Vector -from pyspark.ml.param.shared import * +from pyspark.ml.param.shared import ( + HasAggregationDepth, + HasBlockSize, + HasElasticNetParam, + HasFitIntercept, + HasMaxIter, + HasParallelism, + HasProbabilityCol, + HasRawPredictionCol, + HasRegParam, + HasSeed, + HasSolver, + HasStandardization, + HasStepSize, + HasThreshold, + HasThresholds, + HasTol, + HasWeightCol, +) +from pyspark.ml.regression import _FactorizationMachinesParams from pyspark.ml.tree import ( _DecisionTreeModel, _DecisionTreeParams, - _TreeEnsembleModel, - _RandomForestParams, _GBTParams, _HasVarianceImpurity, + _RandomForestParams, _TreeClassifierParams, - _TreeEnsembleParams, -) -from pyspark.ml.regression import ( - _FactorizationMachinesParams, - DecisionTreeRegressionModel, -) -from pyspark.ml.util import * -from pyspark.ml.wrapper import ( - JavaPredictionModel, - JavaPredictor, - JavaWrapper, - JavaTransformer, + _TreeEnsembleModel, ) +from pyspark.ml.util import HasTrainingSummary, JavaMLReadable, JavaMLWritable +from pyspark.ml.wrapper import JavaPredictionModel, JavaPredictor, JavaWrapper + +from pyspark.ml.linalg import Matrix, Vector +from pyspark.ml.param import Param +from pyspark.ml.regression import DecisionTreeRegressionModel from pyspark.sql.dataframe import DataFrame class _ClassifierParams(HasRawPredictionCol, _PredictorParams): ... diff --git a/third_party/3/pyspark/ml/clustering.pyi b/third_party/3/pyspark/ml/clustering.pyi index e66d1c22..730cc472 100644 --- a/third_party/3/pyspark/ml/clustering.pyi +++ b/third_party/3/pyspark/ml/clustering.pyi @@ -16,15 +16,31 @@ # specific language governing permissions and limitations # under the License. -# Stubs for pyspark.ml.clustering (Python 3) -# - from typing import Any, List, Optional from pyspark.ml.linalg import Matrix, Vector -from pyspark.ml.util import * +from pyspark.ml.util import ( + GeneralJavaMLWritable, + HasTrainingSummary, + JavaMLReadable, + JavaMLWritable, +) from pyspark.ml.wrapper import JavaEstimator, JavaModel, JavaParams, JavaWrapper -from pyspark.ml.param.shared import * +from pyspark.ml.param.shared import ( + HasAggregationDepth, + HasBlockSize, + HasCheckpointInterval, + HasDistanceMeasure, + HasFeaturesCol, + HasMaxIter, + HasPredictionCol, + HasProbabilityCol, + HasSeed, + HasTol, + HasWeightCol, +) + +from pyspark.ml.param import Param from pyspark.ml.stat import MultivariateGaussian from pyspark.sql.dataframe import DataFrame diff --git a/third_party/3/pyspark/ml/evaluation.pyi b/third_party/3/pyspark/ml/evaluation.pyi index 6f9ba736..b6bd7abf 100644 --- a/third_party/3/pyspark/ml/evaluation.pyi +++ b/third_party/3/pyspark/ml/evaluation.pyi @@ -20,7 +20,7 @@ # import abc -from typing import Any, Dict, Optional, Type +from typing import Optional, Type from pyspark.ml._typing import ( ParamMap, BinaryClassificationEvaluatorMetricType, diff --git a/third_party/3/pyspark/ml/feature.pyi b/third_party/3/pyspark/ml/feature.pyi index 7e2b385b..f5b12a5b 100644 --- a/third_party/3/pyspark/ml/feature.pyi +++ b/third_party/3/pyspark/ml/feature.pyi @@ -20,11 +20,27 @@ from typing import overload from typing import Any, Dict, Generic, List, Optional, Tuple from pyspark.ml._typing import JM, P -from pyspark.ml.param.shared import * +from pyspark.ml.param.shared import ( + HasFeaturesCol, + HasHandleInvalid, + HasInputCol, + HasInputCols, + HasLabelCol, + HasMaxIter, + HasNumFeatures, + HasOutputCol, + HasOutputCols, + HasRelativeError, + HasSeed, + HasStepSize, + HasThreshold, + HasThresholds, +) from pyspark.ml.util import JavaMLReadable, JavaMLWritable from pyspark.ml.wrapper import JavaEstimator, JavaModel, JavaParams, JavaTransformer from pyspark.ml.linalg import Vector, DenseVector, DenseMatrix from pyspark.sql.dataframe import DataFrame +from pyspark.ml.param import Param class Binarizer( JavaTransformer, diff --git a/third_party/3/pyspark/ml/fpm.pyi b/third_party/3/pyspark/ml/fpm.pyi index d17d31ec..bfb1c7c4 100644 --- a/third_party/3/pyspark/ml/fpm.pyi +++ b/third_party/3/pyspark/ml/fpm.pyi @@ -21,12 +21,13 @@ from typing import Any, Optional -from pyspark.ml._typing import P -from pyspark.ml.util import * +from pyspark.ml.util import JavaMLReadable, JavaMLWritable from pyspark.ml.wrapper import JavaEstimator, JavaParams, JavaModel -from pyspark.ml.param.shared import * +from pyspark.ml.param.shared import HasPredictionCol from pyspark.sql.dataframe import DataFrame +from pyspark.ml.param import Param + class _FPGrowthParams(HasPredictionCol): itemsCol: Param[str] minSupport: Param[float] diff --git a/third_party/3/pyspark/ml/functions.pyi b/third_party/3/pyspark/ml/functions.pyi index 3a4f7d59..42650e74 100644 --- a/third_party/3/pyspark/ml/functions.pyi +++ b/third_party/3/pyspark/ml/functions.pyi @@ -16,7 +16,7 @@ # specific language governing permissions and limitations # under the License. -from pyspark import SparkContext as SparkContext, since as since +from pyspark import SparkContext as SparkContext, since as since # noqa: F401 from pyspark.sql.column import Column as Column def vector_to_array(col: Column) -> Column: ... diff --git a/third_party/3/pyspark/ml/image.pyi b/third_party/3/pyspark/ml/image.pyi index 42cf6f41..3cee441a 100644 --- a/third_party/3/pyspark/ml/image.pyi +++ b/third_party/3/pyspark/ml/image.pyi @@ -19,7 +19,7 @@ # Stubs for pyspark.ml.image (Python 3) # -from typing import Any, Dict, List +from typing import Dict, List from pyspark.sql.types import Row, StructType diff --git a/third_party/3/pyspark/ml/linalg/__init__.pyi b/third_party/3/pyspark/ml/linalg/__init__.pyi index bd058fff..74efc27e 100644 --- a/third_party/3/pyspark/ml/linalg/__init__.pyi +++ b/third_party/3/pyspark/ml/linalg/__init__.pyi @@ -21,7 +21,7 @@ from typing import overload from typing import Any, Dict, Iterable, List, Optional, Tuple, Union -from pyspark.ml import linalg as newlinalg +from pyspark.ml import linalg as newlinalg # noqa: F401 from pyspark.sql.types import StructType, UserDefinedType from numpy import float64, ndarray # type: ignore[import] diff --git a/third_party/3/pyspark/ml/pipeline.pyi b/third_party/3/pyspark/ml/pipeline.pyi index d9beef79..bdd2cdb1 100644 --- a/third_party/3/pyspark/ml/pipeline.pyi +++ b/third_party/3/pyspark/ml/pipeline.pyi @@ -24,7 +24,17 @@ from pyspark.ml._typing import PipelineStage from pyspark.context import SparkContext from pyspark.ml.base import Estimator, Model, Transformer from pyspark.ml.param import Param -from pyspark.ml.util import * +from pyspark.ml.util import ( # noqa: F401 + DefaultParamsReader as DefaultParamsReader, + DefaultParamsWriter as DefaultParamsWriter, + JavaMLReader as JavaMLReader, + JavaMLWritable as JavaMLWritable, + JavaMLWriter as JavaMLWriter, + MLReadable as MLReadable, + MLReader as MLReader, + MLWritable as MLWritable, + MLWriter as MLWriter, +) class Pipeline(Estimator[PipelineModel], MLReadable[Pipeline], MLWritable): stages: List[PipelineStage] diff --git a/third_party/3/pyspark/ml/recommendation.pyi b/third_party/3/pyspark/ml/recommendation.pyi index 4d523999..390486b4 100644 --- a/third_party/3/pyspark/ml/recommendation.pyi +++ b/third_party/3/pyspark/ml/recommendation.pyi @@ -16,14 +16,24 @@ # specific language governing permissions and limitations # under the License. -# Stubs for pyspark.ml.recommendation (Python 3) -# - from typing import Any, Optional -from pyspark.ml.util import * +import sys # noqa: F401 + +from pyspark import since, keyword_only # noqa: F401 +from pyspark.ml.param.shared import ( + HasBlockSize, + HasCheckpointInterval, + HasMaxIter, + HasPredictionCol, + HasRegParam, + HasSeed, +) from pyspark.ml.wrapper import JavaEstimator, JavaModel -from pyspark.ml.param.shared import * +from pyspark.ml.common import inherit_doc # noqa: F401 +from pyspark.ml.param import Param +from pyspark.ml.util import JavaMLWritable, JavaMLReadable + from pyspark.sql.dataframe import DataFrame class _ALSModelParams(HasPredictionCol, HasBlockSize): diff --git a/third_party/3/pyspark/ml/regression.pyi b/third_party/3/pyspark/ml/regression.pyi index 5464ebdb..42a36b71 100644 --- a/third_party/3/pyspark/ml/regression.pyi +++ b/third_party/3/pyspark/ml/regression.pyi @@ -16,28 +16,45 @@ # specific language governing permissions and limitations # under the License. -# Stubs for pyspark.ml.regression (Python 3) - from typing import Any, List, Optional, Type -from pyspark.ml._typing import JM, M, P, T +from pyspark.ml._typing import JM, M, T import abc from pyspark.ml import PredictionModel, Predictor from pyspark.ml.base import _PredictorParams - -from pyspark.ml.param.shared import * -from pyspark.ml.linalg import Matrix, Vector -from pyspark.ml.util import * +from pyspark.ml.param.shared import ( + HasAggregationDepth, + HasBlockSize, + HasElasticNetParam, + HasFeaturesCol, + HasFitIntercept, + HasLabelCol, + HasLoss, + HasMaxIter, + HasPredictionCol, + HasRegParam, + HasSeed, + HasSolver, + HasStandardization, + HasStepSize, + HasTol, + HasVarianceCol, + HasWeightCol, +) from pyspark.ml.tree import ( _DecisionTreeModel, _DecisionTreeParams, - _TreeEnsembleModel, - _TreeEnsembleParams, - _RandomForestParams, _GBTParams, - _HasVarianceImpurity, + _RandomForestParams, + _TreeEnsembleModel, _TreeRegressorParams, ) +from pyspark.ml.util import ( + GeneralJavaMLWritable, + HasTrainingSummary, + JavaMLReadable, + JavaMLWritable, +) from pyspark.ml.wrapper import ( JavaEstimator, JavaModel, @@ -45,6 +62,9 @@ from pyspark.ml.wrapper import ( JavaPredictor, JavaWrapper, ) + +from pyspark.ml.linalg import Matrix, Vector +from pyspark.ml.param import Param from pyspark.sql.dataframe import DataFrame class Regressor(Predictor[M], _PredictorParams, metaclass=abc.ABCMeta): diff --git a/third_party/3/pyspark/ml/stat.pyi b/third_party/3/pyspark/ml/stat.pyi index 1c350890..83b0f7ea 100644 --- a/third_party/3/pyspark/ml/stat.pyi +++ b/third_party/3/pyspark/ml/stat.pyi @@ -16,9 +16,7 @@ # specific language governing permissions and limitations # under the License. -# Stubs for pyspark.ml.stat (Python 3) - -from typing import Any, Optional +from typing import Optional from pyspark.ml.linalg import Matrix, Vector from pyspark.ml.wrapper import JavaWrapper diff --git a/third_party/3/pyspark/ml/tree.pyi b/third_party/3/pyspark/ml/tree.pyi index 3ee3874c..ff630765 100644 --- a/third_party/3/pyspark/ml/tree.pyi +++ b/third_party/3/pyspark/ml/tree.pyi @@ -16,19 +16,24 @@ # specific language governing permissions and limitations # under the License. -from typing import Any, List, Optional, Sequence +from typing import List, Sequence from pyspark.ml._typing import P, T from pyspark.ml.linalg import Vector -from pyspark.ml.param.shared import * -from pyspark.ml.util import * -from pyspark.ml.wrapper import ( - JavaEstimator as JavaEstimator, - JavaModel as JavaModel, - JavaParams as JavaParams, - JavaPredictionModel as JavaPredictionModel, - JavaPredictor as JavaPredictor, +from pyspark import since as since # noqa: F401 +from pyspark.ml.common import inherit_doc as inherit_doc # noqa: F401 +from pyspark.ml.param import Param, Params as Params +from pyspark.ml.param.shared import ( # noqa: F401 + HasCheckpointInterval as HasCheckpointInterval, + HasMaxIter as HasMaxIter, + HasSeed as HasSeed, + HasStepSize as HasStepSize, + HasValidationIndicatorCol as HasValidationIndicatorCol, + HasWeightCol as HasWeightCol, + Param as Param, + TypeConverters as TypeConverters, ) +from pyspark.ml.wrapper import JavaPredictionModel as JavaPredictionModel class _DecisionTreeModel(JavaPredictionModel[T]): @property diff --git a/third_party/3/pyspark/ml/tuning.pyi b/third_party/3/pyspark/ml/tuning.pyi index ff11f992..3bd5a861 100644 --- a/third_party/3/pyspark/ml/tuning.pyi +++ b/third_party/3/pyspark/ml/tuning.pyi @@ -19,14 +19,14 @@ # Stubs for pyspark.ml.tuning (Python 3) from typing import overload -from typing import Any, Dict, List, Optional, Tuple, Type -from pyspark.ml._typing import P, ParamMap +from typing import Any, List, Optional, Tuple, Type +from pyspark.ml._typing import ParamMap from pyspark.ml import Estimator, Model from pyspark.ml.evaluation import Evaluator from pyspark.ml.param import Param -from pyspark.ml.param.shared import * -from pyspark.ml.util import * +from pyspark.ml.param.shared import HasCollectSubModels, HasParallelism, HasSeed +from pyspark.ml.util import MLReader, MLReadable, MLWriter, MLWritable class ParamGridBuilder: def __init__(self) -> None: ... diff --git a/third_party/3/pyspark/ml/util.pyi b/third_party/3/pyspark/ml/util.pyi index 3f05053a..8d16e153 100644 --- a/third_party/3/pyspark/ml/util.pyi +++ b/third_party/3/pyspark/ml/util.pyi @@ -20,10 +20,10 @@ from typing import Any, Dict, Generic, Optional, Type, TypeVar, Union -from pyspark.context import SparkContext -from pyspark.ml.param import Params -from pyspark.sql.context import SQLContext -from pyspark.sql.session import SparkSession +from pyspark import SparkContext as SparkContext, since as since # noqa: F401 +from pyspark.ml.common import inherit_doc as inherit_doc # noqa: F401 +from pyspark.sql import SparkSession as SparkSession +from pyspark.util import VersionUtils as VersionUtils # noqa: F401 S = TypeVar("S") R = TypeVar("R", bound=MLReadable) diff --git a/third_party/3/pyspark/ml/wrapper.pyi b/third_party/3/pyspark/ml/wrapper.pyi index 9c17827a..48686f68 100644 --- a/third_party/3/pyspark/ml/wrapper.pyi +++ b/third_party/3/pyspark/ml/wrapper.pyi @@ -19,13 +19,12 @@ # Stubs for pyspark.ml.wrapper (Python 3) import abc -from typing import Any, Generic, Optional, Type, TypeVar +from typing import Any, Optional, Type from pyspark.ml._typing import P, T, JM, ParamMap from pyspark.ml import Estimator, Predictor, PredictionModel, Transformer, Model from pyspark.ml.base import _PredictorParams from pyspark.ml.param import Param, Params -from pyspark.ml.param.shared import HasFeaturesCol, HasLabelCol, HasPredictionCol class JavaWrapper: def __init__(self, java_obj: Optional[Any] = ...) -> None: ... diff --git a/third_party/3/pyspark/mllib/_typing.pyi b/third_party/3/pyspark/mllib/_typing.pyi index af0c13da..213a6999 100644 --- a/third_party/3/pyspark/mllib/_typing.pyi +++ b/third_party/3/pyspark/mllib/_typing.pyi @@ -16,8 +16,8 @@ # specific language governing permissions and limitations # under the License. -from typing import Any, Iterable, List, Optional, Tuple, TypeVar, Union +from typing import List, Tuple, Union from pyspark.mllib.linalg import Vector -from numpy import ndarray # type: ignore[import] +from numpy import ndarray # noqa: F401 VectorLike = Union[Vector, List[float], Tuple[float, ...]] diff --git a/third_party/3/pyspark/mllib/classification.pyi b/third_party/3/pyspark/mllib/classification.pyi index a87511c6..c51882c8 100644 --- a/third_party/3/pyspark/mllib/classification.pyi +++ b/third_party/3/pyspark/mllib/classification.pyi @@ -17,7 +17,7 @@ # under the License. from typing import overload -from typing import Any, Optional, Union +from typing import Optional, Union from pyspark.context import SparkContext from pyspark.rdd import RDD diff --git a/third_party/3/pyspark/mllib/clustering.pyi b/third_party/3/pyspark/mllib/clustering.pyi index 1e9ac4cf..42c1a5eb 100644 --- a/third_party/3/pyspark/mllib/clustering.pyi +++ b/third_party/3/pyspark/mllib/clustering.pyi @@ -20,7 +20,7 @@ # from typing import overload -from typing import Any, List, NamedTuple, Optional, Tuple, TypeVar +from typing import List, NamedTuple, Optional, Tuple, TypeVar import array diff --git a/third_party/3/pyspark/mllib/evaluation.pyi b/third_party/3/pyspark/mllib/evaluation.pyi index 7d752c49..2ec034a2 100644 --- a/third_party/3/pyspark/mllib/evaluation.pyi +++ b/third_party/3/pyspark/mllib/evaluation.pyi @@ -19,7 +19,7 @@ # Stubs for pyspark.mllib.evaluation (Python 3.5) # -from typing import Any, List, Optional, Tuple, TypeVar +from typing import List, Optional, Tuple, TypeVar from pyspark.rdd import RDD from pyspark.mllib.common import JavaModelWrapper from pyspark.mllib.linalg import Matrix diff --git a/third_party/3/pyspark/mllib/feature.pyi b/third_party/3/pyspark/mllib/feature.pyi index 7b9838f6..2ba8879f 100644 --- a/third_party/3/pyspark/mllib/feature.pyi +++ b/third_party/3/pyspark/mllib/feature.pyi @@ -20,7 +20,7 @@ # from typing import overload -from typing import Any, Iterable, Hashable, List, Tuple +from typing import Iterable, Hashable, List, Tuple from pyspark.mllib._typing import VectorLike from pyspark.context import SparkContext diff --git a/third_party/3/pyspark/mllib/fpm.pyi b/third_party/3/pyspark/mllib/fpm.pyi index 62e27887..fe0933b1 100644 --- a/third_party/3/pyspark/mllib/fpm.pyi +++ b/third_party/3/pyspark/mllib/fpm.pyi @@ -19,7 +19,7 @@ # Stubs for pyspark.mllib.fpm (Python 3.5) # -from typing import Any, Optional, Generic, List, TypeVar +from typing import Generic, List, TypeVar from pyspark.context import SparkContext from pyspark.rdd import RDD from pyspark.mllib.common import JavaModelWrapper diff --git a/third_party/3/pyspark/mllib/linalg/distributed.pyi b/third_party/3/pyspark/mllib/linalg/distributed.pyi index 3855ffb7..0abb0fa6 100644 --- a/third_party/3/pyspark/mllib/linalg/distributed.pyi +++ b/third_party/3/pyspark/mllib/linalg/distributed.pyi @@ -19,13 +19,13 @@ # Stubs for pyspark.mllib.linalg.distributed (Python 3.5) # -from typing import Any, Generic, Sequence, Optional, Tuple, TypeVar, Union +from typing import Generic, Sequence, Optional, Tuple, TypeVar, Union from pyspark.rdd import RDD from pyspark.storagelevel import StorageLevel from pyspark.mllib.common import JavaModelWrapper from pyspark.mllib.linalg import Vector, Matrix, QRDecomposition from pyspark.mllib.stat import MultivariateStatisticalSummary -from numpy import ndarray # type: ignore[import] +from numpy import ndarray # noqa: F401 VectorLike = Union[Vector, Sequence[Union[float, int]]] diff --git a/third_party/3/pyspark/mllib/random.pyi b/third_party/3/pyspark/mllib/random.pyi index 155027b6..58eda0c3 100644 --- a/third_party/3/pyspark/mllib/random.pyi +++ b/third_party/3/pyspark/mllib/random.pyi @@ -19,7 +19,7 @@ # Stubs for pyspark.mllib.random (Python 3.5) # -from typing import Any, Optional +from typing import Optional from pyspark.context import SparkContext from pyspark.rdd import RDD from pyspark.mllib.linalg import Vector diff --git a/third_party/3/pyspark/mllib/recommendation.pyi b/third_party/3/pyspark/mllib/recommendation.pyi index 863acc9f..6b5d441a 100644 --- a/third_party/3/pyspark/mllib/recommendation.pyi +++ b/third_party/3/pyspark/mllib/recommendation.pyi @@ -19,7 +19,7 @@ # Stubs for pyspark.mllib.recommendation (Python 3.5) # -from typing import Any, List, Optional, Tuple, Union +from typing import List, Optional, Tuple, Union import array from collections import namedtuple diff --git a/third_party/3/pyspark/mllib/regression.pyi b/third_party/3/pyspark/mllib/regression.pyi index fe95de93..9106335a 100644 --- a/third_party/3/pyspark/mllib/regression.pyi +++ b/third_party/3/pyspark/mllib/regression.pyi @@ -20,7 +20,7 @@ # from typing import overload -from typing import Any, Iterable, Optional, Tuple, TypeVar, Union +from typing import Iterable, Optional, Tuple, TypeVar from pyspark.rdd import RDD from pyspark.mllib._typing import VectorLike from pyspark.context import SparkContext diff --git a/third_party/3/pyspark/mllib/stat/KernelDensity.pyi b/third_party/3/pyspark/mllib/stat/KernelDensity.pyi index 5ea6af06..fdd52851 100644 --- a/third_party/3/pyspark/mllib/stat/KernelDensity.pyi +++ b/third_party/3/pyspark/mllib/stat/KernelDensity.pyi @@ -19,7 +19,7 @@ # Stubs for pyspark.mllib.stat.KernelDensity (Python 3.5) # -from typing import Any, Iterable +from typing import Iterable from pyspark.rdd import RDD from numpy import ndarray # type: ignore[import] diff --git a/third_party/3/pyspark/mllib/stat/__init__.pyi b/third_party/3/pyspark/mllib/stat/__init__.pyi index 0ed2561c..bdd080a0 100644 --- a/third_party/3/pyspark/mllib/stat/__init__.pyi +++ b/third_party/3/pyspark/mllib/stat/__init__.pyi @@ -16,15 +16,14 @@ # specific language governing permissions and limitations # under the License. -# Stubs for pyspark.mllib.stat (Python 3.5) -# -# NOTE: This dynamically typed stub was automatically generated by stubgen. - -from pyspark.mllib.stat._statistics import * -from pyspark.mllib.stat.distribution import MultivariateGaussian as MultivariateGaussian -from pyspark.mllib.stat.test import ChiSqTestResult as ChiSqTestResult -from pyspark.mllib.stat.KernelDensity import KernelDensity as KernelDensity - -# Names in __all__ with no definition: -# MultivariateStatisticalSummary -# Statistics +from pyspark.mllib.stat.KernelDensity import ( # noqa: F401 + KernelDensity as KernelDensity, +) +from pyspark.mllib.stat._statistics import ( # noqa: F401 + MultivariateStatisticalSummary as MultivariateStatisticalSummary, + Statistics as Statistics, +) +from pyspark.mllib.stat.distribution import ( # noqa: F401 + MultivariateGaussian as MultivariateGaussian, +) +from pyspark.mllib.stat.test import ChiSqTestResult as ChiSqTestResult # noqa: F401 diff --git a/third_party/3/pyspark/mllib/tree.pyi b/third_party/3/pyspark/mllib/tree.pyi index 92efb03e..597149ab 100644 --- a/third_party/3/pyspark/mllib/tree.pyi +++ b/third_party/3/pyspark/mllib/tree.pyi @@ -20,7 +20,7 @@ # from typing import overload -from typing import Any, Dict, Optional, Tuple +from typing import Dict, Optional, Tuple from pyspark.mllib._typing import VectorLike from pyspark.rdd import RDD from pyspark.mllib.common import JavaModelWrapper diff --git a/third_party/3/pyspark/mllib/util.pyi b/third_party/3/pyspark/mllib/util.pyi index 76721585..b83238c0 100644 --- a/third_party/3/pyspark/mllib/util.pyi +++ b/third_party/3/pyspark/mllib/util.pyi @@ -19,7 +19,7 @@ # Stubs for pyspark.mllib.util (Python 3) # -from typing import Generic, List, Optional, Type, TypeVar +from typing import Generic, List, Optional, TypeVar from pyspark.mllib._typing import VectorLike from pyspark.context import SparkContext diff --git a/third_party/3/pyspark/profiler.pyi b/third_party/3/pyspark/profiler.pyi index 8293a404..7276da52 100644 --- a/third_party/3/pyspark/profiler.pyi +++ b/third_party/3/pyspark/profiler.pyi @@ -16,9 +16,6 @@ # specific language governing permissions and limitations # under the License. -# Stubs for pyspark.profiler (Python 3.5) -# - from typing import Any, Callable, List, Optional, Tuple, Type import pstats diff --git a/third_party/3/pyspark/rdd.pyi b/third_party/3/pyspark/rdd.pyi index 23d0bd6f..35c49e95 100644 --- a/third_party/3/pyspark/rdd.pyi +++ b/third_party/3/pyspark/rdd.pyi @@ -48,7 +48,10 @@ import pyspark.context from pyspark.resultiterable import ResultIterable from pyspark.serializers import Serializer from pyspark.storagelevel import StorageLevel -from pyspark.resource.requests import ExecutorResourceRequests, TaskResourceRequests +from pyspark.resource.requests import ( # noqa: F401 + ExecutorResourceRequests, + TaskResourceRequests, +) from pyspark.resource.profile import ResourceProfile from pyspark.statcounter import StatCounter from pyspark.sql.dataframe import DataFrame diff --git a/third_party/3/pyspark/rddsampler.pyi b/third_party/3/pyspark/rddsampler.pyi index e0f669c9..5cccbabb 100644 --- a/third_party/3/pyspark/rddsampler.pyi +++ b/third_party/3/pyspark/rddsampler.pyi @@ -19,7 +19,7 @@ # Stubs for pyspark.rddsampler (Python 3.5) # -from typing import Any, Dict, Hashable, Iterator, Optional, Tuple, TypeVar +from typing import Any, Dict, Iterator, Optional, Tuple, TypeVar T = TypeVar("T") U = TypeVar("U") diff --git a/third_party/3/pyspark/resource/__init__.pyi b/third_party/3/pyspark/resource/__init__.pyi index 3a2c3faa..87a9b53c 100644 --- a/third_party/3/pyspark/resource/__init__.pyi +++ b/third_party/3/pyspark/resource/__init__.pyi @@ -16,12 +16,14 @@ # specific language governing permissions and limitations # under the License. -from pyspark.resource.information import ResourceInformation as ResourceInformation -from pyspark.resource.profile import ( +from pyspark.resource.information import ( # noqa: F401 + ResourceInformation as ResourceInformation, +) +from pyspark.resource.profile import ( # noqa: F401 ResourceProfile as ResourceProfile, ResourceProfileBuilder as ResourceProfileBuilder, ) -from pyspark.resource.requests import ( +from pyspark.resource.requests import ( # noqa: F401 ExecutorResourceRequest as ExecutorResourceRequest, ExecutorResourceRequests as ExecutorResourceRequests, TaskResourceRequest as TaskResourceRequest, diff --git a/third_party/3/pyspark/resource/profile.pyi b/third_party/3/pyspark/resource/profile.pyi index 9e3a1565..8ce7d93b 100644 --- a/third_party/3/pyspark/resource/profile.pyi +++ b/third_party/3/pyspark/resource/profile.pyi @@ -16,7 +16,7 @@ # specific language governing permissions and limitations # under the License. -from pyspark.resource.requests import ( +from pyspark.resource.requests import ( # noqa: F401 ExecutorResourceRequest as ExecutorResourceRequest, ExecutorResourceRequests as ExecutorResourceRequests, TaskResourceRequest as TaskResourceRequest, diff --git a/third_party/3/pyspark/resultiterable.pyi b/third_party/3/pyspark/resultiterable.pyi index 77f966f8..f48c1b3a 100644 --- a/third_party/3/pyspark/resultiterable.pyi +++ b/third_party/3/pyspark/resultiterable.pyi @@ -20,7 +20,7 @@ # from pyspark._typing import SizedIterable -from typing import Generic, Iterable, Iterator, TypeVar +from typing import Iterator, TypeVar T = TypeVar("T") diff --git a/third_party/3/pyspark/shell.pyi b/third_party/3/pyspark/shell.pyi index c9f96cc7..07603095 100644 --- a/third_party/3/pyspark/shell.pyi +++ b/third_party/3/pyspark/shell.pyi @@ -16,14 +16,16 @@ # specific language governing permissions and limitations # under the License. -from pyspark import SparkConf as SparkConf +from pyspark import SparkConf as SparkConf # noqa: F401 from pyspark.context import SparkContext as SparkContext from pyspark.sql import SQLContext as SQLContext, SparkSession as SparkSession -from typing import Any +from typing import Any, Callable -spark: Any -sc: Any -sql: Any -sqlContext: Any -sqlCtx = sqlContext +from pyspark.sql.dataframe import DataFrame + +spark: SparkSession +sc: SparkContext +sql: Callable[[str], DataFrame] +sqlContext: SQLContext +sqlCtx: SQLContext code: Any diff --git a/third_party/3/pyspark/shuffle.pyi b/third_party/3/pyspark/shuffle.pyi index acf14a61..10648c51 100644 --- a/third_party/3/pyspark/shuffle.pyi +++ b/third_party/3/pyspark/shuffle.pyi @@ -16,14 +16,14 @@ # specific language governing permissions and limitations # under the License. -from pyspark.serializers import ( +from pyspark.serializers import ( # noqa: F401 AutoBatchedSerializer as AutoBatchedSerializer, BatchedSerializer as BatchedSerializer, CompressedSerializer as CompressedSerializer, FlattenedValuesSerializer as FlattenedValuesSerializer, PickleSerializer as PickleSerializer, ) -from pyspark.util import fail_on_stopiteration as fail_on_stopiteration +from pyspark.util import fail_on_stopiteration as fail_on_stopiteration # noqa: F401 from typing import Any, Optional process: Any diff --git a/third_party/3/pyspark/sql/__init__.pyi b/third_party/3/pyspark/sql/__init__.pyi index a4ff548c..787be564 100644 --- a/third_party/3/pyspark/sql/__init__.pyi +++ b/third_party/3/pyspark/sql/__init__.pyi @@ -16,27 +16,26 @@ # specific language governing permissions and limitations # under the License. -# Stubs for pyspark.sql (Python 3.5) -# -# NOTE: This dynamically typed stub was automatically generated by stubgen. - -from pyspark.sql.types import Row as Row -from pyspark.sql.context import ( +from pyspark.sql.catalog import Catalog as Catalog # noqa: F401 +from pyspark.sql.column import Column as Column # noqa: F401 +from pyspark.sql.context import ( # noqa: F401 + HiveContext as HiveContext, SQLContext as SQLContext, UDFRegistration as UDFRegistration, - HiveContext as HiveContext, ) -from pyspark.sql.session import SparkSession as SparkSession -from pyspark.sql.column import Column as Column -from pyspark.sql.dataframe import ( +from pyspark.sql.dataframe import ( # noqa: F401 DataFrame as DataFrame, DataFrameNaFunctions as DataFrameNaFunctions, DataFrameStatFunctions as DataFrameStatFunctions, ) -from pyspark.sql.group import GroupedData as GroupedData -from pyspark.sql.readwriter import ( +from pyspark.sql.group import GroupedData as GroupedData # noqa: F401 +from pyspark.sql.pandas.group_ops import ( # noqa: F401 + PandasCogroupedOps as PandasCogroupedOps, +) +from pyspark.sql.readwriter import ( # noqa: F401 DataFrameReader as DataFrameReader, DataFrameWriter as DataFrameWriter, ) -from pyspark.sql.window import Window as Window, WindowSpec as WindowSpec -from pyspark.sql.pandas.group_ops import PandasCogroupedOps +from pyspark.sql.session import SparkSession as SparkSession # noqa: F401 +from pyspark.sql.types import Row as Row # noqa: F401 +from pyspark.sql.window import Window as Window, WindowSpec as WindowSpec # noqa: F401 diff --git a/third_party/3/pyspark/sql/_typing.pyi b/third_party/3/pyspark/sql/_typing.pyi index 6b6cfab7..799a7320 100644 --- a/third_party/3/pyspark/sql/_typing.pyi +++ b/third_party/3/pyspark/sql/_typing.pyi @@ -18,17 +18,13 @@ from typing import ( Any, - Callable, - Iterable, List, - NewType, Optional, Tuple, TypeVar, Union, ) -from typing_extensions import Protocol, Literal -from types import FunctionType +from typing_extensions import Protocol import datetime import decimal @@ -38,10 +34,6 @@ import pyspark.sql.column import pyspark.sql.types from pyspark.sql.column import Column -from pyspark.sql.pandas._typing import DataFrameLike, SeriesLike -import pandas.core.frame # type: ignore[import] -import pandas.core.series # type: ignore[import] - ColumnOrName = Union[pyspark.sql.column.Column, str] DecimalLiteral = decimal.Decimal DateTimeLiteral = Union[datetime.datetime, datetime.date] diff --git a/third_party/3/pyspark/sql/avro/functions.pyi b/third_party/3/pyspark/sql/avro/functions.pyi index 903c8000..0e7bd333 100644 --- a/third_party/3/pyspark/sql/avro/functions.pyi +++ b/third_party/3/pyspark/sql/avro/functions.pyi @@ -18,7 +18,7 @@ # Stubs for pyspark.sql.avro.functions (Python 3) -from typing import Any, Dict +from typing import Dict from pyspark.sql._typing import ColumnOrName from pyspark.sql.column import Column diff --git a/third_party/3/pyspark/sql/catalog.pyi b/third_party/3/pyspark/sql/catalog.pyi index 88f82f7f..86263fff 100644 --- a/third_party/3/pyspark/sql/catalog.pyi +++ b/third_party/3/pyspark/sql/catalog.pyi @@ -16,11 +16,7 @@ # specific language governing permissions and limitations # under the License. -# Stubs for pyspark.sql.catalog (Python 3.5) -# -# NOTE: This dynamically typed stub was automatically generated by stubgen. - -from typing import Any, Callable, Dict, List, Optional +from typing import Any, Callable, List, Optional from pyspark.sql.dataframe import DataFrame from pyspark.sql.session import SparkSession from pyspark.sql.types import DataType, StructType diff --git a/third_party/3/pyspark/sql/column.pyi b/third_party/3/pyspark/sql/column.pyi index 616d1d35..f4b10bd7 100644 --- a/third_party/3/pyspark/sql/column.pyi +++ b/third_party/3/pyspark/sql/column.pyi @@ -23,7 +23,13 @@ from typing import overload from typing import Any, Union from pyspark.sql._typing import LiteralType, DecimalLiteral, DateTimeLiteral -from pyspark.sql.types import * +from pyspark.sql.types import ( # noqa: F401 + DataType, + StructField, + StructType, + IntegerType, + StringType, +) from pyspark.sql.window import WindowSpec from py4j.java_gateway import JavaObject # type: ignore[import] diff --git a/third_party/3/pyspark/sql/conf.pyi b/third_party/3/pyspark/sql/conf.pyi index 0b92c56d..8a5124a7 100644 --- a/third_party/3/pyspark/sql/conf.pyi +++ b/third_party/3/pyspark/sql/conf.pyi @@ -19,7 +19,7 @@ # Stubs for pyspark.sql.conf (Python 3.5) # -from typing import Any, Optional +from typing import Optional from py4j.java_gateway import JavaObject # type: ignore[import] class RuntimeConfig: diff --git a/third_party/3/pyspark/sql/context.pyi b/third_party/3/pyspark/sql/context.pyi index d606fe78..4150ad8e 100644 --- a/third_party/3/pyspark/sql/context.pyi +++ b/third_party/3/pyspark/sql/context.pyi @@ -28,7 +28,6 @@ from pyspark.sql._typing import ( DateTimeLiteral, LiteralType, DecimalLiteral, - DataTypeOrString, RowLike, ) from pyspark.sql.pandas._typing import DataFrameLike diff --git a/third_party/3/pyspark/sql/dataframe.pyi b/third_party/3/pyspark/sql/dataframe.pyi index 2d1c2cb1..9540fb86 100644 --- a/third_party/3/pyspark/sql/dataframe.pyi +++ b/third_party/3/pyspark/sql/dataframe.pyi @@ -25,16 +25,19 @@ from typing import ( List, Optional, Tuple, - TypeVar, Union, ) -import pandas.core.frame # type: ignore[import] from py4j.java_gateway import JavaObject # type: ignore[import] from pyspark.sql._typing import ColumnOrName, LiteralType, OptionalPrimitiveType -from pyspark.sql.pandas._typing import MapIterPandasUserDefinedFunction -from pyspark.sql.types import * +from pyspark.sql.types import ( # noqa: F401 + StructType, + StructField, + StringType, + IntegerType, + Row, +) # noqa: F401 from pyspark.sql.context import SQLContext from pyspark.sql.group import GroupedData from pyspark.sql.readwriter import DataFrameWriter, DataFrameWriterV2 diff --git a/third_party/3/pyspark/sql/functions.pyi b/third_party/3/pyspark/sql/functions.pyi index 43d2b717..3b0b2030 100644 --- a/third_party/3/pyspark/sql/functions.pyi +++ b/third_party/3/pyspark/sql/functions.pyi @@ -16,25 +16,26 @@ # specific language governing permissions and limitations # under the License. -# Stubs for pyspark.sql.functions (Python 3.5) - from typing import overload from typing import Any, Callable, Dict, List, Optional, Union from pyspark.sql._typing import ( - AtomicDataTypeOrString, ColumnOrName, DataTypeOrString, - UserDefinedFunctionLike, ) -from pyspark.sql.pandas.functions import ( +from pyspark.sql.pandas.functions import ( # noqa: F401 pandas_udf as pandas_udf, PandasUDFType as PandasUDFType, ) from pyspark.sql.column import Column from pyspark.sql.dataframe import DataFrame -from pyspark.sql.types import ArrayType, DataType, StructType -from pyspark.sql.utils import to_str +from pyspark.sql.types import ( # noqa: F401 + ArrayType, + StringType, + StructType, + DataType, +) +from pyspark.sql.utils import to_str # noqa: F401 def approxCountDistinct(col: ColumnOrName, rsd: Optional[float] = ...) -> Column: ... def approx_count_distinct(col: ColumnOrName, rsd: Optional[float] = ...) -> Column: ... diff --git a/third_party/3/pyspark/sql/group.pyi b/third_party/3/pyspark/sql/group.pyi index 47db2eba..478bce69 100644 --- a/third_party/3/pyspark/sql/group.pyi +++ b/third_party/3/pyspark/sql/group.pyi @@ -20,15 +20,13 @@ # from typing import overload -from typing import Any, Callable, Dict, List, Optional +from typing import Dict, List, Optional from pyspark.sql._typing import LiteralType -from pyspark.sql.pandas._typing import GroupedMapPandasUserDefinedFunction from pyspark.sql.context import SQLContext from pyspark.sql.column import Column from pyspark.sql.dataframe import DataFrame from pyspark.sql.pandas.group_ops import PandasGroupedOpsMixin -from pyspark.sql.types import * from py4j.java_gateway import JavaObject # type: ignore[import] class GroupedData(PandasGroupedOpsMixin): diff --git a/third_party/3/pyspark/sql/pandas/_typing/__init__.pyi b/third_party/3/pyspark/sql/pandas/_typing/__init__.pyi index cc2df65e..dda1b334 100644 --- a/third_party/3/pyspark/sql/pandas/_typing/__init__.pyi +++ b/third_party/3/pyspark/sql/pandas/_typing/__init__.pyi @@ -20,12 +20,9 @@ from typing import ( Any, Callable, Iterable, - List, NewType, - Optional, Tuple, Type, - TypeVar, Union, ) from typing_extensions import Protocol, Literal diff --git a/third_party/3/pyspark/sql/pandas/conversion.pyi b/third_party/3/pyspark/sql/pandas/conversion.pyi index 28534459..031852fc 100644 --- a/third_party/3/pyspark/sql/pandas/conversion.pyi +++ b/third_party/3/pyspark/sql/pandas/conversion.pyi @@ -17,17 +17,29 @@ # under the License. from typing import overload -from typing import Any, Iterable, List, Optional, Tuple, Union +from typing import Optional, Union from pyspark.sql.pandas._typing import DataFrameLike -from pyspark import since as since -from pyspark.rdd import RDD +from pyspark import since as since # noqa: F401 +from pyspark.rdd import RDD # noqa: F401 import pyspark.sql.dataframe -from pyspark.sql.pandas.serializers import ( +from pyspark.sql.pandas.serializers import ( # noqa: F401 ArrowCollectSerializer as ArrowCollectSerializer, ) -from pyspark.sql.types import * -from pyspark.traceback_utils import SCCallSiteSync as SCCallSiteSync +from pyspark.sql.types import ( # noqa: F401 + BooleanType as BooleanType, + ByteType as ByteType, + DataType as DataType, + DoubleType as DoubleType, + FloatType as FloatType, + IntegerType as IntegerType, + IntegralType as IntegralType, + LongType as LongType, + ShortType as ShortType, + StructType as StructType, + TimestampType as TimestampType, +) +from pyspark.traceback_utils import SCCallSiteSync as SCCallSiteSync # noqa: F401 class PandasConversionMixin: def toPandas(self) -> DataFrameLike: ... diff --git a/third_party/3/pyspark/sql/pandas/functions.pyi b/third_party/3/pyspark/sql/pandas/functions.pyi index eed635b1..09318e43 100644 --- a/third_party/3/pyspark/sql/pandas/functions.pyi +++ b/third_party/3/pyspark/sql/pandas/functions.pyi @@ -17,12 +17,10 @@ # under the License. from typing import overload -from typing import Any, Optional, Union, Callable +from typing import Union, Callable from pyspark.sql._typing import ( AtomicDataTypeOrString, - ColumnOrName, - DataTypeOrString, UserDefinedFunctionLike, ) from pyspark.sql.pandas._typing import ( @@ -44,10 +42,9 @@ from pyspark.sql.pandas._typing import ( PandasScalarUDFType, ) -from pyspark import since as since -from pyspark.rdd import PythonEvalType as PythonEvalType -from pyspark.sql.column import Column -from pyspark.sql.types import ArrayType, DataType, StructType +from pyspark import since as since # noqa: F401 +from pyspark.rdd import PythonEvalType as PythonEvalType # noqa: F401 +from pyspark.sql.types import ArrayType, StructType class PandasUDFType: SCALAR: PandasScalarUDFType diff --git a/third_party/3/pyspark/sql/pandas/group_ops.pyi b/third_party/3/pyspark/sql/pandas/group_ops.pyi index 6725ef30..2c543e0d 100644 --- a/third_party/3/pyspark/sql/pandas/group_ops.pyi +++ b/third_party/3/pyspark/sql/pandas/group_ops.pyi @@ -20,14 +20,13 @@ from typing import Union from pyspark.sql.pandas._typing import ( GroupedMapPandasUserDefinedFunction, - CogroupedMapPandasUserDefinedFunction, PandasGroupedMapFunction, PandasCogroupedMapFunction, ) -from pyspark import since as since -from pyspark.rdd import PythonEvalType as PythonEvalType -from pyspark.sql.column import Column as Column +from pyspark import since as since # noqa: F401 +from pyspark.rdd import PythonEvalType as PythonEvalType # noqa: F401 +from pyspark.sql.column import Column as Column # noqa: F401 from pyspark.sql.context import SQLContext import pyspark.sql.group from pyspark.sql.dataframe import DataFrame as DataFrame diff --git a/third_party/3/pyspark/sql/pandas/map_ops.pyi b/third_party/3/pyspark/sql/pandas/map_ops.pyi index ae3ae71f..cab88527 100644 --- a/third_party/3/pyspark/sql/pandas/map_ops.pyi +++ b/third_party/3/pyspark/sql/pandas/map_ops.pyi @@ -18,12 +18,9 @@ from typing import Union -from pyspark.sql.pandas._typing import ( - MapIterPandasUserDefinedFunction, - PandasMapIterFunction, -) -from pyspark import since as since -from pyspark.rdd import PythonEvalType as PythonEvalType +from pyspark.sql.pandas._typing import PandasMapIterFunction +from pyspark import since as since # noqa: F401 +from pyspark.rdd import PythonEvalType as PythonEvalType # noqa: F401 from pyspark.sql.types import StructType import pyspark.sql.dataframe diff --git a/third_party/3/pyspark/sql/pandas/serializers.pyi b/third_party/3/pyspark/sql/pandas/serializers.pyi index 633fc820..8be3c0dc 100644 --- a/third_party/3/pyspark/sql/pandas/serializers.pyi +++ b/third_party/3/pyspark/sql/pandas/serializers.pyi @@ -16,7 +16,7 @@ # specific language governing permissions and limitations # under the License. -from pyspark.serializers import ( +from pyspark.serializers import ( # noqa: F401 Serializer as Serializer, UTF8Deserializer as UTF8Deserializer, read_int as read_int, diff --git a/third_party/3/pyspark/sql/pandas/typehints.pyi b/third_party/3/pyspark/sql/pandas/typehints.pyi index 013581d4..eea9c862 100644 --- a/third_party/3/pyspark/sql/pandas/typehints.pyi +++ b/third_party/3/pyspark/sql/pandas/typehints.pyi @@ -16,7 +16,7 @@ # specific language governing permissions and limitations # under the License. -from pyspark.sql.pandas.utils import ( +from pyspark.sql.pandas.utils import ( # noqa: F401 require_minimum_pandas_version as require_minimum_pandas_version, ) from typing import Any, Optional diff --git a/third_party/3/pyspark/sql/pandas/types.pyi b/third_party/3/pyspark/sql/pandas/types.pyi index a83ac504..5ae29bd2 100644 --- a/third_party/3/pyspark/sql/pandas/types.pyi +++ b/third_party/3/pyspark/sql/pandas/types.pyi @@ -16,7 +16,23 @@ # specific language governing permissions and limitations # under the License. -from pyspark.sql.types import * +from pyspark.sql.types import ( # noqa: F401 + ArrayType as ArrayType, + BinaryType as BinaryType, + BooleanType as BooleanType, + ByteType as ByteType, + DateType as DateType, + DecimalType as DecimalType, + DoubleType as DoubleType, + FloatType as FloatType, + IntegerType as IntegerType, + LongType as LongType, + ShortType as ShortType, + StringType as StringType, + StructField as StructField, + StructType as StructType, + TimestampType as TimestampType, +) from typing import Any def to_arrow_type(dt: Any): ... diff --git a/third_party/3/pyspark/sql/readwriter.pyi b/third_party/3/pyspark/sql/readwriter.pyi index 51d66051..509156ed 100644 --- a/third_party/3/pyspark/sql/readwriter.pyi +++ b/third_party/3/pyspark/sql/readwriter.pyi @@ -20,14 +20,14 @@ # from typing import overload -from typing import Any, Dict, List, Optional, Tuple, Union +from typing import Dict, List, Optional, Tuple, Union from pyspark.sql._typing import OptionalPrimitiveType from pyspark.sql.dataframe import DataFrame from pyspark.rdd import RDD from pyspark.sql.column import Column from pyspark.sql.context import SQLContext -from pyspark.sql.types import * +from pyspark.sql.types import StructType PathOrPaths = Union[str, List[str]] TupleOrListOfString = Union[List[str], Tuple[str, ...]] diff --git a/third_party/3/pyspark/sql/session.pyi b/third_party/3/pyspark/sql/session.pyi index 984d0f87..79d719a5 100644 --- a/third_party/3/pyspark/sql/session.pyi +++ b/third_party/3/pyspark/sql/session.pyi @@ -33,7 +33,7 @@ from pyspark.sql.catalog import Catalog from pyspark.sql.conf import RuntimeConfig from pyspark.sql.dataframe import DataFrame from pyspark.sql.pandas.conversion import SparkConversionMixin -from pyspark.sql.types import AtomicType, DataType, StructType +from pyspark.sql.types import AtomicType, StructType from pyspark.sql.readwriter import DataFrameReader from pyspark.sql.streaming import DataStreamReader, StreamingQueryManager from pyspark.sql.udf import UDFRegistration diff --git a/third_party/3/pyspark/sql/utils.pyi b/third_party/3/pyspark/sql/utils.pyi index 6a45555d..e89fcb5e 100644 --- a/third_party/3/pyspark/sql/utils.pyi +++ b/third_party/3/pyspark/sql/utils.pyi @@ -20,7 +20,7 @@ # # NOTE: This dynamically typed stub was automatically generated by stubgen. -from pyspark import SparkContext as SparkContext +from pyspark import SparkContext as SparkContext # noqa: F401 from typing import Any, Optional class CapturedException(Exception): diff --git a/third_party/3/pyspark/sql/window.pyi b/third_party/3/pyspark/sql/window.pyi index 1ffb17ec..4e31d57b 100644 --- a/third_party/3/pyspark/sql/window.pyi +++ b/third_party/3/pyspark/sql/window.pyi @@ -16,10 +16,6 @@ # specific language governing permissions and limitations # under the License. -# Stubs for pyspark.sql.window (Python 3.5) -# - -from typing import Any, Union from pyspark.sql._typing import ColumnOrName from py4j.java_gateway import JavaObject # type: ignore[import] diff --git a/third_party/3/pyspark/status.pyi b/third_party/3/pyspark/status.pyi index 3ccacb9d..0558e245 100644 --- a/third_party/3/pyspark/status.pyi +++ b/third_party/3/pyspark/status.pyi @@ -16,10 +16,7 @@ # specific language governing permissions and limitations # under the License. -# Stubs for pyspark.status (Python 3.5) -# - -from typing import Any, List, NamedTuple, Optional +from typing import List, NamedTuple, Optional from py4j.java_gateway import JavaArray, JavaObject # type: ignore[import] class SparkJobInfo(NamedTuple): diff --git a/third_party/3/pyspark/storagelevel.pyi b/third_party/3/pyspark/storagelevel.pyi index 5849cb6a..2eb05850 100644 --- a/third_party/3/pyspark/storagelevel.pyi +++ b/third_party/3/pyspark/storagelevel.pyi @@ -16,9 +16,7 @@ # specific language governing permissions and limitations # under the License. -# Stubs for pyspark.storagelevel (Python 3.5) -# -from typing import Any, ClassVar +from typing import ClassVar class StorageLevel: DISK_ONLY: ClassVar[StorageLevel] diff --git a/third_party/3/pyspark/streaming/__init__.pyi b/third_party/3/pyspark/streaming/__init__.pyi index 0c5d0094..281c06e5 100644 --- a/third_party/3/pyspark/streaming/__init__.pyi +++ b/third_party/3/pyspark/streaming/__init__.pyi @@ -16,10 +16,8 @@ # specific language governing permissions and limitations # under the License. -# Stubs for pyspark.streaming (Python 3.5) -# -# NOTE: This dynamically typed stub was automatically generated by stubgen. - -from pyspark.streaming.context import StreamingContext as StreamingContext -from pyspark.streaming.dstream import DStream as DStream -from pyspark.streaming.listener import StreamingListener as StreamingListener +from pyspark.streaming.context import StreamingContext as StreamingContext # noqa: F401 +from pyspark.streaming.dstream import DStream as DStream # noqa: F401 +from pyspark.streaming.listener import ( # noqa: F401 + StreamingListener as StreamingListener, +) diff --git a/third_party/3/pyspark/streaming/dstream.pyi b/third_party/3/pyspark/streaming/dstream.pyi index 0a6b025c..828cf3a0 100644 --- a/third_party/3/pyspark/streaming/dstream.pyi +++ b/third_party/3/pyspark/streaming/dstream.pyi @@ -21,7 +21,6 @@ from typing import overload from typing import ( - Any, Callable, Generic, Hashable, diff --git a/third_party/3/pyspark/taskcontext.pyi b/third_party/3/pyspark/taskcontext.pyi index a0f23934..3415c69f 100644 --- a/third_party/3/pyspark/taskcontext.pyi +++ b/third_party/3/pyspark/taskcontext.pyi @@ -16,10 +16,7 @@ # specific language governing permissions and limitations # under the License. -# Stubs for pyspark.taskcontext (Python 3) -# - -from typing import Any, Dict, List +from typing import Dict, List from typing_extensions import Literal from pyspark.resource.information import ResourceInformation diff --git a/third_party/3/pyspark/worker.pyi b/third_party/3/pyspark/worker.pyi index a63417c6..cc264823 100644 --- a/third_party/3/pyspark/worker.pyi +++ b/third_party/3/pyspark/worker.pyi @@ -21,8 +21,19 @@ from pyspark.broadcast import Broadcast as Broadcast from pyspark.files import SparkFiles as SparkFiles from pyspark.java_gateway import local_connect_and_auth as local_connect_and_auth from pyspark.rdd import PythonEvalType as PythonEvalType -from pyspark.resource.information import ResourceInformation -from pyspark.serializers import BatchedSerializer as BatchedSerializer, PickleSerializer as PickleSerializer, SpecialLengths as SpecialLengths, UTF8Deserializer as UTF8Deserializer, read_bool as read_bool, read_int as read_int, read_long as read_long, write_int as write_int, write_long as write_long, write_with_length as write_with_length # type: ignore[attr-defined] +from pyspark.resource import ResourceInformation as ResourceInformation +from pyspark.serializers import ( + BatchedSerializer as BatchedSerializer, + PickleSerializer as PickleSerializer, + SpecialLengths as SpecialLengths, + UTF8Deserializer as UTF8Deserializer, + read_bool as read_bool, + read_int as read_int, + read_long as read_long, + write_int as write_int, + write_long as write_long, + write_with_length as write_with_length, +) from pyspark.sql.pandas.serializers import ( ArrowStreamPandasUDFSerializer as ArrowStreamPandasUDFSerializer, CogroupUDFSerializer as CogroupUDFSerializer,