Revert "[FLINK-25188][python][build] Support m1 chip."

This reverts commit 7e9be78

NOTICE

@@ -17,7 +17,7 @@ See bundled license files for details.
 This project bundles the following dependencies under the BSD license.
 See bundled license files for details.
 
-- cloudpickle:2.0.0
+- cloudpickle:1.2.2
 - net.sf.py4j:py4j:0.10.9.3
 
 This project bundles the following dependencies under SIL OFL 1.1 license (https://opensource.org/licenses/OFL-1.1).

flink-python/dev/dev-requirements.txt

@@ -14,18 +14,17 @@
 # limitations under the License.
 setuptools>=18.0
 wheel
-apache-beam==2.38.0
+apache-beam==2.27.0
 cython==0.29.24
 py4j==0.10.9.3
 python-dateutil==2.8.0
-cloudpickle==2.0.0
+cloudpickle==1.2.2
 avro-python3>=1.8.1,!=1.9.2,<1.10.0
-pandas>=1.3.0
-pyarrow>=5.0.0
+pandas>=1.0,<1.2.0
+pyarrow>=0.15.1,<3.0.0
 pytz>=2018.3
-numpy>=1.21.4
+numpy>=1.14.3,<1.20
 fastavro>=0.21.4,<0.24
 grpcio>=1.29.0,<2
 grpcio-tools>=1.3.5,<=1.14.2
-pemja==0.1.5; python_version >= '3.7'
-httplib2>=0.8,<0.19.0
+pemja==0.1.4; python_version >= '3.7'

flink-python/dev/lint-python.sh

@@ -209,12 +209,6 @@ function install_miniconda() {
  if [ ! -d "$CURRENT_DIR/.conda" ]; then
  print_function "STEP" "installing conda..."
  $CONDA_INSTALL_SH -b -p $CURRENT_DIR/.conda 2>&1 >/dev/null
-
- # orjson depend on pip >= 20.3
- print_function "STEP" "upgrade pip..."
- $CURRENT_DIR/.conda/bin/python -m pip install --upgrade pip 2>&1 >/dev/null
- print_function "STEP" "upgrade pip... [SUCCESS]"
-
  if [ $? -ne 0 ]; then
  echo "install miniconda failed"
  exit $CONDA_INSTALL_STATUS

flink-python/pom.xml

@@ -34,7 +34,7 @@ under the License.
  <packaging>jar</packaging>
 
  <properties>
- <arrow.version>5.0.0</arrow.version>
+ <arrow.version>0.16.0</arrow.version>
  </properties>
 
  <dependencies>
@@ -88,10 +88,6 @@ under the License.
  <groupId>org.apache.beam</groupId>
  <artifactId>beam-vendor-bytebuddy-1_10_8</artifactId>
  </exclusion>
- <exclusion>
- <groupId>com.google.errorprone</groupId>
- <artifactId>error_prone_annotations</artifactId>
- </exclusion>
  </exclusions>
  </dependency>
 
@@ -104,7 +100,7 @@ under the License.
  <dependency>
  <groupId>com.alibaba</groupId>
  <artifactId>pemja</artifactId>
- <version>0.1.5</version>
+ <version>0.1.4</version>
  </dependency>
 
  <!-- Protobuf dependencies -->
@@ -145,11 +141,6 @@ under the License.
  </exclusion>
  </exclusions>
  </dependency>
- <dependency>
- <groupId>org.apache.arrow</groupId>
- <artifactId>arrow-memory-netty</artifactId>
- <version>${arrow.version}</version>
- </dependency>
 
  <!-- test dependencies -->
 
@@ -377,11 +368,11 @@ under the License.
  </excludes>
  </filter>
  <filter>
- <artifact>org.apache.beam:beam-vendor-grpc-1_43_2</artifact>
+ <artifact>org.apache.beam:beam-vendor-grpc-1_26_0</artifact>
  <excludes>
- <exclude>org/apache/beam/vendor/grpc/v1p43p2/org/jboss/**</exclude>
+ <exclude>org/apache/beam/vendor/grpc/v1p26p0/org/jboss/**</exclude>
  <exclude>schema/**</exclude>
- <exclude>org/apache/beam/vendor/grpc/v1p43p2/org/eclipse/jetty/**</exclude>
+ <exclude>org/apache/beam/vendor/grpc/v1p26p0/org/eclipse/jetty/**</exclude>
  </excludes>
  </filter>
  <filter>

flink-python/pyflink/fn_execution/beam/beam_operations.py

@@ -17,7 +17,6 @@
 ################################################################################
 from apache_beam.portability import common_urns
 from apache_beam.portability.api import beam_runner_api_pb2
-from apache_beam.runners import common
 from apache_beam.runners.worker import bundle_processor, operation_specs
 from apache_beam.utils import proto_utils
 
@@ -151,7 +150,6 @@ def _create_user_defined_function_operation(factory, transform_proto, consumers,
  input=None,
  side_inputs=None,
  output_coders=[output_coders[tag] for tag in output_tags])
- name = common.NameContext(transform_proto.unique_name)
 
  serialized_fn = spec.serialized_fn
  if hasattr(serialized_fn, "key_type"):
@@ -174,7 +172,7 @@ def _create_user_defined_function_operation(factory, transform_proto, consumers,
  serialized_fn.map_state_write_cache_size)
 
  return beam_operation_cls(
- name,
+ transform_proto.unique_name,
  spec,
  factory.counter_factory,
  factory.state_sampler,
@@ -191,7 +189,7 @@ def _create_user_defined_function_operation(factory, transform_proto, consumers,
  serialized_fn.map_state_read_cache_size,
  serialized_fn.map_state_write_cache_size)
  return beam_operation_cls(
- name,
+ transform_proto.unique_name,
  spec,
  factory.counter_factory,
  factory.state_sampler,
@@ -200,7 +198,7 @@ def _create_user_defined_function_operation(factory, transform_proto, consumers,
  keyed_state_backend)
  else:
  return beam_operation_cls(
- name,
+ transform_proto.unique_name,
  spec,
  factory.counter_factory,
  factory.state_sampler,

flink-python/pyflink/table/tests/test_dependency.py

@@ -119,7 +119,7 @@ def setUp(self):
  def test_set_requirements_without_cached_directory(self):
  requirements_txt_path = os.path.join(self.tempdir, str(uuid.uuid4()))
  with open(requirements_txt_path, 'w') as f:
- f.write("cloudpickle==2.0.0")
+ f.write("cloudpickle==1.2.2")
  self.st_env.set_python_requirements(requirements_txt_path)
 
  def check_requirements(i):

flink-python/setup.py

@@ -310,12 +310,12 @@ def extracted_output_files(base_dir, file_path, output_directory):
  author='Apache Software Foundation',
  author_email='[email protected]',
  python_requires='>=3.6',
- install_requires=['py4j==0.10.9.3', 'python-dateutil==2.8.0', 'apache-beam==2.38.0',
- 'cloudpickle==2.0.0', 'avro-python3>=1.8.1,!=1.9.2,<1.10.0',
- 'pandas>=1.3.0', 'pyarrow>=5.0.0',
- 'pytz>=2018.3', 'numpy>=1.21.4', 'fastavro>=0.21.4,<0.24',
+ install_requires=['py4j==0.10.9.3', 'python-dateutil==2.8.0', 'apache-beam==2.27.0',
+ 'cloudpickle==1.2.2', 'avro-python3>=1.8.1,!=1.9.2,<1.10.0',
+ 'pandas>=1.0,<1.2.0', 'pyarrow>=0.15.1,<3.0.0',
+ 'pytz>=2018.3', 'numpy>=1.14.3,<1.20', 'fastavro>=0.21.4,<0.24',
  'requests>=2.26.0', 'protobuf<3.18',
- 'pemja==0.1.5;python_full_version >= "3.7"', 'httplib2>=0.8,<0.19.0',
+ 'pemja==0.1.4;python_full_version >= "3.7"',
  apache_flink_libraries_dependency],
  cmdclass={'build_ext': build_ext},
  tests_require=['pytest==4.4.1'],

flink-python/src/main/java/org/apache/beam/runners/fnexecution/state/GrpcStateService.java

@@ -20,9 +20,9 @@
 import org.apache.beam.model.fnexecution.v1.BeamFnApi.StateRequest;
 import org.apache.beam.model.fnexecution.v1.BeamFnApi.StateResponse;
 import org.apache.beam.model.fnexecution.v1.BeamFnStateGrpc;
-import org.apache.beam.sdk.fn.server.FnService;
-import org.apache.beam.vendor.grpc.v1p43p2.io.grpc.stub.ServerCallStreamObserver;
-import org.apache.beam.vendor.grpc.v1p43p2.io.grpc.stub.StreamObserver;
+import org.apache.beam.runners.fnexecution.FnService;
+import org.apache.beam.vendor.grpc.v1p26p0.io.grpc.stub.ServerCallStreamObserver;
+import org.apache.beam.vendor.grpc.v1p26p0.io.grpc.stub.StreamObserver;
 
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.CompletionStage;

flink-python/src/main/java/org/apache/beam/vendor/grpc/v1p43p2/io/grpc/internal/SharedResourceHolder.java → flink-python/src/main/java/org/apache/beam/vendor/grpc/v1p26p0/io/grpc/internal/SharedResourceHolder.java

@@ -14,9 +14,9 @@
  * limitations under the License.
  */
 
-package org.apache.beam.vendor.grpc.v1p43p2.io.grpc.internal;
+package org.apache.beam.vendor.grpc.v1p26p0.io.grpc.internal;
 
-import org.apache.beam.vendor.grpc.v1p43p2.com.google.common.base.Preconditions;
+import org.apache.beam.vendor.grpc.v1p26p0.com.google.common.base.Preconditions;
 
 import javax.annotation.concurrent.ThreadSafe;
 

flink-python/src/main/java/org/apache/flink/streaming/api/runners/python/beam/BeamDataStreamPythonFunctionRunner.java

@@ -179,7 +179,7 @@ private RunnerApi.ParDoPayload createRevisePayload() {
  RunnerApi.FunctionSpec.newBuilder()
  .setUrn(STATELESS_FUNCTION_URN)
  .setPayload(
- org.apache.beam.vendor.grpc.v1p43p2.com.google
+ org.apache.beam.vendor.grpc.v1p26p0.com.google
  .protobuf.ByteString.copyFrom(
  proto.toByteArray()))
  .build());
@@ -196,7 +196,7 @@ private RunnerApi.ParDoPayload createUdfPayload(
  RunnerApi.FunctionSpec.newBuilder()
  .setUrn(urn)
  .setPayload(
- org.apache.beam.vendor.grpc.v1p43p2.com.google
+ org.apache.beam.vendor.grpc.v1p26p0.com.google
  .protobuf.ByteString.copyFrom(
  proto.toByteArray()))
  .build());

flink-python/src/main/java/org/apache/flink/streaming/api/runners/python/beam/BeamPythonFunctionRunner.java

@@ -71,7 +71,7 @@
 import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
 import org.apache.beam.sdk.transforms.windowing.GlobalWindow;
 import org.apache.beam.sdk.util.WindowedValue;
-import org.apache.beam.vendor.grpc.v1p43p2.com.google.protobuf.Struct;
+import org.apache.beam.vendor.grpc.v1p26p0.com.google.protobuf.Struct;
 import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -542,15 +542,15 @@ private ExecutableStage createExecutableStage(RunnerApi.Environment environment)
  RunnerApi.ExecutableStagePayload.WireCoderSetting.newBuilder()
  .setUrn(getUrn(RunnerApi.StandardCoders.Enum.PARAM_WINDOWED_VALUE))
  .setPayload(
- org.apache.beam.vendor.grpc.v1p43p2.com.google.protobuf.ByteString
+ org.apache.beam.vendor.grpc.v1p26p0.com.google.protobuf.ByteString
  .copyFrom(baos.toByteArray()))
  .setInputOrOutputId(INPUT_COLLECTION_ID)
  .build());
  settings.add(
  RunnerApi.ExecutableStagePayload.WireCoderSetting.newBuilder()
  .setUrn(getUrn(RunnerApi.StandardCoders.Enum.PARAM_WINDOWED_VALUE))
  .setPayload(
- org.apache.beam.vendor.grpc.v1p43p2.com.google.protobuf.ByteString
+ org.apache.beam.vendor.grpc.v1p26p0.com.google.protobuf.ByteString
  .copyFrom(baos.toByteArray()))
  .setInputOrOutputId(OUTPUT_COLLECTION_ID)
  .build());
@@ -560,7 +560,7 @@ private ExecutableStage createExecutableStage(RunnerApi.Environment environment)
  RunnerApi.ExecutableStagePayload.WireCoderSetting.newBuilder()
  .setUrn(getUrn(RunnerApi.StandardCoders.Enum.PARAM_WINDOWED_VALUE))
  .setPayload(
- org.apache.beam.vendor.grpc.v1p43p2.com.google.protobuf
+ org.apache.beam.vendor.grpc.v1p26p0.com.google.protobuf
  .ByteString.copyFrom(baos.toByteArray()))
  .setInputOrOutputId(entry.getKey())
  .build());

flink-python/src/main/java/org/apache/flink/streaming/api/runners/python/beam/state/BeamBagStateHandler.java

@@ -26,7 +26,7 @@
 import org.apache.flink.util.Preconditions;
 
 import org.apache.beam.model.fnexecution.v1.BeamFnApi;
-import org.apache.beam.vendor.grpc.v1p43p2.com.google.protobuf.ByteString;
+import org.apache.beam.vendor.grpc.v1p26p0.com.google.protobuf.ByteString;
 
 import javax.annotation.Nullable;
 

flink-python/src/main/java/org/apache/flink/streaming/api/runners/python/beam/state/BeamMapStateHandler.java

@@ -28,7 +28,7 @@
 import org.apache.flink.streaming.api.utils.ByteArrayWrapper;
 
 import org.apache.beam.model.fnexecution.v1.BeamFnApi;
-import org.apache.beam.vendor.grpc.v1p43p2.com.google.protobuf.ByteString;
+import org.apache.beam.vendor.grpc.v1p26p0.com.google.protobuf.ByteString;
 
 import java.util.HashMap;
 import java.util.Iterator;

flink-python/src/main/java/org/apache/flink/streaming/api/runners/python/beam/state/BeamStateRequestHandler.java

@@ -28,8 +28,8 @@
 
 import org.apache.beam.model.fnexecution.v1.BeamFnApi;
 import org.apache.beam.runners.fnexecution.state.StateRequestHandler;
-import org.apache.beam.vendor.grpc.v1p43p2.com.google.common.base.Charsets;
-import org.apache.beam.vendor.grpc.v1p43p2.com.google.protobuf.ByteString;
+import org.apache.beam.vendor.grpc.v1p26p0.com.google.common.base.Charsets;
+import org.apache.beam.vendor.grpc.v1p26p0.com.google.protobuf.ByteString;
 
 import javax.annotation.Nullable;
 

flink-python/src/main/java/org/apache/flink/streaming/api/utils/ProtoUtils.java

@@ -284,7 +284,7 @@ public static RunnerApi.Coder createCoderProto(
  RunnerApi.FunctionSpec.newBuilder()
  .setUrn(FLINK_CODER_URN)
  .setPayload(
- org.apache.beam.vendor.grpc.v1p43p2.com.google.protobuf
+ org.apache.beam.vendor.grpc.v1p26p0.com.google.protobuf
  .ByteString.copyFrom(
  coderInfoDescriptor.toByteArray()))
  .build())

flink-python/src/main/java/org/apache/flink/table/runtime/runners/python/beam/BeamTablePythonFunctionRunner.java

@@ -92,7 +92,7 @@ protected void buildTransforms(RunnerApi.Components.Builder componentsBuilder) {
  RunnerApi.FunctionSpec.newBuilder()
  .setUrn(functionUrn)
  .setPayload(
- org.apache.beam.vendor.grpc.v1p43p2.com.google
+ org.apache.beam.vendor.grpc.v1p26p0.com.google
  .protobuf.ByteString.copyFrom(
  userDefinedFunctionProto.toByteArray()))
  .build())

flink-python/src/main/resources/META-INF/NOTICE

@@ -9,26 +9,25 @@ This project bundles the following dependencies under the Apache Software Licens
 - com.fasterxml.jackson.core:jackson-annotations:2.13.2
 - com.fasterxml.jackson.core:jackson-core:2.13.2
 - com.fasterxml.jackson.core:jackson-databind:2.13.2.2
-- com.google.flatbuffers:flatbuffers-java:1.12.0
+- com.google.flatbuffers:flatbuffers-java:1.9.0
+- io.netty:netty-buffer:4.1.70.Final
+- io.netty:netty-common:4.1.70.Final
 - joda-time:joda-time:2.5
-- org.apache.arrow:arrow-format:5.0.0
-- org.apache.arrow:arrow-memory:5.0.0
-- org.apache.arrow:arrow-memory-core:5.0.0
-- org.apache.arrow:arrow-memory-netty:5.0.0
-- org.apache.arrow:arrow-vector:5.0.0
-- org.apache.beam:beam-model-fn-execution:2.38.0
-- org.apache.beam:beam-model-job-management:2.38.0
-- org.apache.beam:beam-model-pipeline:2.38.0
-- org.apache.beam:beam-runners-core-construction-java:2.38.0
-- org.apache.beam:beam-runners-core-java:2.38.0
-- org.apache.beam:beam-runners-java-fn-execution:2.38.0
-- org.apache.beam:beam-sdks-java-core:2.38.0
-- org.apache.beam:beam-sdks-java-fn-execution:2.38.0
-- org.apache.beam:beam-vendor-bytebuddy-1_11_0:0.1
-- org.apache.beam:beam-vendor-sdks-java-extensions-protobuf:2.38.0
+- org.apache.arrow:arrow-format:0.16.0
+- org.apache.arrow:arrow-memory:0.16.0
+- org.apache.arrow:arrow-vector:0.16.0
+- org.apache.beam:beam-model-fn-execution:2.27.0
+- org.apache.beam:beam-model-job-management:2.27.0
+- org.apache.beam:beam-model-pipeline:2.27.0
+- org.apache.beam:beam-runners-core-construction-java:2.27.0
+- org.apache.beam:beam-runners-core-java:2.27.0
+- org.apache.beam:beam-runners-java-fn-execution:2.27.0
+- org.apache.beam:beam-sdks-java-core:2.27.0
+- org.apache.beam:beam-sdks-java-fn-execution:2.27.0
+- org.apache.beam:beam-vendor-sdks-java-extensions-protobuf:2.27.0
 - org.apache.beam:beam-vendor-guava-26_0-jre:0.1
-- org.apache.beam:beam-vendor-grpc-1_43_2:0.1
-- com.alibaba:pemja:0.1.5
+- org.apache.beam:beam-vendor-grpc-1_26_0:0.3
+- com.alibaba:pemja:0.1.4
 
 This project bundles the following dependencies under the BSD license.
 See bundled license files for details
@@ -48,26 +47,26 @@ The bundled Apache Beam dependencies bundle the following dependencies under the
 - com.google.code.gson:gson:2.8.6
 - com.google.guava:guava:26.0-jre
 - com.ning:compress-lzf:1.0.3
-- io.grpc:grpc-auth:1.43.2
-- io.grpc:grpc-core:1.43.2
-- io.grpc:grpc-context:1.43.2
-- io.grpc:grpc-netty:1.43.2
-- io.grpc:grpc-protobuf:1.43.2
-- io.grpc:grpc-stub:1.43.2
-- io.grpc:grpc-testing:1.43.2
-- io.netty:netty-buffer:4.1.70.Final
-- io.netty:netty-codec:4.1.70.Final
-- io.netty:netty-codec-http:4.1.70.Final
-- io.netty:netty-codec-http2:4.1.70.Final
-- io.netty:netty-codec-socks:4.1.70.Final
-- io.netty:netty-common:4.1.70.Final
-- io.netty:netty-handler:4.1.70.Final
-- io.netty:netty-handler-proxy:4.1.70.Final
-- io.netty:netty-resolver:4.1.70.Final
-- io.netty:netty-transport:4.1.70.Final
-- io.netty:netty-transport-native-epoll:4.1.70.Final
-- io.netty:netty-transport-native-unix-common:4.1.70.Final
-- io.netty:netty-tcnative-boringssl-static:2.0.44.Final
+- io.grpc:grpc-auth:1.26.0
+- io.grpc:grpc-core:1.26.0
+- io.grpc:grpc-context:1.26.0
+- io.grpc:grpc-netty:1.26.0
+- io.grpc:grpc-protobuf:1.26.0
+- io.grpc:grpc-stub:1.26.0
+- io.grpc:grpc-testing:1.26.0
+- io.netty:netty-buffer:4.1.51.Final
+- io.netty:netty-codec:4.1.51.Final
+- io.netty:netty-codec-http:4.1.51.Final
+- io.netty:netty-codec-http2:4.1.51.Final
+- io.netty:netty-codec-socks:4.1.51.Final
+- io.netty:netty-common:4.1.51.Final
+- io.netty:netty-handler:4.1.51.Final
+- io.netty:netty-handler-proxy:4.1.51.Final
+- io.netty:netty-resolver:4.1.51.Final
+- io.netty:netty-transport:4.1.51.Final
+- io.netty:netty-transport-native-epoll:4.1.51.Final
+- io.netty:netty-transport-native-unix-common:4.1.51.Final
+- io.netty:netty-tcnative-boringssl-static:2.0.33.Final
 - io.opencensus:opencensus-api:0.24.0
 - io.opencensus:opencensus-contrib-grpc-metrics:0.24.0
 - io.perfmark:perfmark-api:0.19.0

flink-python/src/test/java/org/apache/flink/table/runtime/utils/PassThroughPythonAggregateFunctionRunner.java

@@ -33,7 +33,7 @@
 import org.apache.flink.table.types.logical.RowType;
 
 import org.apache.beam.runners.fnexecution.control.JobBundleFactory;
-import org.apache.beam.vendor.grpc.v1p43p2.com.google.protobuf.Struct;
+import org.apache.beam.vendor.grpc.v1p26p0.com.google.protobuf.Struct;
 
 import java.util.ArrayList;
 import java.util.LinkedList;