This closes #2920: Move PipelineRunner to user-facing package; make s…

…dk.runners package internal

  Javadoc that the sdk.runners package is internal
  Exclude sdk.runners from javadoc
  Move PipelineRunner to toplevel sdk package (automated refactor)

runners/apex/src/main/java/org/apache/beam/runners/apex/ApexRunner.java

@@ -43,14 +43,14 @@
 import org.apache.beam.runners.core.construction.PrimitiveCreate;
 import org.apache.beam.runners.core.construction.SingleInputOutputOverrideFactory;
 import org.apache.beam.sdk.Pipeline;
+import org.apache.beam.sdk.PipelineRunner;
 import org.apache.beam.sdk.coders.Coder;
 import org.apache.beam.sdk.coders.CoderRegistry;
 import org.apache.beam.sdk.coders.ListCoder;
 import org.apache.beam.sdk.options.PipelineOptions;
 import org.apache.beam.sdk.options.PipelineOptionsValidator;
 import org.apache.beam.sdk.runners.AppliedPTransform;
 import org.apache.beam.sdk.runners.PTransformOverride;
-import org.apache.beam.sdk.runners.PipelineRunner;
 import org.apache.beam.sdk.transforms.Combine;
 import org.apache.beam.sdk.transforms.Combine.GloballyAsSingletonView;
 import org.apache.beam.sdk.transforms.Create;

runners/apex/src/main/java/org/apache/beam/runners/apex/ApexRunnerRegistrar.java

@@ -20,9 +20,9 @@
 
 import com.google.auto.service.AutoService;
 import com.google.common.collect.ImmutableList;
+import org.apache.beam.sdk.PipelineRunner;
 import org.apache.beam.sdk.options.PipelineOptions;
 import org.apache.beam.sdk.options.PipelineOptionsRegistrar;
-import org.apache.beam.sdk.runners.PipelineRunner;
 import org.apache.beam.sdk.runners.PipelineRunnerRegistrar;
 
 /**

runners/apex/src/main/java/org/apache/beam/runners/apex/TestApexRunner.java

@@ -19,9 +19,9 @@
 
 import java.io.IOException;
 import org.apache.beam.sdk.Pipeline;
+import org.apache.beam.sdk.PipelineRunner;
 import org.apache.beam.sdk.options.PipelineOptions;
 import org.apache.beam.sdk.options.PipelineOptionsValidator;
-import org.apache.beam.sdk.runners.PipelineRunner;
 import org.joda.time.Duration;
 
 /**

runners/core-java/src/main/java/org/apache/beam/runners/core/OldDoFn.java

@@ -18,9 +18,9 @@
 package org.apache.beam.runners.core;
 
 import java.io.Serializable;
+import org.apache.beam.sdk.PipelineRunner;
 import org.apache.beam.sdk.annotations.Experimental;
 import org.apache.beam.sdk.options.PipelineOptions;
-import org.apache.beam.sdk.runners.PipelineRunner;
 import org.apache.beam.sdk.transforms.DoFn;
 import org.apache.beam.sdk.transforms.DoFnTester;
 import org.apache.beam.sdk.transforms.ParDo;
@@ -71,7 +71,7 @@ public abstract class Context {
 
  /**
  * Returns the {@code PipelineOptions} specified with the
- * {@link org.apache.beam.sdk.runners.PipelineRunner}
+ * {@link PipelineRunner}
  * invoking this {@code OldDoFn}. The {@code PipelineOptions} will
  * be the default running via {@link DoFnTester}.
  */

runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectRegistrar.java

@@ -19,9 +19,9 @@
 
 import com.google.auto.service.AutoService;
 import com.google.common.collect.ImmutableList;
+import org.apache.beam.sdk.PipelineRunner;
 import org.apache.beam.sdk.options.PipelineOptions;
 import org.apache.beam.sdk.options.PipelineOptionsRegistrar;
-import org.apache.beam.sdk.runners.PipelineRunner;
 import org.apache.beam.sdk.runners.PipelineRunnerRegistrar;
 
 /**

runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectRunner.java

@@ -34,12 +34,12 @@
 import org.apache.beam.sdk.Pipeline;
 import org.apache.beam.sdk.Pipeline.PipelineExecutionException;
 import org.apache.beam.sdk.PipelineResult;
+import org.apache.beam.sdk.PipelineRunner;
 import org.apache.beam.sdk.io.Read;
 import org.apache.beam.sdk.metrics.MetricResults;
 import org.apache.beam.sdk.metrics.MetricsEnvironment;
 import org.apache.beam.sdk.options.PipelineOptions;
 import org.apache.beam.sdk.runners.PTransformOverride;
-import org.apache.beam.sdk.runners.PipelineRunner;
 import org.apache.beam.sdk.testing.TestStream;
 import org.apache.beam.sdk.transforms.GroupByKey;
 import org.apache.beam.sdk.transforms.PTransform;

runners/direct-java/src/main/java/org/apache/beam/runners/direct/DoFnLifecycleManager.java

@@ -26,7 +26,7 @@
 import java.util.Collection;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ConcurrentMap;
-import org.apache.beam.sdk.runners.PipelineRunner;
+import org.apache.beam.sdk.PipelineRunner;
 import org.apache.beam.sdk.transforms.DoFn;
 import org.apache.beam.sdk.transforms.DoFn.Setup;
 import org.apache.beam.sdk.transforms.DoFn.Teardown;

runners/direct-java/src/test/java/org/apache/beam/runners/direct/DirectRunnerTest.java

@@ -43,6 +43,7 @@
 import org.apache.beam.sdk.Pipeline;
 import org.apache.beam.sdk.PipelineResult;
 import org.apache.beam.sdk.PipelineResult.State;
+import org.apache.beam.sdk.PipelineRunner;
 import org.apache.beam.sdk.coders.Coder;
 import org.apache.beam.sdk.coders.CoderException;
 import org.apache.beam.sdk.coders.CustomCoder;
@@ -55,7 +56,6 @@
 import org.apache.beam.sdk.io.Read;
 import org.apache.beam.sdk.options.PipelineOptions;
 import org.apache.beam.sdk.options.PipelineOptionsFactory;
-import org.apache.beam.sdk.runners.PipelineRunner;
 import org.apache.beam.sdk.testing.PAssert;
 import org.apache.beam.sdk.testing.TestPipeline;
 import org.apache.beam.sdk.transforms.Count;

runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkRunner.java

@@ -31,10 +31,10 @@
 import java.util.TreeSet;
 import org.apache.beam.sdk.Pipeline;
 import org.apache.beam.sdk.PipelineResult;
+import org.apache.beam.sdk.PipelineRunner;
 import org.apache.beam.sdk.metrics.MetricsEnvironment;
 import org.apache.beam.sdk.options.PipelineOptions;
 import org.apache.beam.sdk.options.PipelineOptionsValidator;
-import org.apache.beam.sdk.runners.PipelineRunner;
 import org.apache.beam.sdk.runners.TransformHierarchy;
 import org.apache.beam.sdk.transforms.PTransform;
 import org.apache.beam.sdk.transforms.View;

runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkRunnerRegistrar.java

@@ -20,9 +20,9 @@
 
 import com.google.auto.service.AutoService;
 import com.google.common.collect.ImmutableList;
+import org.apache.beam.sdk.PipelineRunner;
 import org.apache.beam.sdk.options.PipelineOptions;
 import org.apache.beam.sdk.options.PipelineOptionsRegistrar;
-import org.apache.beam.sdk.runners.PipelineRunner;
 import org.apache.beam.sdk.runners.PipelineRunnerRegistrar;
 
 

runners/flink/src/main/java/org/apache/beam/runners/flink/TestFlinkRunner.java

@@ -20,10 +20,10 @@
 import org.apache.beam.sdk.Pipeline;
 import org.apache.beam.sdk.Pipeline.PipelineExecutionException;
 import org.apache.beam.sdk.PipelineResult;
+import org.apache.beam.sdk.PipelineRunner;
 import org.apache.beam.sdk.options.PipelineOptions;
 import org.apache.beam.sdk.options.PipelineOptionsFactory;
 import org.apache.beam.sdk.options.PipelineOptionsValidator;
-import org.apache.beam.sdk.runners.PipelineRunner;
 import org.apache.beam.sdk.util.UserCodeException;
 
 /**

runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowPipelineRegistrar.java

@@ -20,9 +20,9 @@
 import com.google.auto.service.AutoService;
 import com.google.common.collect.ImmutableList;
 import org.apache.beam.runners.dataflow.options.DataflowPipelineOptions;
+import org.apache.beam.sdk.PipelineRunner;
 import org.apache.beam.sdk.options.PipelineOptions;
 import org.apache.beam.sdk.options.PipelineOptionsRegistrar;
-import org.apache.beam.sdk.runners.PipelineRunner;
 import org.apache.beam.sdk.runners.PipelineRunnerRegistrar;
 
 /**

runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowRunner.java

@@ -77,6 +77,7 @@
 import org.apache.beam.sdk.Pipeline;
 import org.apache.beam.sdk.Pipeline.PipelineVisitor;
 import org.apache.beam.sdk.PipelineResult.State;
+import org.apache.beam.sdk.PipelineRunner;
 import org.apache.beam.sdk.annotations.Experimental;
 import org.apache.beam.sdk.coders.Coder;
 import org.apache.beam.sdk.coders.Coder.NonDeterministicException;
@@ -102,7 +103,6 @@
 import org.apache.beam.sdk.runners.AppliedPTransform;
 import org.apache.beam.sdk.runners.PTransformOverride;
 import org.apache.beam.sdk.runners.PTransformOverrideFactory;
-import org.apache.beam.sdk.runners.PipelineRunner;
 import org.apache.beam.sdk.runners.TransformHierarchy;
 import org.apache.beam.sdk.runners.TransformHierarchy.Node;
 import org.apache.beam.sdk.transforms.Combine;

runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/testing/TestDataflowRunner.java

@@ -38,8 +38,8 @@
 import org.apache.beam.runners.dataflow.util.MonitoringUtil.JobMessagesHandler;
 import org.apache.beam.sdk.Pipeline;
 import org.apache.beam.sdk.PipelineResult.State;
+import org.apache.beam.sdk.PipelineRunner;
 import org.apache.beam.sdk.options.PipelineOptions;
-import org.apache.beam.sdk.runners.PipelineRunner;
 import org.apache.beam.sdk.testing.PAssert;
 import org.apache.beam.sdk.testing.TestPipeline;
 import org.apache.beam.sdk.testing.TestPipelineOptions;

runners/spark/src/main/java/org/apache/beam/runners/spark/SparkRunner.java

@@ -41,13 +41,13 @@
 import org.apache.beam.runners.spark.translation.streaming.SparkRunnerStreamingContextFactory;
 import org.apache.beam.runners.spark.util.GlobalWatermarkHolder.WatermarksListener;
 import org.apache.beam.sdk.Pipeline;
+import org.apache.beam.sdk.PipelineRunner;
 import org.apache.beam.sdk.io.Read;
 import org.apache.beam.sdk.metrics.MetricsEnvironment;
 import org.apache.beam.sdk.options.PipelineOptions;
 import org.apache.beam.sdk.options.PipelineOptionsFactory;
 import org.apache.beam.sdk.options.PipelineOptionsValidator;
 import org.apache.beam.sdk.runners.AppliedPTransform;
-import org.apache.beam.sdk.runners.PipelineRunner;
 import org.apache.beam.sdk.runners.TransformHierarchy;
 import org.apache.beam.sdk.transforms.Combine;
 import org.apache.beam.sdk.transforms.PTransform;

runners/spark/src/main/java/org/apache/beam/runners/spark/SparkRunnerDebugger.java

@@ -25,9 +25,9 @@
 import org.apache.beam.runners.spark.translation.TransformTranslator;
 import org.apache.beam.runners.spark.translation.streaming.StreamingTransformTranslator;
 import org.apache.beam.sdk.Pipeline;
+import org.apache.beam.sdk.PipelineRunner;
 import org.apache.beam.sdk.options.PipelineOptions;
 import org.apache.beam.sdk.options.PipelineOptionsValidator;
-import org.apache.beam.sdk.runners.PipelineRunner;
 import org.apache.spark.api.java.JavaSparkContext;
 import org.apache.spark.streaming.api.java.JavaStreamingContext;
 import org.joda.time.Duration;

runners/spark/src/main/java/org/apache/beam/runners/spark/SparkRunnerRegistrar.java

@@ -20,9 +20,9 @@
 
 import com.google.auto.service.AutoService;
 import com.google.common.collect.ImmutableList;
+import org.apache.beam.sdk.PipelineRunner;
 import org.apache.beam.sdk.options.PipelineOptions;
 import org.apache.beam.sdk.options.PipelineOptionsRegistrar;
-import org.apache.beam.sdk.runners.PipelineRunner;
 import org.apache.beam.sdk.runners.PipelineRunnerRegistrar;
 
 /**

runners/spark/src/main/java/org/apache/beam/runners/spark/TestSparkRunner.java

@@ -39,13 +39,13 @@
 import org.apache.beam.runners.spark.util.GlobalWatermarkHolder;
 import org.apache.beam.sdk.Pipeline;
 import org.apache.beam.sdk.PipelineResult;
+import org.apache.beam.sdk.PipelineRunner;
 import org.apache.beam.sdk.io.BoundedReadFromUnboundedSource;
 import org.apache.beam.sdk.options.PipelineOptions;
 import org.apache.beam.sdk.options.PipelineOptionsValidator;
 import org.apache.beam.sdk.runners.AppliedPTransform;
 import org.apache.beam.sdk.runners.PTransformOverride;
 import org.apache.beam.sdk.runners.PTransformOverrideFactory;
-import org.apache.beam.sdk.runners.PipelineRunner;
 import org.apache.beam.sdk.transforms.PTransform;
 import org.apache.beam.sdk.transforms.ParDo;
 import org.apache.beam.sdk.values.PBegin;

sdks/java/core/src/main/java/org/apache/beam/sdk/Pipeline.java

@@ -38,7 +38,6 @@
 import org.apache.beam.sdk.runners.PTransformOverrideFactory;
 import org.apache.beam.sdk.runners.PTransformOverrideFactory.PTransformReplacement;
 import org.apache.beam.sdk.runners.PTransformOverrideFactory.ReplacementOutput;
-import org.apache.beam.sdk.runners.PipelineRunner;
 import org.apache.beam.sdk.runners.TransformHierarchy;
 import org.apache.beam.sdk.runners.TransformHierarchy.Node;
 import org.apache.beam.sdk.transforms.Create;

sdks/java/core/src/main/java/org/apache/beam/sdk/runners/PipelineRunner.java → sdks/java/core/src/main/java/org/apache/beam/sdk/PipelineRunner.java

@@ -15,12 +15,10 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.beam.sdk.runners;
+package org.apache.beam.sdk;
 
 import static com.google.common.base.Preconditions.checkNotNull;
 
-import org.apache.beam.sdk.Pipeline;
-import org.apache.beam.sdk.PipelineResult;
 import org.apache.beam.sdk.io.FileSystems;
 import org.apache.beam.sdk.options.PipelineOptions;
 import org.apache.beam.sdk.options.PipelineOptionsValidator;

sdks/java/core/src/main/java/org/apache/beam/sdk/coders/Coder.java

@@ -29,6 +29,7 @@
 import java.util.Arrays;
 import java.util.List;
 import javax.annotation.Nullable;
+import org.apache.beam.sdk.PipelineRunner;
 import org.apache.beam.sdk.annotations.Experimental;
 import org.apache.beam.sdk.annotations.Experimental.Kind;
 import org.apache.beam.sdk.util.common.ElementByteSizeObserver;
@@ -243,7 +244,7 @@ public abstract T decode(InputStream inStream, Context context)
  * constant time (or lazily).
  *
  * <p>Not intended to be called by user code, but instead by
- * {@link org.apache.beam.sdk.runners.PipelineRunner}
+ * {@link PipelineRunner}
  * implementations.
  */
  public abstract boolean isRegisterByteSizeObserverCheap(T value);
@@ -255,7 +256,7 @@ public abstract T decode(InputStream inStream, Context context)
  * constant time (or lazily).
  *
  * <p>Not intended to be called by user code, but instead by
- * {@link org.apache.beam.sdk.runners.PipelineRunner}
+ * {@link PipelineRunner}
  * implementations.
  */
  @Deprecated
@@ -266,7 +267,7 @@ public abstract T decode(InputStream inStream, Context context)
  * of the encoded value using this {@code Coder}.
  *
  * <p>Not intended to be called by user code, but instead by
- * {@link org.apache.beam.sdk.runners.PipelineRunner}
+ * {@link PipelineRunner}
  * implementations.
  */
  public abstract void registerByteSizeObserver(
@@ -278,7 +279,7 @@ public abstract void registerByteSizeObserver(
  * of the encoded value using this {@code Coder}.
  *
  * <p>Not intended to be called by user code, but instead by
- * {@link org.apache.beam.sdk.runners.PipelineRunner}
+ * {@link PipelineRunner}
  * implementations.
  */
  @Deprecated

sdks/java/core/src/main/java/org/apache/beam/sdk/coders/package-info.java

@@ -27,7 +27,7 @@
  * machines.
  *
  * <p>Exactly when PCollection elements are encoded during execution depends on which
- * {@link org.apache.beam.sdk.runners.PipelineRunner} is being used and how that runner
+ * {@link org.apache.beam.sdk.PipelineRunner} is being used and how that runner
  * chooses to execute the pipeline. As such, Beam requires that all PCollections have an
  * appropriate Coder in case it becomes necessary. In many cases, the Coder can be inferred from
  * the available Java type

sdks/java/core/src/main/java/org/apache/beam/sdk/io/AvroSource.java

@@ -48,12 +48,12 @@
 import org.apache.avro.io.DecoderFactory;
 import org.apache.avro.reflect.ReflectData;
 import org.apache.avro.reflect.ReflectDatumReader;
+import org.apache.beam.sdk.PipelineRunner;
 import org.apache.beam.sdk.annotations.Experimental;
 import org.apache.beam.sdk.coders.AvroCoder;
 import org.apache.beam.sdk.io.fs.MatchResult.Metadata;
 import org.apache.beam.sdk.io.fs.ResourceId;
 import org.apache.beam.sdk.options.PipelineOptions;
-import org.apache.beam.sdk.runners.PipelineRunner;
 import org.apache.beam.sdk.values.PCollection;
 import org.apache.commons.compress.compressors.bzip2.BZip2CompressorInputStream;
 import org.apache.commons.compress.compressors.snappy.SnappyCompressorInputStream;

sdks/java/core/src/main/java/org/apache/beam/sdk/options/PipelineOptions.java

@@ -30,9 +30,9 @@
 import java.util.concurrent.atomic.AtomicLong;
 import javax.annotation.concurrent.ThreadSafe;
 import org.apache.beam.sdk.Pipeline;
+import org.apache.beam.sdk.PipelineRunner;
 import org.apache.beam.sdk.options.ProxyInvocationHandler.Deserializer;
 import org.apache.beam.sdk.options.ProxyInvocationHandler.Serializer;
-import org.apache.beam.sdk.runners.PipelineRunner;
 import org.apache.beam.sdk.transforms.DoFn;
 import org.apache.beam.sdk.transforms.display.HasDisplayData;
 import org.joda.time.DateTimeUtils;

sdks/java/core/src/main/java/org/apache/beam/sdk/options/PipelineOptionsFactory.java

@@ -73,8 +73,8 @@
 import java.util.TreeMap;
 import java.util.TreeSet;
 import javax.annotation.Nonnull;
+import org.apache.beam.sdk.PipelineRunner;
 import org.apache.beam.sdk.options.Validation.Required;
-import org.apache.beam.sdk.runners.PipelineRunner;
 import org.apache.beam.sdk.runners.PipelineRunnerRegistrar;
 import org.apache.beam.sdk.transforms.display.DisplayData;
 import org.apache.beam.sdk.util.StringUtils;

sdks/java/core/src/main/java/org/apache/beam/sdk/package-info.java

@@ -27,7 +27,7 @@
  * the PTransforms consume and produce.
  *
  * <p>Each Pipeline has a
- * {@link org.apache.beam.sdk.runners.PipelineRunner} to specify
+ * {@link org.apache.beam.sdk.PipelineRunner} to specify
  * where and how it should run after pipeline construction is complete.
  *
  */

sdks/java/core/src/main/java/org/apache/beam/sdk/runners/PTransformOverrideFactory.java

@@ -21,6 +21,7 @@
 
 import com.google.auto.value.AutoValue;
 import java.util.Map;
+import org.apache.beam.sdk.PipelineRunner;
 import org.apache.beam.sdk.annotations.Experimental;
 import org.apache.beam.sdk.annotations.Experimental.Kind;
 import org.apache.beam.sdk.annotations.Internal;

sdks/java/core/src/main/java/org/apache/beam/sdk/runners/PipelineRunnerRegistrar.java

@@ -19,6 +19,7 @@
 
 import com.google.auto.service.AutoService;
 import java.util.ServiceLoader;
+import org.apache.beam.sdk.PipelineRunner;
 import org.apache.beam.sdk.annotations.Internal;
 
 /**

sdks/java/core/src/main/java/org/apache/beam/sdk/runners/package-info.java

@@ -16,8 +16,8 @@
  * limitations under the License.
  */
 /**
- * Defines different data processing backends (aka, runners) for executing Beam Pipelines.
+ * <b>For internal use only; no backwards compatibility guarantees.</b>
 
- * <p>The runner is specified as part of the {@link org.apache.beam.sdk.options.PipelineOptions}.
+ * <p>Internals for use by runners.
  */
 package org.apache.beam.sdk.runners;

sdks/java/core/src/main/java/org/apache/beam/sdk/state/StateContext.java

@@ -17,6 +17,7 @@
  */
 package org.apache.beam.sdk.state;
 
+import org.apache.beam.sdk.PipelineRunner;
 import org.apache.beam.sdk.annotations.Internal;
 import org.apache.beam.sdk.options.PipelineOptions;
 import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
@@ -30,7 +31,7 @@
 public interface StateContext<W extends BoundedWindow> {
  /**
  * Returns the {@code PipelineOptions} specified with the
- * {@link org.apache.beam.sdk.runners.PipelineRunner}.
+ * {@link PipelineRunner}.
  */
  PipelineOptions getPipelineOptions();