[FLINK-12127][network] Consolidate network options in NetworkEnvironm…

…entOptions

docs/_includes/generated/network_environment_configuration.html

@@ -0,0 +1,61 @@
+<table class="table table-bordered">
+ <thead>
+ <tr>
+ <th class="text-left" style="width: 20%">Key</th>
+ <th class="text-left" style="width: 15%">Default</th>
+ <th class="text-left" style="width: 65%">Description</th>
+ </tr>
+ </thead>
+ <tbody>
+ <tr>
+ <td><h5>taskmanager.data.port</h5></td>
+ <td style="word-wrap: break-word;">0</td>
+ <td>The task manager’s port used for data exchange operations.</td>
+ </tr>
+ <tr>
+ <td><h5>taskmanager.data.ssl.enabled</h5></td>
+ <td style="word-wrap: break-word;">true</td>
+ <td>Enable SSL support for the taskmanager data transport. This is applicable only when the global flag for internal SSL (security.ssl.internal.enabled) is set to true</td>
+ </tr>
+ <tr>
+ <td><h5>taskmanager.network.detailed-metrics</h5></td>
+ <td style="word-wrap: break-word;">false</td>
+ <td>Boolean flag to enable/disable more detailed metrics about inbound/outbound network queue lengths.</td>
+ </tr>
+ <tr>
+ <td><h5>taskmanager.network.memory.buffers-per-channel</h5></td>
+ <td style="word-wrap: break-word;">2</td>
+ <td>Maximum number of network buffers to use for each outgoing/incoming channel (subpartition/input channel).In credit-based flow control mode, this indicates how many credits are exclusive in each input channel. It should be configured at least 2 for good performance. 1 buffer is for receiving in-flight data in the subpartition and 1 buffer is for parallel serialization.</td>
+ </tr>
+ <tr>
+ <td><h5>taskmanager.network.memory.floating-buffers-per-gate</h5></td>
+ <td style="word-wrap: break-word;">8</td>
+ <td>Number of extra network buffers to use for each outgoing/incoming gate (result partition/input gate). In credit-based flow control mode, this indicates how many floating credits are shared among all the input channels. The floating buffers are distributed based on backlog (real-time output buffers in the subpartition) feedback, and can help relieve back-pressure caused by unbalanced data distribution among the subpartitions. This value should be increased in case of higher round trip times between nodes and/or larger number of machines in the cluster.</td>
+ </tr>
+ <tr>
+ <td><h5>taskmanager.network.memory.fraction</h5></td>
+ <td style="word-wrap: break-word;">0.1</td>
+ <td>Fraction of JVM memory to use for network buffers. This determines how many streaming data exchange channels a TaskManager can have at the same time and how well buffered the channels are. If a job is rejected or you get a warning that the system has not enough buffers available, increase this value or the min/max values below. Also note, that "taskmanager.network.memory.min"` and "taskmanager.network.memory.max" may override this fraction.</td>
+ </tr>
+ <tr>
+ <td><h5>taskmanager.network.memory.max</h5></td>
+ <td style="word-wrap: break-word;">"1gb"</td>
+ <td>Maximum memory size for network buffers.</td>
+ </tr>
+ <tr>
+ <td><h5>taskmanager.network.memory.min</h5></td>
+ <td style="word-wrap: break-word;">"64mb"</td>
+ <td>Minimum memory size for network buffers.</td>
+ </tr>
+ <tr>
+ <td><h5>taskmanager.network.request-backoff.initial</h5></td>
+ <td style="word-wrap: break-word;">100</td>
+ <td>Minimum backoff in milliseconds for partition requests of input channels.</td>
+ </tr>
+ <tr>
+ <td><h5>taskmanager.network.request-backoff.max</h5></td>
+ <td style="word-wrap: break-word;">10000</td>
+ <td>Maximum backoff in milliseconds for partition requests of input channels.</td>
+ </tr>
+ </tbody>
+</table>

docs/_includes/generated/task_manager_configuration.html

@@ -27,16 +27,6 @@
  <td style="word-wrap: break-word;">-1</td>
  <td>The maximum number of bytes that a checkpoint alignment may buffer. If the checkpoint alignment buffers more than the configured amount of data, the checkpoint is aborted (skipped). A value of -1 indicates that there is no limit.</td>
  </tr>
- <tr>
- <td><h5>taskmanager.data.port</h5></td>
- <td style="word-wrap: break-word;">0</td>
- <td>The task manager’s port used for data exchange operations.</td>
- </tr>
- <tr>
- <td><h5>taskmanager.data.ssl.enabled</h5></td>
- <td style="word-wrap: break-word;">true</td>
- <td>Enable SSL support for the taskmanager data transport. This is applicable only when the global flag for internal SSL (security.ssl.internal.enabled) is set to true</td>
- </tr>
  <tr>
  <td><h5>taskmanager.debug.memory.log</h5></td>
  <td style="word-wrap: break-word;">false</td>
@@ -73,46 +63,6 @@
  <td>The automatic address binding policy used by the TaskManager if "taskmanager.host" is not set. The value should be one of the following:
 <ul><li>"name" - uses hostname as binding address</li><li>"ip" - uses host's ip address as binding address</li></ul></td>
  </tr>
- <tr>
- <td><h5>taskmanager.network.detailed-metrics</h5></td>
- <td style="word-wrap: break-word;">false</td>
- <td>Boolean flag to enable/disable more detailed metrics about inbound/outbound network queue lengths.</td>
- </tr>
- <tr>
- <td><h5>taskmanager.network.memory.buffers-per-channel</h5></td>
- <td style="word-wrap: break-word;">2</td>
- <td>Maximum number of network buffers to use for each outgoing/incoming channel (subpartition/input channel).In credit-based flow control mode, this indicates how many credits are exclusive in each input channel. It should be configured at least 2 for good performance. 1 buffer is for receiving in-flight data in the subpartition and 1 buffer is for parallel serialization.</td>
- </tr>
- <tr>
- <td><h5>taskmanager.network.memory.floating-buffers-per-gate</h5></td>
- <td style="word-wrap: break-word;">8</td>
- <td>Number of extra network buffers to use for each outgoing/incoming gate (result partition/input gate). In credit-based flow control mode, this indicates how many floating credits are shared among all the input channels. The floating buffers are distributed based on backlog (real-time output buffers in the subpartition) feedback, and can help relieve back-pressure caused by unbalanced data distribution among the subpartitions. This value should be increased in case of higher round trip times between nodes and/or larger number of machines in the cluster.</td>
- </tr>
- <tr>
- <td><h5>taskmanager.network.memory.fraction</h5></td>
- <td style="word-wrap: break-word;">0.1</td>
- <td>Fraction of JVM memory to use for network buffers. This determines how many streaming data exchange channels a TaskManager can have at the same time and how well buffered the channels are. If a job is rejected or you get a warning that the system has not enough buffers available, increase this value or the min/max values below. Also note, that "taskmanager.network.memory.min"` and "taskmanager.network.memory.max" may override this fraction.</td>
- </tr>
- <tr>
- <td><h5>taskmanager.network.memory.max</h5></td>
- <td style="word-wrap: break-word;">"1gb"</td>
- <td>Maximum memory size for network buffers.</td>
- </tr>
- <tr>
- <td><h5>taskmanager.network.memory.min</h5></td>
- <td style="word-wrap: break-word;">"64mb"</td>
- <td>Minimum memory size for network buffers.</td>
- </tr>
- <tr>
- <td><h5>taskmanager.network.request-backoff.initial</h5></td>
- <td style="word-wrap: break-word;">100</td>
- <td>Minimum backoff in milliseconds for partition requests of input channels.</td>
- </tr>
- <tr>
- <td><h5>taskmanager.network.request-backoff.max</h5></td>
- <td style="word-wrap: break-word;">10000</td>
- <td>Maximum backoff in milliseconds for partition requests of input channels.</td>
- </tr>
  <tr>
  <td><h5>taskmanager.numberOfTaskSlots</h5></td>
  <td style="word-wrap: break-word;">1</td>

docs/ops/config.md

@@ -100,11 +100,15 @@ The default fraction for managed memory can be adjusted using the taskmanager.me
 
 {% include generated/security_configuration.html %}
 
-### Network communication (via Netty)
+### Network Environment
+
+{% include generated/network_environment_configuration.html %}
+
+### Network Communication (via Netty)
 
 These parameters allow for advanced tuning. The default values are sufficient when running concurrent high-throughput jobs on a large cluster.
 
-{% include generated/netty_configuration.html %}
+{% include generated/network_netty_configuration.html %}
 
 ### Web Frontend
 

flink-connectors/flink-hbase/src/test/java/org/apache/flink/addons/hbase/HBaseConnectorITCase.java

@@ -27,7 +27,7 @@
 import org.apache.flink.api.java.LocalEnvironment;
 import org.apache.flink.api.java.tuple.Tuple1;
 import org.apache.flink.configuration.Configuration;
-import org.apache.flink.configuration.TaskManagerOptions;
+import org.apache.flink.configuration.NetworkEnvironmentOptions;
 import org.apache.flink.table.api.Table;
 import org.apache.flink.table.api.TableConfig;
 import org.apache.flink.table.api.java.BatchTableEnvironment;
@@ -362,7 +362,7 @@ private abstract static class LimitNetworkBuffersTestEnvironment extends Executi
  public static void setAsContext() {
  Configuration config = new Configuration();
  // the default network buffers size (10% of heap max =~ 150MB) seems to much for this test case
- config.setString(TaskManagerOptions.NETWORK_BUFFERS_MEMORY_MAX, String.valueOf(80L << 20)); // 80 MB
+ config.setString(NetworkEnvironmentOptions.NETWORK_BUFFERS_MEMORY_MAX, String.valueOf(80L << 20)); // 80 MB
  final LocalEnvironment le = new LocalEnvironment(config);
 
  initializeContextEnvironment(new ExecutionEnvironmentFactory() {

flink-core/src/main/java/org/apache/flink/configuration/ConfigConstants.java

@@ -204,15 +204,15 @@ public final class ConfigConstants {
  public static final String TASK_MANAGER_IPC_PORT_KEY = "taskmanager.rpc.port";
 
  /**
- * @deprecated use {@link TaskManagerOptions#DATA_PORT} instead
+ * @deprecated use {@link NetworkEnvironmentOptions#DATA_PORT} instead
  */
  @Deprecated
  public static final String TASK_MANAGER_DATA_PORT_KEY = "taskmanager.data.port";
 
  /**
  * Config parameter to override SSL support for taskmanager's data transport.
  *
- * @deprecated use {@link TaskManagerOptions#DATA_SSL_ENABLED} instead
+ * @deprecated use {@link NetworkEnvironmentOptions#DATA_SSL_ENABLED} instead
  */
  @Deprecated
  public static final String TASK_MANAGER_DATA_SSL_ENABLED = "taskmanager.data.ssl.enabled";
@@ -270,7 +270,7 @@ public final class ConfigConstants {
  * The config parameter defining the number of buffers used in the network stack. This defines the
  * number of possible tasks and shuffles.
  *
- * @deprecated Use {@link TaskManagerOptions#NETWORK_NUM_BUFFERS} instead
+ * @deprecated Use {@link NetworkEnvironmentOptions#NETWORK_NUM_BUFFERS} instead
  */
  @Deprecated
  public static final String TASK_MANAGER_NETWORK_NUM_BUFFERS_KEY = "taskmanager.network.numberOfBuffers";
@@ -1392,15 +1392,15 @@ public final class ConfigConstants {
  * The default network port the task manager expects to receive transfer envelopes on. The {@code 0} means that
  * the TaskManager searches for a free port.
  *
- * @deprecated use {@link TaskManagerOptions#DATA_PORT} instead
+ * @deprecated use {@link NetworkEnvironmentOptions#DATA_PORT} instead
  */
  @Deprecated
  public static final int DEFAULT_TASK_MANAGER_DATA_PORT = 0;
 
  /**
  * The default value to override ssl support for task manager's data transport.
  *
- * @deprecated use {@link TaskManagerOptions#DATA_SSL_ENABLED} instead
+ * @deprecated use {@link NetworkEnvironmentOptions#DATA_SSL_ENABLED} instead
  */
  @Deprecated
  public static final boolean DEFAULT_TASK_MANAGER_DATA_SSL_ENABLED = true;
@@ -1424,7 +1424,7 @@ public final class ConfigConstants {
  /**
  * Config key has been deprecated. Therefore, no default value required.
  *
- * @deprecated {@link TaskManagerOptions#NETWORK_NUM_BUFFERS} provides the default value now
+ * @deprecated {@link NetworkEnvironmentOptions#NETWORK_NUM_BUFFERS} provides the default value now
  */
  @Deprecated
  public static final int DEFAULT_TASK_MANAGER_NETWORK_NUM_BUFFERS = 2048;