Skip to content

Commit

Permalink
[FLINK-7408] [conf] Create WebOptions for WebRuntimeMonitor
Browse files Browse the repository at this point in the history
This commit moves the WebRuntimeMonitor related configuration options from
JobManagerOptions to WebOptions. Moreover, it removes the prefix jobmanager.

This closes apache#4512.
  • Loading branch information
tillrohrmann committed Aug 11, 2017
1 parent d6126e7 commit c685251
Show file tree
Hide file tree
Showing 19 changed files with 225 additions and 176 deletions.
22 changes: 11 additions & 11 deletions docs/ops/config.md
Original file line number Diff line number Diff line change
Expand Up @@ -367,27 +367,27 @@ These parameters allow for advanced tuning. The default values are sufficient wh

- `taskmanager.net.transport`: The Netty transport type, either "nio" or "epoll" (DEFAULT: **nio**).

### JobManager Web Frontend
### Web Frontend

- `jobmanager.web.port`: Port of the JobManager's web interface that displays status of running jobs and execution time breakdowns of finished jobs (DEFAULT: 8081). Setting this value to `-1` disables the web frontend.
- `web.port`: Port of the web interface that displays status of running jobs and execution time breakdowns of finished jobs (DEFAULT: 8081). Setting this value to `-1` disables the web frontend.

- `jobmanager.web.history`: The number of latest jobs that the JobManager's web front-end in its history (DEFAULT: 5).
- `web.history`: The number of latest jobs that the web front-end in its history (DEFAULT: 5).

- `jobmanager.web.checkpoints.disable`: Disables checkpoint statistics (DEFAULT: `false`).
- `web.checkpoints.disable`: Disables checkpoint statistics (DEFAULT: `false`).

- `jobmanager.web.checkpoints.history`: Number of checkpoint statistics to remember (DEFAULT: `10`).
- `web.checkpoints.history`: Number of checkpoint statistics to remember (DEFAULT: `10`).

- `jobmanager.web.backpressure.cleanup-interval`: Time after which cached stats are cleaned up if not accessed (DEFAULT: `600000`, 10 mins).
- `web.backpressure.cleanup-interval`: Time after which cached stats are cleaned up if not accessed (DEFAULT: `600000`, 10 mins).

- `jobmanager.web.backpressure.refresh-interval`: Time after which available stats are deprecated and need to be refreshed (DEFAULT: `60000`, 1 min).
- `web.backpressure.refresh-interval`: Time after which available stats are deprecated and need to be refreshed (DEFAULT: `60000`, 1 min).

- `jobmanager.web.backpressure.num-samples`: Number of stack trace samples to take to determine back pressure (DEFAULT: `100`).
- `web.backpressure.num-samples`: Number of stack trace samples to take to determine back pressure (DEFAULT: `100`).

- `jobmanager.web.backpressure.delay-between-samples`: Delay between stack trace samples to determine back pressure (DEFAULT: `50`, 50 ms).
- `web.backpressure.delay-between-samples`: Delay between stack trace samples to determine back pressure (DEFAULT: `50`, 50 ms).

- `jobmanager.web.ssl.enabled`: Enable https access to the web frontend. This is applicable only when the global ssl flag security.ssl.enabled is set to true (DEFAULT: `true`).
- `web.ssl.enabled`: Enable https access to the web frontend. This is applicable only when the global ssl flag security.ssl.enabled is set to true (DEFAULT: `true`).

- `jobmanager.web.access-control-allow-origin`: Enable custom access control parameter for allow origin header, default is `*`.
- `web.access-control-allow-origin`: Enable custom access control parameter for allow origin header, default is `*`.

### File Systems

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@

import org.apache.flink.api.common.JobSubmissionResult;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.configuration.JobManagerOptions;
import org.apache.flink.configuration.WebOptions;
import org.apache.flink.runtime.clusterframework.messages.GetClusterStatus;
import org.apache.flink.runtime.clusterframework.messages.GetClusterStatusResponse;
import org.apache.flink.runtime.highavailability.HighAvailabilityServices;
Expand Down Expand Up @@ -54,7 +54,7 @@ public void waitForClusterToBeReady() {}
@Override
public String getWebInterfaceURL() {
String host = getJobManagerAddress().getHostString();
int port = getFlinkConfiguration().getInteger(JobManagerOptions.WEB_PORT);
int port = getFlinkConfiguration().getInteger(WebOptions.PORT);
return "http:https://" + host + ":" + port;
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -156,7 +156,7 @@ private static String toHtmlString(final ConfigOption<?> option) {
// This is a temporary hack that should be removed once FLINK-6490 is resolved.
// These options use System.getProperty("java.io.tmpdir") as the default.
// As a result the generated table contains an actual path as the default, which is simply wrong.
if (option == JobManagerOptions.WEB_TMP_DIR || option.key().equals("python.dc.tmp.dir")) {
if (option == WebOptions.TMP_DIR || option.key().equals("python.dc.tmp.dir")) {
defaultValue = null;
}
return "<tr>" +
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -89,117 +89,6 @@ public class JobManagerOptions {
key("jobmanager.resourcemanager.reconnect-interval")
.defaultValue(2000L);

// ------------------------------------------------------------------------
// JobManager web UI
// ------------------------------------------------------------------------

/**
* Config parameter defining the runtime monitor web-frontend server address.
*/
public static final ConfigOption<String> WEB_FRONTEND_ADDRESS =
key("jobmanager.web.address")
.noDefaultValue();

/**
* The port for the runtime monitor web-frontend server.
*/
public static final ConfigOption<Integer> WEB_PORT =
key("jobmanager.web.port")
.defaultValue(8081);

/**
* The config parameter defining the Access-Control-Allow-Origin header for all
* responses from the web-frontend.
*/
public static final ConfigOption<String> WEB_ACCESS_CONTROL_ALLOW_ORIGIN =
key("jobmanager.web.access-control-allow-origin")
.defaultValue("*");

/**
* The config parameter defining the refresh interval for the web-frontend.
*/
public static final ConfigOption<Long> WEB_REFRESH_INTERVAL =
key("jobmanager.web.refresh-interval")
.defaultValue(3000L);

/**
* Config parameter to override SSL support for the JobManager Web UI
*/
public static final ConfigOption<Boolean> WEB_SSL_ENABLED =
key("jobmanager.web.ssl.enabled")
.defaultValue(true);

/**
* The config parameter defining the flink web directory to be used by the webmonitor.
*/
public static final ConfigOption<String> WEB_TMP_DIR =
key("jobmanager.web.tmpdir")
.defaultValue(System.getProperty("java.io.tmpdir"));

/**
* The config parameter defining the directory for uploading the job jars. If not specified a dynamic directory
* will be used under the directory specified by JOB_MANAGER_WEB_TMPDIR_KEY.
*/
public static final ConfigOption<String> WEB_UPLOAD_DIR =
key("jobmanager.web.upload.dir")
.noDefaultValue();

/**
* The config parameter defining the number of archived jobs for the jobmanager.
*/
public static final ConfigOption<Integer> WEB_ARCHIVE_COUNT =
key("jobmanager.web.history")
.defaultValue(5);

/**
* The log file location (may be in /log for standalone but under log directory when using YARN).
*/
public static final ConfigOption<String> WEB_LOG_PATH =
key("jobmanager.web.log.path")
.noDefaultValue();

/**
* Config parameter indicating whether jobs can be uploaded and run from the web-frontend.
*/
public static final ConfigOption<Boolean> WEB_SUBMIT_ENABLE =
key("jobmanager.web.submit.enable")
.defaultValue(true);

/**
* Config parameter defining the number of checkpoints to remember for recent history.
*/
public static final ConfigOption<Integer> WEB_CHECKPOINTS_HISTORY_SIZE =
key("jobmanager.web.checkpoints.history")
.defaultValue(10);

/**
* Time after which cached stats are cleaned up if not accessed.
*/
public static final ConfigOption<Integer> WEB_BACKPRESSURE_CLEANUP_INTERVAL =
key("jobmanager.web.backpressure.cleanup-interval")
.defaultValue(10 * 60 * 1000);

/**
* Time after which available stats are deprecated and need to be refreshed (by resampling).
*/
public static final ConfigOption<Integer> WEB_BACKPRESSURE_REFRESH_INTERVAL =
key("jobmanager.web.backpressure.refresh-interval")
.defaultValue(60 * 1000);

/**
* Number of stack trace samples to take to determine back pressure.
*/
public static final ConfigOption<Integer> WEB_BACKPRESSURE_NUM_SAMPLES =
key("jobmanager.web.backpressure.num-samples")
.defaultValue(100);

/**
* Delay between stack trace samples to determine back pressure.
*/
public static final ConfigOption<Integer> WEB_BACKPRESSURE_DELAY =
key("jobmanager.web.backpressure.delay-between-samples")
.defaultValue(50);

/**
* The location where the JobManager stores the archives of completed jobs.
*/
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,156 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http:https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package org.apache.flink.configuration;

import org.apache.flink.annotation.PublicEvolving;

import static org.apache.flink.configuration.ConfigOptions.key;

/**
* Configuration options for the WebRuntimeMonitor.
*/
@PublicEvolving
public class WebOptions {
/**
* Config parameter defining the runtime monitor web-frontend server address.
*/
public static final ConfigOption<String> ADDRESS =
key("web.address")
.noDefaultValue()
.withDeprecatedKeys("jobmanager.web.address");

/**
* The port for the runtime monitor web-frontend server.
*/
public static final ConfigOption<Integer> PORT =
key("web.port")
.defaultValue(8081)
.withDeprecatedKeys("jobmanager.web.port");

/**
* The config parameter defining the Access-Control-Allow-Origin header for all
* responses from the web-frontend.
*/
public static final ConfigOption<String> ACCESS_CONTROL_ALLOW_ORIGIN =
key("web.access-control-allow-origin")
.defaultValue("*")
.withDeprecatedKeys("jobmanager.web.access-control-allow-origin");

/**
* The config parameter defining the refresh interval for the web-frontend.
*/
public static final ConfigOption<Long> REFRESH_INTERVAL =
key("web.refresh-interval")
.defaultValue(3000L)
.withDeprecatedKeys("jobmanager.web.refresh-interval");

/**
* Config parameter to override SSL support for the JobManager Web UI
*/
public static final ConfigOption<Boolean> SSL_ENABLED =
key("web.ssl.enabled")
.defaultValue(true)
.withDeprecatedKeys("jobmanager.web.ssl.enabled");

/**
* The config parameter defining the flink web directory to be used by the webmonitor.
*/
public static final ConfigOption<String> TMP_DIR =
key("web.tmpdir")
.defaultValue(System.getProperty("java.io.tmpdir"))
.withDeprecatedKeys("jobmanager.web.tmpdir");

/**
* The config parameter defining the directory for uploading the job jars. If not specified a dynamic directory
* will be used under the directory specified by JOB_MANAGER_WEB_TMPDIR_KEY.
*/
public static final ConfigOption<String> UPLOAD_DIR =
key("web.upload.dir")
.noDefaultValue()
.withDeprecatedKeys("jobmanager.web.upload.dir");

/**
* The config parameter defining the number of archived jobs for the jobmanager.
*/
public static final ConfigOption<Integer> ARCHIVE_COUNT =
key("web.history")
.defaultValue(5)
.withDeprecatedKeys("jobmanager.web.history");

/**
* The log file location (may be in /log for standalone but under log directory when using YARN).
*/
public static final ConfigOption<String> LOG_PATH =
key("web.log.path")
.noDefaultValue()
.withDeprecatedKeys("jobmanager.web.log.path");

/**
* Config parameter indicating whether jobs can be uploaded and run from the web-frontend.
*/
public static final ConfigOption<Boolean> SUBMIT_ENABLE =
key("web.submit.enable")
.defaultValue(true)
.withDeprecatedKeys("jobmanager.web.submit.enable");

/**
* Config parameter defining the number of checkpoints to remember for recent history.
*/
public static final ConfigOption<Integer> CHECKPOINTS_HISTORY_SIZE =
key("web.checkpoints.history")
.defaultValue(10)
.withDeprecatedKeys("jobmanager.web.checkpoints.history");

/**
* Time after which cached stats are cleaned up if not accessed.
*/
public static final ConfigOption<Integer> BACKPRESSURE_CLEANUP_INTERVAL =
key("web.backpressure.cleanup-interval")
.defaultValue(10 * 60 * 1000)
.withDeprecatedKeys("jobmanager.web.backpressure.cleanup-interval");

/**
* Time after which available stats are deprecated and need to be refreshed (by resampling).
*/
public static final ConfigOption<Integer> BACKPRESSURE_REFRESH_INTERVAL =
key("web.backpressure.refresh-interval")
.defaultValue(60 * 1000)
.withDeprecatedKeys("jobmanager.web.backpressure.refresh-interval");

/**
* Number of stack trace samples to take to determine back pressure.
*/
public static final ConfigOption<Integer> BACKPRESSURE_NUM_SAMPLES =
key("web.backpressure.num-samples")
.defaultValue(100)
.withDeprecatedKeys("jobmanager.web.backpressure.num-samples");

/**
* Delay between stack trace samples to determine back pressure.
*/
public static final ConfigOption<Integer> BACKPRESSURE_DELAY =
key("web.backpressure.delay-between-samples")
.defaultValue(50)
.withDeprecatedKeys("jobmanager.web.backpressure.delay-between-samples");


private WebOptions() {
throw new IllegalAccessError();
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@
package org.apache.flink.runtime.webmonitor;

import org.apache.flink.configuration.Configuration;
import org.apache.flink.configuration.JobManagerOptions;
import org.apache.flink.configuration.WebOptions;

/**
* Configuration object for {@link WebMonitor}.
Expand All @@ -37,22 +37,22 @@ public WebMonitorConfig(Configuration config) {
}

public String getWebFrontendAddress() {
return config.getValue(JobManagerOptions.WEB_FRONTEND_ADDRESS);
return config.getValue(WebOptions.ADDRESS);
}

public int getWebFrontendPort() {
return config.getInteger(JobManagerOptions.WEB_PORT);
return config.getInteger(WebOptions.PORT);
}

public long getRefreshInterval() {
return config.getLong(JobManagerOptions.WEB_REFRESH_INTERVAL);
return config.getLong(WebOptions.REFRESH_INTERVAL);
}

public boolean isProgramSubmitEnabled() {
return config.getBoolean(JobManagerOptions.WEB_SUBMIT_ENABLE);
return config.getBoolean(WebOptions.SUBMIT_ENABLE);
}

public String getAllowOrigin() {
return config.getString(JobManagerOptions.WEB_ACCESS_CONTROL_ALLOW_ORIGIN);
return config.getString(WebOptions.ACCESS_CONTROL_ALLOW_ORIGIN);
}
}
Loading

0 comments on commit c685251

Please sign in to comment.