[hotfix][docs] Refactor ConfigOptionsDocGenerator

laubersder · Apr 12, 2018 · 1f21af6 · 1f21af6
1 parent 21a66ea
commit 1f21af6
Show file tree

Hide file tree

Showing 4 changed files with 134 additions and 71 deletions.
diff --git a/flink-docs/README.md b/flink-docs/README.md
@@ -39,7 +39,7 @@ The documentation must be regenerated whenever
 
 The `ConfigOptionsDocGenerator` can be use to generate a reference of `ConfigOptions`. By default, a separate file is generated for each `*Options` class found in `org.apache.flink.configuration` and `org.apache.flink.yarn.configuration`. The `@ConfigGroups` annotation can be used to generate multiple files from a single class.
 
-To integrate an `*Options` class from another package, add another module-package argument pair to the `maven-antrun-plugin` configuration in the `generate-config-docs` profile.
+To integrate an `*Options` class from another package, add another module-package argument pair to `ConfigOptionsDocGenerator#LOCATIONS`.
 
 The files can be generated by running `mvn package -Dgenerate-config-docs -pl flink-docs -am -nsu`, and can be integrated into the documentation using `{% include generated/<file-name>.html %}`.
 

diff --git a/flink-docs/pom.xml b/flink-docs/pom.xml
@@ -118,6 +118,15 @@ under the License.
  </executions>
  </plugin>
  </plugins>
+
+ <pluginManagement>
+ <plugins>
+ <plugin>
+ <artifactId>maven-antrun-plugin</artifactId>
+ <version>1.7</version>
+ </plugin>
+ </plugins>
+ </pluginManagement>
  </build>
 
  <profiles>
@@ -132,24 +141,24 @@ under the License.
  <plugins>
  <plugin>
  <artifactId>maven-antrun-plugin</artifactId>
- <version>1.7</version>
  <executions>
  <execution>
+ <id>generate-rest-docs</id>
  <phase>package</phase>
  <goals>
  <goal>run</goal>
  </goals>
+ <configuration>
+ <target>
+ <mkdir dir="${rootDir}/${generated.docs.dir}"/>
+ <java classname="org.apache.flink.docs.rest.RestAPIDocGenerator" fork="true" failonerror="true">
+ <classpath refid="maven.compile.classpath"/>
+ <arg value="${rootDir}/${generated.docs.dir}/"/>
+ </java>
+ </target>
+ </configuration>
  </execution>
  </executions>
- <configuration>
- <target>
- <mkdir dir="${rootDir}/${generated.docs.dir}"/>
- <java classname="org.apache.flink.docs.rest.RestAPIDocGenerator" fork="true">
- <classpath refid="maven.compile.classpath"/>
- <arg value="${rootDir}/${generated.docs.dir}/"/>
- </java>
- </target>
- </configuration>
  </plugin>
  </plugins>
  </build>
@@ -165,36 +174,25 @@ under the License.
  <plugins>
  <plugin>
  <artifactId>maven-antrun-plugin</artifactId>
- <version>1.7</version>
  <executions>
  <execution>
+ <id>generate-config-docs</id>
  <phase>package</phase>
  <goals>
  <goal>run</goal>
  </goals>
+ <configuration>
+ <target>
+ <mkdir dir="${rootDir}/${generated.docs.dir}"/>
+ <java classname="org.apache.flink.docs.configuration.ConfigOptionsDocGenerator" fork="true" failonerror="true">
+ <classpath refid="maven.compile.classpath" />
+ <arg value="${rootDir}/${generated.docs.dir}/" />
+ <arg value="${rootDir}" />
+ </java>
+ </target>
+ </configuration>
  </execution>
  </executions>
- <configuration>
- <target>
- <mkdir dir="${rootDir}/${generated.docs.dir}"/>
- <java classname="org.apache.flink.docs.configuration.ConfigOptionsDocGenerator" fork="true">
- <classpath refid="maven.compile.classpath" />
- <arg value="${rootDir}/${generated.docs.dir}/" />
- <arg value="${rootDir}" />
- <!--packages with configuration classes-->
- <arg value="flink-core" />
- <arg value="org.apache.flink.configuration" />
- <arg value="flink-runtime" />
- <arg value="org.apache.flink.runtime.io.network.netty" />
- <arg value="flink-yarn" />
- <arg value="org.apache.flink.yarn.configuration" />
- <arg value="flink-mesos" />
- <arg value="org.apache.flink.mesos.configuration" />
- <arg value="flink-mesos" />
- <arg value="org.apache.flink.mesos.runtime.clusterframework" />
- </java>
- </target>
- </configuration>
  </plugin>
  </plugins>
  </build>

diff --git a/flink-docs/src/main/java/org/apache/flink/docs/configuration/ConfigOptionsDocGenerator.java b/flink-docs/src/main/java/org/apache/flink/docs/configuration/ConfigOptionsDocGenerator.java
@@ -25,6 +25,7 @@
 import org.apache.flink.configuration.ConfigOption;
 import org.apache.flink.configuration.CoreOptions;
 import org.apache.flink.configuration.WebOptions;
+import org.apache.flink.util.function.ThrowingConsumer;
 
 import java.io.IOException;
 import java.lang.reflect.Field;
@@ -35,6 +36,7 @@
 import java.nio.file.Paths;
 import java.util.ArrayList;
 import java.util.Collection;
+import java.util.Collections;
 import java.util.Comparator;
 import java.util.HashMap;
 import java.util.List;
@@ -47,6 +49,18 @@
  */
 public class ConfigOptionsDocGenerator {
 
+ private static final OptionsClassLocation[] LOCATIONS = new OptionsClassLocation[]{
+ new OptionsClassLocation("flink-core", "org.apache.flink.configuration"),
+ new OptionsClassLocation("flink-runtime", "org.apache.flink.runtime.io.network.netty"),
+ new OptionsClassLocation("flink-yarn", "org.apache.flink.yarn.configuration"),
+ new OptionsClassLocation("flink-mesos", "org.apache.flink.mesos.configuration"),
+ new OptionsClassLocation("flink-mesos", "org.apache.flink.mesos.runtime.clusterframework"),
+ };
+
+ private static final String CLASS_NAME_GROUP = "className";
+ private static final String CLASS_PREFIX_GROUP = "classPrefix";
+ private static final Pattern CLASS_NAME_PATTERN = Pattern.compile("(?<" + CLASS_NAME_GROUP + ">(?<" + CLASS_PREFIX_GROUP + ">[a-zA-Z]*)(?:Options|Config|Parameters))(?:\\.java)?");
+
  /**
  * This method generates html tables from set of classes containing {@link ConfigOption ConfigOptions}.
  *
@@ -57,39 +71,47 @@ public class ConfigOptionsDocGenerator {
  * @param args
  * [0] output directory for the generated files
  * [1] project root directory
- * [x] module containing an *Options class
- * [x+1] package to the * Options.classes
  */
  public static void main(String[] args) throws IOException, ClassNotFoundException {
  String outputDirectory = args[0];
  String rootDir = args[1];
- for (int x = 2; x + 1 < args.length; x += 2) {
- createTable(rootDir, args[x], args[x + 1], outputDirectory);
+
+ for (OptionsClassLocation location : LOCATIONS) {
+ createTable(rootDir, location.getModule(), location.getPackage(), outputDirectory);
  }
  }
 
  private static void createTable(String rootDir, String module, String packageName, String outputDirectory) throws IOException, ClassNotFoundException {
+ processConfigOptions(rootDir, module, packageName, optionsClass -> {
+ List<Tuple2<ConfigGroup, String>> tables = generateTablesForClass(optionsClass);
+ for (Tuple2<ConfigGroup, String> group : tables) {
+ String name;
+ if (group.f0 == null) {
+ Matcher matcher = CLASS_NAME_PATTERN.matcher(optionsClass.getSimpleName());
+ if (!matcher.matches()) {
+ throw new RuntimeException("Pattern did not match for " + optionsClass.getSimpleName() + '.');
+ }
+ name = matcher.group(CLASS_PREFIX_GROUP).replaceAll("(.)(\\p{Upper})", "$1_$2").toLowerCase();
+ } else {
+ name = group.f0.name().replaceAll("(.)(\\p{Upper})", "$1_$2").toLowerCase();
+ }
+
+ String outputFile = name + "_configuration.html";
+ Files.write(Paths.get(outputDirectory, outputFile), group.f1.getBytes(StandardCharsets.UTF_8));
+ }
+ });
+ }
+
+ private static void processConfigOptions(String rootDir, String module, String packageName, ThrowingConsumer<Class<?>, IOException> classConsumer) throws IOException, ClassNotFoundException {
  Path configDir = Paths.get(rootDir, module, "src/main/java", packageName.replaceAll("\\.", "/"));
 
- Pattern p = Pattern.compile("(([a-zA-Z]*)(Options|Config|Parameters))\\.java");
  try (DirectoryStream<Path> stream = Files.newDirectoryStream(configDir)) {
  for (Path entry : stream) {
  String fileName = entry.getFileName().toString();
- Matcher matcher = p.matcher(fileName);
+ Matcher matcher = CLASS_NAME_PATTERN.matcher(fileName);
  if (!fileName.equals("ConfigOptions.java") && matcher.matches()) {
- Class<?> optionsClass = Class.forName(packageName + "." + matcher.group(1));
- List<Tuple2<ConfigGroup, String>> tables = generateTablesForClass(optionsClass);
- if (tables.size() > 0) {
- for (Tuple2<ConfigGroup, String> group : tables) {
-
- String name = group.f0 == null
- ? matcher.group(2).replaceAll("(.)(\\p{Upper})", "$1_$2").toLowerCase()
- : group.f0.name().replaceAll("(.)(\\p{Upper})", "$1_$2").toLowerCase();
-
- String outputFile = name + "_configuration.html";
- Files.write(Paths.get(outputDirectory, outputFile), group.f1.getBytes(StandardCharsets.UTF_8));
- }
- }
+ Class<?> optionsClass = Class.forName(packageName + '.' + matcher.group(CLASS_NAME_GROUP));
+ classConsumer.accept(optionsClass);
  }
  }
  }
@@ -98,40 +120,41 @@ private static void createTable(String rootDir, String module, String packageNam
  @VisibleForTesting
  static List<Tuple2<ConfigGroup, String>> generateTablesForClass(Class<?> optionsClass) {
  ConfigGroups configGroups = optionsClass.getAnnotation(ConfigGroups.class);
- List<Tuple2<ConfigGroup, String>> tables = new ArrayList<>();
- List<ConfigOption> allOptions = extractConfigOptions(optionsClass);
+ List<ConfigOption<?>> allOptions = extractConfigOptions(optionsClass);
 
+ List<Tuple2<ConfigGroup, String>> tables;
  if (configGroups != null) {
+ tables = new ArrayList<>(configGroups.groups().length + 1);
  Tree tree = new Tree(configGroups.groups(), allOptions);
 
  for (ConfigGroup group : configGroups.groups()) {
- List<ConfigOption> configOptions = tree.findConfigOptions(group);
+ List<ConfigOption<?>> configOptions = tree.findConfigOptions(group);
  sortOptions(configOptions);
  tables.add(Tuple2.of(group, toHtmlTable(configOptions)));
  }
- List<ConfigOption> configOptions = tree.getDefaultOptions();
+ List<ConfigOption<?>> configOptions = tree.getDefaultOptions();
  sortOptions(configOptions);
  tables.add(Tuple2.of(null, toHtmlTable(configOptions)));
  } else {
  sortOptions(allOptions);
- tables.add(Tuple2.of(null, toHtmlTable(allOptions)));
+ tables = Collections.singletonList(Tuple2.of(null, toHtmlTable(allOptions)));
  }
  return tables;
  }
 
- private static List<ConfigOption> extractConfigOptions(Class<?> clazz) {
+ private static List<ConfigOption<?>> extractConfigOptions(Class<?> clazz) {
  try {
- List<ConfigOption> configOptions = new ArrayList<>();
+ List<ConfigOption<?>> configOptions = new ArrayList<>(8);
  Field[] fields = clazz.getFields();
  for (Field field : fields) {
  if (field.getType().equals(ConfigOption.class) && field.getAnnotation(Deprecated.class) == null) {
- configOptions.add((ConfigOption) field.get(null));
+ configOptions.add((ConfigOption<?>) field.get(null));
  }
  }
 
  return configOptions;
  } catch (Exception e) {
- throw new RuntimeException("Failed to extract config options from class " + clazz + ".", e);
+ throw new RuntimeException("Failed to extract config options from class " + clazz + '.', e);
  }
  }
 
@@ -142,7 +165,7 @@ private static List<ConfigOption> extractConfigOptions(Class<?> clazz) {
  * @param options list of options to include in this group
  * @return string containing HTML formatted table
  */
- private static String toHtmlTable(final List<ConfigOption> options) {
+ private static String toHtmlTable(final List<ConfigOption<?>> options) {
  StringBuilder htmlTable = new StringBuilder();
  htmlTable.append("<table class=\"table table-bordered\">\n");
  htmlTable.append(" <thead>\n");
@@ -154,7 +177,7 @@ private static String toHtmlTable(final List<ConfigOption> options) {
  htmlTable.append(" </thead>\n");
  htmlTable.append(" <tbody>\n");
 
- for (ConfigOption option : options) {
+ for (ConfigOption<?> option : options) {
  htmlTable.append(toHtmlString(option));
  }
 
@@ -203,7 +226,7 @@ private static String escapeCharacters(String value) {
  .replaceAll(">", "&#62;");
  }
 
- private static void sortOptions(List<ConfigOption> configOptions) {
+ private static void sortOptions(List<ConfigOption<?>> configOptions) {
  configOptions.sort(Comparator.comparing(ConfigOption::key));
  }
 
@@ -214,7 +237,7 @@ private static void sortOptions(List<ConfigOption> configOptions) {
  private static class Tree {
  private final Node root = new Node();
 
- Tree(ConfigGroup[] groups, Collection<ConfigOption> options) {
+ Tree(ConfigGroup[] groups, Collection<ConfigOption<?>> options) {
  // generate a tree based on all key prefixes
  for (ConfigGroup group : groups) {
  String[] keyComponents = group.keyPrefix().split("\\.");
@@ -232,12 +255,12 @@ private static class Tree {
  }
  }
 
- List<ConfigOption> findConfigOptions(ConfigGroup configGroup) {
+ List<ConfigOption<?>> findConfigOptions(ConfigGroup configGroup) {
  Node groupRoot = findGroupRoot(configGroup.keyPrefix());
  return groupRoot.getConfigOptions();
  }
 
- List<ConfigOption> getDefaultOptions() {
+ List<ConfigOption<?>> getDefaultOptions() {
  return root.getConfigOptions();
  }
 
@@ -251,8 +274,8 @@ private Node findGroupRoot(String key) {
  }
 
  private static class Node {
- private final List<ConfigOption> configOptions = new ArrayList<>();
- private final Map<String, Node> children = new HashMap<>();
+ private final List<ConfigOption<?>> configOptions = new ArrayList<>(8);
+ private final Map<String, Node> children = new HashMap<>(8);
  private boolean isGroupRoot = false;
 
  private Node addChild(String keyComponent) {
@@ -272,7 +295,7 @@ private Node findChild(String keyComponent) {
  return child;
  }
 
- private void assignOption(ConfigOption option) {
+ private void assignOption(ConfigOption<?> option) {
  configOptions.add(option);
  }
 
@@ -284,7 +307,7 @@ private void markAsGroupRoot() {
  this.isGroupRoot = true;
  }
 
- private List<ConfigOption> getConfigOptions() {
+ private List<ConfigOption<?>> getConfigOptions() {
  return configOptions;
  }
  }

diff --git a/flink-docs/src/main/java/org/apache/flink/docs/configuration/OptionsClassLocation.java b/flink-docs/src/main/java/org/apache/flink/docs/configuration/OptionsClassLocation.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http:https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.docs.configuration;
+
+import org.apache.flink.configuration.ConfigOption;
+
+/**
+ * Simple descriptor for the location of a class containing {@link ConfigOption ConfigOptions}.
+ */
+class OptionsClassLocation {
+ private final String module;
+ private final String pckg;
+
+ OptionsClassLocation(String module, String pckg) {
+ this.module = module;
+ this.pckg = pckg;
+ }
+
+ public String getModule() {
+ return module;
+ }
+
+ public String getPackage() {
+ return pckg;
+ }
+}