support complex domain host for datasource

tis-plugin/src/main/java/com/qlangtech/tis/db/parser/DBConfigParser.java

@@ -182,7 +182,7 @@ private boolean dbPrimaryInfo() {
  if (t.isTokenType(TokenTypes.TT_COLON)) {
  tokenBuffer.popToken();
  t = tokenBuffer.nextToken();
- if (t.isTokenType(TokenTypes.TT_IDENTIFIER)) {
+ if (t.isTokenType(TokenTypes.TT_PASSWORD_VALUE)) {
  tokenBuffer.popToken();
  // this.dbConfigResult.setPassword(t.getContent());
  } else {

tis-plugin/src/main/java/com/qlangtech/tis/db/parser/DBTokenizer.java

@@ -1,19 +1,19 @@
 /**
- *  Licensed to the Apache Software Foundation (ASF) under one
- *  or more contributor license agreements. See the NOTICE file
- *  distributed with this work for additional information
- *  regarding copyright ownership. The ASF licenses this file
- *  to you under the Apache License, Version 2.0 (the
- *  "License"); you may not use this file except in compliance
- *  with the License. You may obtain a copy of the License at
- *
- *  http:https://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http:https://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  */
 package com.qlangtech.tis.db.parser;
 
@@ -74,18 +74,21 @@ private boolean matchToken(Iterator<ScanRecognizer> patternIterator) {
  matcher = pattern.matcher(scannerBuffer);
  if (matcher.find()) {
  // System.out.println(matcher.group());
- if (recognizer.isOutputToken()) {
- content = recognizer.getToken().getGourpIndex() > 0
- ? matcher.group(recognizer.getToken().getGourpIndex()) : matcher.group();
- tokenList.add(new Token(recognizer.getToken(), content));
+
+ content = recognizer.getToken().getGourpIndex() > 0
+ ? matcher.group(recognizer.getToken().getGourpIndex()) : matcher.group();
+ if (recognizer.getToken().furtherChecker.apply(content)) {
+ if (recognizer.isOutputToken()) {
+ tokenList.add(new Token(recognizer.getToken(), content));
+ }
+ tokenMatch = true;
+ scannerBuffer = scannerBuffer.substring(matcher.end());
  }
- tokenMatch = true;
- scannerBuffer = scannerBuffer.substring(matcher.end());
- // System.out.println(scannerBuffer);
+ // System.out.println(scannerBuffer);
  }
  } while (patternIterator.hasNext() && !tokenMatch);
  if (// || (matcher.end() == scannerBuffer.length())
- !tokenMatch) {
+  !tokenMatch) {
  result = false;
  }
  return result;

tis-plugin/src/main/java/com/qlangtech/tis/db/parser/ScanRecognizer.java

@@ -1,19 +1,19 @@
 /**
- *  Licensed to the Apache Software Foundation (ASF) under one
- *  or more contributor license agreements. See the NOTICE file
- *  distributed with this work for additional information
- *  regarding copyright ownership. The ASF licenses this file
- *  to you under the Apache License, Version 2.0 (the
- *  "License"); you may not use this file except in compliance
- *  with the License. You may obtain a copy of the License at
- *
- *  http:https://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http:https://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  */
 package com.qlangtech.tis.db.parser;
 
@@ -51,4 +51,13 @@ public Pattern getPattern() {
  public boolean isOutputToken() {
  return this.outputToken;
  }
+
+
+ @Override
+ public String toString() {
+ return "{ token=" + tokenTypes +
+ ", pattern=" + pattern +
+ ", output=" + outputToken +
+ '}';
+ }
 }

tis-plugin/src/main/java/com/qlangtech/tis/db/parser/ScannerPatterns.java

@@ -17,9 +17,15 @@
  */
 package com.qlangtech.tis.db.parser;
 
+import com.google.common.collect.Lists;
+import com.qlangtech.tis.plugin.ValidatorCommons;
+
 import java.util.ArrayList;
 import java.util.Collections;
+import java.util.Comparator;
 import java.util.List;
+import java.util.function.Function;
+import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
 /**
@@ -29,10 +35,11 @@
 public class ScannerPatterns {
 
  public static final String HOST_KEY = "host";
+ private static final Pattern a2zPattern = Pattern.compile("[a-z]");
 
  public enum TokenTypes {
  //
- TT_HOST_DESC("^" + HOST_KEY, true),
+ TT_HOST_DESC("^" + HOST_KEY, true, 999),
  //
  TT_PASSWORD("^password", true),
  //
@@ -55,37 +62,58 @@ public enum TokenTypes {
  TT_RANGE_MINUS("^-", true),
  // http:https://www.regular-expressions.info/ip.html
  TT_IP(//
- "^(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[1-9]?[0-9])\\.(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[1-9]?[0-9])\\.(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[1-9]?[0-9])\\.(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[1-9]?[0-9])", true),
+ "^(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[1-9]?[0-9])\\.(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[1-9]?[0-9])\\.(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[1-9]?[0-9])\\.(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[1-9]?[0-9])", true, 995),
  //
  TT_RANGE_NUMBER("^\\d+", true),
  //
  TT_WHITESPACE("^(\\s)+", false),
  TT_HOST(//
- "(^([a-z0-9][a-z0-9\\-]{0,61}[a-z0-9])(\\.([a-z0-9][a-z0-9\\-]{0,61}[a-z0-9]))+)|^localhost", true),
+ "(^([a-z0-9][a-z0-9\\-]{0,61}[a-z0-9])(\\.([a-z0-9][a-z0-9\\-]{0,61}[a-z0-9]))+)|^localhost"
+ , true, new Group(-1), 996, (testToken) -> {
+ Matcher matcher = a2zPattern.matcher(testToken);
+ // 存在 a-z的字符 说明是一个host地址
+ return matcher.find();
+ }),
  //
- TT_COLON("^:", true),
+ TT_COLON("^:", true, 998),
  //
- TT_IDENTIFIER("^(\\S+)[\\r|\\n]?", true, 1),
+ TT_IDENTIFIER("^(" + ValidatorCommons.pattern_identity.pattern() + ")[\\r|\\n]?", true, new Group(1)),
+ TT_PASSWORD_VALUE("^(\\S+)[\\r|\\n]?", true, new Group(1)),
  TT_EOF("^EOF", false);
 
  private final String regExpattern;
 
  private final boolean outputToken;
 
- private final int gourpIndex;
+ private final Group gourpIndex;
+ private final int priority;
+ /**
+ * 有些情况需要进一步确认，例如 TT_HOST，"192.168.28" 会作为合法的host地址，但是其实它应该是一个不合法的ip地址
+ */
+ public final Function<String, Boolean> furtherChecker;
 
  public int getGourpIndex() {
- return gourpIndex;
+ return gourpIndex.gourpIndex;
  }
 
  TokenTypes(String regExpattern, boolean outputToken) {
- this(regExpattern, outputToken, -1);
+ this(regExpattern, outputToken, 0);
+ }
+
+ TokenTypes(String regExpattern, boolean outputToken, int priority) {
+ this(regExpattern, outputToken, new Group(-1), priority, (test) -> true);
  }
 
- TokenTypes(String regExpattern, boolean outputToken, int group) {
+ TokenTypes(String regExpattern, boolean outputToken, Group group) {
+ this(regExpattern, outputToken, group, 0, (test) -> true);
+ }
+
+ TokenTypes(String regExpattern, boolean outputToken, Group group, int priority, Function<String, Boolean> furtherChecker) {
  this.regExpattern = regExpattern;
  this.outputToken = outputToken;
  this.gourpIndex = group;
+ this.priority = priority;
+ this.furtherChecker = furtherChecker;
  }
 
  public Pattern createPattern() {
@@ -94,6 +122,14 @@ public Pattern createPattern() {
 
  }
 
+ private static class Group {
+ final int gourpIndex;
+
+ public Group(int group) {
+ this.gourpIndex = group;
+ }
+ }
+
  private static List<ScanRecognizer> patternMatchers;
 
  public static List<ScanRecognizer> loadPatterns() {
@@ -102,7 +138,15 @@ public static List<ScanRecognizer> loadPatterns() {
  synchronized (ScannerPatterns.class) {
  if (patternMatchers == null) {
  patternMatchers = new ArrayList<ScanRecognizer>();
- for (TokenTypes t : TokenTypes.values()) {
+
+ ArrayList<TokenTypes> tokens = Lists.newArrayList(TokenTypes.values());
+ Collections.sort(tokens, new Comparator<TokenTypes>() {
+ @Override
+ public int compare(TokenTypes t1, TokenTypes t2) {
+ return t2.priority - t1.priority;
+ }
+ });
+ for (TokenTypes t : tokens) {
  pattern = t.createPattern();
  patternMatchers.add(new ScanRecognizer(t, pattern, t.outputToken));
  }

tis-plugin/src/main/java/com/qlangtech/tis/extension/impl/PropertyType.java

@@ -561,7 +561,7 @@ public List<? extends Descriptor> getApplicableDescriptors() {
  String script = " package " + pkg + " ;\n" //
  + "import java.util.function.Function;\n" //
  + "import java.util.List;\n" //
- + "import com.qlangtech.tis.extension.Descriptor;\n"
+ + "import "+com.qlangtech.tis.extension.Descriptor.class.getName()+";\n"
  // + "import com.qlangtech.plugins.incr.flink.chunjun.sink.SinkTabPropsExtends;\n"
  + "class " + className + " implements Function<List<? extends Descriptor>,List<? extends " //
  + "Descriptor>> { \n" //

tis-plugin/src/main/java/com/qlangtech/tis/util/HeteroEnum.java

@@ -49,6 +49,7 @@
 import com.qlangtech.tis.plugin.k8s.K8sImage.ImageCategory;
 import org.apache.commons.lang.StringUtils;
 
+import java.util.Arrays;
 import java.util.Collections;
 import java.util.List;
 import java.util.Objects;
@@ -117,11 +118,22 @@ public IPluginStore getPluginStore(IPluginContext pluginContext, UploadPluginMet
  private static class DockerImageHeteroEnum extends HeteroEnum<K8sImage> {
  private ImageCategory imageCategory;
 
+ private static String parseCaption(String token) {
+ String[] tokens = StringUtils.split(token, "-");
+ return Arrays.stream(tokens).map((t) -> StringUtils.capitalize(t)).collect(Collectors.joining("-"));
+ }
+
  public DockerImageHeteroEnum(ImageCategory imageCategory) {
- super(K8sImage.class, imageCategory.token, KEY_K8S_IMAGES);
+ super(K8sImage.class, imageCategory.token, parseCaption(imageCategory.token));
  this.imageCategory = imageCategory;
  }
 
+ @Override
+ public List<Descriptor<K8sImage>> descriptors() {
+ List<Descriptor<K8sImage>> descs = getPluginStore(null, null).allDescriptor();
+ return descs.stream().filter((desc) -> imageCategory.token.equals(desc.getDisplayName())).collect(Collectors.toList());
+ }
+
  @Override
  public IPluginStore getPluginStore(IPluginContext pluginContext, UploadPluginMeta pluginMeta) {
  // UploadPluginMeta.TargetDesc targetDesc = pluginMeta.getTargetDesc();
@@ -147,6 +159,7 @@ public IPluginStore getPluginStore(IPluginContext pluginContext, UploadPluginMet
  public IPluginStore getPluginStore(IPluginContext pluginContext, UploadPluginMeta pluginMeta) {
  return super.getPluginStore(pluginContext, pluginMeta);
  }
+
  @Override
  public List<DataXJobWorker> getPlugins(IPluginContext pluginContext, UploadPluginMeta pluginMeta) {
  return super.getPlugins(pluginContext, pluginMeta);

tis-plugin/src/test/java/com/qlangtech/tis/db/parser/TestDBConfigParser.java

@@ -18,6 +18,7 @@
 package com.qlangtech.tis.db.parser;
 
 import com.qlangtech.tis.common.utils.Assert;
+import com.qlangtech.tis.db.parser.ScannerPatterns.TokenTypes;
 import com.qlangtech.tis.plugin.ds.DBConfig;
 import junit.framework.TestCase;
 import org.apache.commons.collections.MapUtils;
@@ -29,6 +30,8 @@
 import java.text.DecimalFormat;
 import java.util.List;
 import java.util.Map;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
 
 /**
  * @author 百岁（[email protected]）
@@ -45,8 +48,25 @@ public void testJustParseDBHost() throws Exception {
  validateDbEnum(dbName, dbEnum);
  }
 
+ public void testK8SReplicaSetDomain() {
+ String mysqlHost = "mysql-0.mysql-svc.default";
+ String dbName = "dbname";
+ Pattern hostPattern = TokenTypes.TT_HOST.createPattern();
+ Matcher matcher = hostPattern.matcher(mysqlHost);
+ Assert.assertTrue(String.valueOf(hostPattern), matcher.matches());
+
+ Map<String, List<String>> dbname = DBConfigParser.parseDBEnum(dbName, mysqlHost);
+ Assert.assertEquals(1, dbname.size());
+ List<String> dbs = dbname.get(mysqlHost);
+ Assert.assertEquals(1, dbs.size());
+ Assert.assertTrue(dbs.contains(dbName));
+ }
+
  public void testParserErrorDBNodeDesc() throws Exception {
  Map<String, List<String>> dbname = DBConfigParser.parseDBEnum("dbname", "192.168.28。200");
+
+ TokenTypes.TT_IP.createPattern();
+
  Assert.assertNotNull(dbname);
  Assert.assertTrue("dbname shall be empty", MapUtils.isEmpty(dbname));
  }