diff --git a/tis-assemble/src/test/java/StartAssembleWeb.java b/tis-assemble/src/test/java/StartAssembleWeb.java index 16a06ad6c..29a3f90cd 100644 --- a/tis-assemble/src/test/java/StartAssembleWeb.java +++ b/tis-assemble/src/test/java/StartAssembleWeb.java @@ -42,7 +42,7 @@ public class StartAssembleWeb extends TestCase { public void testStart() throws Exception { - System.out.println(this.getClass().getClassLoader().loadClass("com.alibaba.datax.common.statistics.PerfTrace")); + System.out.println(this.getClass().getClassLoader().loadClass("com.alibaba.datax.common.statistics.PerfTrace")); // System.setProperty(com.qlangtech.tis.fullbuild.indexbuild.IRemoteTaskTrigger.KEY_DELTA_STREM_DEBUG, "true"); System.setProperty(DataxUtils.EXEC_TIMESTAMP, String.valueOf(TimeFormat.getCurrentTimeStamp())); diff --git a/tis-base-test/src/main/java/com/qlangtech/tis/selectedtab/TabApplicationCreator.java b/tis-base-test/src/main/java/com/qlangtech/tis/selectedtab/TabApplicationCreator.java new file mode 100644 index 000000000..4932d953a --- /dev/null +++ b/tis-base-test/src/main/java/com/qlangtech/tis/selectedtab/TabApplicationCreator.java @@ -0,0 +1,59 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.qlangtech.tis.selectedtab; + +import com.google.common.collect.Lists; +import com.qlangtech.tis.datax.IDataxProcessor; +import com.qlangtech.tis.plugin.datax.SelectedTab; +import com.qlangtech.tis.plugin.ds.CMeta; +import com.qlangtech.tis.plugin.ds.DataXReaderColType; + +import java.util.List; +import java.util.function.Consumer; + +public class TabApplicationCreator { + public static IDataxProcessor.TableMap getTabApplication(Consumer>... colsProcess) { + String tableName = "application"; + SelectedTab tab = new SelectedTab(); + tab.name = tableName; + + List sourceCols = Lists.newArrayList(); + CMeta col = new CMeta(); + col.setPk(true); + col.setName("user_id"); + col.setType(DataXReaderColType.Long.dataType); + tab.primaryKeys = Lists.newArrayList(col.getName()); + sourceCols.add(col); + + col = new CMeta(); + col.setName("user_name"); + col.setType(DataXReaderColType.STRING.dataType); + sourceCols.add(col); + + for (Consumer> p : colsProcess) { + p.accept(sourceCols); + } + tab.cols.addAll(sourceCols); + IDataxProcessor.TableMap tableMap = new IDataxProcessor.TableMap(tab); + tableMap.setFrom(tableName); + tableMap.setTo("application_alias"); + //tableMap.setSourceCols(sourceCols); + return tableMap; + } +} diff --git a/tis-manage-pojo/src/main/java/com/qlangtech/tis/plugin/ValidatorCommons.java b/tis-manage-pojo/src/main/java/com/qlangtech/tis/plugin/ValidatorCommons.java index 12ea19bc6..7e2b87102 100644 --- a/tis-manage-pojo/src/main/java/com/qlangtech/tis/plugin/ValidatorCommons.java +++ b/tis-manage-pojo/src/main/java/com/qlangtech/tis/plugin/ValidatorCommons.java @@ -25,8 +25,7 @@ * @create: 2020-06-12 14:40 */ public interface ValidatorCommons { - - String _host = "[-A-Za-z0-9+&@#%?=~_|!,.;]+[-A-Za-z0-9+&@#/%=~_|]"; + String _host = "[-A-Za-z0-9+&@#/%?=~_|!,.;]+[-A-Za-z0-9+&@#/%=~_|]"; Pattern PATTERN_URL = Pattern.compile("(https?|hdfs)://" + _host + "(:\\d+)?"); diff --git a/tis-manage-pojo/src/test/java/com/qlangtech/tis/plugin/TestValidatorCommons.java b/tis-manage-pojo/src/test/java/com/qlangtech/tis/plugin/TestValidatorCommons.java index 25817d13c..03d63afab 100644 --- a/tis-manage-pojo/src/test/java/com/qlangtech/tis/plugin/TestValidatorCommons.java +++ b/tis-manage-pojo/src/test/java/com/qlangtech/tis/plugin/TestValidatorCommons.java @@ -34,6 +34,14 @@ public void testUrlPattern() { Matcher matcher = ValidatorCommons.PATTERN_URL.matcher(url); Assert.assertTrue(url, matcher.matches()); + url = "http://service.cn-hangzhou.maxcompute.aliyun.com/api"; + matcher = ValidatorCommons.PATTERN_URL.matcher(url); + Assert.assertTrue(url, matcher.matches()); + + url = "https://service.cn-hangzhou.maxcompute.aliyun.com/api"; + matcher = ValidatorCommons.PATTERN_URL.matcher(url); + Assert.assertTrue(url, matcher.matches()); + url = "https://192.168.64.3"; matcher = ValidatorCommons.PATTERN_URL.matcher(url); Assert.assertTrue(url, matcher.matches()); diff --git a/tis-plugin/src/main/java/com/qlangtech/tis/config/flink/IFlinkCluster.java b/tis-plugin/src/main/java/com/qlangtech/tis/config/flink/IFlinkCluster.java index 784c9640f..1ef4e945e 100644 --- a/tis-plugin/src/main/java/com/qlangtech/tis/config/flink/IFlinkCluster.java +++ b/tis-plugin/src/main/java/com/qlangtech/tis/config/flink/IFlinkCluster.java @@ -33,10 +33,11 @@ public interface IFlinkCluster extends IdentityName, IFlinkClusterConfig { String PLUGIN_DEPENDENCY_FLINK_DEPENDENCY = "tis-flink-dependency"; String PLUGIN_SKIP_FLINK_EXTENDS = "tis-flink-extends-plugin"; + String PLUGIN_TIS_DATAX_LOCAL_EXECOTOR = "tis-datax-local-executor"; String SKIP_CLASSLOADER_FACTORY_CREATION = "skip_classloader_factory_creation"; String KEY_DISPLAY_NAME = "Flink-Cluster"; Set SKIP_PLUGIN_NAMES = Collections.unmodifiableSet( Sets.newHashSet(IFlinkCluster.PLUGIN_DEPENDENCY_FLINK_DEPENDENCY// , IFlinkCluster.PLUGIN_SKIP_FLINK_EXTENDS // - , "tis-datax-local-executor")); + , PLUGIN_TIS_DATAX_LOCAL_EXECOTOR)); } diff --git a/tis-plugin/src/main/java/com/qlangtech/tis/extension/impl/PropertyType.java b/tis-plugin/src/main/java/com/qlangtech/tis/extension/impl/PropertyType.java index bd419c379..cec4f7481 100644 --- a/tis-plugin/src/main/java/com/qlangtech/tis/extension/impl/PropertyType.java +++ b/tis-plugin/src/main/java/com/qlangtech/tis/extension/impl/PropertyType.java @@ -197,13 +197,30 @@ public Void process(Class targetClass, Void v) { String placeholder = fieldExtraProps.getPlaceholder(); Object dftVal = fieldExtraProps.getDftVal(); String help = fieldExtraProps.getHelpContent(); + JSONObject props = fieldExtraProps.getProps(); + // Object disabled = props.get(PluginExtraProps.KEY_DISABLE); + +// if (disabled != null) { +// if (StringUtils.startsWith(String.valueOf(disabled), IMessageHandler.TSEARCH_PACKAGE)) { +// props.put(PluginExtraProps.KEY_DISABLE, GroovyShellEvaluate.scriptEval(String.valueOf(disabled))); +// } +// } +// disabled = props.get(PluginExtraProps.KEY_DISABLE); +// if (disabled != null) { +// if ((disabled instanceof JsonUtil.UnCacheString && ((JsonUtil.UnCacheString) disabled).getValue()) +// || (disabled instanceof Boolean && (Boolean) disabled) +// ) { +// propMapper.remove(f.getName()); +// continue; +// } +// } if (fieldExtraProps.getBoolean(PluginExtraProps.KEY_DISABLE)) { propMapper.remove(f.getName()); continue; //return null; } - JSONObject props = fieldExtraProps.getProps(); + if (StringUtils.isNotEmpty(help) && StringUtils.startsWith(help, IMessageHandler.TSEARCH_PACKAGE)) { props.put(PluginExtraProps.Props.KEY_HELP, GroovyShellUtil.eval(help)); @@ -561,7 +578,7 @@ public List getApplicableDescriptors() { String script = " package " + pkg + " ;\n" // + "import java.util.function.Function;\n" // + "import java.util.List;\n" // - + "import "+com.qlangtech.tis.extension.Descriptor.class.getName()+";\n" + + "import " + com.qlangtech.tis.extension.Descriptor.class.getName() + ";\n" // + "import com.qlangtech.plugins.incr.flink.chunjun.sink.SinkTabPropsExtends;\n" + "class " + className + " implements Function,List> { \n" //