diff --git a/datax-config/src/main/java/com/alibaba/datax/common/element/ColumnAwareRecord.java b/datax-config/src/main/java/com/alibaba/datax/common/element/ColumnAwareRecord.java
new file mode 100644
index 000000000..75441a120
--- /dev/null
+++ b/datax-config/src/main/java/com/alibaba/datax/common/element/ColumnAwareRecord.java
@@ -0,0 +1,53 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.alibaba.datax.common.element;
+
+import java.util.Map;
+
+/**
+ *
+ * @author: 百岁（baisui@qlangtech.com）
+ * @create: 2024-06-18 17:40
+ **/
+public interface ColumnAwareRecord {
+
+ /**
+ * 设置列名称到列所在index的位置
+ *
+ * @param mapper
+ */
+ public void setCol2Index(Map mapper);
+
+ /**
+ * @param field 字段名称
+ * @param colVal
+ */
+ public void setColumn(String field, final ColValType colVal);
+
+
+ public void setString(String field, final String val);
+
+ /**
+ * @param field 字段名称
+ * @return
+ */
+ public ColValType getColumn(String field);
+
+ public String getString(String field);
+}
diff --git a/datax-config/src/main/java/com/alibaba/datax/common/element/Record.java b/datax-config/src/main/java/com/alibaba/datax/common/element/Record.java
index a7a5c3d00..2ef9695f3 100755
--- a/datax-config/src/main/java/com/alibaba/datax/common/element/Record.java
+++ b/datax-config/src/main/java/com/alibaba/datax/common/element/Record.java
@@ -1,42 +1,51 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
*/
package com.alibaba.datax.common.element;
-import com.alibaba.datax.common.element.Column;
+import java.util.Map;
+
/**
* Created by jingxing on 14-8-24.
*/
-public interface Record {
+public interface Record extends ColumnAwareRecord {
+
+
+ public void addColumn(Column column);
+
+ public void setColumn(int i, final Column column);
- public void addColumn(Column column);
- public void setColumn(int i, final Column column);
+ public Column getColumn(int i);
- public Column getColumn(int i);
+ @Override
+ default String getString(String field) {
+ Column colVal = getColumn(field);
+ return colVal != null ? colVal.asString() : null;
+ }
- public String toString();
+ public String toString();
- public int getColumnNumber();
+ public int getColumnNumber();
- public int getByteSize();
+ public int getByteSize();
- public int getMemorySize();
+ public int getMemorySize();
}
diff --git a/datax-config/src/main/java/com/alibaba/datax/core/job/IJobContainerContext.java b/datax-config/src/main/java/com/alibaba/datax/core/job/IJobContainerContext.java
index 4ad9dd9cd..c07e4be8f 100644
--- a/datax-config/src/main/java/com/alibaba/datax/core/job/IJobContainerContext.java
+++ b/datax-config/src/main/java/com/alibaba/datax/core/job/IJobContainerContext.java
@@ -3,10 +3,12 @@
import com.qlangtech.tis.datax.IDataXNameAware;
import com.qlangtech.tis.datax.IDataXTaskRelevant;
+import java.util.Optional;
+
/**
* @author: 百岁（baisui@qlangtech.com）
* @create: 2023-02-23 10:06
**/
public interface IJobContainerContext extends IDataXTaskRelevant, IDataXNameAware {
-
+ Optional getTransformerBuildCfg();
}
diff --git a/tis-plugin/src/main/java/com/qlangtech/tis/plugin/datax/transformer/jdbcprop/VirtualColType.java b/datax-config/src/main/java/com/alibaba/datax/core/job/ITransformerBuildInfo.java
similarity index 70%
rename from tis-plugin/src/main/java/com/qlangtech/tis/plugin/datax/transformer/jdbcprop/VirtualColType.java
rename to datax-config/src/main/java/com/alibaba/datax/core/job/ITransformerBuildInfo.java
index 3b59cfb1b..9d5c04070 100644
--- a/tis-plugin/src/main/java/com/qlangtech/tis/plugin/datax/transformer/jdbcprop/VirtualColType.java
+++ b/datax-config/src/main/java/com/alibaba/datax/core/job/ITransformerBuildInfo.java
@@ -16,24 +16,15 @@
* limitations under the License.
*/
-package com.qlangtech.tis.plugin.datax.transformer.jdbcprop;
+package com.alibaba.datax.core.job;
-import com.qlangtech.tis.plugin.ds.TypeBase;
+import java.util.List;
/**
+ *
* @author: 百岁（baisui@qlangtech.com）
- * @create: 2024-06-10 10:14
+ * @create: 2024-06-15 12:34
**/
-public class VirtualColType extends TypeBase {
-
- private String name;
-
- @Override
- public String getName() {
- return this.name;
- }
-
- public void setName(String name) {
- this.name = name;
- }
+public interface ITransformerBuildInfo {
+ List relevantOutterColKeys();
}
diff --git a/datax-config/src/main/java/com/alibaba/datax/core/util/container/TransformerConstant.java b/datax-config/src/main/java/com/alibaba/datax/core/util/container/TransformerConstant.java
new file mode 100644
index 000000000..d3d38df73
--- /dev/null
+++ b/datax-config/src/main/java/com/alibaba/datax/core/util/container/TransformerConstant.java
@@ -0,0 +1,11 @@
+package com.alibaba.datax.core.util.container;
+
+/**
+ * @author: 百岁（baisui@qlangtech.com）
+ * @create: 2024-06-15 08:22
+ **/
+public interface TransformerConstant {
+ String JOB_TRANSFORMER = "transformer";
+ String JOB_TRANSFORMER_NAME = "name";
+ String JOB_TRANSFORMER_RELEVANT_KEYS = "cols";
+}
diff --git a/datax-config/src/main/java/com/qlangtech/tis/datax/IDataXNameAware.java b/datax-config/src/main/java/com/qlangtech/tis/datax/IDataXNameAware.java
index 131edae99..f872c12ed 100644
--- a/datax-config/src/main/java/com/qlangtech/tis/datax/IDataXNameAware.java
+++ b/datax-config/src/main/java/com/qlangtech/tis/datax/IDataXNameAware.java
@@ -18,16 +18,30 @@
* limitations under the License.
*/
+import org.apache.commons.lang3.StringUtils;
+
/**
* @author: 百岁（baisui@qlangtech.com）
* @create: 2023-02-23 10:35
**/
public interface IDataXNameAware {
+ /**
+ * 是否在索引
+ *
+ * @return
+ */
+ default boolean isCollectionAware() {
+ return StringUtils.isNotEmpty(getCollectionName());
+ }
+
+ String getCollectionName();
/**
* 取得DataX名称
*
* @return
*/
- public String getTISDataXName();
+ public default String getTISDataXName() {
+ return this.getCollectionName();
+ }
}
diff --git a/tis-base-test/src/main/java/com/qlangtech/tis/test/TISEasyMock.java b/tis-base-test/src/main/java/com/qlangtech/tis/test/TISEasyMock.java
index b1e5115ab..667ec0670 100644
--- a/tis-base-test/src/main/java/com/qlangtech/tis/test/TISEasyMock.java
+++ b/tis-base-test/src/main/java/com/qlangtech/tis/test/TISEasyMock.java
@@ -17,15 +17,13 @@
*/
package com.qlangtech.tis.test;
-import org.junit.Before;
-
/**
* @author 百岁（baisui@qlangtech.com）
* @date 2021-03-04 12:41
*/
public interface TISEasyMock {
- @Before
+
default void clearMocks() {
EasyMockUtil.clearMocks();
}
diff --git a/tis-builder-api/src/main/java/com/qlangtech/tis/realtime/transfer/UnderlineUtils.java b/tis-builder-api/src/main/java/com/qlangtech/tis/realtime/transfer/UnderlineUtils.java
index 291d315b9..e3bb00f83 100644
--- a/tis-builder-api/src/main/java/com/qlangtech/tis/realtime/transfer/UnderlineUtils.java
+++ b/tis-builder-api/src/main/java/com/qlangtech/tis/realtime/transfer/UnderlineUtils.java
@@ -1,19 +1,19 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
*/
package com.qlangtech.tis.realtime.transfer;
@@ -48,13 +48,28 @@ public static StringBuffer removeUnderline(String value) {
}
public static StringBuffer addUnderline(String value) {
+ //return UnderlineUtils.addUnderline(value);
StringBuffer parsedName = new StringBuffer();
char[] nameAry = value.toCharArray();
+ boolean firstAppend = true;
+ boolean previousUnderline = false;
for (int i = 0; i < nameAry.length; i++) {
if (Character.isUpperCase(nameAry[i])) {
- parsedName.append('_').append(Character.toLowerCase(nameAry[i]));
+ if (firstAppend) {
+ parsedName.append(Character.toLowerCase(nameAry[i]));
+ firstAppend = false;
+ } else {
+ if (!previousUnderline) {
+ parsedName.append('_');
+ }
+ parsedName.append(Character.toLowerCase(nameAry[i]));
+ previousUnderline = true;
+ }
} else {
parsedName.append(nameAry[i]);
+ firstAppend = false;
+ previousUnderline = false;
+ // .append(Character.toLowerCase());
}
}
return parsedName;
diff --git a/tis-builder-api/src/test/java/com/qlangtech/tis/TestAll.java b/tis-builder-api/src/test/java/com/qlangtech/tis/TestAll.java
index fde9d6a98..188470d6c 100644
--- a/tis-builder-api/src/test/java/com/qlangtech/tis/TestAll.java
+++ b/tis-builder-api/src/test/java/com/qlangtech/tis/TestAll.java
@@ -20,6 +20,7 @@
import com.qlangtech.tis.fs.TestIPath;
import com.qlangtech.tis.manage.common.TestConfig;
+import com.qlangtech.tis.realtime.transfer.TestUnderlineUtils;
import com.qlangtech.tis.utils.TestTisMetaProps;
import com.qlangtech.tis.utils.TestUtils;
import junit.framework.Test;
@@ -41,6 +42,7 @@ public static Test suite() {
suite.addTestSuite(TestUtils.class);
suite.addTestSuite(TestTisMetaProps.class);
suite.addTestSuite(TestIPath.class);
+ suite.addTestSuite(TestUnderlineUtils.class);
return suite;
}
diff --git a/tis-builder-api/src/test/java/com/qlangtech/tis/realtime/transfer/TestUnderlineUtils.java b/tis-builder-api/src/test/java/com/qlangtech/tis/realtime/transfer/TestUnderlineUtils.java
new file mode 100644
index 000000000..ae907ac7e
--- /dev/null
+++ b/tis-builder-api/src/test/java/com/qlangtech/tis/realtime/transfer/TestUnderlineUtils.java
@@ -0,0 +1,34 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.qlangtech.tis.realtime.transfer;
+
+import junit.framework.TestCase;
+import org.junit.Assert;
+
+/**
+ * @author: 百岁（baisui@qlangtech.com）
+ * @create: 2024-06-18 10:18
+ **/
+public class TestUnderlineUtils extends TestCase {
+
+ public void testAddUnderline() {
+ StringBuffer copyValUDF = UnderlineUtils.addUnderline("CopyValUDF");
+ Assert.assertEquals("copy_val_udf", copyValUDF.toString());
+ }
+}
diff --git a/tis-console/src/main/java/com/qlangtech/tis/config/module/action/CollectionAction.java b/tis-console/src/main/java/com/qlangtech/tis/config/module/action/CollectionAction.java
index 8effe77cd..2ca720112 100644
--- a/tis-console/src/main/java/com/qlangtech/tis/config/module/action/CollectionAction.java
+++ b/tis-console/src/main/java/com/qlangtech/tis/config/module/action/CollectionAction.java
@@ -40,6 +40,7 @@
import com.qlangtech.tis.runtime.module.action.CreateIndexConfirmModel;
import com.qlangtech.tis.runtime.module.action.SchemaAction;
import com.qlangtech.tis.runtime.module.action.SysInitializeAction;
+import com.qlangtech.tis.runtime.module.misc.IControlMsgHandler;
import com.qlangtech.tis.solrdao.ISchemaField;
import com.qlangtech.tis.solrdao.ISchemaPluginContext;
import com.qlangtech.tis.solrdao.SchemaResult;
@@ -560,7 +561,7 @@ public AppDomainInfo getAppDomain() {
private TargetColumnMeta getTargetColumnMeta(
- IPluginContext pluginContext, Context context, JSONObject post, String targetTable, PluginItems dataSourceItems)
+ IControlMsgHandler pluginContext, Context context, JSONObject post, String targetTable, PluginItems dataSourceItems)
throws TableNotFoundException {
TargetColumnMeta columnMeta = new TargetColumnMeta(targetTable);
Map colMetas = null;
@@ -572,7 +573,7 @@ private TargetColumnMeta getTargetColumnMeta(
// if (!vals.validate(this, context, false).isValid()) {
// return columnMeta.invalid();
// }
- DataSourceFactory dsFactory = (DataSourceFactory) vals.createDescribable(pluginContext).getInstance();
+ DataSourceFactory dsFactory = (DataSourceFactory) vals.createDescribable(pluginContext, context).getInstance();
List tableMetadata = null;
tableMetadata = dsFactory.getTableMetadata(false, EntityName.parse(targetTable));
@@ -823,7 +824,7 @@ private boolean createIncrSyncChannel(Context context //, WorkFlow df
// 生成DAO脚本
HeteroEnum pluginType = HeteroEnum.MQ;
UploadPluginMeta pluginMeta = UploadPluginMeta.parse(pluginType.identity + ":" + UploadPluginMeta.KEY_REQUIRE);
- PluginItems incrPluginItems = getPluginItems(incrCfg, pluginType, pluginMeta);
+ PluginItems incrPluginItems = getPluginItems(context, incrCfg, pluginType, pluginMeta);
if (incrPluginItems.items.size() < 1) {
throw new IllegalStateException("incr plugin item size can not small than 1");
}
@@ -866,15 +867,15 @@ private boolean createIncrSyncChannel(Context context //, WorkFlow df
return true;
}
- private PluginItems getDataSourceItems(JSONObject datasource) {
- HeteroEnum pluginType = HeteroEnum.DATASOURCE;
- UploadPluginMeta pluginMeta = UploadPluginMeta.parse(pluginType.identity
- + ":" + UploadPluginMeta.KEY_REQUIRE + "," + PostedDSProp.KEY_TYPE + "_detailed,update_false");
- return getPluginItems(datasource, pluginType, pluginMeta);
- }
+// private PluginItems getDataSourceItems(JSONObject datasource) {
+// HeteroEnum pluginType = HeteroEnum.DATASOURCE;
+// UploadPluginMeta pluginMeta = UploadPluginMeta.parse(pluginType.identity
+// + ":" + UploadPluginMeta.KEY_REQUIRE + "," + PostedDSProp.KEY_TYPE + "_detailed,update_false");
+// return getPluginItems(datasource, pluginType, pluginMeta);
+// }
- private PluginItems getPluginItems(JSONObject pluginCfg, HeteroEnum pluginType, UploadPluginMeta pluginMeta) {
+ private PluginItems getPluginItems(Context context, JSONObject pluginCfg, HeteroEnum pluginType, UploadPluginMeta pluginMeta) {
Map dsParams = Maps.newHashMap();
for (String dsKey : pluginCfg.keySet()) {
dsParams.put(dsKey, pluginCfg.getString(dsKey));
@@ -893,7 +894,7 @@ private PluginItems getPluginItems(JSONObject pluginCfg, HeteroEnum pluginType,
dsDescriptpr = pluginDesc.get();
- PluginItems items = new PluginItems(new DftPluginContext(pluginType), pluginMeta);
+ PluginItems items = new PluginItems(new DftPluginContext(pluginType), context, pluginMeta);
JSONArray itemsArray = new JSONArray();
JSONObject item = new JSONObject();
JSONObject vals = new JSONObject();
diff --git a/tis-console/src/main/java/com/qlangtech/tis/coredefine/module/action/DataxAction.java b/tis-console/src/main/java/com/qlangtech/tis/coredefine/module/action/DataxAction.java
index a32a25f79..410da6f1f 100644
--- a/tis-console/src/main/java/com/qlangtech/tis/coredefine/module/action/DataxAction.java
+++ b/tis-console/src/main/java/com/qlangtech/tis/coredefine/module/action/DataxAction.java
@@ -1434,7 +1434,7 @@ public boolean validate(IFieldErrorHandler msgHandler, Context context, String f
return false;
}
- CMeta.ParsePostMCols postMCols = (new IdlistElementCreatorFactory()).parsePostMCols(null, msgHandler,
+ CMeta.ParsePostMCols postMCols = (new IdlistElementCreatorFactory()).parsePostMCols(null, (IControlMsgHandler) msgHandler,
context, fieldKey /*MultiItemsViewType.keyColsMeta*/, targetCols);
// Map existCols = Maps.newHashMap();
diff --git a/tis-console/src/main/java/com/qlangtech/tis/coredefine/module/action/PluginAction.java b/tis-console/src/main/java/com/qlangtech/tis/coredefine/module/action/PluginAction.java
index 480605f14..ca0b22369 100644
--- a/tis-console/src/main/java/com/qlangtech/tis/coredefine/module/action/PluginAction.java
+++ b/tis-console/src/main/java/com/qlangtech/tis/coredefine/module/action/PluginAction.java
@@ -604,7 +604,7 @@ public void run() {
} catch (InterruptedException e) {
}
// 为了让Assemble等节点的uberClassLoader重新加载一次，需要主动向Assemble等节点发送一个指令
- // notifyPluginUpdate2AssembleNode(TIS.KEY_ACTION_CLEAN_TIS + "=true", "TIS");
+ // notifyPluginUpdate2AssembleNode(TIS.KEY_ACTION_CLEAN_TIS + "=true", "TIS");
InstallUtil.proceedToNextStateFrom(InstallState.INITIAL_PLUGINS_INSTALLING);
}
}
@@ -948,7 +948,7 @@ public static PluginItemsParser parsePluginItems(BasicModule module, UploadPlugi
Optional subFormFilter = pluginMeta.getSubFormFilter();
IPluginEnum hEnum = pluginMeta.getHeteroEnum();
- PluginItems pluginItems = new PluginItems(module, pluginMeta);
+ PluginItems pluginItems = new PluginItems(module, context, pluginMeta);
List describableAttrValMapList = AttrValMap.describableAttrValMapList(itemsArray, subFormFilter);
if (pluginMeta.isRequired() && describableAttrValMapList.size() < 1) {
module.addErrorMessage(context, "请设置'" + hEnum.getCaption() + "'表单内容");
@@ -977,7 +977,7 @@ public static PluginItemsParser parsePluginItems(BasicModule module, UploadPlugi
for (IdentityName p : plugins) {
desc = ((Describable) p).getDescriptor();
Descriptor.PluginValidateResult r = new Descriptor.PluginValidateResult(new Descriptor.PostFormVals(desc,
- module, AttrValMap.IAttrVals.rootForm(Collections.emptyMap())), pluginIndex, newAddItemsCount++);
+ module, context, AttrValMap.IAttrVals.rootForm(Collections.emptyMap())), pluginIndex, newAddItemsCount++);
r.setDescriptor(desc);
identityUniqueMap.put(p.identityValue(), r);
}
diff --git a/tis-console/src/main/java/com/qlangtech/tis/manage/common/TisActionMapper.java b/tis-console/src/main/java/com/qlangtech/tis/manage/common/TisActionMapper.java
index 7bad80067..fad8258a8 100644
--- a/tis-console/src/main/java/com/qlangtech/tis/manage/common/TisActionMapper.java
+++ b/tis-console/src/main/java/com/qlangtech/tis/manage/common/TisActionMapper.java
@@ -1,23 +1,24 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
*/
package com.qlangtech.tis.manage.common;
import com.opensymphony.xwork2.config.ConfigurationManager;
+import com.qlangtech.tis.realtime.transfer.UnderlineUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.struts2.dispatcher.mapper.ActionMapping;
import org.apache.struts2.dispatcher.mapper.DefaultActionMapper;
@@ -53,25 +54,25 @@ protected void parseNameAndNamespace(String uri, ActionMapping mapping, Configur
public static StringBuffer addUnderline(String value) {
- //return UnderlineUtils.addUnderline(value);
- StringBuffer parsedName = new StringBuffer();
- char[] nameAry = value.toCharArray();
- boolean firstAppend = true;
- for (int i = 0; i < nameAry.length; i++) {
- if (Character.isUpperCase(nameAry[i])) {
- if (firstAppend) {
- parsedName.append(Character.toLowerCase(nameAry[i]));
- firstAppend = false;
- } else {
- parsedName.append('_').append(Character.toLowerCase(nameAry[i]));
- }
- } else {
- parsedName.append(nameAry[i]);
- firstAppend = false;
- // .append(Character.toLowerCase());
- }
- }
- return parsedName;
+ return UnderlineUtils.addUnderline(value);
+// StringBuffer parsedName = new StringBuffer();
+// char[] nameAry = value.toCharArray();
+// boolean firstAppend = true;
+// for (int i = 0; i < nameAry.length; i++) {
+// if (Character.isUpperCase(nameAry[i])) {
+// if (firstAppend) {
+// parsedName.append(Character.toLowerCase(nameAry[i]));
+// firstAppend = false;
+// } else {
+// parsedName.append('_').append(Character.toLowerCase(nameAry[i]));
+// }
+// } else {
+// parsedName.append(nameAry[i]);
+// firstAppend = false;
+// // .append(Character.toLowerCase());
+// }
+// }
+// return parsedName;
}
diff --git a/tis-console/src/main/java/com/qlangtech/tis/util/PluginItems.java b/tis-console/src/main/java/com/qlangtech/tis/util/PluginItems.java
index f6103e10e..2d27c6238 100644
--- a/tis-console/src/main/java/com/qlangtech/tis/util/PluginItems.java
+++ b/tis-console/src/main/java/com/qlangtech/tis/util/PluginItems.java
@@ -67,6 +67,7 @@ public class PluginItems {
private final IPluginEnum heteroEnum;
private final UploadPluginMeta pluginMeta;
private final IPluginContext pluginContext;
+ private final Context context;
public List items;
@@ -78,7 +79,7 @@ public static void addPluginItemsSaveObserver(PluginItemsSaveObserver obsv) {
observable.addObserver(obsv);
}
- public PluginItems(IPluginContext pluginContext, UploadPluginMeta pluginMeta) {
+ public PluginItems(IPluginContext pluginContext, Context context, UploadPluginMeta pluginMeta) {
this.heteroEnum = pluginMeta.getHeteroEnum();
this.pluginMeta = pluginMeta;
this.pluginContext = pluginMeta.isDisableBizSet() ? new AdapterPluginContext(pluginContext) {
@@ -87,6 +88,7 @@ public void setBizResult(Context context, Object result) {
//super.setBizResult(context, result);
}
} : pluginContext;
+ this.context = context;
}
@@ -383,7 +385,7 @@ private List> getPlugins(List descri
/**====================================================
* 将客户端POST数据包装
======================================================*/
- describable = attrValMap.createDescribable(pluginContext);
+ describable = attrValMap.createDescribable((IControlMsgHandler) pluginContext, this.context);
dlist.add(describable);
if (!describable.subFormFields) {
describableList.add((Describable) describable.getInstance());
diff --git a/tis-console/src/test/java/com/qlangtech/tis/util/TestPluginItems.java b/tis-console/src/test/java/com/qlangtech/tis/util/TestPluginItems.java
index d6e441169..25e3ed742 100644
--- a/tis-console/src/test/java/com/qlangtech/tis/util/TestPluginItems.java
+++ b/tis-console/src/test/java/com/qlangtech/tis/util/TestPluginItems.java
@@ -1,19 +1,19 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
*/
package com.qlangtech.tis.util;
@@ -157,6 +157,7 @@ private void validatePropertyValue(Map propertiesType, Str
private void validateSubFormSave() {
IPluginContext pluginContext = EasyMock.createMock("pluginContext", IPluginContext.class);
+
EasyMock.expect(pluginContext.isDataSourceAware()).andReturn(false);
EasyMock.expect(pluginContext.isCollectionAware()).andReturn(false).anyTimes();
EasyMock.expect(pluginContext.getRequestHeader(DataxReader.HEAD_KEY_REFERER)).andReturn("/x/" + dataXName + "/config").times(2);
@@ -165,7 +166,7 @@ private void validateSubFormSave() {
Optional subFormFilter = subFieldPluginMeta.getSubFormFilter();
assertTrue("subFormFilter.isPresent():true", subFormFilter.isPresent());
- PluginItems pluginItems = new PluginItems(pluginContext, subFieldPluginMeta);
+ PluginItems pluginItems = new PluginItems(pluginContext, context, subFieldPluginMeta);
IControlMsgHandler fieldErrorHandler = EasyMock.createMock("fieldErrorHandler", IControlMsgHandler.class);
JSONArray jsonArray = IOUtils.loadResourceFromClasspath(TestPluginItems.class
@@ -175,7 +176,7 @@ private void validateSubFormSave() {
JSONArray itemsArray = jsonArray.getJSONArray(0);
// Optional.empty();
- List items = AttrValMap.describableAttrValMapList( itemsArray, subFormFilter);
+ List items = AttrValMap.describableAttrValMapList(itemsArray, subFormFilter);
pluginItems.items = items;
EasyMock.replay(pluginContext, context, fieldErrorHandler);
@@ -195,7 +196,7 @@ private void validateRootFormSave() {
//targetDescriptorName_MySQL,subFormFieldName_selectedTabs
UploadPluginMeta pluginMeta = UploadPluginMeta.parse("dataxReader:require,dataxName_" + dataXName);
- PluginItems pluginItems = new PluginItems(pluginContext, pluginMeta);
+ PluginItems pluginItems = new PluginItems(pluginContext, context, pluginMeta);
IControlMsgHandler fieldErrorHandler = EasyMock.createMock("fieldErrorHandler", IControlMsgHandler.class);
JSONArray jsonArray = IOUtils.loadResourceFromClasspath(TestPluginItems.class, "datax_reader_mysql_post.json", true, (input) -> {
@@ -205,7 +206,7 @@ private void validateRootFormSave() {
JSONArray itemsArray = jsonArray.getJSONArray(0);
Optional subFormFilter = pluginMeta.getSubFormFilter();
assertFalse("subFormFilter.isPresent():false", subFormFilter.isPresent());
- List items = AttrValMap.describableAttrValMapList( itemsArray, subFormFilter);
+ List items = AttrValMap.describableAttrValMapList(itemsArray, subFormFilter);
pluginItems.items = items;
EasyMock.replay(pluginContext, context, fieldErrorHandler);
diff --git a/tis-manage-pojo/src/main/java/com/qlangtech/tis/plugin/ds/CMeta.java b/tis-manage-pojo/src/main/java/com/qlangtech/tis/plugin/ds/CMeta.java
index 0bc5ee691..a5f4a3f56 100644
--- a/tis-manage-pojo/src/main/java/com/qlangtech/tis/plugin/ds/CMeta.java
+++ b/tis-manage-pojo/src/main/java/com/qlangtech/tis/plugin/ds/CMeta.java
@@ -181,7 +181,7 @@ public Class getDescribleClass() {
return IdentityName.super.getDescribleClass();
}
- public static class ParsePostMCols {
+ public static class ParsePostMCols {
public List writerCols = Lists.newArrayList();
public boolean validateFaild = false;
public boolean pkHasSelected = false;
diff --git a/tis-manage-pojo/src/main/java/com/qlangtech/tis/plugin/ds/ElementCreatorFactory.java b/tis-manage-pojo/src/main/java/com/qlangtech/tis/plugin/ds/ElementCreatorFactory.java
index 8fae68a26..901459607 100644
--- a/tis-manage-pojo/src/main/java/com/qlangtech/tis/plugin/ds/ElementCreatorFactory.java
+++ b/tis-manage-pojo/src/main/java/com/qlangtech/tis/plugin/ds/ElementCreatorFactory.java
@@ -22,11 +22,11 @@
import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import com.qlangtech.tis.extension.IPropertyType;
-import com.qlangtech.tis.runtime.module.misc.IFieldErrorHandler;
+import com.qlangtech.tis.runtime.module.misc.IControlMsgHandler;
import java.util.function.BiConsumer;
-public interface ElementCreatorFactory {
+public interface ElementCreatorFactory {
/**
@@ -39,8 +39,8 @@ public interface ElementCreatorFactory {
* @return
*/
public CMeta.ParsePostMCols parsePostMCols(IPropertyType propertyType,
- IFieldErrorHandler msgHandler, Context context, String keyColsMeta,
- JSONArray targetCols);
+ IControlMsgHandler msgHandler, Context context, String keyColsMeta,
+ JSONArray targetCols);
default ViewContent getViewContentType() {
return ViewContent.MongoCols;
diff --git a/tis-manage-pojo/src/main/java/com/qlangtech/tis/plugin/ds/IMultiElement.java b/tis-manage-pojo/src/main/java/com/qlangtech/tis/plugin/ds/IMultiElement.java
new file mode 100644
index 000000000..351370d0a
--- /dev/null
+++ b/tis-manage-pojo/src/main/java/com/qlangtech/tis/plugin/ds/IMultiElement.java
@@ -0,0 +1,28 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.qlangtech.tis.plugin.ds;
+
+/**
+ *
+ * @author: 百岁（baisui@qlangtech.com）
+ * @create: 2024-06-16 08:44
+ **/
+public interface IMultiElement {
+ String getName();
+}
diff --git a/tis-manage-pojo/src/main/java/com/qlangtech/tis/plugin/ds/IdlistElementCreatorFactory.java b/tis-manage-pojo/src/main/java/com/qlangtech/tis/plugin/ds/IdlistElementCreatorFactory.java
index fb77430d3..9862fd10d 100644
--- a/tis-manage-pojo/src/main/java/com/qlangtech/tis/plugin/ds/IdlistElementCreatorFactory.java
+++ b/tis-manage-pojo/src/main/java/com/qlangtech/tis/plugin/ds/IdlistElementCreatorFactory.java
@@ -23,12 +23,11 @@
import com.alibaba.fastjson.JSONObject;
import com.qlangtech.tis.extension.IPropertyType;
import com.qlangtech.tis.plugin.ds.CMeta.ParsePostMCols;
-import com.qlangtech.tis.runtime.module.misc.IFieldErrorHandler;
+import com.qlangtech.tis.runtime.module.misc.IControlMsgHandler;
import java.util.function.BiConsumer;
/**
- *
* //@see com.qlangtech.tis.extension.util.MultiItemsViewType.ViewFormatType
*/
public class IdlistElementCreatorFactory implements ElementCreatorFactory {
@@ -50,7 +49,7 @@ public CMeta create(JSONObject targetCol, BiConsumer errorProces
@Override
public ParsePostMCols parsePostMCols(IPropertyType propertyType,
- IFieldErrorHandler msgHandler, Context context, String keyColsMeta, JSONArray targetCols) {
- throw new UnsupportedOperationException();
+ IControlMsgHandler msgHandler, Context context, String keyColsMeta, JSONArray targetCols) {
+ throw new UnsupportedOperationException();
}
}
diff --git a/tis-manage-pojo/src/main/java/com/qlangtech/tis/plugin/ds/TypeBase.java b/tis-manage-pojo/src/main/java/com/qlangtech/tis/plugin/ds/TypeBase.java
index a55e0b430..7ff307ecc 100644
--- a/tis-manage-pojo/src/main/java/com/qlangtech/tis/plugin/ds/TypeBase.java
+++ b/tis-manage-pojo/src/main/java/com/qlangtech/tis/plugin/ds/TypeBase.java
@@ -23,7 +23,7 @@
/**
* 有Jdbc类型相关的类型
*/
-public abstract class TypeBase implements JDBCColumnProp {
+public abstract class TypeBase implements JDBCColumnProp, IMultiElement {
private DataType type;
public final DataType getType() {
diff --git a/tis-manage-pojo/src/main/java/com/qlangtech/tis/runtime/module/misc/IControlMsgHandler.java b/tis-manage-pojo/src/main/java/com/qlangtech/tis/runtime/module/misc/IControlMsgHandler.java
index 9fa41e6b7..cb8632fb3 100644
--- a/tis-manage-pojo/src/main/java/com/qlangtech/tis/runtime/module/misc/IControlMsgHandler.java
+++ b/tis-manage-pojo/src/main/java/com/qlangtech/tis/runtime/module/misc/IControlMsgHandler.java
@@ -17,6 +17,8 @@
*/
package com.qlangtech.tis.runtime.module.misc;
+import com.alibaba.citrus.turbine.Context;
+import com.qlangtech.tis.datax.IDataXNameAware;
import com.qlangtech.tis.runtime.module.action.IParamGetter;
import java.io.PrintWriter;
@@ -25,7 +27,67 @@
* @author 百岁（baisui@qlangtech.com）
* @date 2020/09/25
*/
-public interface IControlMsgHandler extends IFieldErrorHandler, IMessageHandler, IParamGetter {
+public interface IControlMsgHandler extends IFieldErrorHandler, IMessageHandler, IParamGetter, IDataXNameAware {
+
+ public static IControlMsgHandler namedContext(String collectionName) {
+ return new IControlMsgHandler() {
+ @Override
+ public PrintWriter getEventStreamWriter() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public String getCollectionName() {
+ return collectionName;
+ }
+
+ @Override
+ public String getString(String key) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public String getString(String key, String dftVal) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public boolean getBoolean(String key) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void addFieldError(Context context, String fieldName, String msg, Object... params) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public boolean validateBizLogic(BizLogic logicType, Context context, String fieldName, String value) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void errorsPageShow(Context context) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void addActionMessage(Context context, String msg) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void setBizResult(Context context, Object result, boolean overwriteable) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void addErrorMessage(Context context, String msg) {
+ throw new UnsupportedOperationException();
+ }
+ };
+ }
+
/**
* 获得EventStream 类型的Writer，用于在UI流程中显示执行进度
*
diff --git a/tis-manage-pojo/src/main/java/com/qlangtech/tis/runtime/module/misc/impl/AdaperControlMsgHandler.java b/tis-manage-pojo/src/main/java/com/qlangtech/tis/runtime/module/misc/impl/AdaperControlMsgHandler.java
new file mode 100644
index 000000000..e8e77f7ec
--- /dev/null
+++ b/tis-manage-pojo/src/main/java/com/qlangtech/tis/runtime/module/misc/impl/AdaperControlMsgHandler.java
@@ -0,0 +1,119 @@
+///**
+// * Licensed to the Apache Software Foundation (ASF) under one
+// * or more contributor license agreements. See the NOTICE file
+// * distributed with this work for additional information
+// * regarding copyright ownership. The ASF licenses this file
+// * to you under the Apache License, Version 2.0 (the
+// * "License"); you may not use this file except in compliance
+// * with the License. You may obtain a copy of the License at
+// *
+// * http://www.apache.org/licenses/LICENSE-2.0
+// *
+// * Unless required by applicable law or agreed to in writing, software
+// * distributed under the License is distributed on an "AS IS" BASIS,
+// * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// * See the License for the specific language governing permissions and
+// * limitations under the License.
+// */
+//
+//package com.qlangtech.tis.runtime.module.misc.impl;
+//
+//import com.alibaba.citrus.turbine.Context;
+//import com.qlangtech.tis.runtime.module.misc.IControlMsgHandler;
+//import com.qlangtech.tis.runtime.module.misc.IFieldErrorHandler;
+//
+//import java.io.PrintWriter;
+//
+///**
+// * @author: 百岁（baisui@qlangtech.com）
+// * @create: 2024-06-16 20:11
+// **/
+//public class AdaperControlMsgHandler implements IControlMsgHandler {
+//
+//
+//
+// private final IControlMsgHandler delegate;
+//
+// public AdaperControlMsgHandler(IFieldErrorHandler msgHandler) {
+// this((IControlMsgHandler) msgHandler);
+// }
+//
+// public AdaperControlMsgHandler(IControlMsgHandler delegate) {
+// this.delegate = delegate;
+// }
+//
+// @Override
+// public boolean getBoolean(String key) {
+// return delegate.getBoolean(key);
+// }
+//
+// @Override
+// public String getString(String key, String dftVal) {
+// return delegate.getString(key, dftVal);
+// }
+//
+// @Override
+// public String getString(String key) {
+// return delegate.getString(key);
+// }
+//
+// @Override
+// public String getRequestHeader(String key) {
+// return delegate.getRequestHeader(key);
+// }
+//
+// @Override
+// public void addErrorMessage(Context context, String msg) {
+// delegate.addErrorMessage(context, msg);
+// }
+//
+// @Override
+// public void setBizResult(Context context, Object result, boolean overwriteable) {
+// delegate.setBizResult(context, result, overwriteable);
+// }
+//
+// @Override
+// public void setBizResult(Context context, Object result) {
+// delegate.setBizResult(context, result);
+// }
+//
+// @Override
+// public void addActionMessage(Context context, String msg) {
+// delegate.addActionMessage(context, msg);
+// }
+//
+// @Override
+// public void errorsPageShow(Context context) {
+// delegate.errorsPageShow(context);
+// }
+//
+// @Override
+// public boolean validateBizLogic(BizLogic logicType, Context context, String fieldName, String value) {
+// return delegate.validateBizLogic(logicType, context, fieldName, value);
+// }
+//
+// @Override
+// public void addFieldError(Context context, String fieldName, String msg, Object... params) {
+// delegate.addFieldError(context, fieldName, msg, params);
+// }
+//
+//
+// @Override
+// public PrintWriter getEventStreamWriter() {
+// return delegate.getEventStreamWriter();
+// }
+// @Override
+// public String getTISDataXName() {
+// return delegate.getTISDataXName();
+// }
+//
+// @Override
+// public String getCollectionName() {
+// return delegate.getCollectionName();
+// }
+//
+// @Override
+// public boolean isCollectionAware() {
+// return delegate.isCollectionAware();
+// }
+//}
diff --git a/tis-manage-pojo/src/main/java/com/qlangtech/tis/runtime/module/misc/impl/BasicDelegateMsgHandler.java b/tis-manage-pojo/src/main/java/com/qlangtech/tis/runtime/module/misc/impl/BasicDelegateMsgHandler.java
index a7e0dd5d0..00d7a32c1 100644
--- a/tis-manage-pojo/src/main/java/com/qlangtech/tis/runtime/module/misc/impl/BasicDelegateMsgHandler.java
+++ b/tis-manage-pojo/src/main/java/com/qlangtech/tis/runtime/module/misc/impl/BasicDelegateMsgHandler.java
@@ -19,6 +19,9 @@
import com.alibaba.citrus.turbine.Context;
import com.qlangtech.tis.runtime.module.misc.IControlMsgHandler;
+import com.qlangtech.tis.runtime.module.misc.IFieldErrorHandler;
+
+import java.io.PrintWriter;
/**
* @author 百岁（baisui@qlangtech.com）
@@ -26,19 +29,24 @@
*/
public abstract class BasicDelegateMsgHandler implements IControlMsgHandler {
+
private final IControlMsgHandler delegate;
public BasicDelegateMsgHandler(IControlMsgHandler delegate) {
this.delegate = delegate;
}
+ public BasicDelegateMsgHandler(IFieldErrorHandler msgHandler) {
+ this((IControlMsgHandler) msgHandler);
+ }
+
@Override
public boolean validateBizLogic(BizLogic logicType, Context context, String fieldName, String value) {
throw new UnsupportedOperationException();
}
@Override
- public final void addFieldError(Context context, String fieldName, String msg, Object... params) {
+ public void addFieldError(Context context, String fieldName, String msg, Object... params) {
delegate.addFieldError(context, fieldName, msg, params);
}
@@ -61,4 +69,39 @@ public final void setBizResult(Context context, Object result, boolean overwrite
public final void addErrorMessage(Context context, String msg) {
delegate.addErrorMessage(context, msg);
}
+
+ @Override
+ public boolean isCollectionAware() {
+ return delegate.isCollectionAware();
+ }
+
+ @Override
+ public String getCollectionName() {
+ return delegate.getCollectionName();
+ }
+
+ @Override
+ public String getTISDataXName() {
+ return delegate.getTISDataXName();
+ }
+
+ @Override
+ public PrintWriter getEventStreamWriter() {
+ return delegate.getEventStreamWriter();
+ }
+
+ @Override
+ public String getString(String key) {
+ return delegate.getString(key);
+ }
+
+ @Override
+ public String getString(String key, String dftVal) {
+ return delegate.getString(key, dftVal);
+ }
+
+ @Override
+ public boolean getBoolean(String key) {
+ return delegate.getBoolean(key);
+ }
}
diff --git a/tis-manage-pojo/src/main/java/com/qlangtech/tis/util/DescriptorsJSONResult.java b/tis-manage-pojo/src/main/java/com/qlangtech/tis/util/DescriptorsJSONResult.java
index ea9cd22bb..9172975fe 100644
--- a/tis-manage-pojo/src/main/java/com/qlangtech/tis/util/DescriptorsJSONResult.java
+++ b/tis-manage-pojo/src/main/java/com/qlangtech/tis/util/DescriptorsJSONResult.java
@@ -41,11 +41,11 @@ public static T getRootDescInstance() {
return (T) Objects.requireNonNull(rootDescriptorLocal.get(), "rootDescriptorLocal element can not be null");
}
- Map> descs = Maps.newHashMap();
+ Map> descs = Maps.newHashMap();
/**
* 由于describe 可以嵌套，此标志位可以判断 是否是根元素
*/
- final boolean rootDesc;
+ final boolean rootDesc;
public DescriptorsJSONResult(boolean rootDesc) {
this.rootDesc = rootDesc;
@@ -56,9 +56,9 @@ public void addDesc(String id, JSONObject descJson, Object desc) {
}
-
public JSONObject getJSONObject(String descId) {
return Objects.requireNonNull(descs.get(descId)
, "descId:" + descId + " relevant desc can not be null").getLeft();
}
+
}
diff --git a/tis-plugin/src/main/java/com/qlangtech/tis/async/message/client/consumer/IConsumerHandle.java b/tis-plugin/src/main/java/com/qlangtech/tis/async/message/client/consumer/IConsumerHandle.java
index 4f935944d..bcb711c9a 100644
--- a/tis-plugin/src/main/java/com/qlangtech/tis/async/message/client/consumer/IConsumerHandle.java
+++ b/tis-plugin/src/main/java/com/qlangtech/tis/async/message/client/consumer/IConsumerHandle.java
@@ -1,19 +1,19 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
*/
package com.qlangtech.tis.async.message.client.consumer;
@@ -31,5 +31,6 @@ public interface IConsumerHandle {
*
* @param asyncMsg
*/
- FLINK_RESULT consume(TargetResName dataxName, AsyncMsg asyncMsg, IDataxProcessor dataXProcessor) throws Exception;
+ FLINK_RESULT consume(TargetResName dataxName, AsyncMsg asyncMsg
+ , IDataxProcessor dataXProcessor, IFlinkColCreator flinkColCreator) throws Exception;
}
diff --git a/tis-plugin/src/main/java/com/qlangtech/tis/async/message/client/consumer/IFlinkColCreator.java b/tis-plugin/src/main/java/com/qlangtech/tis/async/message/client/consumer/IFlinkColCreator.java
new file mode 100644
index 000000000..4d1554fbb
--- /dev/null
+++ b/tis-plugin/src/main/java/com/qlangtech/tis/async/message/client/consumer/IFlinkColCreator.java
@@ -0,0 +1,29 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.qlangtech.tis.async.message.client.consumer;
+
+import com.qlangtech.tis.plugin.ds.IColMetaGetter;
+
+/**
+ * @author: 百岁（baisui@qlangtech.com）
+ * @create: 2024-06-19 11:05
+ **/
+public interface IFlinkColCreator {
+ FlinkColType build(IColMetaGetter meta, int colIndex);
+}
diff --git a/tis-plugin/src/main/java/com/qlangtech/tis/datax/impl/DataXCfgGenerator.java b/tis-plugin/src/main/java/com/qlangtech/tis/datax/impl/DataXCfgGenerator.java
index b67f4a247..22d1183ab 100644
--- a/tis-plugin/src/main/java/com/qlangtech/tis/datax/impl/DataXCfgGenerator.java
+++ b/tis-plugin/src/main/java/com/qlangtech/tis/datax/impl/DataXCfgGenerator.java
@@ -18,6 +18,7 @@
package com.qlangtech.tis.datax.impl;
+import com.alibaba.datax.core.util.container.TransformerConstant;
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
@@ -28,6 +29,7 @@
import com.qlangtech.tis.datax.DBDataXChildTask;
import com.qlangtech.tis.datax.DataXCfgFile;
import com.qlangtech.tis.datax.DataXJobInfo;
+import com.qlangtech.tis.datax.IDataXNameAware;
import com.qlangtech.tis.datax.IDataXPluginMeta;
import com.qlangtech.tis.datax.IDataxContext;
import com.qlangtech.tis.datax.IDataxGlobalCfg;
@@ -40,6 +42,7 @@
import com.qlangtech.tis.manage.common.TisUTF8;
import com.qlangtech.tis.offline.DataxUtils;
import com.qlangtech.tis.plugin.datax.CreateTableSqlBuilder;
+import com.qlangtech.tis.plugin.datax.transformer.RecordTransformerRules;
import com.qlangtech.tis.plugin.ds.CMeta;
import com.qlangtech.tis.plugin.ds.ISelectedTab;
import com.qlangtech.tis.plugin.trigger.JobTrigger;
@@ -70,7 +73,7 @@
* @author: baisui 百岁
* @create: 2021-04-20 18:06
**/
-public class DataXCfgGenerator {
+public class DataXCfgGenerator implements IDataXNameAware {
private transient static final VelocityEngine velocityEngine;
@@ -95,6 +98,11 @@ public class DataXCfgGenerator {
private final String dataxName;
private final IPluginContext pluginCtx;
+ @Override
+ public String getCollectionName() {
+ return this.dataxName;
+ }
+
public DataXCfgGenerator(IPluginContext pluginCtx, String dataxName, IDataxProcessor dataxProcessor) {
Objects.requireNonNull(dataxProcessor, "dataXprocessor can not be null");
IDataxGlobalCfg dataXGlobalCfg = dataxProcessor.getDataXGlobalCfg();
@@ -105,22 +113,35 @@ public DataXCfgGenerator(IPluginContext pluginCtx, String dataxName, IDataxProce
this.pluginCtx = pluginCtx;
}
- protected String getTemplateContent(IDataxReader reader, IDataxWriter writer) {
+ protected String getTemplateContent(IDataxReaderContext readerContext, IDataxReader reader, IDataxWriter writer) {
final String tpl = globalCfg.getTemplate();
// List readers = dataxProcessor.getReaders(pluginCtx);
// for (IDataxReader reader : readers) {
//IDataxReader reader = dataxProcessor.getReader(pluginCtx);
// IDataxWriter writer = dataxProcessor.getWriter(pluginCtx);
- String readerTpl = reader.getTemplate();
- String writerTpl = writer.getTemplate();
- if (StringUtils.isEmpty(readerTpl)) {
+ if (StringUtils.isEmpty(reader.getTemplate())) {
throw new IllegalStateException("readerTpl of '" + reader.getDataxMeta().getName() + "' can not be null");
}
+ StringBuffer readerTpl = new StringBuffer(reader.getTemplate());
+ String writerTpl = writer.getTemplate();
+
if (StringUtils.isEmpty(writerTpl)) {
throw new IllegalStateException("writerTpl of '" + writer.getDataxMeta().getName() + "' can not be null");
}
- String template = StringUtils.replace(tpl, "", readerTpl);
+
+ RecordTransformerRules transformerRules
+ = RecordTransformerRules.loadTransformerRules(this, readerContext.getSourceEntityName());
+ if (transformerRules != null) {
+ readerTpl.append(",\n\t\"")
+ .append(TransformerConstant.JOB_TRANSFORMER).append("\":\t\t\n {\"").append(TransformerConstant.JOB_TRANSFORMER_NAME)
+ .append("\":\"").append(readerContext.getSourceEntityName()).append("\",\n\"")
+ .append(TransformerConstant.JOB_TRANSFORMER_RELEVANT_KEYS).append("\":").append("[")
+ .append(transformerRules.relevantColKeys().stream().map((col) -> "\"" + col + "\"").collect(Collectors.joining(",")))
+ .append("]").append("}");
+ }
+
+ String template = StringUtils.replace(tpl, "", readerTpl.toString());
template = StringUtils.replace(template, "", writerTpl);
return template;
// }
@@ -306,6 +327,7 @@ public GenerateCfgs startGenerateCfg(IGenerateScriptFile scriptFileGenerator) th
return cfgs;
}
+
public interface IGenerateScriptFile {
void generateScriptFile(IDataxReader reader, IDataxWriter writer, IDataxReaderContext readerContext,
Set createDDLFiles, Optional tableMapper) throws IOException;
@@ -540,7 +562,9 @@ public String generateDataxConfig(
Optional tableMap) throws IOException {
Objects.requireNonNull(writer, "writer can not be null");
StringWriter writerContent = null;
- final String tpl = getTemplateContent(reader, writer);
+
+
+ final String tpl = getTemplateContent(readerContext, reader, writer);
if (StringUtils.isEmpty(tpl)) {
throw new IllegalStateException("velocity template content can not be null");
}
diff --git a/tis-plugin/src/main/java/com/qlangtech/tis/extension/Descriptor.java b/tis-plugin/src/main/java/com/qlangtech/tis/extension/Descriptor.java
index 022e5c0a4..6e55d2a40 100644
--- a/tis-plugin/src/main/java/com/qlangtech/tis/extension/Descriptor.java
+++ b/tis-plugin/src/main/java/com/qlangtech/tis/extension/Descriptor.java
@@ -18,6 +18,7 @@
package com.qlangtech.tis.extension;
import com.alibaba.citrus.turbine.Context;
+import com.alibaba.citrus.turbine.impl.DefaultContext;
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
@@ -47,6 +48,7 @@
import com.qlangtech.tis.plugin.annotation.SubForm;
import com.qlangtech.tis.plugin.annotation.Validator;
import com.qlangtech.tis.plugin.ds.CMeta;
+import com.qlangtech.tis.plugin.ds.IMultiElement;
import com.qlangtech.tis.plugin.ds.ISelectedTab;
import com.qlangtech.tis.plugin.ds.TypeBase;
import com.qlangtech.tis.runtime.module.misc.IControlMsgHandler;
@@ -478,7 +480,15 @@ public final PluginValidateResult verify(IControlMsgHandler msgHandler, Context
, boolean verify //
, AttrVals formData, Optional pTypes, Optional subFormFilter) {
-
+ if (context == null) {
+ throw new IllegalArgumentException("param contenxt can not be null");
+ }
+ if (msgHandler == null) {
+ throw new IllegalArgumentException("param msgHandler can not be null");
+ }
+ if (msgHandler == null) {
+ throw new IllegalArgumentException("formData msgHandler can not be null");
+ }
try {
// IRepositoryTargetFile.TARGET_FILE_CONTEXT.set(targetFile);
// final PluginFormProperties /** * fieldname */
@@ -490,7 +500,7 @@ public final PluginValidateResult verify(IControlMsgHandler msgHandler, Context
@Override
public PluginValidateResult visit(RootFormProperties props) {
- PostFormVals postFormVals = new PostFormVals(Descriptor.this, props, subFormFilter, msgHandler, formData);
+ PostFormVals postFormVals = new PostFormVals(Descriptor.this, props, subFormFilter, msgHandler, context, formData);
PluginValidateResult validateResult = new PluginValidateResult(postFormVals,
(Integer) context.get(DefaultFieldErrorHandler.KEY_VALIDATE_PLUGIN_INDEX),
@@ -498,6 +508,7 @@ public PluginValidateResult visit(RootFormProperties props) {
boolean valid = validatePostFormVals(postFormVals, Optional.empty());
validateResult.valid = valid;
+ validateResult.setDescriptor(Descriptor.this);
return validateResult;
}
@@ -543,7 +554,7 @@ public PluginValidateResult visit(BaseSubFormProperties props) {
if (filter.subformDetailView) {
// 校验的时候子表单是{key1:val1,key2:val2} 的格式
- PostFormVals formVals = new PostFormVals(Descriptor.this, props, subFormFilter, msgHandler, formData);
+ PostFormVals formVals = new PostFormVals(Descriptor.this, props, subFormFilter, msgHandler, context, formData);
// boolean valid = isValid(msgHandler, context, verify, subFormFilter, propertyTypes, formVals);
boolean valid = validatePostFormVals(formVals, subFormFilter);
if (!valid) {
@@ -829,7 +840,7 @@ public PostFormVals getItemForm() {
return this.itemForm;
}
- public T newInstance(IControlMsgHandler msgHandler) {
+ public T newInstance() {
if (this.descriptor == null) {
throw new IllegalStateException("descriptor can not be null");
}
@@ -873,17 +884,17 @@ private void addFieldRequiredError(IFieldErrorHandler msgHandler, Context contex
}
- public ParseDescribable newInstance(IPluginContext pluginContext, //
+ public ParseDescribable newInstance(IControlMsgHandler pluginContext, Context context, //
FormData formData //
) {
- return newInstance(pluginContext, formData, Optional.empty());
+ return newInstance(pluginContext, context, formData, Optional.empty());
}
public ParseDescribable newInstance(String appName, //
FormData formData //
) {
- return newInstance(IPluginContext.namedContext(appName), formData, Optional.empty());
+ return newInstance(IControlMsgHandler.namedContext(appName), new DefaultContext(), formData, Optional.empty());
}
public static class FormData extends AttrVals {
@@ -916,11 +927,11 @@ public JSONObject addSubForm(String key, String formImpl, FormData form) {
}
}
- public ParseDescribable newInstance(IPluginContext pluginContext, //
+ public ParseDescribable newInstance(IControlMsgHandler pluginContext, Context context, //
AttrValMap.IAttrVals formData, //
Optional subFormFilter) {
try {
- return parseDescribable(pluginContext, formData, Optional.empty(), subFormFilter);
+ return parseDescribable(pluginContext, context, formData, Optional.empty(), subFormFilter);
} catch (Exception e) {
throw new RuntimeException("class:" + this.clazz.getName(), e);
}
@@ -931,8 +942,8 @@ public ParseDescribable newInstance(IPluginContext pluginContext, /
// return parseDescribable(pluginContext, keyValMap, , subFormFilter);
// }
- public ParseDescribable parseDescribable(IPluginContext pluginContext
- , AttrValMap.IAttrVals keyValMap, Optional pTypes, Optional subFormFilter) {
+ public ParseDescribable parseDescribable(IControlMsgHandler pluginContext
+ , Context context, AttrValMap.IAttrVals keyValMap, Optional pTypes, Optional subFormFilter) {
PluginFormProperties propertyTypes = getPropertyTypes(pTypes, subFormFilter);
@@ -947,7 +958,7 @@ public ParseDescribable visit(RootFormProperties props) {
private ParseDescribable createPluginInstance() {
try {
ParseDescribable result = new ParseDescribable<>(clazz.newInstance());
- Descriptor.this.buildPluginInstance(pluginContext, keyValMap.asRootFormVals(), result,
+ Descriptor.this.buildPluginInstance(pluginContext, context, keyValMap.asRootFormVals(), result,
propertyTypes);
return result;
} catch (Exception e) {
@@ -984,7 +995,7 @@ public ParseDescribable visit(BaseSubFormProperties props) {
props.visitAllSubDetailed(null, keyValMap, new SuFormProperties.ISubDetailedProcess() {
public Void process(String subFormId, AttrValMap attrVals) {
- ParseDescribable r = attrVals.createDescribable(pluginContext, Optional.of(props.convertRootFormProps()));
+ ParseDescribable r = attrVals.createDescribable(pluginContext, context, Optional.of(props.convertRootFormProps()));
Describable plugin = setParentPluginClass(r);
// if (plugin instanceof IParentHostPluginAware) {
@@ -1034,8 +1045,9 @@ private PluginFormProperties getPropertyTypes(Optional pTy
});
}
- private > TARGET buildPluginInstance(IPluginContext pluginContext, Map keyValMap, ParseDescribable result, PluginFormProperties propertyTypes) {
+ private > TARGET buildPluginInstance(IControlMsgHandler pluginContext, Context context //
+ , Map keyValMap //
+ , ParseDescribable result, PluginFormProperties propertyTypes) {
TARGET describable = result.getInstance();
String attr;
PropertyType attrDesc;
@@ -1049,7 +1061,7 @@ private > TARGET buildPluginInstance(IPluginC
valJ = keyValMap.get(attr);
// attrDesc.getExtraProps(PluginExtraProps.KEY_DISABLE);
if (valJ == null && attrDesc.isInputRequired()) {
- throw new IllegalStateException("prop:" + attr + " can not be empty");
+ throw new IllegalStateException("prop:" + attr + " can not be empty,desc :" + this.clazz.getSimpleName());
}
if (valJ == null) {
valJ = new JSONObject();
@@ -1062,14 +1074,14 @@ private > TARGET buildPluginInstance(IPluginC
if (descriptor == null) {
throw new IllegalStateException("impl:" + impl + " relevant descripotor can not be null");
}
- ParseDescribable vals = descriptor.newInstance(pluginContext,
+ ParseDescribable vals = descriptor.newInstance(pluginContext, context,
parseAttrValMap(descVal.get(AttrValMap.PLUGIN_EXTENSION_VALS)), Optional.empty());
attrDesc.setVal(describable, vals.getInstance());
} else {
if (attrDesc.typeIdentity() == FormFieldType.MULTI_SELECTABLE.getIdentity()) {
- List selectedItems = getSelectedMultiItems(null, null, attrDesc, valJ);
- List multi = selectedItems.stream().filter((item) -> item.isChecked()).map((item) -> {
+ List selectedItems = getSelectedMultiItems(pluginContext, context, attrDesc, valJ);
+ List multi = selectedItems.stream().filter((item) -> item.isChecked()).map((item) -> {
if (item.getCmeta() != null) {
return item.getCmeta();
} else {
@@ -1083,7 +1095,7 @@ private > TARGET buildPluginInstance(IPluginC
if (attrDesc.isCollectionType()) {
attrDesc.setVal(describable, multi);
} else {
- for (TypeBase type : multi) {
+ for (IMultiElement type : multi) {
attrDesc.setVal(describable, type);
}
}
@@ -1094,7 +1106,7 @@ private > TARGET buildPluginInstance(IPluginC
valJ.containsKey(KEY_primaryVal) && StringUtils.isNotBlank(valJ.getString(KEY_primaryVal));
// describable
if (!containVal && attrDesc.isInputRequired()) {
- throw new IllegalStateException("prop:" + attr + " can not be empty ,descriptor:" + describable.getDescriptor());
+ throw new IllegalStateException("prop:" + attr + " can not be empty ,descriptor:" + this.clazz.getSimpleName());
}
if (containVal) {
attrVal = valJ.get(KEY_primaryVal);
@@ -1185,7 +1197,18 @@ public static AttrVals parseAttrValMap(Object vals) {
// Object vals = jsonObject.get("vals");
if (vals instanceof Map) {
((Map) vals).forEach((attrName, val) -> {
- attrValMap.put(attrName, (JSON) val);
+ try {
+ attrValMap.put(attrName, (JSON) val);
+ } catch (Exception e) {
+ // 在multiSelectItem的场景下，可能存在提交的itemProperty没有使用‘ItemPropVal’包装的情况
+ if (val instanceof String) {
+ JSONObject o = new JSONObject();
+ o.put(Descriptor.KEY_primaryVal, val);
+ attrValMap.put(attrName, o);
+ } else {
+ throw new RuntimeException("attrName:" + attrName + ",valType:" + val.getClass().getSimpleName(), e);
+ }
+ }
});
}
return new AttrVals(attrValMap);
@@ -1308,6 +1331,7 @@ public static class PostFormVals {
private final Optional subFormFilter;
private final IControlMsgHandler msgHandler;
+ private final Context context;
private Describable instance;
@@ -1315,7 +1339,7 @@ public T newInstance() {
if (instance == null) {
ParseDescribable plugin = desc.parseDescribable(
- (IPluginContext) msgHandler, this.rawFormData, Optional.of(this.formProperties), subFormFilter);
+ msgHandler, this.context, this.rawFormData, Optional.of(this.formProperties), subFormFilter);
// ParseDescribable plugin = desc.newInstance((IPluginContext) msgHandler, this.rawFormData
// , Optional.empty());
@@ -1325,19 +1349,20 @@ public T newInstance() {
}
public PostFormVals(Descriptor desc //
- , IControlMsgHandler msgHandler, AttrValMap.IAttrVals rawFormData) {
+ , IControlMsgHandler msgHandler, Context context, AttrValMap.IAttrVals rawFormData) {
// Optional subFormFilter = Optional.empty();
- this(desc, desc.getPluginFormPropertyTypes(Optional.empty()), Optional.empty(), msgHandler, rawFormData);
+ this(desc, desc.getPluginFormPropertyTypes(Optional.empty()), Optional.empty(), msgHandler, context, rawFormData);
}
public PostFormVals(Descriptor desc, PluginFormProperties formProperties //
, Optional subFormFilter //
- , IControlMsgHandler msgHandler, AttrValMap.IAttrVals rawFormData) {
+ , IControlMsgHandler msgHandler, Context context, AttrValMap.IAttrVals rawFormData) {
this.rawFormData = rawFormData;
this.desc = desc;
this.formProperties = formProperties;
this.subFormFilter = subFormFilter;
this.msgHandler = msgHandler;
+ this.context = context;
}
private Map fieldVals = Maps.newHashMap();
diff --git a/tis-plugin/src/main/java/com/qlangtech/tis/extension/util/MultiItemsViewType.java b/tis-plugin/src/main/java/com/qlangtech/tis/extension/util/MultiItemsViewType.java
index 91aba49fc..68e93df38 100644
--- a/tis-plugin/src/main/java/com/qlangtech/tis/extension/util/MultiItemsViewType.java
+++ b/tis-plugin/src/main/java/com/qlangtech/tis/extension/util/MultiItemsViewType.java
@@ -32,10 +32,14 @@
import com.qlangtech.tis.plugin.ds.IdlistElementCreatorFactory;
import com.qlangtech.tis.plugin.ds.ViewContent;
import com.qlangtech.tis.runtime.module.misc.IControlMsgHandler;
+import org.apache.commons.beanutils.BeanUtilsBean;
import org.apache.commons.beanutils.BeanUtilsBean2;
+import org.apache.commons.beanutils.PropertyUtilsBean;
import org.apache.commons.lang.StringUtils;
+import java.beans.PropertyDescriptor;
import java.util.Collections;
+import java.util.HashSet;
import java.util.List;
import java.util.Objects;
import java.util.function.Function;
@@ -112,7 +116,14 @@ public void appendExternalJsonProp(JSONObject biz) {
public List getElementPropertyKeys() {
if (elementPropertyKeys == null) {
try {
- elementPropertyKeys = Lists.newArrayList(BeanUtilsBean2.getInstance().describe(tupleFactory.createDefault()).keySet());
+// BeanUtilsBean.getInstance().getPropertyUtils()
+// PropertyUtilsBean
+ PropertyDescriptor[] propertyDescs = BeanUtilsBean2.getInstance().getPropertyUtils().getPropertyDescriptors(tupleFactory.createDefault());
+ elementPropertyKeys = Lists.newArrayList();
+ for (PropertyDescriptor desc : propertyDescs) {
+ elementPropertyKeys.add(desc.getName());
+ }
+ elementPropertyKeys = Collections.unmodifiableList(this.elementPropertyKeys);
} catch (Exception e) {
throw new RuntimeException(e);
}
@@ -193,7 +204,7 @@ public enum ViewFormatType {
// String keyColsMeta = StringUtils.EMPTY;
JSONArray mcols = eprops.getJSONObject(Descriptor.KEY_ENUM_PROP).getJSONArray("_mcols");
- CMeta.ParsePostMCols parsePostMCols = elementCreator.parsePostMCols(attrDesc,msgHandler, context, attrDesc.f.getName(), mcols);
+ CMeta.ParsePostMCols parsePostMCols = elementCreator.parsePostMCols(attrDesc, msgHandler, context, attrDesc.f.getName(), mcols);
if (parsePostMCols.validateFaild) {
return Collections.emptyList();
}
diff --git a/tis-plugin/src/main/java/com/qlangtech/tis/plugin/PluginAndCfgsSnapshot.java b/tis-plugin/src/main/java/com/qlangtech/tis/plugin/PluginAndCfgsSnapshot.java
index d5ad50ed0..d2ece67ed 100644
--- a/tis-plugin/src/main/java/com/qlangtech/tis/plugin/PluginAndCfgsSnapshot.java
+++ b/tis-plugin/src/main/java/com/qlangtech/tis/plugin/PluginAndCfgsSnapshot.java
@@ -23,6 +23,7 @@
import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
import com.qlangtech.tis.TIS;
+import com.qlangtech.tis.async.message.client.consumer.IFlinkColCreator;
import com.qlangtech.tis.async.message.client.consumer.impl.MQListenerFactory;
import com.qlangtech.tis.config.flink.IFlinkCluster;
import com.qlangtech.tis.coredefine.module.action.TargetResName;
@@ -38,6 +39,7 @@
import com.qlangtech.tis.manage.common.HttpUtils;
import com.qlangtech.tis.manage.common.TisUTF8;
import com.qlangtech.tis.maven.plugins.tpi.PluginClassifier;
+import com.qlangtech.tis.plugin.ds.IColMetaGetter;
import com.qlangtech.tis.plugin.incr.TISSinkFactory;
import com.qlangtech.tis.trigger.util.JsonUtil;
import com.qlangtech.tis.util.HeteroEnum;
@@ -306,7 +308,12 @@ public static Manifest createFlinkIncrJobManifestCfgAttrs(TargetResName collecti
// 先收集plugmeta，特别是通过dataXWriter的dataSource关联的元数据
IDataxProcessor processor = DataxProcessor.load(null, resourceType, collection.getName());
TISSinkFactory incrSinKFactory = TISSinkFactory.getIncrSinKFactory(collection.getName());
- incrSinKFactory.createSinkFunction(processor);
+ incrSinKFactory.createSinkFunction(processor, new IFlinkColCreator(){
+ @Override
+ public Object build(IColMetaGetter meta, int colIndex) {
+ return null;
+ }
+ });
});
return createFlinkIncrJobManifestCfgAttrs(resourceType, collection, timestamp, pluginMetas);
}
diff --git a/tis-plugin/src/main/java/com/qlangtech/tis/plugin/annotation/FormFieldType.java b/tis-plugin/src/main/java/com/qlangtech/tis/plugin/annotation/FormFieldType.java
index 1af19f819..612dafb74 100644
--- a/tis-plugin/src/main/java/com/qlangtech/tis/plugin/annotation/FormFieldType.java
+++ b/tis-plugin/src/main/java/com/qlangtech/tis/plugin/annotation/FormFieldType.java
@@ -23,6 +23,7 @@
import com.qlangtech.tis.extension.impl.PropertyType;
import com.qlangtech.tis.extension.impl.PropertyType.PropVal;
import com.qlangtech.tis.manage.common.Option;
+import com.qlangtech.tis.plugin.ds.IMultiElement;
import com.qlangtech.tis.plugin.ds.TypeBase;
import com.qlangtech.tis.runtime.module.misc.IFieldErrorHandler;
import org.apache.commons.lang.StringUtils;
@@ -202,7 +203,7 @@ public static class SelectedItem extends Option {
// 是否选中了
private boolean checked;
- private TypeBase cmeta;
+ private IMultiElement cmeta;
public SelectedItem(String name, String value, boolean checked) {
super(name, value);
@@ -210,13 +211,13 @@ public SelectedItem(String name, String value, boolean checked) {
this.checked = checked;
}
- public SelectedItem(TypeBase cmeta) {
+ public SelectedItem(IMultiElement cmeta) {
this(cmeta.getName(), cmeta.getName(), true // !cmeta.isDisable()
);
this.cmeta = cmeta;
}
- public TypeBase getCmeta() {
+ public IMultiElement getCmeta() {
return cmeta;
}
diff --git a/tis-plugin/src/main/java/com/qlangtech/tis/plugin/datax/transformer/RecordTransformer.java b/tis-plugin/src/main/java/com/qlangtech/tis/plugin/datax/transformer/RecordTransformer.java
index 5989aa477..0a27ceb35 100644
--- a/tis-plugin/src/main/java/com/qlangtech/tis/plugin/datax/transformer/RecordTransformer.java
+++ b/tis-plugin/src/main/java/com/qlangtech/tis/plugin/datax/transformer/RecordTransformer.java
@@ -18,24 +18,22 @@
package com.qlangtech.tis.plugin.datax.transformer;
-import com.qlangtech.tis.plugin.ds.TypeBase;
+import com.qlangtech.tis.plugin.ds.IMultiElement;
/**
* 定义一条 记录处理规则
*/
-public class RecordTransformer extends TypeBase {
+public class RecordTransformer implements IMultiElement {
-
- public final String getName() {
- // return target;
- return null;
+ @Override
+ public String getName() {
+ return "transformer-rule";
}
/**
* 自定义规则
*/
private UDFDefinition udf;
- // private String target;
public UDFDefinition getUdf() {
return udf;
@@ -44,12 +42,4 @@ public UDFDefinition getUdf() {
public void setUdf(UDFDefinition udf) {
this.udf = udf;
}
-
-// public String getTarget() {
-// return target;
-// }
-//
-// public void setTarget(String target) {
-// this.target = target;
-// }
}
diff --git a/tis-plugin/src/main/java/com/qlangtech/tis/plugin/datax/transformer/RecordTransformerRules.java b/tis-plugin/src/main/java/com/qlangtech/tis/plugin/datax/transformer/RecordTransformerRules.java
index 5bbf43e3a..4722bd6b8 100644
--- a/tis-plugin/src/main/java/com/qlangtech/tis/plugin/datax/transformer/RecordTransformerRules.java
+++ b/tis-plugin/src/main/java/com/qlangtech/tis/plugin/datax/transformer/RecordTransformerRules.java
@@ -19,18 +19,24 @@
package com.qlangtech.tis.plugin.datax.transformer;
import com.google.common.collect.Lists;
+import com.qlangtech.tis.datax.IDataXNameAware;
import com.qlangtech.tis.extension.Describable;
import com.qlangtech.tis.extension.Descriptor;
import com.qlangtech.tis.extension.TISExtension;
-import com.qlangtech.tis.plugin.IdentityName;
+import com.qlangtech.tis.extension.impl.SuFormProperties;
import com.qlangtech.tis.plugin.annotation.FormField;
import com.qlangtech.tis.plugin.annotation.FormFieldType;
import com.qlangtech.tis.plugin.annotation.Validator;
-import com.qlangtech.tis.plugin.datax.transformer.impl.CopyValUDF;
-import com.qlangtech.tis.plugin.ds.DataType;
+import com.qlangtech.tis.plugin.datax.transformer.jdbcprop.TargetColType;
+import com.qlangtech.tis.util.HeteroEnum;
+import com.qlangtech.tis.util.IPluginContext;
+import com.qlangtech.tis.util.UploadPluginMeta;
import org.apache.commons.lang3.StringUtils;
import java.util.List;
+import java.util.Objects;
+import java.util.function.Function;
+import java.util.stream.Collectors;
/**
* 为一条记录Record定义的 Transformer 转化规则
@@ -38,17 +44,56 @@
* @author: 百岁（baisui@qlangtech.com）
* @create: 2023-05-07 12:35
**/
-public class RecordTransformerRules implements Describable, IdentityName {
+public class RecordTransformerRules implements Describable {
- @FormField(ordinal = 0, identity = true, type = FormFieldType.INPUTTEXT, validate = {Validator.require})
- public String name;
+
+ public static Function transformerRulesLoader4Test;
+
+ /**
+ * 加载基于数据通道的表转换（Transformer）规则
+ *
+ * @param collectionName
+ * @param tableName
+ * @return
+ */
+ public static RecordTransformerRules loadTransformerRules(IDataXNameAware collectionName, String tableName) {
+
+ if (StringUtils.isEmpty(tableName)) {
+ throw new IllegalArgumentException("param tableName can not be empty");
+ }
+
+ if (transformerRulesLoader4Test != null) {
+ return transformerRulesLoader4Test.apply(tableName);
+ }
+
+ String rawContent = HeteroEnum.TRANSFORMER_RULES.identity + ":require,"
+ + SuFormProperties.SuFormGetterContext.FIELD_SUBFORM_ID + "_" + tableName;
+
+
+ for (RecordTransformerRules trule
+ : HeteroEnum.TRANSFORMER_RULES.getPlugins(
+ IPluginContext.namedContext(Objects.requireNonNull(collectionName, "collectionName can not be null").getTISDataXName())
+ , UploadPluginMeta.parse(rawContent))) {
+ return trule;
+ }
+
+ return null;
+ }
@FormField(ordinal = 1, type = FormFieldType.MULTI_SELECTABLE, validate = {Validator.require})
public List rules = Lists.newArrayList();
- @Override
- public String identityValue() {
- return this.name;
+ /**
+ * udf集合相关的列
+ *
+ * @return
+ */
+ public final List relevantColKeys() {
+ return this.rules.stream().flatMap((r) -> r.getUdf().outParameters().stream().map((tcol) -> tcol.getName())).collect(Collectors.toList());
+ }
+
+ public final List relevantTypedColKeys() {
+ return this.rules.stream().flatMap((r) -> r.getUdf().outParameters().stream()).collect(Collectors.toList());
}
public static List getRules() {
diff --git a/tis-plugin/src/main/java/com/qlangtech/tis/plugin/datax/transformer/TargetColumn.java b/tis-plugin/src/main/java/com/qlangtech/tis/plugin/datax/transformer/TargetColumn.java
index 76f69515a..fb11d1c73 100644
--- a/tis-plugin/src/main/java/com/qlangtech/tis/plugin/datax/transformer/TargetColumn.java
+++ b/tis-plugin/src/main/java/com/qlangtech/tis/plugin/datax/transformer/TargetColumn.java
@@ -32,8 +32,15 @@
* @author: 百岁（baisui@qlangtech.com）
* @create: 2024-06-09 13:43
**/
+
public abstract class TargetColumn implements Describable, AfterPluginSaved, PluginLiteriaDesc, IdentityName {
+ /**
+ * 是否是虚拟列（通过原表记录值计算之后新增加的列）
+ *
+ * @return
+ */
+ public abstract boolean isVirtual();
public abstract String getName();
diff --git a/tis-plugin/src/main/java/com/qlangtech/tis/plugin/datax/transformer/UDFDefinition.java b/tis-plugin/src/main/java/com/qlangtech/tis/plugin/datax/transformer/UDFDefinition.java
index b8baa3587..b891c6f39 100644
--- a/tis-plugin/src/main/java/com/qlangtech/tis/plugin/datax/transformer/UDFDefinition.java
+++ b/tis-plugin/src/main/java/com/qlangtech/tis/plugin/datax/transformer/UDFDefinition.java
@@ -19,28 +19,52 @@
package com.qlangtech.tis.plugin.datax.transformer;
import com.alibaba.citrus.turbine.Context;
+import com.alibaba.datax.common.element.ColumnAwareRecord;
+import com.alibaba.datax.common.element.Record;
import com.alibaba.fastjson2.annotation.JSONField;
import com.qlangtech.tis.extension.Describable;
import com.qlangtech.tis.extension.Descriptor;
import com.qlangtech.tis.plugin.IPluginStore.AfterPluginSaved;
+import com.qlangtech.tis.plugin.datax.transformer.jdbcprop.TargetColType;
import com.qlangtech.tis.util.IPluginContext;
+import java.io.Serializable;
import java.util.List;
import java.util.Optional;
/**
* DataX 的UDF 定义
+ * https://www.processon.com/diagraming/66580c3ee57102599640d679
*
* @author: 百岁（baisui@qlangtech.com）
* @create: 2023-05-07 12:35
**/
-public abstract class UDFDefinition implements Describable, AfterPluginSaved,PluginLiteriaDesc {
+public abstract class UDFDefinition implements Describable, AfterPluginSaved, PluginLiteriaDesc, Serializable {
+
+ protected static final String KEY_FROM = "from";
+ protected static final String KEY_TO = "to";
public final String getImpl() {
return this.getClass().getName();
}
+ /**
+ * 函数的出参
+ *
+ * @return
+ */
+ public abstract List outParameters();
+
+ /**
+ * 对记录进行处理
+ *
+ * @param record
+ */
+ public abstract void evaluate(ColumnAwareRecord record);
+
+
@Override
public final void afterSaved(IPluginContext pluginContext, Optional context) {
try {
@@ -58,7 +82,6 @@ public final void afterSaved(IPluginContext pluginContext, Optional con
}
-
@JSONField(serialize = false)
@Override
public final Descriptor getDescriptor() {
diff --git a/tis-plugin/src/main/java/com/qlangtech/tis/plugin/datax/transformer/UDFDesc.java b/tis-plugin/src/main/java/com/qlangtech/tis/plugin/datax/transformer/UDFDesc.java
index 23cec6913..8b7eae7f3 100644
--- a/tis-plugin/src/main/java/com/qlangtech/tis/plugin/datax/transformer/UDFDesc.java
+++ b/tis-plugin/src/main/java/com/qlangtech/tis/plugin/datax/transformer/UDFDesc.java
@@ -32,11 +32,19 @@
public class UDFDesc {
private List pairs = Lists.newArrayList();
- public UDFDesc(String key, Object content) {
+ public UDFDesc(String key, String content) {
this.addPair(key, content);
}
- public void addPair(String key, Object content) {
+ public UDFDesc(String key,List content){
+ this.addPair(key, content);
+ }
+
+ public void addPair(String key, String content) {
+ this.pairs.add(new Option(key, content));
+ }
+
+ public void addPair(String key, List content) {
this.pairs.add(new Option(key, content));
}
diff --git a/tis-plugin/src/main/java/com/qlangtech/tis/plugin/datax/transformer/impl/AbstractFromColumnUDFDefinition.java b/tis-plugin/src/main/java/com/qlangtech/tis/plugin/datax/transformer/impl/AbstractFromColumnUDFDefinition.java
deleted file mode 100644
index 81b840968..000000000
--- a/tis-plugin/src/main/java/com/qlangtech/tis/plugin/datax/transformer/impl/AbstractFromColumnUDFDefinition.java
+++ /dev/null
@@ -1,53 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.qlangtech.tis.plugin.datax.transformer.impl;
-
-import com.google.common.collect.Lists;
-import com.qlangtech.tis.plugin.IdentityName;
-import com.qlangtech.tis.plugin.annotation.FormField;
-import com.qlangtech.tis.plugin.annotation.FormFieldType;
-import com.qlangtech.tis.plugin.annotation.Validator;
-import com.qlangtech.tis.plugin.datax.SelectedTab;
-import com.qlangtech.tis.plugin.datax.transformer.UDFDefinition;
-import com.qlangtech.tis.plugin.datax.transformer.UDFDesc;
-import com.qlangtech.tis.plugin.ds.CMeta;
-
-import java.util.List;
-import java.util.stream.Collectors;
-
-/**
- *
- */
-public abstract class AbstractFromColumnUDFDefinition extends UDFDefinition {
-
- @FormField(ordinal = 1, type = FormFieldType.ENUM, validate = {Validator.require})
- public String from;
-
- @Override
- public List getLiteria() {
- return Lists.newArrayList(new UDFDesc("from", this.from));
- }
-
-
- public static List colsCandidate() {
- List colsCandidate = SelectedTab.getColsCandidate();
- return colsCandidate.stream().collect(Collectors.toList());
- }
-
-}
diff --git a/tis-plugin/src/main/java/com/qlangtech/tis/plugin/datax/transformer/impl/CopyValUDF.java b/tis-plugin/src/main/java/com/qlangtech/tis/plugin/datax/transformer/impl/CopyValUDF.java
deleted file mode 100644
index ea9793d49..000000000
--- a/tis-plugin/src/main/java/com/qlangtech/tis/plugin/datax/transformer/impl/CopyValUDF.java
+++ /dev/null
@@ -1,63 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.qlangtech.tis.plugin.datax.transformer.impl;
-
-import com.google.common.collect.Lists;
-import com.qlangtech.tis.extension.Descriptor;
-import com.qlangtech.tis.extension.TISExtension;
-import com.qlangtech.tis.plugin.annotation.FormField;
-import com.qlangtech.tis.plugin.annotation.FormFieldType;
-import com.qlangtech.tis.plugin.annotation.Validator;
-import com.qlangtech.tis.plugin.datax.transformer.UDFDefinition;
-import com.qlangtech.tis.plugin.datax.transformer.UDFDesc;
-import com.qlangtech.tis.plugin.datax.transformer.jdbcprop.TargetColType;
-
-import java.util.List;
-
-/**
- * 用于使用虚拟列
- */
-public class CopyValUDF extends AbstractFromColumnUDFDefinition {
-
- @FormField(ordinal = 2, type = FormFieldType.MULTI_SELECTABLE, validate = {Validator.require})
- public TargetColType to;
-
- public static List getCols() {
- return Lists.newArrayList();
- }
-
- @Override
- public List getLiteria() {
- List literia = super.getLiteria();
- literia.add(new UDFDesc("to", to.getLiteria()));
- return literia;
- }
-
- @TISExtension
- public static final class DefaultDescriptor extends Descriptor {
- public DefaultDescriptor() {
- super();
- }
-
- @Override
- public String getDisplayName() {
- return "CopyVal";
- }
- }
-}
diff --git a/tis-plugin/src/main/java/com/qlangtech/tis/plugin/datax/transformer/impl/ExistTargetCoumn.java b/tis-plugin/src/main/java/com/qlangtech/tis/plugin/datax/transformer/impl/ExistTargetCoumn.java
index c4637dd37..c03f3fa18 100644
--- a/tis-plugin/src/main/java/com/qlangtech/tis/plugin/datax/transformer/impl/ExistTargetCoumn.java
+++ b/tis-plugin/src/main/java/com/qlangtech/tis/plugin/datax/transformer/impl/ExistTargetCoumn.java
@@ -38,6 +38,11 @@ public class ExistTargetCoumn extends TargetColumn {
@FormField(ordinal = 1, identity = true, type = FormFieldType.ENUM, validate = {Validator.require})
public String name;
+ @Override
+ public boolean isVirtual() {
+ return false;
+ }
+
@Override
public String getName() {
return this.name;
diff --git a/tis-plugin/src/main/java/com/qlangtech/tis/plugin/datax/transformer/impl/JSONSplitterUDF.java b/tis-plugin/src/main/java/com/qlangtech/tis/plugin/datax/transformer/impl/JSONSplitterUDF.java
deleted file mode 100644
index dae364663..000000000
--- a/tis-plugin/src/main/java/com/qlangtech/tis/plugin/datax/transformer/impl/JSONSplitterUDF.java
+++ /dev/null
@@ -1,72 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.qlangtech.tis.plugin.datax.transformer.impl;
-
-import com.google.common.collect.Lists;
-import com.qlangtech.tis.extension.Descriptor;
-import com.qlangtech.tis.extension.TISExtension;
-import com.qlangtech.tis.plugin.annotation.FormField;
-import com.qlangtech.tis.plugin.annotation.FormFieldType;
-import com.qlangtech.tis.plugin.annotation.Validator;
-import com.qlangtech.tis.plugin.datax.transformer.UDFDefinition;
-import com.qlangtech.tis.plugin.datax.transformer.UDFDesc;
-import com.qlangtech.tis.plugin.datax.transformer.jdbcprop.TargetColType;
-
-import java.util.List;
-
-/**
- * 将某列JSON内容拆解成多个字段
- *
- * @author: 百岁（baisui@qlangtech.com）
- * @create: 2024-06-11 14:38
- **/
-public class JSONSplitterUDF extends AbstractFromColumnUDFDefinition {
-
- @FormField(ordinal = 2, type = FormFieldType.MULTI_SELECTABLE, validate = {Validator.require})
- public List to;
-
- @Override
- public List getLiteria() {
- List result = Lists.newArrayList(super.getLiteria());
-
- List toDesc = Lists.newArrayList();
- this.to.forEach((colType) -> {
- toDesc.addAll(colType.getLiteria());
- });
- result.add(new UDFDesc("to", toDesc));
- return result;
- }
-
- public static List getCols() {
- return Lists.newArrayList();
- }
-
- @TISExtension
- public static final class DefaultDescriptor extends Descriptor {
- public DefaultDescriptor() {
- super();
- }
-
- @Override
- public String getDisplayName() {
- return "JSON Splitter";
- }
- }
-
-}
diff --git a/tis-plugin/src/main/java/com/qlangtech/tis/plugin/datax/transformer/impl/TransformerRuleElementCreatorFactory.java b/tis-plugin/src/main/java/com/qlangtech/tis/plugin/datax/transformer/impl/TransformerRuleElementCreatorFactory.java
index 5ba9ddc5f..a628600f2 100644
--- a/tis-plugin/src/main/java/com/qlangtech/tis/plugin/datax/transformer/impl/TransformerRuleElementCreatorFactory.java
+++ b/tis-plugin/src/main/java/com/qlangtech/tis/plugin/datax/transformer/impl/TransformerRuleElementCreatorFactory.java
@@ -21,20 +21,16 @@
import com.alibaba.citrus.turbine.Context;
import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
-import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.qlangtech.tis.extension.Descriptor.ParseDescribable;
import com.qlangtech.tis.extension.IPropertyType;
import com.qlangtech.tis.plugin.ValidatorCommons;
-import com.qlangtech.tis.plugin.annotation.Validator;
-import com.qlangtech.tis.plugin.datax.SelectedTab;
import com.qlangtech.tis.plugin.datax.transformer.RecordTransformer;
import com.qlangtech.tis.plugin.datax.transformer.UDFDefinition;
-import com.qlangtech.tis.plugin.ds.CMeta;
import com.qlangtech.tis.plugin.ds.CMeta.ParsePostMCols;
-import com.qlangtech.tis.plugin.ds.DataType;
import com.qlangtech.tis.plugin.ds.ElementCreatorFactory;
import com.qlangtech.tis.plugin.ds.ViewContent;
+import com.qlangtech.tis.runtime.module.misc.IControlMsgHandler;
import com.qlangtech.tis.runtime.module.misc.IFieldErrorHandler;
import com.qlangtech.tis.util.AttrValMap;
@@ -74,7 +70,7 @@ public RecordTransformer create(JSONObject targetCol, BiConsumer
@Override
public ParsePostMCols parsePostMCols(IPropertyType propertyType,
- IFieldErrorHandler msgHandler, Context context, String _keyColsMeta, JSONArray targetCols) {
+ IControlMsgHandler msgHandler, Context context, String _keyColsMeta, JSONArray targetCols) {
final String keyColsMeta = "rules";
final String keyTarget = "target";
final String keyUdf = "udf";
@@ -141,10 +137,10 @@ public ParsePostMCols parsePostMCols(IPropertyType propertyTy
valsMap = AttrValMap.parseDescribableMap(Optional.empty(), udfObj);
- ParseDescribable describable = valsMap.createDescribable(null);
+ ParseDescribable describable = valsMap.createDescribable(msgHandler, context);
udf = (UDFDefinition) describable.getInstance();
transformerRule.setUdf(udf);
- // transformerRule.setType(dataType);
+ // transformerRule.setType(dataType);
// transformerRule.setTarget(targetColName);
postMCols.writerCols.add(transformerRule);
}
diff --git a/tis-plugin/src/main/java/com/qlangtech/tis/plugin/datax/transformer/impl/VirtualTargetColumn.java b/tis-plugin/src/main/java/com/qlangtech/tis/plugin/datax/transformer/impl/VirtualTargetColumn.java
index 2fe397296..3bf8c7591 100644
--- a/tis-plugin/src/main/java/com/qlangtech/tis/plugin/datax/transformer/impl/VirtualTargetColumn.java
+++ b/tis-plugin/src/main/java/com/qlangtech/tis/plugin/datax/transformer/impl/VirtualTargetColumn.java
@@ -38,6 +38,11 @@ public class VirtualTargetColumn extends TargetColumn {
@FormField(ordinal = 1, identity = true, type = FormFieldType.INPUTTEXT, validate = {Validator.require, Validator.db_col_name})
public String name;
+ @Override
+ public boolean isVirtual() {
+ return true;
+ }
+
@Override
public String getName() {
return this.name;
@@ -50,12 +55,12 @@ public String identityValue() {
@Override
public List getLiteria() {
- return Collections.singletonList(new UDFDesc("virtual col", this.name));
+ return Collections.singletonList(new UDFDesc("col", this.name));
}
@TISExtension
- public static class DefaultDesc extends Descriptor {
- public DefaultDesc() {
+ public static class VirtualTargetColumnDesc extends Descriptor {
+ public VirtualTargetColumnDesc() {
super();
}
diff --git a/tis-plugin/src/main/java/com/qlangtech/tis/plugin/datax/transformer/jdbcprop/JdbcPropertyElementCreatorFactory.java b/tis-plugin/src/main/java/com/qlangtech/tis/plugin/datax/transformer/jdbcprop/JdbcPropertyElementCreatorFactory.java
index 23d7c8c09..6e7fd2ac8 100644
--- a/tis-plugin/src/main/java/com/qlangtech/tis/plugin/datax/transformer/jdbcprop/JdbcPropertyElementCreatorFactory.java
+++ b/tis-plugin/src/main/java/com/qlangtech/tis/plugin/datax/transformer/jdbcprop/JdbcPropertyElementCreatorFactory.java
@@ -24,23 +24,24 @@
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.qlangtech.tis.extension.Descriptor.ParseDescribable;
+import com.qlangtech.tis.extension.Descriptor.PluginValidateResult;
import com.qlangtech.tis.extension.IPropertyType;
-import com.qlangtech.tis.extension.impl.PropertyType;
import com.qlangtech.tis.plugin.IdentityName;
import com.qlangtech.tis.plugin.ValidatorCommons;
import com.qlangtech.tis.plugin.datax.SelectedTab;
import com.qlangtech.tis.plugin.datax.transformer.TargetColumn;
-import com.qlangtech.tis.plugin.datax.transformer.UDFDefinition;
import com.qlangtech.tis.plugin.datax.transformer.impl.VirtualTargetColumn;
+import com.qlangtech.tis.plugin.datax.transformer.impl.VirtualTargetColumn.VirtualTargetColumnDesc;
import com.qlangtech.tis.plugin.ds.CMeta;
import com.qlangtech.tis.plugin.ds.CMeta.ParsePostMCols;
import com.qlangtech.tis.plugin.ds.DataType;
import com.qlangtech.tis.plugin.ds.ElementCreatorFactory;
-import com.qlangtech.tis.plugin.ds.TypeBase;
import com.qlangtech.tis.plugin.ds.ViewContent;
+import com.qlangtech.tis.runtime.module.misc.IControlMsgHandler;
import com.qlangtech.tis.runtime.module.misc.IFieldErrorHandler;
+import com.qlangtech.tis.runtime.module.misc.impl.BasicDelegateMsgHandler;
import com.qlangtech.tis.util.AttrValMap;
-import org.apache.commons.lang3.StringUtils;
+import com.qlangtech.tis.util.DescriptorsJSON;
import java.util.Collection;
import java.util.Collections;
@@ -53,7 +54,7 @@
* @author: 百岁（baisui@qlangtech.com）
* @create: 2024-06-10 09:20
**/
-public class JdbcPropertyElementCreatorFactory implements ElementCreatorFactory {
+public class JdbcPropertyElementCreatorFactory implements ElementCreatorFactory {
private static final String KEY_TARGET_COL = "name";
@@ -65,6 +66,9 @@ public ViewContent getViewContentType() {
@Override
public void appendExternalJsonProp(IPropertyType propertyType, JSONObject biz) {
biz.put("isList", propertyType.isCollectionType());
+ if (propertyType.isCollectionType()) {
+ setPropertyInCollectionFieldType(biz);
+ }
List colsCandidate = SelectedTab.getColsCandidate();
biz.put("sourceTabCols", colsCandidate);
biz.put("dftStrType", DataType.createVarChar(32));
@@ -72,41 +76,94 @@ public void appendExternalJsonProp(IPropertyType propertyType, JSONObject biz) {
// ElementCreatorFactory.super.appendExternalJsonProp(propertyType, biz);
}
+ protected void setPropertyInCollectionFieldType(JSONObject biz) {
+ biz.put("dftListElementDesc", DescriptorsJSON.desc(new VirtualTargetColumnDesc()));
+ }
+
@Override
- public ParsePostMCols parsePostMCols(IPropertyType propertyType,
- IFieldErrorHandler msgHandler, Context context, String keyColsMeta, JSONArray targetCols) {
+ public ParsePostMCols parsePostMCols(IPropertyType propertyType,
+ IControlMsgHandler msgHandler, Context context, String keyColsMeta, JSONArray targetCols) {
boolean isCollection = propertyType.isCollectionType();
- ParsePostMCols result = new ParsePostMCols<>();
+ ParsePostMCols result = new ParsePostMCols<>();
if (isCollection) {
final int[] index = new int[1];
Map> duplicateCols = Maps.newHashMap();
for (Object e : targetCols) {
try {
JSONObject element = (JSONObject) e;
- String target = element.getString(KEY_TARGET_COL);
- if (StringUtils.isEmpty(target)) {
- msgHandler.addFieldError(context //
- , IFieldErrorHandler.joinField(keyColsMeta, Collections.singletonList(index[0]), KEY_TARGET_COL) //
- , ValidatorCommons.MSG_EMPTY_INPUT_ERROR);
- result.validateFaild = true;
+ Object targetCol = element.get(KEY_TARGET_COL);
+
+ /**
+ * 这里处理分为两种场景 1，2
+ */
+ VirtualTargetColumn vcol = null;
+ if (targetCol instanceof String) {
+ throw new UnsupportedOperationException();
+// /***************************************
+// * 1：JSONSplitterUDF 表单提交
+// **************************************/
+// final String target = (String) targetCol;
+//
+// if (StringUtils.isEmpty(target)) {
+// msgHandler.addFieldError(context //
+// , IFieldErrorHandler.joinField(keyColsMeta, Collections.singletonList(index[0]), KEY_TARGET_COL) //
+// , ValidatorCommons.MSG_EMPTY_INPUT_ERROR);
+// result.validateFaild = true;
+// }
+//
+// Collection sameKeys = duplicateCols.get(target);
+// if (sameKeys == null) {
+// sameKeys = Lists.newArrayList();
+// duplicateCols.put(target, sameKeys);
+// }
+// sameKeys.add(index[0]);
+//
+// vcol = new VirtualTargetColumn();
+// vcol.name = target;
+ } else if (targetCol instanceof JSONObject) {
+ /***************************************
+ * 2：RecordTransformerRules 表单提交
+ **************************************/
+ AttrValMap attrVals = AttrValMap.parseDescribableMap(Optional.empty(), (JSONObject) targetCol);
+
+ BasicDelegateMsgHandler msgHandle = new BasicDelegateMsgHandler(msgHandler) {
+ @Override
+ public void addFieldError(Context context, String fieldName, String msg, Object... params) {
+ super.addFieldError(context
+ , IFieldErrorHandler.joinField(keyColsMeta, Collections.singletonList(index[0]), fieldName)
+ , msg, params);
+ }
+ };
+
+ PluginValidateResult targetColValidate = attrVals.validate(msgHandle, context, false);
+ if (targetColValidate.isValid()) {
+ vcol = targetColValidate.newInstance();
+ Collection sameKeys = duplicateCols.get(vcol.getName());
+ if (sameKeys == null) {
+ sameKeys = Lists.newArrayList();
+ duplicateCols.put(vcol.getName(), sameKeys);
+ }
+ sameKeys.add(index[0]);
+ } else {
+ result.validateFaild = true;
+ }
+
+
+ } else {
+ throw new IllegalStateException("unsupported type:" + (targetCol == null ? "null" : targetCol.getClass().getSimpleName()));
}
- Collection sameKeys = duplicateCols.get(target);
- if (sameKeys == null) {
- sameKeys = Lists.newArrayList();
- duplicateCols.put(target, sameKeys);
- }
- sameKeys.add(index[0]);
+
DataType type = CMeta.parseType(element, (propKey, errMsg) -> {
msgHandler.addFieldError(context
, IFieldErrorHandler.joinField(keyColsMeta, Collections.singletonList(index[0]), propKey)
, errMsg);
result.validateFaild = true;
});
- TargetColType targetColType = new TargetColType();
- PainTargetColumn vcol = new PainTargetColumn(target);
+ TargetColType targetColType = createDefault();
+
targetColType.setTarget(vcol);
targetColType.setType(type);
result.writerCols.add(targetColType);
@@ -126,18 +183,23 @@ public ParsePostMCols parsePostMCols(IPropertyType propertyType,
}
}
} else {
- this.parseSinglePojo(targetCols, result);
+ this.parseSinglePojo(msgHandler, context, keyColsMeta, targetCols, result);
}
return result;
}
- private void parseSinglePojo(JSONArray targetCols, ParsePostMCols result) {
+ private void parseSinglePojo(IControlMsgHandler pluginContext
+ , Context context, String keyColsMeta, JSONArray targetCols, ParsePostMCols result) {
for (Object e : targetCols) {
JSONObject element = (JSONObject) e;
// JSONObject type = element.getJSONObject("type");
JSONObject target = element.getJSONObject(KEY_TARGET_COL);
-
+ if (target == null) {
+ pluginContext.addFieldError(context, keyColsMeta, ValidatorCommons.MSG_EMPTY_INPUT_ERROR);
+ result.validateFaild = true;
+ break;
+ }
DataType type = CMeta.parseType(element, (propKey, errMsg) -> {
// msgHandler.addFieldError(context
// , IFieldErrorHandler.joinField(keyColsMeta, Collections.singletonList(index), propKey)
@@ -146,9 +208,10 @@ private void parseSinglePojo(JSONArray targetCols, ParsePostMCols resu
});
AttrValMap valsMap = AttrValMap.parseDescribableMap(Optional.empty(), target);
- ParseDescribable describable = valsMap.createDescribable(null);
+
+ ParseDescribable describable = valsMap.createDescribable(pluginContext, context);
TargetColumn targetCol = (TargetColumn) describable.getInstance();
- TargetColType targetColType = new TargetColType();
+ TargetColType targetColType = this.createDefault();
targetColType.setTarget(targetCol);
targetColType.setType(type);
result.writerCols.add(targetColType);
@@ -161,15 +224,14 @@ private void parseSinglePojo(JSONArray targetCols, ParsePostMCols resu
*
* @return
* @see com.qlangtech.tis.plugin.datax.transformer.jdbcprop.TargetColType
- * @see com.qlangtech.tis.plugin.datax.transformer.jdbcprop.VirtualColType
*/
@Override
- public TypeBase createDefault() {
- return null;
+ public TargetColType createDefault() {
+ return new TargetColType();
}
@Override
- public TypeBase create(JSONObject targetCol, BiConsumer errorProcess) {
+ public TargetColType create(JSONObject targetCol, BiConsumer errorProcess) {
return null;
}
}
diff --git a/tis-plugin/src/main/java/com/qlangtech/tis/plugin/datax/transformer/jdbcprop/PainTargetColumnSerializer.java b/tis-plugin/src/main/java/com/qlangtech/tis/plugin/datax/transformer/jdbcprop/JdbcPropertyElementSelectFromExistFieldCreatorFactory.java
similarity index 54%
rename from tis-plugin/src/main/java/com/qlangtech/tis/plugin/datax/transformer/jdbcprop/PainTargetColumnSerializer.java
rename to tis-plugin/src/main/java/com/qlangtech/tis/plugin/datax/transformer/jdbcprop/JdbcPropertyElementSelectFromExistFieldCreatorFactory.java
index 95c05e270..f7a248977 100644
--- a/tis-plugin/src/main/java/com/qlangtech/tis/plugin/datax/transformer/jdbcprop/PainTargetColumnSerializer.java
+++ b/tis-plugin/src/main/java/com/qlangtech/tis/plugin/datax/transformer/jdbcprop/JdbcPropertyElementSelectFromExistFieldCreatorFactory.java
@@ -18,27 +18,18 @@
package com.qlangtech.tis.plugin.datax.transformer.jdbcprop;
-import com.alibaba.fastjson.serializer.JSONSerializer;
-import com.alibaba.fastjson.serializer.ObjectSerializer;
-import com.alibaba.fastjson2.JSONWriter;
-
-import java.io.IOException;
-import java.io.UnsupportedEncodingException;
-import java.lang.reflect.Type;
+import com.alibaba.fastjson.JSONObject;
/**
+ * 从已有的数据表中选择一个已有的数据列
+ *
* @author: 百岁（baisui@qlangtech.com）
- * @create: 2024-06-12 16:56
+ * @create: 2024-06-17 10:21
**/
-public class PainTargetColumnSerializer implements ObjectSerializer {
- @Override
- public void write(JSONWriter jsonWriter, Object object, Object fieldName, Type fieldType, long features) {
- PainTargetColumn targetColumn = (PainTargetColumn) object;
- jsonWriter.writeRaw("\"" + targetColumn.getName() + "\"");
- }
-
+public class JdbcPropertyElementSelectFromExistFieldCreatorFactory extends JdbcPropertyElementCreatorFactory {
@Override
- public void write(JSONSerializer jsonSerializer, Object o, Object o1, Type type, int i) throws IOException {
- throw new UnsupportedEncodingException();
+ protected void setPropertyInCollectionFieldType(JSONObject biz) {
+ super.setPropertyInCollectionFieldType(biz);
+ biz.put("selectFromExistField", true);
}
}
diff --git a/tis-plugin/src/main/java/com/qlangtech/tis/plugin/datax/transformer/jdbcprop/TargetColType.java b/tis-plugin/src/main/java/com/qlangtech/tis/plugin/datax/transformer/jdbcprop/TargetColType.java
index 44777ddb0..3a3301363 100644
--- a/tis-plugin/src/main/java/com/qlangtech/tis/plugin/datax/transformer/jdbcprop/TargetColType.java
+++ b/tis-plugin/src/main/java/com/qlangtech/tis/plugin/datax/transformer/jdbcprop/TargetColType.java
@@ -23,21 +23,33 @@
import com.qlangtech.tis.plugin.datax.transformer.PluginLiteria;
import com.qlangtech.tis.plugin.datax.transformer.TargetColumn;
import com.qlangtech.tis.plugin.datax.transformer.UDFDesc;
+import com.qlangtech.tis.plugin.ds.IColMetaGetter;
import com.qlangtech.tis.plugin.ds.TypeBase;
import java.util.List;
+import java.util.Objects;
/**
* https://github.com/alibaba/fastjson/wiki/JSONType_serializer
*
* @author: 百岁（baisui@qlangtech.com）
* @create: 2024-06-10 10:10
+ * @see JdbcPropertyElementCreatorFactory
**/
@JSONType(serializer = TargetColTypeSerializer.class)
-public class TargetColType extends TypeBase implements PluginLiteria {
+public final class TargetColType extends TypeBase implements PluginLiteria, IColMetaGetter {
TargetColumn target;
+ public boolean isVirtual() {
+ return Objects.requireNonNull(target, "prop target can not be null").isVirtual();
+ }
+
+ @Override
+ public boolean isPk() {
+ return false;
+ }
+
@Override
public List getLiteria() {
diff --git a/tis-plugin/src/main/java/com/qlangtech/tis/plugin/datax/transformer/jdbcprop/TargetColTypeSerializer.java b/tis-plugin/src/main/java/com/qlangtech/tis/plugin/datax/transformer/jdbcprop/TargetColTypeSerializer.java
index 932298b64..61ff03e60 100644
--- a/tis-plugin/src/main/java/com/qlangtech/tis/plugin/datax/transformer/jdbcprop/TargetColTypeSerializer.java
+++ b/tis-plugin/src/main/java/com/qlangtech/tis/plugin/datax/transformer/jdbcprop/TargetColTypeSerializer.java
@@ -18,9 +18,11 @@
package com.qlangtech.tis.plugin.datax.transformer.jdbcprop;
+import com.alibaba.fastjson.JSONObject;
import com.alibaba.fastjson.serializer.JSONSerializer;
import com.alibaba.fastjson.serializer.ObjectSerializer;
import com.google.common.collect.Maps;
+import com.qlangtech.tis.util.DescribableJSON;
import java.io.IOException;
import java.lang.reflect.Type;
@@ -30,6 +32,7 @@
/**
* @author: 百岁（baisui@qlangtech.com）
* @create: 2024-06-12 11:43
+ * @see TargetColType
**/
public class TargetColTypeSerializer implements ObjectSerializer {
@Override
@@ -38,10 +41,18 @@ public void write(JSONSerializer serializer, Object object, Object fieldName, Ty
}
- private Map serialize(TargetColType colType) {
- Map result = Maps.newHashMap();
- result.put("name", Objects.requireNonNull(colType.target, "target can not be null"));
- result.put("type", colType.getType());
- return result;
+ private Map serialize(TargetColType colType) {
+ try {
+ Map result = Maps.newHashMap();
+ DescribableJSON target = new DescribableJSON(Objects.requireNonNull(colType.target, "target can not be null"));
+ JSONObject itemJson = target.getItemJson();
+ itemJson.put("literia", colType.target.getLiteria());
+
+ result.put("name", itemJson);
+ result.put("type", colType.getType());
+ return result;
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
}
}
diff --git a/tis-plugin/src/main/java/com/qlangtech/tis/plugin/datax/transformer/jdbcprop/PainTargetColumn.java b/tis-plugin/src/main/java/com/qlangtech/tis/plugin/incr/CreatedSinkFunction.java
similarity index 50%
rename from tis-plugin/src/main/java/com/qlangtech/tis/plugin/datax/transformer/jdbcprop/PainTargetColumn.java
rename to tis-plugin/src/main/java/com/qlangtech/tis/plugin/incr/CreatedSinkFunction.java
index ac735a967..560447eb8 100644
--- a/tis-plugin/src/main/java/com/qlangtech/tis/plugin/datax/transformer/jdbcprop/PainTargetColumn.java
+++ b/tis-plugin/src/main/java/com/qlangtech/tis/plugin/incr/CreatedSinkFunction.java
@@ -16,39 +16,25 @@
* limitations under the License.
*/
-package com.qlangtech.tis.plugin.datax.transformer.jdbcprop;
+package com.qlangtech.tis.plugin.incr;
-import com.alibaba.fastjson.annotation.JSONType;
-import com.google.common.collect.Lists;
-import com.qlangtech.tis.plugin.datax.transformer.TargetColumn;
-import com.qlangtech.tis.plugin.datax.transformer.UDFDesc;
+import com.google.common.collect.Maps;
+import com.qlangtech.tis.datax.TableAlias;
-import java.util.List;
+import java.util.Map;
/**
* @author: 百岁（baisui@qlangtech.com）
- * @create: 2024-06-12 16:56
+ * @create: 2024-06-19 14:10
**/
-@JSONType(serializer = PainTargetColumnSerializer.class)
-public class PainTargetColumn extends TargetColumn {
- private final String colName;
+public class CreatedSinkFunction {
- public PainTargetColumn(String colName) {
- this.colName = colName;
- }
-
- @Override
- public String getName() {
- return this.colName;
- }
+ public final Map tabSinkFunc = Maps.newHashMap();
- @Override
- public String identityValue() {
- throw new UnsupportedOperationException();
+ public CreatedSinkFunction() {
}
- @Override
- public List getLiteria() {
- return Lists.newArrayList(new UDFDesc("column", this.colName));
+ public void addTabSinkFunc(TableAlias tab, SinkFunc sinkFunc) {
+ this.tabSinkFunc.put(tab, sinkFunc);
}
}
diff --git a/tis-plugin/src/main/java/com/qlangtech/tis/plugin/incr/TISSinkFactory.java b/tis-plugin/src/main/java/com/qlangtech/tis/plugin/incr/TISSinkFactory.java
index 6e1b40d59..b42247901 100644
--- a/tis-plugin/src/main/java/com/qlangtech/tis/plugin/incr/TISSinkFactory.java
+++ b/tis-plugin/src/main/java/com/qlangtech/tis/plugin/incr/TISSinkFactory.java
@@ -20,8 +20,10 @@
import com.qlangtech.tis.TIS;
import com.qlangtech.tis.annotation.Public;
+import com.qlangtech.tis.async.message.client.consumer.IFlinkColCreator;
import com.qlangtech.tis.async.message.client.consumer.impl.MQListenerFactory;
import com.qlangtech.tis.compiler.incr.ICompileAndPackage;
+import com.qlangtech.tis.datax.IDataXNameAware;
import com.qlangtech.tis.datax.IDataXPluginMeta;
import com.qlangtech.tis.datax.IDataxProcessor;
import com.qlangtech.tis.datax.TableAlias;
@@ -36,6 +38,7 @@
import com.qlangtech.tis.util.IPluginContext;
import com.qlangtech.tis.util.Selectable;
import org.apache.commons.io.IOUtils;
+import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -52,7 +55,7 @@
* @see MQListenerFactory
**/
@Public
-public abstract class TISSinkFactory implements Describable, KeyedPluginStore.IPluginKeyAware {
+public abstract class TISSinkFactory implements Describable, KeyedPluginStore.IPluginKeyAware, IDataXNameAware {
// public static final String KEY_FLINK_STREAM_APP_NAME_PREFIX = "flink_stream_";
public static final String KEY_PLUGIN_TPI_CHILD_PATH = "flink/";
private static final Logger logger = LoggerFactory.getLogger(TISSinkFactory.class);
@@ -83,7 +86,7 @@ public static void main(String[] args) throws Exception {
}
@TISExtension
- public static final HeteroEnum sinkFactory = new HeteroEnum(//
+ public static final HeteroEnum sinkFactory = new HeteroEnum<>(//
TISSinkFactory.class, //
"sinkFactory", //
"Incr Sink Factory", //
@@ -116,6 +119,14 @@ public static TISSinkFactory getIncrSinKFactory(IPluginContext pluginContext) {
public transient String dataXName;
+ @Override
+ public final String getCollectionName() {
+ if (StringUtils.isEmpty(this.dataXName)) {
+ throw new IllegalStateException("param dataXName can not be empty");
+ }
+ return this.dataXName;
+ }
+
/**
* 取得增量执行单元，脚本编译器
*
@@ -134,7 +145,8 @@ public void setKey(KeyedPluginStore.Key key) {
* @param dataxProcessor
* @return
*/
- public abstract Map createSinkFunction(IDataxProcessor dataxProcessor);
+ public abstract Map createSinkFunction(
+ IDataxProcessor dataxProcessor, IFlinkColCreator flinkColCreator);
@Override
@@ -157,7 +169,7 @@ public static abstract class BaseSinkFunctionDescriptor extends Descriptor getExtractProps() {
Map vals = super.getExtractProps();
EndType targetType = this.getTargetType();
- // this.getEndType().appendProps(vals);
+ // this.getEndType().appendProps(vals);
vals.put(IDataXPluginMeta.END_TARGET_TYPE, targetType.getVal());
vals.put(ISelectedTabExtendFactory.KEY_EXTEND_SELECTED_TAB_PROP
, (this instanceof ISelectedTabExtendFactory)
diff --git a/tis-plugin/src/main/java/com/qlangtech/tis/util/AttrValMap.java b/tis-plugin/src/main/java/com/qlangtech/tis/util/AttrValMap.java
index 2fc972907..33ae8377e 100644
--- a/tis-plugin/src/main/java/com/qlangtech/tis/util/AttrValMap.java
+++ b/tis-plugin/src/main/java/com/qlangtech/tis/util/AttrValMap.java
@@ -127,12 +127,12 @@ public String createOrGetNotebook(IControlMsgHandler msgHandler, Context context
throw new IllegalStateException("desc:" + this.descriptor.getClass().getName() + " must be instance of " + INotebookable.class.getSimpleName());
}
INotebookable notebook = (INotebookable) this.descriptor;
- return notebook.createOrGetNotebook((new Descriptor.PostFormVals(this.descriptor, msgHandler,
+ return notebook.createOrGetNotebook((new Descriptor.PostFormVals(this.descriptor, msgHandler, context,
this.attrValMap)).newInstance());
}
- public Descriptor.ParseDescribable createDescribable(IPluginContext pluginContext) {
- return this.createDescribable(pluginContext, Optional.empty());
+ public Descriptor.ParseDescribable createDescribable(IControlMsgHandler pluginContext, Context context) {
+ return this.createDescribable(pluginContext, context, Optional.empty());
}
/**
@@ -140,8 +140,8 @@ public Descriptor.ParseDescribable createDescribable(IPluginContext pluginContex
*
* @return
*/
- public Descriptor.ParseDescribable createDescribable(IPluginContext pluginContext, Optional formProperties) {
- return this.descriptor.parseDescribable(pluginContext, this.attrValMap, (formProperties), this.subFormFilter);
+ public Descriptor.ParseDescribable createDescribable(IControlMsgHandler pluginContext, Context context, Optional formProperties) {
+ return this.descriptor.parseDescribable(pluginContext, context, this.attrValMap, (formProperties), this.subFormFilter);
}
public int size() {
diff --git a/tis-plugin/src/main/java/com/qlangtech/tis/util/IPluginContext.java b/tis-plugin/src/main/java/com/qlangtech/tis/util/IPluginContext.java
index 427aadaa8..6332df78e 100644
--- a/tis-plugin/src/main/java/com/qlangtech/tis/util/IPluginContext.java
+++ b/tis-plugin/src/main/java/com/qlangtech/tis/util/IPluginContext.java
@@ -18,6 +18,7 @@
package com.qlangtech.tis.util;
import com.alibaba.citrus.turbine.Context;
+import com.qlangtech.tis.datax.IDataXNameAware;
import com.qlangtech.tis.extension.Descriptor;
import com.qlangtech.tis.extension.impl.SuFormProperties.SuFormGetterContext;
import com.qlangtech.tis.plugin.ds.DataSourceFactory;
@@ -29,7 +30,7 @@
* @author 百岁（baisui@qlangtech.com）
* @date 2020/04/13
*/
-public interface IPluginContext extends IMessageHandler {
+public interface IPluginContext extends IMessageHandler, IDataXNameAware {
@@ -94,13 +95,10 @@ public void addErrorMessage(Context context, String msg) {
*/
String getExecId();
- /**
- * 是否在索引
- *
- * @return
- */
- boolean isCollectionAware();
- String getCollectionName();
+
+
+
+
/**
* 是否和数据源相关
*
diff --git a/tis-plugin/src/main/resources/com/qlangtech/tis/plugin/datax/transformer/impl/AbstractFromColumnUDFDefinition.json b/tis-plugin/src/main/resources/com/qlangtech/tis/plugin/datax/transformer/impl/AbstractFromColumnUDFDefinition.json
deleted file mode 100644
index e8c2e7a64..000000000
--- a/tis-plugin/src/main/resources/com/qlangtech/tis/plugin/datax/transformer/impl/AbstractFromColumnUDFDefinition.json
+++ /dev/null
@@ -1,5 +0,0 @@
-{
- "from": {
- "enum": "com.qlangtech.tis.plugin.datax.transformer.impl.AbstractFromColumnUDFDefinition.colsCandidate():uncache_true"
- }
-}
\ No newline at end of file
diff --git a/tis-plugin/src/main/resources/com/qlangtech/tis/plugin/datax/transformer/impl/CopyValUDF.json b/tis-plugin/src/main/resources/com/qlangtech/tis/plugin/datax/transformer/impl/CopyValUDF.json
deleted file mode 100644
index 45942ab4f..000000000
--- a/tis-plugin/src/main/resources/com/qlangtech/tis/plugin/datax/transformer/impl/CopyValUDF.json
+++ /dev/null
@@ -1,7 +0,0 @@
-{
- "to": {
- "elementCreator": "com.qlangtech.tis.plugin.datax.transformer.jdbcprop.JdbcPropertyElementCreatorFactory",
- "enum": "com.qlangtech.tis.plugin.datax.transformer.impl.CopyValUDF.getCols():uncache_true",
- "viewtype": "tuplelist"
- }
-}
\ No newline at end of file
diff --git a/tis-plugin/src/main/resources/com/qlangtech/tis/plugin/datax/transformer/impl/JSONSplitterUDF.json b/tis-plugin/src/main/resources/com/qlangtech/tis/plugin/datax/transformer/impl/JSONSplitterUDF.json
deleted file mode 100644
index c4d7230ee..000000000
--- a/tis-plugin/src/main/resources/com/qlangtech/tis/plugin/datax/transformer/impl/JSONSplitterUDF.json
+++ /dev/null
@@ -1,7 +0,0 @@
-{
- "to": {
- "elementCreator": "com.qlangtech.tis.plugin.datax.transformer.jdbcprop.JdbcPropertyElementCreatorFactory",
- "enum": "com.qlangtech.tis.plugin.datax.transformer.impl.JSONSplitterUDF.getCols():uncache_true",
- "viewtype": "tuplelist"
- }
-}
\ No newline at end of file
diff --git a/tis-plugin/src/test/java/TestAll.java b/tis-plugin/src/test/java/TestAll.java
index cdff43281..df1a0fd4a 100644
--- a/tis-plugin/src/test/java/TestAll.java
+++ b/tis-plugin/src/test/java/TestAll.java
@@ -17,6 +17,7 @@
*/
import com.qlangtech.tis.TestTIS;
+import com.qlangtech.tis.datax.impl.TestDataXCfgGenerator;
import com.qlangtech.tis.datax.impl.TestDataxReader;
import com.qlangtech.tis.datax.impl.TestTableAlias;
import com.qlangtech.tis.datax.job.TestServerLaunchToken;
@@ -40,7 +41,7 @@
import com.qlangtech.tis.plugin.datax.TestSelectedTab;
import com.qlangtech.tis.plugin.datax.transformer.impl.TestCopyValUDF;
import com.qlangtech.tis.plugin.datax.transformer.impl.TestTransformerRuleElementCreatorFactory;
-import com.qlangtech.tis.plugin.datax.transformer.jdbcprop.TestPainTargetColumn;
+//import com.qlangtech.tis.plugin.datax.transformer.jdbcprop.TestPainTargetColumn;
import com.qlangtech.tis.plugin.datax.transformer.jdbcprop.TestTargetColType;
import com.qlangtech.tis.plugin.ds.TestDataSourceFactoryPluginStore;
import com.qlangtech.tis.plugin.ds.TestTableInDB;
@@ -92,9 +93,10 @@ public static Test suite() {
suite.addTestSuite(TestK8sImage.class);
suite.addTestSuite(TestServerLaunchToken.class);
suite.addTestSuite(TestTransformerRuleElementCreatorFactory.class);
- suite.addTestSuite(TestCopyValUDF.class);
+ // suite.addTestSuite(TestCopyValUDF.class);
suite.addTestSuite(TestTargetColType.class);
- suite.addTestSuite(TestPainTargetColumn.class);
+ // suite.addTestSuite(TestPainTargetColumn.class);
+ suite.addTestSuite(TestDataXCfgGenerator.class);
return suite;
}
diff --git a/tis-plugin/src/test/java/com/qlangtech/tis/datax/impl/TestDataXCfgGenerator.java b/tis-plugin/src/test/java/com/qlangtech/tis/datax/impl/TestDataXCfgGenerator.java
new file mode 100644
index 000000000..3fa2c7631
--- /dev/null
+++ b/tis-plugin/src/test/java/com/qlangtech/tis/datax/impl/TestDataXCfgGenerator.java
@@ -0,0 +1,124 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.qlangtech.tis.datax.impl;
+
+import com.qlangtech.tis.common.utils.Assert;
+import com.qlangtech.tis.datax.IDataXPluginMeta.DataXMeta;
+import com.qlangtech.tis.datax.IDataxContext;
+import com.qlangtech.tis.datax.IDataxGlobalCfg;
+import com.qlangtech.tis.datax.IDataxProcessor;
+import com.qlangtech.tis.datax.IDataxReader;
+import com.qlangtech.tis.datax.IDataxReaderContext;
+import com.qlangtech.tis.datax.IDataxWriter;
+import com.qlangtech.tis.plugin.datax.transformer.RecordTransformer;
+import com.qlangtech.tis.plugin.datax.transformer.RecordTransformerRules;
+import com.qlangtech.tis.plugin.datax.transformer.impl.TestCopyValUDF;
+import com.qlangtech.tis.plugin.datax.transformer.impl.VirtualTargetColumn;
+import com.qlangtech.tis.plugin.datax.transformer.jdbcprop.TargetColType;
+import com.qlangtech.tis.trigger.util.JsonUtil;
+import com.qlangtech.tis.util.IPluginContext;
+import junit.framework.TestCase;
+import org.easymock.EasyMock;
+
+import java.util.Optional;
+
+/**
+ * @author: 百岁（baisui@qlangtech.com）
+ * @create: 2024-06-15 08:48
+ **/
+public class TestDataXCfgGenerator extends TestCase {
+
+ /**
+ * 测试Transformer的json生成
+ *
+ * @throws Exception
+ */
+ public void testGenerateDataxConfigForTransformer() throws Exception {
+ String dataxName = "test";
+ String transformerTable = "base";
+
+ RecordTransformerRules.transformerRulesLoader4Test = (tab) -> {
+ RecordTransformerRules tRules = new RecordTransformerRules();
+ RecordTransformer transformer = new RecordTransformer();
+ TestCopyValUDF cpUDF = new TestCopyValUDF();
+ cpUDF.from = "base_id";
+ TargetColType targetColType = new TargetColType();
+ VirtualTargetColumn targetColumn = new VirtualTargetColumn();
+ final String baseId = "new_base_id";
+ targetColumn.name = baseId;
+ targetColType.setTarget(targetColumn);
+ cpUDF.to = targetColType;
+ transformer.setUdf(cpUDF);
+ tRules.rules.add(transformer);
+
+ //
+ Assert.assertEquals(baseId, String.join(",", tRules.relevantColKeys()));
+ return tRules;
+ };
+
+ IPluginContext pluginCtx = IPluginContext.namedContext(dataxName);
+
+ IDataxProcessor dataxProcessor = EasyMock.createMock("processor", IDataxProcessor.class);
+
+ IDataxGlobalCfg globalCfg = EasyMock.createMock("globalCfg", IDataxGlobalCfg.class);
+
+ EasyMock.expect(globalCfg.getTemplate())
+ .andReturn("{\n" +
+ " \"job\": {\n" +
+ " \"content\": [\n" +
+ " {\n" +
+ " \"reader\": ,\n" +
+ " \"writer\": \n" +
+ " }\n" +
+ " ]\n" +
+ " }\n" +
+ "}");
+
+ EasyMock.expect(dataxProcessor.getDataXGlobalCfg()).andReturn(globalCfg);
+
+ IDataxReaderContext readerContext = EasyMock.createMock("readerContext", IDataxReaderContext.class);
+ EasyMock.expect(readerContext.getSourceEntityName()).andReturn(transformerTable).anyTimes();
+
+ IDataxWriter writer = EasyMock.createMock("write", IDataxWriter.class);
+ IDataxReader reader = EasyMock.createMock("reader", IDataxReader.class);
+ EasyMock.expect(reader.getTemplate()).andReturn("{}").times(2);
+ EasyMock.expect(writer.getTemplate()).andReturn("{}");
+ EasyMock.expect(writer.getSubTask(Optional.empty())).andReturn(new IDataxContext() {
+ });
+ DataXMeta writeDataXMeta = new DataXMeta();
+ EasyMock.expect(writer.getDataxMeta()).andReturn(writeDataXMeta);
+ DataXMeta readerDataXMeta = new DataXMeta();
+ EasyMock.expect(reader.getDataxMeta()).andReturn(readerDataXMeta);
+ Optional tableMap = Optional.empty();
+
+ EasyMock.replay(writer, reader, readerContext, dataxProcessor, globalCfg);
+
+ DataXCfgGenerator generator = new DataXCfgGenerator(pluginCtx, dataxName, dataxProcessor);
+ String dataXCfg = generator.generateDataxConfig(readerContext, writer, reader, tableMap);
+
+ String assertFileName = "generate-datax-config-for-transformer.json";
+
+ JsonUtil.assertJSONEqual(TestDataXCfgGenerator.class, assertFileName, dataXCfg, (message, expected, actual) -> {
+ Assert.assertEquals(message, expected, actual);
+ });
+
+ EasyMock.verify(writer, reader, readerContext, dataxProcessor, globalCfg);
+ }
+
+}
diff --git a/tis-plugin/src/test/java/com/qlangtech/tis/extension/util/TestPluginExtraProps.java b/tis-plugin/src/test/java/com/qlangtech/tis/extension/util/TestPluginExtraProps.java
index f90961d81..bc788edb1 100644
--- a/tis-plugin/src/test/java/com/qlangtech/tis/extension/util/TestPluginExtraProps.java
+++ b/tis-plugin/src/test/java/com/qlangtech/tis/extension/util/TestPluginExtraProps.java
@@ -27,6 +27,7 @@
import com.qlangtech.tis.plugin.ds.CMeta;
import com.qlangtech.tis.plugin.ds.CMeta.ParsePostMCols;
import com.qlangtech.tis.plugin.ds.ElementCreatorFactory;
+import com.qlangtech.tis.runtime.module.misc.IControlMsgHandler;
import com.qlangtech.tis.runtime.module.misc.IFieldErrorHandler;
import com.qlangtech.tis.trigger.util.JsonUtil;
import com.qlangtech.tis.util.DescriptorsJSON;
@@ -44,7 +45,7 @@ public class TestPluginExtraProps extends TestCase {
public void testParsePostMCols() {
ElementCreatorFactory elementCreator = new TestElementCreatorFactory();
- IFieldErrorHandler msgHandler = null;
+ IControlMsgHandler msgHandler = null;
Context context = null;
String keyColsMeta = null;
JSONArray targetCols = null;
@@ -64,7 +65,7 @@ public CMeta create(JSONObject targetCol, BiConsumer errorProces
}
@Override
- public ParsePostMCols parsePostMCols(IPropertyType propertyType,IFieldErrorHandler msgHandler, Context context, String keyColsMeta, JSONArray targetCols) {
+ public ParsePostMCols parsePostMCols(IPropertyType propertyType, IControlMsgHandler msgHandler, Context context, String keyColsMeta, JSONArray targetCols) {
throw new UnsupportedOperationException();
}
// @Override
diff --git a/tis-plugin/src/test/java/com/qlangtech/tis/plugin/datax/transformer/impl/TestCopyValUDF.java b/tis-plugin/src/test/java/com/qlangtech/tis/plugin/datax/transformer/impl/TestCopyValUDF.java
index 8dace4e12..1b11d7bd9 100644
--- a/tis-plugin/src/test/java/com/qlangtech/tis/plugin/datax/transformer/impl/TestCopyValUDF.java
+++ b/tis-plugin/src/test/java/com/qlangtech/tis/plugin/datax/transformer/impl/TestCopyValUDF.java
@@ -18,19 +18,74 @@
package com.qlangtech.tis.plugin.datax.transformer.impl;
-import com.qlangtech.tis.extension.util.impl.DefaultGroovyShellFactory;
-import com.qlangtech.tis.plugin.datax.transformer.TestRecordTransformerRules;
-import junit.framework.TestCase;
+import com.alibaba.datax.common.element.ColumnAwareRecord;
+import com.alibaba.datax.common.element.Record;
+import com.google.common.collect.Lists;
+import com.qlangtech.tis.extension.Descriptor;
+import com.qlangtech.tis.extension.TISExtension;
+import com.qlangtech.tis.plugin.IdentityName;
+import com.qlangtech.tis.plugin.annotation.FormField;
+import com.qlangtech.tis.plugin.annotation.FormFieldType;
+import com.qlangtech.tis.plugin.annotation.Validator;
+import com.qlangtech.tis.plugin.datax.SelectedTab;
+import com.qlangtech.tis.plugin.datax.transformer.UDFDefinition;
+import com.qlangtech.tis.plugin.datax.transformer.UDFDesc;
+import com.qlangtech.tis.plugin.datax.transformer.jdbcprop.TargetColType;
+import com.qlangtech.tis.plugin.ds.CMeta;
+
+import java.util.Collections;
+import java.util.List;
+import java.util.stream.Collectors;
/**
- *
- * @author: 百岁（baisui@qlangtech.com）
- * @create: 2024-06-10 11:38
- **/
-public class TestCopyValUDF extends TestCase {
-
- public void testDescJsonGen() {
- DefaultGroovyShellFactory.setInConsoleModule();
- TestRecordTransformerRules.assertDesc(CopyValUDF.class,"record-transformer-rules-descriptor.json");
+ * 用于使用虚拟列
+ */
+public class TestCopyValUDF extends UDFDefinition {
+
+ @FormField(ordinal = 1, type = FormFieldType.ENUM, validate = {Validator.require})
+ public String from;
+
+ @Override
+ public List getLiteria() {
+ List literia = Lists.newArrayList(new UDFDesc("from", this.from));
+ literia.add(new UDFDesc("to", to.getLiteria()));
+ return literia;
+ }
+
+
+ public static List colsCandidate() {
+ List colsCandidate = SelectedTab.getColsCandidate();
+ return colsCandidate.stream().collect(Collectors.toList());
+ }
+
+ @FormField(ordinal = 2, type = FormFieldType.MULTI_SELECTABLE, validate = {Validator.require})
+ public TargetColType to;
+
+ public static List getCols() {
+ return Lists.newArrayList();
+ }
+
+ @Override
+ public List outParameters() {
+ return Collections.singletonList(this.to);
+ }
+
+ @Override
+ public void evaluate(ColumnAwareRecord record) {
+
+ record.setColumn(this.to.getName(), record.getColumn(this.from));
+ }
+
+
+ @TISExtension
+ public static final class DefaultDescriptor extends Descriptor {
+ public DefaultDescriptor() {
+ super();
+ }
+
+ @Override
+ public String getDisplayName() {
+ return "TestCopyVal";
+ }
}
}
diff --git a/tis-plugin/src/test/java/com/qlangtech/tis/plugin/datax/transformer/impl/TestTransformerRuleElementCreatorFactory.java b/tis-plugin/src/test/java/com/qlangtech/tis/plugin/datax/transformer/impl/TestTransformerRuleElementCreatorFactory.java
index 4350917bf..441428511 100644
--- a/tis-plugin/src/test/java/com/qlangtech/tis/plugin/datax/transformer/impl/TestTransformerRuleElementCreatorFactory.java
+++ b/tis-plugin/src/test/java/com/qlangtech/tis/plugin/datax/transformer/impl/TestTransformerRuleElementCreatorFactory.java
@@ -25,7 +25,7 @@
import com.qlangtech.tis.extension.impl.IOUtils;
import com.qlangtech.tis.plugin.datax.transformer.RecordTransformer;
import com.qlangtech.tis.plugin.ds.CMeta.ParsePostMCols;
-import com.qlangtech.tis.runtime.module.misc.IFieldErrorHandler;
+import com.qlangtech.tis.runtime.module.misc.IControlMsgHandler;
import junit.framework.TestCase;
import org.easymock.EasyMock;
@@ -34,7 +34,7 @@ public class TestTransformerRuleElementCreatorFactory extends TestCase {
public void testParsePostMCols() {
TransformerRuleElementCreatorFactory creatorFactory = new TransformerRuleElementCreatorFactory();
IPropertyType propertyType = null;
- IFieldErrorHandler msgHandler = EasyMock.createMock("msgHandler", IFieldErrorHandler.class);
+ IControlMsgHandler msgHandler = EasyMock.createMock("msgHandler", IControlMsgHandler.class);
Context context = EasyMock.createMock("context", Context.class);
String keyColsMeta = "colsMeta";
JSONArray jsonArray
@@ -43,24 +43,25 @@ public void testParsePostMCols() {
EasyMock.replay(msgHandler, context);
ParsePostMCols postResult
- = creatorFactory.parsePostMCols(propertyType,msgHandler, context, keyColsMeta, jsonArray);
+ = creatorFactory.parsePostMCols(propertyType, msgHandler, context, keyColsMeta, jsonArray);
Assert.assertFalse(postResult.validateFaild);
Assert.assertEquals(1, postResult.writerCols.size());
- // DataType type = null;
- CopyValUDF udf = null;
+ // DataType type = null;
+ TestCopyValUDF udf = null;
for (RecordTransformer transformer : postResult.writerCols) {
- // Assert.assertEquals("sort_num", transformer.getTarget());
+ // Assert.assertEquals("sort_num", transformer.getTarget());
// type = transformer.getType();
// Assert.assertNotNull("type can not be null", type);
// Assert.assertEquals(-5, type.getType());
- Assert.assertTrue(transformer.getUdf() instanceof CopyValUDF);
- udf = (CopyValUDF) transformer.getUdf();
+ Assert.assertTrue(transformer.getUdf() instanceof TestCopyValUDF);
+ udf = (TestCopyValUDF) transformer.getUdf();
Assert.assertNotNull(udf);
Assert.assertEquals("emp_id", udf.from);
-
+ Assert.assertNotNull("udf.to can not be null", udf.to);
+ Assert.assertEquals("new_add_field", udf.to.getName());
}
EasyMock.verify(msgHandler, context);
diff --git a/tis-plugin/src/test/java/com/qlangtech/tis/plugin/datax/transformer/jdbcprop/TestPainTargetColumn.java b/tis-plugin/src/test/java/com/qlangtech/tis/plugin/datax/transformer/jdbcprop/TestPainTargetColumn.java
deleted file mode 100644
index e4c8cb6f2..000000000
--- a/tis-plugin/src/test/java/com/qlangtech/tis/plugin/datax/transformer/jdbcprop/TestPainTargetColumn.java
+++ /dev/null
@@ -1,52 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.qlangtech.tis.plugin.datax.transformer.jdbcprop;
-
-import com.qlangtech.tis.common.utils.Assert;
-import com.qlangtech.tis.manage.common.Option;
-import com.qlangtech.tis.plugin.datax.transformer.UDFDesc;
-import com.qlangtech.tis.trigger.util.JsonUtil;
-import junit.framework.TestCase;
-import org.junit.Test;
-
-import java.util.List;
-
-/**
- * @author: 百岁（baisui@qlangtech.com）
- * @create: 2024-06-12 17:01
- **/
-public class TestPainTargetColumn extends TestCase {
-
- @Test
- public void testSerialize() {
- String colName = "user_name";
- PainTargetColumn targetColumn = new PainTargetColumn(colName);
- Assert.assertEquals("\"" + colName + "\"", JsonUtil.toString(targetColumn, true));
-
- List descs = targetColumn.getLiteria();
- Assert.assertEquals(1, descs.size());
- for (UDFDesc desc : descs) {
- Assert.assertEquals(1, desc.getPairs().size());
- for (Option opt : desc.getPairs()) {
- Assert.assertEquals("column", opt.getName());
- Assert.assertEquals(colName, opt.getValue());
- }
- }
- }
-}
diff --git a/tis-plugin/src/test/java/com/qlangtech/tis/util/TestAttrValMap.java b/tis-plugin/src/test/java/com/qlangtech/tis/util/TestAttrValMap.java
index 7c356dcfa..afd1f1a3d 100644
--- a/tis-plugin/src/test/java/com/qlangtech/tis/util/TestAttrValMap.java
+++ b/tis-plugin/src/test/java/com/qlangtech/tis/util/TestAttrValMap.java
@@ -1,23 +1,24 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
*/
package com.qlangtech.tis.util;
+import com.alibaba.citrus.turbine.Context;
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONObject;
import com.qlangtech.tis.extension.Descriptor;
@@ -38,17 +39,18 @@ public class TestAttrValMap extends TestCase {
public void testCreateDescribable() {
IControlMsgHandler fieldErrorHandler = EasyMock.createMock("fieldErrorHandler", IControlMsgHandler.class);
- IPluginContext pluginContext = EasyMock.createMock("pluginContext", IPluginContext.class);
+ IControlMsgHandler pluginContext = EasyMock.createMock("pluginContext", IControlMsgHandler.class);
+ Context context = EasyMock.createMock("context", Context.class);
JSONObject jsonObject = IOUtils.loadResourceFromClasspath(TestPluginImpl.class
, "testPluginImpl-post-content.json", true, (input) -> {
return JSON.parseObject(org.apache.commons.io.IOUtils.toString(input, TisUTF8.get()));
});
- EasyMock.replay(fieldErrorHandler, pluginContext);
- AttrValMap attrValMap = AttrValMap.parseDescribableMap( Optional.empty(), jsonObject);
+ EasyMock.replay(fieldErrorHandler, pluginContext, context);
+ AttrValMap attrValMap = AttrValMap.parseDescribableMap(Optional.empty(), jsonObject);
- Descriptor.ParseDescribable describable = attrValMap.createDescribable(pluginContext);
+ Descriptor.ParseDescribable describable = attrValMap.createDescribable(pluginContext, context);
assertNotNull(describable);
TestPluginImpl testPlugin = (TestPluginImpl) describable.getInstance();
assertNotNull(testPlugin);
@@ -56,6 +58,6 @@ public void testCreateDescribable() {
assertTrue("testPlugin.connectionsPerHost must be null", testPlugin.connectionsPerHost == null);
assertEquals(12, (int) testPlugin.maxPendingPerConnection);
- EasyMock.verify(fieldErrorHandler, pluginContext);
+ EasyMock.verify(fieldErrorHandler, pluginContext, context);
}
}
diff --git a/tis-plugin/src/test/resources/com/qlangtech/tis/datax/impl/generate-datax-config-for-transformer.json b/tis-plugin/src/test/resources/com/qlangtech/tis/datax/impl/generate-datax-config-for-transformer.json
new file mode 100644
index 000000000..3eb654958
--- /dev/null
+++ b/tis-plugin/src/test/resources/com/qlangtech/tis/datax/impl/generate-datax-config-for-transformer.json
@@ -0,0 +1,18 @@
+{
+ "job": {
+ "content": [
+ {
+ "transformer": {
+ "cols": [
+ "new_base_id"
+ ],
+ "name": "base"
+ },
+ "reader": {
+ },
+ "writer": {
+ }
+ }
+ ]
+ }
+}
\ No newline at end of file
diff --git a/tis-plugin/src/test/resources/com/qlangtech/tis/plugin/datax/transformer/impl/copy-val-udf-descriptor.json b/tis-plugin/src/test/resources/com/qlangtech/tis/plugin/datax/transformer/impl/copy-val-udf-descriptor.json
deleted file mode 100644
index 9e26dfeeb..000000000
--- a/tis-plugin/src/test/resources/com/qlangtech/tis/plugin/datax/transformer/impl/copy-val-udf-descriptor.json
+++ /dev/null
@@ -1 +0,0 @@
-{}
\ No newline at end of file
diff --git a/tis-plugin/src/test/resources/com/qlangtech/tis/plugin/datax/transformer/impl/transformer-rule-sample.json b/tis-plugin/src/test/resources/com/qlangtech/tis/plugin/datax/transformer/impl/transformer-rule-sample.json
index f628baaad..5a83841e1 100644
--- a/tis-plugin/src/test/resources/com/qlangtech/tis/plugin/datax/transformer/impl/transformer-rule-sample.json
+++ b/tis-plugin/src/test/resources/com/qlangtech/tis/plugin/datax/transformer/impl/transformer-rule-sample.json
@@ -10,162 +10,110 @@
"disabled": false,
"key": "from",
"pk": false,
+ "placeholder": "",
+ "dateTimeFormat": "yyyy-MM-dd HH:mm:ss",
+ "required": true,
+ "type": 5
+ },
+ "to": {
+ "updateModel": true,
+ "has_set_primaryVal": false,
+ "disabled": false,
+ "key": "to",
+ "pk": false,
"_eprops": {
- "enum": [{
- "val": "id",
- "label": "id"
- }, {
- "val": "emp_id",
- "label": "emp_id"
- }, {
- "val": "photo_sn",
- "label": "photo_sn"
- }, {
- "val": "photo_file",
- "label": "photo_file"
- }, {
- "val": "photo_url",
- "label": "photo_url"
- }, {
- "val": "photo_data",
- "label": "photo_data"
- }, {
- "val": "photo_sign",
- "label": "photo_sign"
- }, {
- "val": "is_analyse",
- "label": "is_analyse"
- }, {
- "val": "photo_time",
- "label": "photo_time"
- }, {
- "val": "is_active",
- "label": "is_active"
- }, {
- "val": "quality_score",
- "label": "quality_score"
- }, {
- "val": "create_by",
- "label": "create_by"
- }, {
- "val": "create_time",
- "label": "create_time"
- }, {
- "val": "modify_by",
- "label": "modify_by"
- }, {
- "val": "modify_time",
- "label": "modify_time"
- }, {
- "val": "use_scene",
- "label": "use_scene"
- }, {
- "val": "is_deleted",
- "label": "is_deleted"
- }, {
- "val": "delete_time",
- "label": "delete_time"
- }, {
- "val": "sort_num",
- "label": "sort_num"
- }, {
- "val": "time_stamp",
- "label": "time_stamp"
- }]
+ "viewtype": "tuplelist",
+ "elementCreator": "com.qlangtech.tis.plugin.datax.transformer.jdbcprop.JdbcPropertyElementCreatorFactory",
+ "enum": {
+ "_mcols": [{
+ "error_name": {
+ "impl": "com.qlangtech.tis.plugin.datax.transformer.impl.VirtualTargetColumn",
+ "literia": [{
+ "pairs": [{
+ "name": "virtual col",
+ "value": "test"
+ }]
+ }],
+ "name": "test"
+ },
+
+ "name": {
+ "updateModel": false,
+ "impl": "com.qlangtech.tis.plugin.datax.transformer.impl.VirtualTargetColumn",
+ "vals": {
+ "name": {
+ "updateModel": true,
+ "_primaryVal": "new_add_field",
+ "has_set_primaryVal": false,
+ "disabled": false,
+ "key": "name",
+ "pk": true,
+ "_eprops": {},
+ "placeholder": "",
+ "dateTimeFormat": "yyyy-MM-dd HH:mm:ss",
+ "required": true,
+ "type": 1
+ }
+ },
+ "displayName": "",
+ "showAllField": false,
+ "dspt": {
+ "impl": "com.qlangtech.tis.plugin.datax.transformer.impl.VirtualTargetColumn",
+ "pkField": "name",
+ "implUrl": "http://tis.pub/docs/plugin/plugins/#comqlangtechtisplugindataxtransformerimplvirtualtargetcolumn",
+ "displayName": "Virtual Column",
+ "extendPoint": "com.qlangtech.tis.plugin.datax.transformer.TargetColumn",
+ "containAdvance": false,
+ "veriflable": false,
+ "extractProps": {
+ "notebook": {
+ "activate": false,
+ "ability": false
+ }
+ },
+ "attrs": [{
+ "ord": 1,
+ "describable": false,
+ "pk": true,
+ "type": 1,
+ "key": "name",
+ "required": true
+ }]
+ }
+ },
+ "type": {
+ "columnSize": 32,
+ "decimalDigits": -1,
+ "type": 12,
+ "typeDesc": "varchar(32)",
+ "typeName": "VARCHAR"
+ },
+ "ip": {
+ "updateModel": false,
+ "has_set_primaryVal": false,
+ "disabled": false
+ }
+ }],
+ "isCollection": false,
+ "_typeMetas": [],
+ "tabColsMapper": {},
+ "dftType": {
+ "columnSize": 32,
+ "decimalDigits": -1,
+ "type": 12,
+ "typeDesc": "varchar(32)",
+ "typeName": "VARCHAR"
+ }
+ }
},
"placeholder": "",
"dateTimeFormat": "yyyy-MM-dd HH:mm:ss",
"required": true,
- "type": 5
+ "type": 8,
+ "_tupleViewType": "jdbcTypeProps"
}
},
"displayName": "CopyVal",
- "showAllField": false,
- "dspt": {
- "impl": "com.qlangtech.tis.plugin.datax.transformer.impl.CopyValUDF",
- "implUrl": "http://tis.pub/docs/plugin/plugins/#comqlangtechtisplugindataxtransformerimplcopyvaludf",
- "displayName": "CopyVal",
- "extendPoint": "com.qlangtech.tis.plugin.datax.transformer.UDFDefinition",
- "containAdvance": false,
- "veriflable": false,
- "extractProps": {
- "notebook": {
- "activate": false,
- "ability": false
- }
- },
- "attrs": [{
- "ord": 200,
- "eprops": {
- "enum": [{
- "val": "id",
- "label": "id"
- }, {
- "val": "emp_id",
- "label": "emp_id"
- }, {
- "val": "photo_sn",
- "label": "photo_sn"
- }, {
- "val": "photo_file",
- "label": "photo_file"
- }, {
- "val": "photo_url",
- "label": "photo_url"
- }, {
- "val": "photo_data",
- "label": "photo_data"
- }, {
- "val": "photo_sign",
- "label": "photo_sign"
- }, {
- "val": "is_analyse",
- "label": "is_analyse"
- }, {
- "val": "photo_time",
- "label": "photo_time"
- }, {
- "val": "is_active",
- "label": "is_active"
- }, {
- "val": "quality_score",
- "label": "quality_score"
- }, {
- "val": "create_by",
- "label": "create_by"
- }, {
- "val": "create_time",
- "label": "create_time"
- }, {
- "val": "modify_by",
- "label": "modify_by"
- }, {
- "val": "modify_time",
- "label": "modify_time"
- }, {
- "val": "use_scene",
- "label": "use_scene"
- }, {
- "val": "is_deleted",
- "label": "is_deleted"
- }, {
- "val": "delete_time",
- "label": "delete_time"
- }, {
- "val": "sort_num",
- "label": "sort_num"
- }, {
- "val": "time_stamp",
- "label": "time_stamp"
- }]
- },
- "describable": false,
- "pk": false,
- "type": 5,
- "key": "from",
- "required": true
- }]
- },
- "implUrl": "http://tis.pub/docs/plugin/plugins/#comqlangtechtisplugindataxtransformerimplcopyvaludf"
+ "showAllField": false
}
}]
\ No newline at end of file
diff --git a/tis-scala-compiler/src/test/java/com/qlangtech/tis/compiler/streamcode/TestGenerateDAOAndIncrScript.java b/tis-scala-compiler/src/test/java/com/qlangtech/tis/compiler/streamcode/TestGenerateDAOAndIncrScript.java
index 521ebca6c..2ccd3771f 100644
--- a/tis-scala-compiler/src/test/java/com/qlangtech/tis/compiler/streamcode/TestGenerateDAOAndIncrScript.java
+++ b/tis-scala-compiler/src/test/java/com/qlangtech/tis/compiler/streamcode/TestGenerateDAOAndIncrScript.java
@@ -126,5 +126,9 @@ public void setBizResult(Context context, Object result, boolean o) {
public void addErrorMessage(Context context, String msg) {
}
+ @Override
+ public String getCollectionName() {
+ throw new UnsupportedOperationException();
+ }
}
}