From 3d8fbe294602d3babad463cea155c654553b2d6f Mon Sep 17 00:00:00 2001 From: mozhenghua Date: Wed, 12 Jun 2024 18:32:36 +0800 Subject: [PATCH] add transformer for TIS,and upgrade fastjson version 2.0.51 --- pom.xml | 2 +- .../impl/StubSuFormGetterContext.java | 7 +- .../qlangtech/tis/plugin/IdentityName.java | 1 + .../module/action/CollectionAction.java | 2 +- .../coredefine/module/action/DataxAction.java | 9 +- .../module/action/PluginAction.java | 10 +- .../action/OfflineDatasourceAction.java | 12 +- .../action/TestOfflineDatasourceAction.java | 2 +- .../qlangtech/tis/util/TestPluginItems.java | 8 +- .../tis/extension/IPropertyType.java | 30 ++ .../tis/manage/common/HttpUtils.java | 2 +- .../qlangtech/tis/plugin/JDBCColumnProp.java | 30 +- .../tis/plugin/ds/ElementCreatorFactory.java | 5 +- .../ds/IdlistElementCreatorFactory.java | 3 +- .../qlangtech/tis/trigger/util/JsonUtil.java | 104 +++--- .../tis/trigger/util/UnCacheString.java | 44 +++ .../trigger/util/UnCacheStringSerializer.java | 50 +++ .../tis/util/DescriptorsJSONResult.java | 32 +- .../util/DescriptorsJSONResultSerializer.java | 79 ++++ .../qlangtech/tis/datax/impl/DataxReader.java | 8 +- .../qlangtech/tis/extension/Descriptor.java | 38 +- .../tis/extension/IPropertyType.java | 133 ------- .../tis/extension/SubFormFilter.java | 127 +++++++ .../extension/impl/BaseSubFormProperties.java | 7 +- .../impl/IncrSourceExtendSelected.java | 4 +- .../tis/extension/impl/PropertyType.java | 12 +- .../tis/extension/impl/SuFormProperties.java | 13 +- .../tis/extension/model/UpdateSite.java | 6 +- .../extension/util/GroovyShellEvaluate.java | 5 +- .../extension/util/MultiItemsViewType.java | 5 +- .../tis/plugin/annotation/FormFieldType.java | 6 +- .../tis/plugin/annotation/SubForm.java | 4 +- .../tis/plugin/datax/SelectedTab.java | 8 +- .../tis/plugin/datax/SelectedTabExtend.java | 3 +- .../plugin/datax/transformer/PluginImpl.java | 28 ++ .../datax/transformer/PluginLiteria.java | 35 ++ .../datax/transformer/PluginLiteriaDesc.java | 13 +- .../datax/transformer/RecordTransformer.java | 19 +- .../datax/transformer/TargetColumn.java | 9 +- .../tis/plugin/datax/transformer/UDFDesc.java | 46 +++ .../impl/AbstractFromColumnUDFDefinition.java | 11 +- .../datax/transformer/impl/CopyValUDF.java | 8 + .../transformer/impl/ExistTargetCoumn.java | 12 +- .../transformer/impl/JSONSplitterUDF.java | 72 ++++ .../TransformerRuleElementCreatorFactory.java | 57 +-- .../transformer/impl/VirtualTargetColumn.java | 12 +- .../JdbcPropertyElementCreatorFactory.java | 149 +++++++- .../jdbcprop/PainTargetColumn.java | 54 +++ .../jdbcprop/PainTargetColumnSerializer.java | 44 +++ .../transformer/jdbcprop/TargetColType.java | 22 +- .../jdbcprop/TargetColTypeSerializer.java | 47 +++ .../com/qlangtech/tis/util/AttrValMap.java | 12 +- .../qlangtech/tis/util/DescribableJSON.java | 5 +- .../qlangtech/tis/util/DescriptorsJSON.java | 14 +- .../com/qlangtech/tis/util/HeteroEnum.java | 7 +- .../com/qlangtech/tis/util/HeteroList.java | 6 +- .../qlangtech/tis/util/UploadPluginMeta.java | 15 +- .../transformer/impl/JSONSplitterUDF.json | 7 + tis-plugin/src/test/java/TestAll.java | 6 + .../coredefine/module/action/DataxAction.java | 4 +- .../tis/extension/TestDescriptor.java | 4 +- .../extension/impl/TestSuFormProperties.java | 4 +- .../extension/util/TestPluginExtraProps.java | 9 +- .../TestRecordTransformerRules.java | 4 +- ...tTransformerRuleElementCreatorFactory.java | 16 +- .../jdbcprop/TestPainTargetColumn.java | 52 +++ .../jdbcprop/TestTargetColType.java | 51 +++ .../qlangtech/tis/util/TestHeteroList.java | 4 +- .../tis/util/TestUploadPluginMeta.java | 8 +- .../impl/transformer-rule-sample.json | 8 - .../jdbcprop/pain-target-column.json | 0 .../transformer/jdbcprop/target-col-type.json | 16 + .../record-transformer-rules-descriptor.json | 346 +++++++++++++++++- 73 files changed, 1585 insertions(+), 462 deletions(-) create mode 100644 tis-manage-pojo/src/main/java/com/qlangtech/tis/extension/IPropertyType.java create mode 100644 tis-manage-pojo/src/main/java/com/qlangtech/tis/trigger/util/UnCacheString.java create mode 100644 tis-manage-pojo/src/main/java/com/qlangtech/tis/trigger/util/UnCacheStringSerializer.java create mode 100644 tis-manage-pojo/src/main/java/com/qlangtech/tis/util/DescriptorsJSONResultSerializer.java delete mode 100644 tis-plugin/src/main/java/com/qlangtech/tis/extension/IPropertyType.java create mode 100644 tis-plugin/src/main/java/com/qlangtech/tis/extension/SubFormFilter.java create mode 100644 tis-plugin/src/main/java/com/qlangtech/tis/plugin/datax/transformer/PluginImpl.java create mode 100644 tis-plugin/src/main/java/com/qlangtech/tis/plugin/datax/transformer/PluginLiteria.java create mode 100644 tis-plugin/src/main/java/com/qlangtech/tis/plugin/datax/transformer/UDFDesc.java create mode 100644 tis-plugin/src/main/java/com/qlangtech/tis/plugin/datax/transformer/impl/JSONSplitterUDF.java create mode 100644 tis-plugin/src/main/java/com/qlangtech/tis/plugin/datax/transformer/jdbcprop/PainTargetColumn.java create mode 100644 tis-plugin/src/main/java/com/qlangtech/tis/plugin/datax/transformer/jdbcprop/PainTargetColumnSerializer.java create mode 100644 tis-plugin/src/main/java/com/qlangtech/tis/plugin/datax/transformer/jdbcprop/TargetColTypeSerializer.java create mode 100644 tis-plugin/src/main/resources/com/qlangtech/tis/plugin/datax/transformer/impl/JSONSplitterUDF.json create mode 100644 tis-plugin/src/test/java/com/qlangtech/tis/plugin/datax/transformer/jdbcprop/TestPainTargetColumn.java create mode 100644 tis-plugin/src/test/java/com/qlangtech/tis/plugin/datax/transformer/jdbcprop/TestTargetColType.java create mode 100644 tis-plugin/src/test/resources/com/qlangtech/tis/plugin/datax/transformer/jdbcprop/pain-target-column.json create mode 100644 tis-plugin/src/test/resources/com/qlangtech/tis/plugin/datax/transformer/jdbcprop/target-col-type.json diff --git a/pom.xml b/pom.xml index 85b179cec..4916ad9b5 100644 --- a/pom.xml +++ b/pom.xml @@ -541,7 +541,7 @@ com.alibaba fastjson - 2.0.17 + 2.0.51 diff --git a/tis-base-test/src/main/java/com/qlangtech/tis/extension/impl/StubSuFormGetterContext.java b/tis-base-test/src/main/java/com/qlangtech/tis/extension/impl/StubSuFormGetterContext.java index 800e4302b..500078014 100644 --- a/tis-base-test/src/main/java/com/qlangtech/tis/extension/impl/StubSuFormGetterContext.java +++ b/tis-base-test/src/main/java/com/qlangtech/tis/extension/impl/StubSuFormGetterContext.java @@ -20,7 +20,7 @@ import com.google.common.collect.Lists; import com.qlangtech.tis.extension.Describable; -import com.qlangtech.tis.extension.IPropertyType; +import com.qlangtech.tis.extension.SubFormFilter; import com.qlangtech.tis.plugin.ds.ColumnMetaData; import com.qlangtech.tis.plugin.ds.DataSourceMeta; import com.qlangtech.tis.plugin.ds.JDBCTypes; @@ -30,7 +30,6 @@ import com.qlangtech.tis.util.UploadPluginMeta; import org.easymock.EasyMock; -import java.sql.Types; import java.util.List; /** @@ -60,10 +59,10 @@ private void initSuForm(boolean withUserIdPk) { this.metaPlugin = easyMock.mock("metaPlugin", MockPlugin.class); this.param = easyMock.mock("param", UploadPluginMeta.class); - param.putExtraParams(IPropertyType.SubFormFilter.PLUGIN_META_SUBFORM_DETAIL_ID_VALUE, id1); + param.putExtraParams(SubFormFilter.PLUGIN_META_SUBFORM_DETAIL_ID_VALUE, id1); EasyMock.expectLastCall().times(1); EasyMock.expect(param.getExtraParam( - IPropertyType.SubFormFilter.PLUGIN_META_SUBFORM_DETAIL_ID_VALUE)).andReturn(id1); + SubFormFilter.PLUGIN_META_SUBFORM_DETAIL_ID_VALUE)).andReturn(id1); List cols = Lists.newArrayList(); // (int index, String key, DataType type, boolean pk) diff --git a/tis-builder-api/src/main/java/com/qlangtech/tis/plugin/IdentityName.java b/tis-builder-api/src/main/java/com/qlangtech/tis/plugin/IdentityName.java index 17454987d..de8e059b5 100644 --- a/tis-builder-api/src/main/java/com/qlangtech/tis/plugin/IdentityName.java +++ b/tis-builder-api/src/main/java/com/qlangtech/tis/plugin/IdentityName.java @@ -57,6 +57,7 @@ public String identityValue() { // return des.getIdentityValue(plugin); //} + default Class getDescribleClass() { return this.getClass(); } diff --git a/tis-console/src/main/java/com/qlangtech/tis/config/module/action/CollectionAction.java b/tis-console/src/main/java/com/qlangtech/tis/config/module/action/CollectionAction.java index 8b5fb90a4..8effe77cd 100644 --- a/tis-console/src/main/java/com/qlangtech/tis/config/module/action/CollectionAction.java +++ b/tis-console/src/main/java/com/qlangtech/tis/config/module/action/CollectionAction.java @@ -801,7 +801,7 @@ private Map getTargetCols(JSONObject post) { }); } JSONArray targetCols = colMeta.getJSONArray("columns"); - Map targetColMap = ((Stream) targetCols.stream()).map((c) -> { + Map targetColMap = ((Stream) targetCols.stream()).map((c) -> { JSONObject o = (JSONObject) c; TargetCol targetCol = new TargetCol(o.getString("name")); Boolean indexable = o.getBoolean("search"); diff --git a/tis-console/src/main/java/com/qlangtech/tis/coredefine/module/action/DataxAction.java b/tis-console/src/main/java/com/qlangtech/tis/coredefine/module/action/DataxAction.java index 9517b8e8a..a32a25f79 100644 --- a/tis-console/src/main/java/com/qlangtech/tis/coredefine/module/action/DataxAction.java +++ b/tis-console/src/main/java/com/qlangtech/tis/coredefine/module/action/DataxAction.java @@ -55,7 +55,7 @@ import com.qlangtech.tis.datax.job.SubJobResName; import com.qlangtech.tis.extension.Descriptor; import com.qlangtech.tis.extension.DescriptorExtensionList; -import com.qlangtech.tis.extension.IPropertyType; +import com.qlangtech.tis.extension.SubFormFilter; import com.qlangtech.tis.extension.util.MultiItemsViewType; import com.qlangtech.tis.fullbuild.IFullBuildContext; import com.qlangtech.tis.lang.TisException; @@ -85,7 +85,6 @@ import com.qlangtech.tis.plugin.ds.DataTypeMeta; import com.qlangtech.tis.plugin.ds.DataTypeMeta.IMultiItemsView; import com.qlangtech.tis.plugin.ds.DefaultTab; -import com.qlangtech.tis.plugin.ds.ElementCreatorFactory; import com.qlangtech.tis.plugin.ds.ISelectedTab; import com.qlangtech.tis.plugin.ds.IdlistElementCreatorFactory; import com.qlangtech.tis.plugin.trigger.JobTrigger; @@ -1435,8 +1434,8 @@ public boolean validate(IFieldErrorHandler msgHandler, Context context, String f return false; } - CMeta.ParsePostMCols postMCols = (new IdlistElementCreatorFactory()).parsePostMCols(msgHandler, - context, MultiItemsViewType.keyColsMeta, targetCols); + CMeta.ParsePostMCols postMCols = (new IdlistElementCreatorFactory()).parsePostMCols(null, msgHandler, + context, fieldKey /*MultiItemsViewType.keyColsMeta*/, targetCols); // Map existCols = Maps.newHashMap(); // boolean validateFaild = false; @@ -1581,7 +1580,7 @@ public void doGetDataXMeta(Context context) { } - public static List getTablesInDB(IPropertyType.SubFormFilter filter) { + public static List getTablesInDB(SubFormFilter filter) { DataxReader reader = DataxReader.getDataxReader(filter); return reader.getTablesInDB().getTabs(); } diff --git a/tis-console/src/main/java/com/qlangtech/tis/coredefine/module/action/PluginAction.java b/tis-console/src/main/java/com/qlangtech/tis/coredefine/module/action/PluginAction.java index fdbeafe8a..480605f14 100644 --- a/tis-console/src/main/java/com/qlangtech/tis/coredefine/module/action/PluginAction.java +++ b/tis-console/src/main/java/com/qlangtech/tis/coredefine/module/action/PluginAction.java @@ -33,10 +33,10 @@ import com.qlangtech.tis.extension.Descriptor; import com.qlangtech.tis.extension.Descriptor.PluginValidateResult; import com.qlangtech.tis.extension.INotebookable; -import com.qlangtech.tis.extension.IPropertyType; import com.qlangtech.tis.extension.PluginFormProperties; import com.qlangtech.tis.extension.PluginManager; import com.qlangtech.tis.extension.PluginWrapper; +import com.qlangtech.tis.extension.SubFormFilter; import com.qlangtech.tis.extension.impl.PropertyType; import com.qlangtech.tis.extension.impl.RootFormProperties; import com.qlangtech.tis.extension.impl.SuFormProperties; @@ -47,9 +47,6 @@ import com.qlangtech.tis.install.InstallState; import com.qlangtech.tis.install.InstallUtil; import com.qlangtech.tis.manage.IAppSource; -import com.qlangtech.tis.manage.common.Config; -import com.qlangtech.tis.manage.common.ConfigFileContext; -import com.qlangtech.tis.manage.common.HttpUtils; import com.qlangtech.tis.manage.common.Option; import com.qlangtech.tis.maven.plugins.tpi.PluginClassifier; import com.qlangtech.tis.offline.module.manager.impl.OfflineManager; @@ -70,7 +67,6 @@ import com.qlangtech.tis.util.PluginItems; import com.qlangtech.tis.util.Selectable; import com.qlangtech.tis.util.UploadPluginMeta; -import com.qlangtech.tis.web.start.TisAppLaunch; import com.qlangtech.tis.workflow.pojo.DatasourceDb; import com.qlangtech.tis.workflow.pojo.DatasourceDbCriteria; import org.apache.commons.codec.digest.DigestUtils; @@ -88,8 +84,6 @@ import org.springframework.beans.factory.annotation.Autowired; import javax.servlet.http.HttpServletRequest; -import java.io.InputStream; -import java.net.URL; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -951,7 +945,7 @@ public static PluginItemsParser parsePluginItems(BasicModule module, UploadPlugi int pluginIndex, JSONArray itemsArray, boolean verify) { context.put(UploadPluginMeta.KEY_PLUGIN_META, pluginMeta); // List items = Lists.newArrayList(); - Optional subFormFilter = pluginMeta.getSubFormFilter(); + Optional subFormFilter = pluginMeta.getSubFormFilter(); IPluginEnum hEnum = pluginMeta.getHeteroEnum(); PluginItems pluginItems = new PluginItems(module, pluginMeta); diff --git a/tis-console/src/main/java/com/qlangtech/tis/offline/module/action/OfflineDatasourceAction.java b/tis-console/src/main/java/com/qlangtech/tis/offline/module/action/OfflineDatasourceAction.java index 00aef91f7..eb32595f7 100644 --- a/tis-console/src/main/java/com/qlangtech/tis/offline/module/action/OfflineDatasourceAction.java +++ b/tis-console/src/main/java/com/qlangtech/tis/offline/module/action/OfflineDatasourceAction.java @@ -26,12 +26,10 @@ import com.google.common.collect.Sets; import com.koubei.web.tag.pager.Pager; import com.qlangtech.tis.TIS; -import com.qlangtech.tis.assemble.FullbuildPhase; import com.qlangtech.tis.coredefine.module.action.DataxAction; import com.qlangtech.tis.coredefine.module.action.PluginDescMeta; import com.qlangtech.tis.coredefine.module.action.TriggerBuildResult; import com.qlangtech.tis.datax.DataXJobSubmit; -import com.qlangtech.tis.datax.IDataXPowerJobSubmit; import com.qlangtech.tis.datax.IDataxProcessor; import com.qlangtech.tis.datax.impl.DataXBasicProcessMeta; import com.qlangtech.tis.datax.impl.DataxProcessor; @@ -50,7 +48,6 @@ import com.qlangtech.tis.manage.PermissionConstant; import com.qlangtech.tis.manage.biz.dal.pojo.Application; import com.qlangtech.tis.manage.common.AppDomainInfo; -import com.qlangtech.tis.manage.common.HttpUtils.PostParam; import com.qlangtech.tis.manage.common.IUser; import com.qlangtech.tis.manage.common.Option; import com.qlangtech.tis.manage.servlet.BasicServlet; @@ -85,6 +82,7 @@ import com.qlangtech.tis.sql.parser.meta.*; import com.qlangtech.tis.sql.parser.tuple.creator.EntityName; import com.qlangtech.tis.trigger.util.JsonUtil; +import com.qlangtech.tis.trigger.util.UnCacheString; import com.qlangtech.tis.util.DescriptorsJSON; import com.qlangtech.tis.util.HeteroList; import com.qlangtech.tis.util.IPluginContext; @@ -1163,7 +1161,7 @@ public void doGetDsTabsVals(Context context) throws IOException { if (tabs == null) { throw new IllegalArgumentException("initialize Tabs can not be null"); } - List selectedTabs = ((Stream) tabs.stream()).map((tab) -> (String) tab).collect(Collectors.toList()); + List selectedTabs = ((Stream) tabs.stream()).map((tab) -> (String) tab).collect(Collectors.toList()); UploadPluginMeta pluginMeta = Objects.requireNonNull(getPluginMeta(body), "pluginMeta can not be null"); @@ -1193,7 +1191,7 @@ public void doGetDsTabsVals(Context context) throws IOException { allNewTabs.add(createNewSelectedTab(pluginFormPropertyTypes, tab2cols)); // 需要将desc中的取option列表解析一下(JsonUtil.UnCacheString) tabDesc.put(tab2cols.getKey(), - JSON.parseObject(desc2Json.getDescriptorsJSON(pluginMeta.getSubFormFilter()).toJSONString())); + JSON.parseObject(JsonUtil.toString(desc2Json.getDescriptorsJSON(pluginMeta.getSubFormFilter())))); } finally { SuFormProperties.subFormGetterProcessThreadLocal.remove(); } @@ -1338,8 +1336,8 @@ private Describable createPluginByDefaultVals(StringBuffer propPath, final Set) enumPp).getValue(); + } else if (enumPp instanceof UnCacheString) { + enums = ((UnCacheString) enumPp).getValue(); } else { throw new IllegalStateException("unsupport type:" + pp.getClass().getName()); } diff --git a/tis-console/src/test/java/com/qlangtech/tis/offline/module/action/TestOfflineDatasourceAction.java b/tis-console/src/test/java/com/qlangtech/tis/offline/module/action/TestOfflineDatasourceAction.java index 2e3b90a39..c625606f8 100644 --- a/tis-console/src/test/java/com/qlangtech/tis/offline/module/action/TestOfflineDatasourceAction.java +++ b/tis-console/src/test/java/com/qlangtech/tis/offline/module/action/TestOfflineDatasourceAction.java @@ -20,7 +20,7 @@ import com.opensymphony.xwork2.ActionProxy; import com.qlangtech.tis.BasicActionTestCase; -import com.qlangtech.tis.extension.IPropertyType.SubFormFilter; +import com.qlangtech.tis.extension.SubFormFilter; import com.qlangtech.tis.manage.common.TisUTF8; import com.qlangtech.tis.manage.common.valve.AjaxValve; import com.qlangtech.tis.offline.DataxUtils; diff --git a/tis-console/src/test/java/com/qlangtech/tis/util/TestPluginItems.java b/tis-console/src/test/java/com/qlangtech/tis/util/TestPluginItems.java index f5485d23d..d6e441169 100644 --- a/tis-console/src/test/java/com/qlangtech/tis/util/TestPluginItems.java +++ b/tis-console/src/test/java/com/qlangtech/tis/util/TestPluginItems.java @@ -24,11 +24,11 @@ import com.google.common.collect.Lists; import com.google.common.collect.Sets; import com.qlangtech.tis.coredefine.module.action.DataxAction; +import com.qlangtech.tis.extension.SubFormFilter; import com.qlangtech.tis.extension.impl.*; import com.qlangtech.tis.plugin.ds.CMeta; import com.qlangtech.tis.plugin.ds.ISelectedTab; import com.qlangtech.tis.datax.impl.DataxReader; -import com.qlangtech.tis.extension.IPropertyType; import com.qlangtech.tis.extension.PluginFormProperties; import com.qlangtech.tis.manage.common.*; import com.qlangtech.tis.offline.module.action.OfflineDatasourceAction; @@ -98,7 +98,7 @@ public Boolean visit(RootFormProperties props) { return true; } })); - Optional subFormFilter = subFieldPluginMeta.getSubFormFilter(); + Optional subFormFilter = subFieldPluginMeta.getSubFormFilter(); assertTrue(subFormFilter.isPresent()); PluginFormProperties pluginFormPropertyTypes = reader.getDescriptor().getPluginFormPropertyTypes(subFormFilter); assertTrue("get SuFormProperties process result", pluginFormPropertyTypes.accept(new PluginFormProperties.IVisitor() { @@ -163,7 +163,7 @@ private void validateSubFormSave() { Context context = EasyMock.createMock("context", Context.class); //targetDescriptorName_MySQL,subFormFieldName_selectedTabs - Optional subFormFilter = subFieldPluginMeta.getSubFormFilter(); + Optional subFormFilter = subFieldPluginMeta.getSubFormFilter(); assertTrue("subFormFilter.isPresent():true", subFormFilter.isPresent()); PluginItems pluginItems = new PluginItems(pluginContext, subFieldPluginMeta); IControlMsgHandler fieldErrorHandler = EasyMock.createMock("fieldErrorHandler", IControlMsgHandler.class); @@ -203,7 +203,7 @@ private void validateRootFormSave() { }); JSONArray itemsArray = jsonArray.getJSONArray(0); - Optional subFormFilter = pluginMeta.getSubFormFilter(); + Optional subFormFilter = pluginMeta.getSubFormFilter(); assertFalse("subFormFilter.isPresent():false", subFormFilter.isPresent()); List items = AttrValMap.describableAttrValMapList( itemsArray, subFormFilter); pluginItems.items = items; diff --git a/tis-manage-pojo/src/main/java/com/qlangtech/tis/extension/IPropertyType.java b/tis-manage-pojo/src/main/java/com/qlangtech/tis/extension/IPropertyType.java new file mode 100644 index 000000000..dab8fbbd0 --- /dev/null +++ b/tis-manage-pojo/src/main/java/com/qlangtech/tis/extension/IPropertyType.java @@ -0,0 +1,30 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.qlangtech.tis.extension; + +/** + * @author 百岁(baisui@qlangtech.com) + * @date 2021-04-11 12:11 + */ +public interface IPropertyType { + /** + * 对应的property 是否是集合属性 + * @return + */ + boolean isCollectionType(); +} diff --git a/tis-manage-pojo/src/main/java/com/qlangtech/tis/manage/common/HttpUtils.java b/tis-manage-pojo/src/main/java/com/qlangtech/tis/manage/common/HttpUtils.java index 41c28664c..f2d1b943d 100644 --- a/tis-manage-pojo/src/main/java/com/qlangtech/tis/manage/common/HttpUtils.java +++ b/tis-manage-pojo/src/main/java/com/qlangtech/tis/manage/common/HttpUtils.java @@ -216,7 +216,7 @@ public AjaxResult p(int status, InputStream stream, Map> } } msgs = result.getJSONArray(IAjaxResult.KEY_MSG); - r.setMsg(((Stream) msgs.stream()).map((m) -> (String) m).collect(Collectors.toList())); + r.setMsg(((Stream) msgs.stream()).map((m) -> (String) m).collect(Collectors.toList())); if (clazz == Void.class) { return r; } diff --git a/tis-manage-pojo/src/main/java/com/qlangtech/tis/plugin/JDBCColumnProp.java b/tis-manage-pojo/src/main/java/com/qlangtech/tis/plugin/JDBCColumnProp.java index 4a403c933..c9890f418 100644 --- a/tis-manage-pojo/src/main/java/com/qlangtech/tis/plugin/JDBCColumnProp.java +++ b/tis-manage-pojo/src/main/java/com/qlangtech/tis/plugin/JDBCColumnProp.java @@ -1,19 +1,19 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package com.qlangtech.tis.plugin; diff --git a/tis-manage-pojo/src/main/java/com/qlangtech/tis/plugin/ds/ElementCreatorFactory.java b/tis-manage-pojo/src/main/java/com/qlangtech/tis/plugin/ds/ElementCreatorFactory.java index 6f8146885..8fae68a26 100644 --- a/tis-manage-pojo/src/main/java/com/qlangtech/tis/plugin/ds/ElementCreatorFactory.java +++ b/tis-manage-pojo/src/main/java/com/qlangtech/tis/plugin/ds/ElementCreatorFactory.java @@ -21,6 +21,7 @@ import com.alibaba.citrus.turbine.Context; import com.alibaba.fastjson.JSONArray; import com.alibaba.fastjson.JSONObject; +import com.qlangtech.tis.extension.IPropertyType; import com.qlangtech.tis.runtime.module.misc.IFieldErrorHandler; import java.util.function.BiConsumer; @@ -37,7 +38,7 @@ public interface ElementCreatorFactory { * @param targetCols * @return */ - public CMeta.ParsePostMCols parsePostMCols( + public CMeta.ParsePostMCols parsePostMCols(IPropertyType propertyType, IFieldErrorHandler msgHandler, Context context, String keyColsMeta, JSONArray targetCols); @@ -61,7 +62,7 @@ default T create(JSONObject targetCol) { * * @param biz */ - default void appendExternalJsonProp(JSONObject biz) { + default void appendExternalJsonProp(IPropertyType propertyType, JSONObject biz) { } } diff --git a/tis-manage-pojo/src/main/java/com/qlangtech/tis/plugin/ds/IdlistElementCreatorFactory.java b/tis-manage-pojo/src/main/java/com/qlangtech/tis/plugin/ds/IdlistElementCreatorFactory.java index 56b4707b3..fb77430d3 100644 --- a/tis-manage-pojo/src/main/java/com/qlangtech/tis/plugin/ds/IdlistElementCreatorFactory.java +++ b/tis-manage-pojo/src/main/java/com/qlangtech/tis/plugin/ds/IdlistElementCreatorFactory.java @@ -21,6 +21,7 @@ import com.alibaba.citrus.turbine.Context; import com.alibaba.fastjson.JSONArray; import com.alibaba.fastjson.JSONObject; +import com.qlangtech.tis.extension.IPropertyType; import com.qlangtech.tis.plugin.ds.CMeta.ParsePostMCols; import com.qlangtech.tis.runtime.module.misc.IFieldErrorHandler; @@ -48,7 +49,7 @@ public CMeta create(JSONObject targetCol, BiConsumer errorProces } @Override - public ParsePostMCols parsePostMCols( + public ParsePostMCols parsePostMCols(IPropertyType propertyType, IFieldErrorHandler msgHandler, Context context, String keyColsMeta, JSONArray targetCols) { throw new UnsupportedOperationException(); } diff --git a/tis-manage-pojo/src/main/java/com/qlangtech/tis/trigger/util/JsonUtil.java b/tis-manage-pojo/src/main/java/com/qlangtech/tis/trigger/util/JsonUtil.java index 8725d5a21..7965eec0d 100644 --- a/tis-manage-pojo/src/main/java/com/qlangtech/tis/trigger/util/JsonUtil.java +++ b/tis-manage-pojo/src/main/java/com/qlangtech/tis/trigger/util/JsonUtil.java @@ -23,8 +23,6 @@ import com.alibaba.fastjson.serializer.ObjectSerializer; import com.alibaba.fastjson.serializer.SerializeConfig; import com.alibaba.fastjson.serializer.SerializerFeature; -import com.alibaba.fastjson2.JSONWriter; -import com.alibaba.fastjson2.writer.ObjectWriter; import com.google.common.collect.Lists; import com.qlangtech.tis.extension.impl.IOUtils; import com.qlangtech.tis.util.DescriptorsJSONResult; @@ -43,7 +41,6 @@ import java.util.Map; import java.util.Objects; import java.util.Set; -import java.util.concurrent.Callable; /** * @author 百岁(baisui@qlangtech.com) @@ -54,6 +51,7 @@ public class JsonUtil { private JsonUtil() { } + /** * 将map中的内容序反列化成一个map结果 * @@ -90,37 +88,57 @@ public static String serialize(Map param) { static { - com.alibaba.fastjson.serializer.ObjectSerializer serializer = new ObjectSerializer() { - @Override - public void write(JSONSerializer serializer, Object object, Object fieldName, Type fieldType, int features) throws IOException { - try { - // SerializeWriter out = serializer.out; - - UnCacheString value = (UnCacheString) object; - Objects.requireNonNull(value, "callable of " + fieldName + " can not be null"); - - // out.writeString(value.getValue()); - - serializer.write(value.getValue()); - - } catch (Exception e) { - throw new IOException(e); - } - } - }; - - ObjectWriter descSerializer = new ObjectWriter() { - @Override - public void write(JSONWriter jsonWriter, Object object, Object fieldName, Type fieldType, long features) { - DescriptorsJSONResult value = (DescriptorsJSONResult) object; - Objects.requireNonNull(value, "callable of " + fieldName + " can not be null"); - jsonWriter.writeRaw(value.toJSONString()); - } - - }; - - com.alibaba.fastjson2.JSON.register(DescriptorsJSONResult.class, descSerializer); - SerializeConfig.global.put(UnCacheString.class, serializer); +// com.alibaba.fastjson.serializer.ObjectSerializer serializer = new ObjectSerializer() { +// @Override +// public void write(JSONSerializer serializer, Object object, Object fieldName, Type fieldType, int features) throws IOException { +// try { +// // SerializeWriter out = serializer.out; +// +// UnCacheString value = (UnCacheString) object; +// Objects.requireNonNull(value, "callable of " + fieldName + " can not be null"); +// +// // out.writeString(value.getValue()); +// +// serializer.write(value.getValue()); +// +// } catch (Exception e) { +// throw new IOException(e); +// } +// } +// }; + +// com.alibaba.fastjson.serializer.ObjectSerializer jsonSerializer = new ObjectSerializer() { +// @Override +// public void write(JSONSerializer serializer, Object object, Object fieldName, Type fieldType, int features) throws IOException { +// try { +// // SerializeWriter out = serializer.out; +// +// JsonSerializer value = (JsonSerializer) object; +// Objects.requireNonNull(value, "callable of " + fieldName + " can not be null"); +// +// // out.writeString(value.getValue()); +// +// serializer.write(Objects.requireNonNull(value.serialize(), "serialize object can not be null")); +// +// } catch (Exception e) { +// throw new IOException(e); +// } +// } +// }; + +// ObjectWriter descSerializer = new ObjectWriter() { +// @Override +// public void write(JSONWriter jsonWriter, Object object, Object fieldName, Type fieldType, long features) { +// DescriptorsJSONResult value = (DescriptorsJSONResult) object; +// Objects.requireNonNull(value, "callable of " + fieldName + " can not be null"); +// jsonWriter.writeRaw(value.toJSONString()); +// } +// +// }; + + // com.alibaba.fastjson2.JSON.register(DescriptorsJSONResult.class, descSerializer); + // SerializeConfig.global.put(UnCacheString.class, serializer); + // SerializeConfig.global.put(JsonSerializer.class, jsonSerializer); // SerializeConfig.global.put(DescriptorsJSONResult.class, descSerializer); } @@ -187,22 +205,6 @@ private static boolean compareEqual(Object prop1, Object prop2, Set igno } - public static final class UnCacheString { - private final Callable valGetter; - - public UnCacheString(Callable valGetter) { - this.valGetter = valGetter; - } - - public T getValue() { - try { - return this.valGetter.call(); - } catch (Exception e) { - throw new RuntimeException(e); - } - } - } - public static String toString(Object json, boolean prettyFormat) { List features = Lists.newArrayList(SerializerFeature.DisableCircularReferenceDetect); @@ -219,7 +221,7 @@ public static String toString(Object json) { } public static void assertJSONEqual(Class invokeClass, String assertFileName, DescriptorsJSONResult actual, IAssert azzert) { - assertJSONEqual(invokeClass, assertFileName, actual.toJSONString(), azzert); + assertJSONEqual(invokeClass, assertFileName, JsonUtil.toString(actual,true), azzert); } public static void assertJSONEqual(Class invokeClass, String assertFileName, String actual, IAssert azzert) { diff --git a/tis-manage-pojo/src/main/java/com/qlangtech/tis/trigger/util/UnCacheString.java b/tis-manage-pojo/src/main/java/com/qlangtech/tis/trigger/util/UnCacheString.java new file mode 100644 index 000000000..938abfb9a --- /dev/null +++ b/tis-manage-pojo/src/main/java/com/qlangtech/tis/trigger/util/UnCacheString.java @@ -0,0 +1,44 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.qlangtech.tis.trigger.util; + +import com.alibaba.fastjson.annotation.JSONType; + +import java.util.concurrent.Callable; + +/** + * @author: 百岁(baisui@qlangtech.com) + * @create: 2024-06-12 13:44 + **/ +@JSONType(serializer = UnCacheStringSerializer.class) +public final class UnCacheString { + private final Callable valGetter; + + public UnCacheString(Callable valGetter) { + this.valGetter = valGetter; + } + + public T getValue() { + try { + return this.valGetter.call(); + } catch (Exception e) { + throw new RuntimeException(e); + } + } +} diff --git a/tis-manage-pojo/src/main/java/com/qlangtech/tis/trigger/util/UnCacheStringSerializer.java b/tis-manage-pojo/src/main/java/com/qlangtech/tis/trigger/util/UnCacheStringSerializer.java new file mode 100644 index 000000000..4d1df7c59 --- /dev/null +++ b/tis-manage-pojo/src/main/java/com/qlangtech/tis/trigger/util/UnCacheStringSerializer.java @@ -0,0 +1,50 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.qlangtech.tis.trigger.util; + +import com.alibaba.fastjson.serializer.JSONSerializer; +import com.alibaba.fastjson.serializer.ObjectSerializer; + +import java.io.IOException; +import java.lang.reflect.Type; +import java.util.Objects; + +/** + * + * @author: 百岁(baisui@qlangtech.com) + * @create: 2024-06-12 13:45 + **/ +public class UnCacheStringSerializer implements ObjectSerializer { + @Override + public void write(JSONSerializer serializer, Object object, Object fieldName, Type type, int i) throws IOException { + try { + // SerializeWriter out = serializer.out; + + UnCacheString value = (UnCacheString) object; + Objects.requireNonNull(value, "callable of " + fieldName + " can not be null"); + + // out.writeString(value.getValue()); + + serializer.write(value.getValue()); + + } catch (Exception e) { + throw new IOException(e); + } + } +} diff --git a/tis-manage-pojo/src/main/java/com/qlangtech/tis/util/DescriptorsJSONResult.java b/tis-manage-pojo/src/main/java/com/qlangtech/tis/util/DescriptorsJSONResult.java index 4e65dbe60..ea9cd22bb 100644 --- a/tis-manage-pojo/src/main/java/com/qlangtech/tis/util/DescriptorsJSONResult.java +++ b/tis-manage-pojo/src/main/java/com/qlangtech/tis/util/DescriptorsJSONResult.java @@ -19,6 +19,7 @@ package com.qlangtech.tis.util; import com.alibaba.fastjson.JSONObject; +import com.alibaba.fastjson.annotation.JSONType; import com.google.common.collect.Maps; import com.qlangtech.tis.trigger.util.JsonUtil; import com.qlangtech.tis.web.start.TisAppLaunch; @@ -32,6 +33,7 @@ * @create: 2024-03-25 12:41 * @seee DescriptorsJSON **/ +@JSONType(serializer = DescriptorsJSONResultSerializer.class) public class DescriptorsJSONResult { public static final ThreadLocal rootDescriptorLocal = new ThreadLocal(); @@ -39,11 +41,11 @@ public static T getRootDescInstance() { return (T) Objects.requireNonNull(rootDescriptorLocal.get(), "rootDescriptorLocal element can not be null"); } - private Map> descs = Maps.newHashMap(); + Map> descs = Maps.newHashMap(); /** * 由于describe 可以嵌套,此标志位可以判断 是否是根元素 */ - private final boolean rootDesc; + final boolean rootDesc; public DescriptorsJSONResult(boolean rootDesc) { this.rootDesc = rootDesc; @@ -53,31 +55,7 @@ public void addDesc(String id, JSONObject descJson, Object desc) { descs.put(id, Pair.of(descJson, desc)); } - public String toJSONString() { - JSONObject o = new JSONObject(); - final int fieldSize = descs.size(); - StringBuffer json = new StringBuffer(); - json.append("{\n"); - int fieldIndex = 0; - for (Map.Entry> entry : descs.entrySet()) { - try { - if (this.rootDesc) { - rootDescriptorLocal.set(entry.getValue().getValue()); - } - json.append("\t\"").append(entry.getKey()).append("\":") - .append(JsonUtil.toString(entry.getValue().getLeft(), TisAppLaunch.isTestMock())); - if (++fieldIndex < fieldSize) { - json.append(","); - } - } finally { - if (this.rootDesc) { - rootDescriptorLocal.remove(); - } - } - } - json.append("\n}"); - return json.toString(); - } + public JSONObject getJSONObject(String descId) { return Objects.requireNonNull(descs.get(descId) diff --git a/tis-manage-pojo/src/main/java/com/qlangtech/tis/util/DescriptorsJSONResultSerializer.java b/tis-manage-pojo/src/main/java/com/qlangtech/tis/util/DescriptorsJSONResultSerializer.java new file mode 100644 index 000000000..440266feb --- /dev/null +++ b/tis-manage-pojo/src/main/java/com/qlangtech/tis/util/DescriptorsJSONResultSerializer.java @@ -0,0 +1,79 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.qlangtech.tis.util; + +import com.alibaba.fastjson.JSONObject; +import com.alibaba.fastjson.serializer.JSONSerializer; +import com.alibaba.fastjson.serializer.ObjectSerializer; +import com.alibaba.fastjson2.JSONWriter; +import com.qlangtech.tis.trigger.util.JsonUtil; +import com.qlangtech.tis.web.start.TisAppLaunch; +import org.apache.commons.lang3.tuple.Pair; + +import java.io.IOException; +import java.io.UnsupportedEncodingException; +import java.lang.reflect.Type; +import java.util.Map; +import java.util.Objects; + +/** + * @author: 百岁(baisui@qlangtech.com) + * @create: 2024-06-12 13:30 + **/ +public class DescriptorsJSONResultSerializer implements ObjectSerializer { + + @Override + public void write(JSONWriter jsonWriter, Object object, Object fieldName, Type fieldType, long features) { + + DescriptorsJSONResult value = (DescriptorsJSONResult) object; + Objects.requireNonNull(value, "callable of " + fieldName + " can not be null"); + jsonWriter.writeRaw(toJSONString(value)); + } + + public String toJSONString(DescriptorsJSONResult value) { + JSONObject o = new JSONObject(); + final int fieldSize = value.descs.size(); + StringBuffer json = new StringBuffer(); + json.append("{\n"); + int fieldIndex = 0; + for (Map.Entry> entry : value.descs.entrySet()) { + try { + if (value.rootDesc) { + DescriptorsJSONResult.rootDescriptorLocal.set(entry.getValue().getValue()); + } + json.append("\t\"").append(entry.getKey()).append("\":") + .append(JsonUtil.toString(entry.getValue().getLeft(), TisAppLaunch.isTestMock())); + if (++fieldIndex < fieldSize) { + json.append(","); + } + } finally { + if (value.rootDesc) { + DescriptorsJSONResult.rootDescriptorLocal.remove(); + } + } + } + json.append("\n}"); + return json.toString(); + } + + @Override + public void write(JSONSerializer jsonSerializer, Object o, Object o1, Type type, int i) throws IOException { + throw new UnsupportedEncodingException(); + } +} diff --git a/tis-plugin/src/main/java/com/qlangtech/tis/datax/impl/DataxReader.java b/tis-plugin/src/main/java/com/qlangtech/tis/datax/impl/DataxReader.java index 7c2a38449..2a590df33 100644 --- a/tis-plugin/src/main/java/com/qlangtech/tis/datax/impl/DataxReader.java +++ b/tis-plugin/src/main/java/com/qlangtech/tis/datax/impl/DataxReader.java @@ -27,8 +27,8 @@ import com.qlangtech.tis.datax.IDataxReader; import com.qlangtech.tis.extension.Describable; import com.qlangtech.tis.extension.Descriptor; -import com.qlangtech.tis.extension.IPropertyType; import com.qlangtech.tis.extension.PluginFormProperties; +import com.qlangtech.tis.extension.SubFormFilter; import com.qlangtech.tis.extension.impl.BaseSubFormProperties; import com.qlangtech.tis.extension.impl.IncrSourceExtendSelected; import com.qlangtech.tis.extension.impl.SuFormProperties; @@ -80,7 +80,7 @@ public static DataxReader getThreadBingDataXReader() { return reader; } - public static T getDataxReader(IPropertyType.SubFormFilter filter) { + public static T getDataxReader(SubFormFilter filter) { return getDataxReader(filter.uploadPluginMeta); } @@ -366,8 +366,8 @@ protected Class getExpect public static abstract class BaseDataxReaderDescriptor extends Descriptor implements IDataXEndTypeGetter { @Override - public PluginFormProperties getPluginFormPropertyTypes(Optional subFormFilter) { - IPropertyType.SubFormFilter filter = null; + public PluginFormProperties getPluginFormPropertyTypes(Optional subFormFilter) { + SubFormFilter filter = null; if (subFormFilter.isPresent()) { filter = subFormFilter.get(); if (filter.isIncrProcessExtend()) { diff --git a/tis-plugin/src/main/java/com/qlangtech/tis/extension/Descriptor.java b/tis-plugin/src/main/java/com/qlangtech/tis/extension/Descriptor.java index 462c1b7f3..022e5c0a4 100644 --- a/tis-plugin/src/main/java/com/qlangtech/tis/extension/Descriptor.java +++ b/tis-plugin/src/main/java/com/qlangtech/tis/extension/Descriptor.java @@ -342,9 +342,9 @@ public PluginFormProperties getPluginFormPropertyTypes() { return getPluginFormPropertyTypes(Optional.empty()); } - public PluginFormProperties getPluginFormPropertyTypes(Optional subFormFilter) { + public PluginFormProperties getPluginFormPropertyTypes(Optional subFormFilter) { - IPropertyType.SubFormFilter filter = null; + SubFormFilter filter = null; SuFormProperties subPluginFormPropertyTypes; if (subFormFilter.isPresent()) { filter = subFormFilter.get(); @@ -476,7 +476,7 @@ public JSON getInstancePropsJson(Object instance) { */ public final PluginValidateResult verify(IControlMsgHandler msgHandler, Context context // , boolean verify // - , AttrVals formData, Optional pTypes, Optional subFormFilter) { + , AttrVals formData, Optional pTypes, Optional subFormFilter) { try { @@ -502,7 +502,7 @@ public PluginValidateResult visit(RootFormProperties props) { } private boolean validatePostFormVals(PostFormVals postFormVals, - Optional subFormFilter) { + Optional subFormFilter) { boolean valid = isValid(msgHandler, context, verify, subFormFilter, propertyTypes, postFormVals); if (valid && verify) { @@ -520,7 +520,7 @@ private boolean validatePostFormVals(PostFormVals postFormVals, return valid; } - private boolean validateSubformByParent(Optional subFormFilter, + private boolean validateSubformByParent(Optional subFormFilter, PostFormVals postFormVals) { if (subFormFilter.isPresent()) { Descriptor parentDesc = subFormFilter.get().getTargetDescriptor(); @@ -539,7 +539,7 @@ public PluginValidateResult visit(BaseSubFormProperties props) { if (!subFormFilter.isPresent()) { throw new IllegalStateException("subFormFilter must be present"); } - IPropertyType.SubFormFilter filter = subFormFilter.get(); + SubFormFilter filter = subFormFilter.get(); if (filter.subformDetailView) { // 校验的时候子表单是{key1:val1,key2:val2} 的格式 @@ -637,7 +637,7 @@ public boolean isValid() { private boolean isValid(IControlMsgHandler msgHandler, Context context, boolean bizValidate, - Optional subFormFilter, PluginFormProperties propertyTypes, + Optional subFormFilter, PluginFormProperties propertyTypes, PostFormVals postFormVals) { Objects.requireNonNull(postFormVals, "postFormVals can not be null"); @@ -918,7 +918,7 @@ public JSONObject addSubForm(String key, String formImpl, FormData form) { public ParseDescribable newInstance(IPluginContext pluginContext, // AttrValMap.IAttrVals formData, // - Optional subFormFilter) { + Optional subFormFilter) { try { return parseDescribable(pluginContext, formData, Optional.empty(), subFormFilter); } catch (Exception e) { @@ -932,7 +932,7 @@ public ParseDescribable newInstance(IPluginContext pluginContext, / // } public ParseDescribable parseDescribable(IPluginContext pluginContext - , AttrValMap.IAttrVals keyValMap, Optional pTypes, Optional subFormFilter) { + , AttrValMap.IAttrVals keyValMap, Optional pTypes, Optional subFormFilter) { PluginFormProperties propertyTypes = getPropertyTypes(pTypes, subFormFilter); @@ -969,7 +969,7 @@ public ParseDescribable visit(BaseSubFormProperties props) { if (!subFormFilter.isPresent()) { throw new IllegalStateException("subFormFilter must be present"); } - IPropertyType.SubFormFilter filter = subFormFilter.get(); + SubFormFilter filter = subFormFilter.get(); if (filter.subformDetailView) { return new ParseDescribable<>(setParentPluginClass(createPluginInstance())); } else { @@ -1026,7 +1026,7 @@ public Void process(String subFormId, AttrValMap attrVals) { }); } - private PluginFormProperties getPropertyTypes(Optional pTypes, Optional subFormFilter) { + private PluginFormProperties getPropertyTypes(Optional pTypes, Optional subFormFilter) { return pTypes.map((pp) -> { return (Descriptor.this == pp.getDescriptor()) ? pp : null; }).orElseGet(() -> { @@ -1078,8 +1078,16 @@ private > TARGET buildPluginInstance(IPluginC return c; } }).collect(Collectors.toList()); + //attrDesc.is + + if (attrDesc.isCollectionType()) { + attrDesc.setVal(describable, multi); + } else { + for (TypeBase type : multi) { + attrDesc.setVal(describable, type); + } + } - attrDesc.setVal(describable, multi); } else { boolean containVal = @@ -1297,7 +1305,7 @@ public static class PostFormVals { public final AttrValMap.IAttrVals rawFormData; private final Descriptor desc; private final PluginFormProperties formProperties; - private final Optional subFormFilter; + private final Optional subFormFilter; private final IControlMsgHandler msgHandler; @@ -1323,7 +1331,7 @@ public PostFormVals(Descriptor desc // } public PostFormVals(Descriptor desc, PluginFormProperties formProperties // - , Optional subFormFilter // + , Optional subFormFilter // , IControlMsgHandler msgHandler, AttrValMap.IAttrVals rawFormData) { this.rawFormData = rawFormData; this.desc = desc; @@ -1350,7 +1358,7 @@ public PluginExtraProps.Props addFieldDescriptor(String fieldName, Object dftVal JSONObject c = new JSONObject(); c.put(PluginExtraProps.KEY_DFTVAL_PROP, dftVal); PluginExtraProps.Props props = new PluginExtraProps.Props(c); - if(StringUtils.isNotEmpty(helperContent)){ + if (StringUtils.isNotEmpty(helperContent)) { props.tagAsynHelp(new StringBuffer(helperContent)); } if (enums.isPresent()) { diff --git a/tis-plugin/src/main/java/com/qlangtech/tis/extension/IPropertyType.java b/tis-plugin/src/main/java/com/qlangtech/tis/extension/IPropertyType.java deleted file mode 100644 index fb0c7eee3..000000000 --- a/tis-plugin/src/main/java/com/qlangtech/tis/extension/IPropertyType.java +++ /dev/null @@ -1,133 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.qlangtech.tis.extension; - -import com.qlangtech.tis.util.IPluginContext; -import com.qlangtech.tis.util.UploadPluginMeta; -import org.apache.commons.lang.StringUtils; - -import java.util.Optional; - -/** - * @author 百岁(baisui@qlangtech.com) - * @date 2021-04-11 12:11 - */ -public interface IPropertyType { - - /** - * plugin form 某一个field为集合类型,且字段内为一个javabean类型,需要填写多个字段 - */ - public class SubFormFilter { - - public static final String KEY_INCR_PROCESS_EXTEND = "incr_process_extend"; - - /** - * 对DataX Reader端selected tab 属性进行扩展 - */ - public static final String KEY_BATCH_SOURCE_PROCESS_EXTEND = "batch_source_process_extend"; - - /** - * 表明点击进入子表单显示 - */ - public static String PLUGIN_META_SUBFORM_DETAIL_ID_VALUE = "subformDetailIdValue"; - - - public static String PLUGIN_META_SUB_FORM_FIELD = "subFormFieldName"; - public final UploadPluginMeta.TargetDesc targetDesc; - //public final String targetDescImpl; - public final String subFieldName; - public final UploadPluginMeta uploadPluginMeta; - // 是否显示子表单内容 - public final boolean subformDetailView; - public final String subformDetailId; - - public Descriptor getTargetDescriptor() { - return this.targetDesc.getTargetDescriptor(); - // Descriptor parentDesc = Objects.requireNonNull(TIS.get().getDescriptor(this.targetDesc.impl) - // , this.targetDesc + "->" + this.targetDesc.impl + " relevant desc can not be null"); - // return parentDesc; - } - - /** - * 增量流程中需要对已经选的表进程属性设置? - * - * @return - */ - public boolean isIncrProcessExtend() { - return this.targetDesc.isNameMatch(KEY_INCR_PROCESS_EXTEND); - } - - public boolean isBatchSourceProcessExtend() { - return this.targetDesc.isNameMatch(KEY_BATCH_SOURCE_PROCESS_EXTEND); - } - - public boolean match(Descriptor desc) { - //return targetDesc.descDisplayName(desc.getDisplayName()); - return StringUtils.equals(desc.getDisplayName(), this.targetDesc.descDisplayName); - // return StringUtils.equals(desc.getDisplayName(), this.targetDescriptorName.getName()); - } - - - public final String param(String key) { - return uploadPluginMeta.getExtraParam(key); - } - - /** - * 取得子表单的宿主plugin - * - * @param pluginContext - * @param - * @return - */ - public T getOwnerPlugin(IPluginContext pluginContext) { - Optional first = this.uploadPluginMeta.getHeteroEnum().getPlugins(pluginContext, - this.uploadPluginMeta).stream().findFirst(); - if (!first.isPresent()) { - throw new IllegalStateException("can not find owner plugin:" + uploadPluginMeta.toString()); - } - return (T) first.get(); - } - - public SubFormFilter(UploadPluginMeta uploadPluginMeta, UploadPluginMeta.TargetDesc targetDescriptorName //, - // String targetDescImpl - , String subFieldName) { - if ((targetDescriptorName) == null) { - throw new IllegalArgumentException("param fieldName can not be empty"); - } - if (StringUtils.isEmpty(subFieldName)) { - throw new IllegalArgumentException("param subFieldName can not be empty"); - } - this.targetDesc = targetDescriptorName; - //this.targetDescImpl = targetDescImpl; - this.subFieldName = subFieldName; - this.uploadPluginMeta = uploadPluginMeta; - this.subformDetailView = StringUtils.isNotEmpty(subformDetailId = - uploadPluginMeta.getExtraParam(PLUGIN_META_SUBFORM_DETAIL_ID_VALUE)); - } - - public boolean useCache() { - return this.uploadPluginMeta.isUseCache(); - } - - @Override - public String toString() { - return "{" + " descName='" + targetDesc + '\'' + ", subFieldName='" + subFieldName + '\'' + ", " + - "subformDetailId='" + subformDetailId + '\'' + '}'; - } - } -} diff --git a/tis-plugin/src/main/java/com/qlangtech/tis/extension/SubFormFilter.java b/tis-plugin/src/main/java/com/qlangtech/tis/extension/SubFormFilter.java new file mode 100644 index 000000000..bc1daecf0 --- /dev/null +++ b/tis-plugin/src/main/java/com/qlangtech/tis/extension/SubFormFilter.java @@ -0,0 +1,127 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.qlangtech.tis.extension; + +import com.qlangtech.tis.util.IPluginContext; +import com.qlangtech.tis.util.UploadPluginMeta; +import org.apache.commons.lang.StringUtils; + +import java.util.Optional; + +/** + * plugin form 某一个field为集合类型,且字段内为一个javabean类型,需要填写多个字段 + */ +public class SubFormFilter { + + public static final String KEY_INCR_PROCESS_EXTEND = "incr_process_extend"; + + /** + * 对DataX Reader端selected tab 属性进行扩展 + */ + public static final String KEY_BATCH_SOURCE_PROCESS_EXTEND = "batch_source_process_extend"; + + /** + * 表明点击进入子表单显示 + */ + public static String PLUGIN_META_SUBFORM_DETAIL_ID_VALUE = "subformDetailIdValue"; + + + public static String PLUGIN_META_SUB_FORM_FIELD = "subFormFieldName"; + public final UploadPluginMeta.TargetDesc targetDesc; + //public final String targetDescImpl; + public final String subFieldName; + public final UploadPluginMeta uploadPluginMeta; + // 是否显示子表单内容 + public final boolean subformDetailView; + public final String subformDetailId; + + public Descriptor getTargetDescriptor() { + return this.targetDesc.getTargetDescriptor(); + // Descriptor parentDesc = Objects.requireNonNull(TIS.get().getDescriptor(this.targetDesc.impl) + // , this.targetDesc + "->" + this.targetDesc.impl + " relevant desc can not be null"); + // return parentDesc; + } + + /** + * 增量流程中需要对已经选的表进程属性设置? + * + * @return + */ + public boolean isIncrProcessExtend() { + return this.targetDesc.isNameMatch(KEY_INCR_PROCESS_EXTEND); + } + + public boolean isBatchSourceProcessExtend() { + return this.targetDesc.isNameMatch(KEY_BATCH_SOURCE_PROCESS_EXTEND); + } + + public boolean match(Descriptor desc) { + //return targetDesc.descDisplayName(desc.getDisplayName()); + return StringUtils.equals(desc.getDisplayName(), this.targetDesc.descDisplayName); + // return StringUtils.equals(desc.getDisplayName(), this.targetDescriptorName.getName()); + } + + + public final String param(String key) { + return uploadPluginMeta.getExtraParam(key); + } + + /** + * 取得子表单的宿主plugin + * + * @param pluginContext + * @param + * @return + */ + public T getOwnerPlugin(IPluginContext pluginContext) { + Optional first = this.uploadPluginMeta.getHeteroEnum().getPlugins(pluginContext, + this.uploadPluginMeta).stream().findFirst(); + if (!first.isPresent()) { + throw new IllegalStateException("can not find owner plugin:" + uploadPluginMeta.toString()); + } + return (T) first.get(); + } + + public SubFormFilter(UploadPluginMeta uploadPluginMeta, UploadPluginMeta.TargetDesc targetDescriptorName //, + // String targetDescImpl + , String subFieldName) { + if ((targetDescriptorName) == null) { + throw new IllegalArgumentException("param fieldName can not be empty"); + } + if (StringUtils.isEmpty(subFieldName)) { + throw new IllegalArgumentException("param subFieldName can not be empty"); + } + this.targetDesc = targetDescriptorName; + //this.targetDescImpl = targetDescImpl; + this.subFieldName = subFieldName; + this.uploadPluginMeta = uploadPluginMeta; + this.subformDetailView = StringUtils.isNotEmpty(subformDetailId = + uploadPluginMeta.getExtraParam(PLUGIN_META_SUBFORM_DETAIL_ID_VALUE)); + } + + public boolean useCache() { + return this.uploadPluginMeta.isUseCache(); + } + + @Override + public String toString() { + return "{" + " descName='" + targetDesc + '\'' + ", subFieldName='" + subFieldName + '\'' + ", " + + "subformDetailId='" + subformDetailId + '\'' + '}'; + } +} diff --git a/tis-plugin/src/main/java/com/qlangtech/tis/extension/impl/BaseSubFormProperties.java b/tis-plugin/src/main/java/com/qlangtech/tis/extension/impl/BaseSubFormProperties.java index ecdf1bd48..b84f2ec49 100644 --- a/tis-plugin/src/main/java/com/qlangtech/tis/extension/impl/BaseSubFormProperties.java +++ b/tis-plugin/src/main/java/com/qlangtech/tis/extension/impl/BaseSubFormProperties.java @@ -26,6 +26,7 @@ import com.qlangtech.tis.extension.Descriptor; import com.qlangtech.tis.extension.IPropertyType; import com.qlangtech.tis.extension.PluginFormProperties; +import com.qlangtech.tis.extension.SubFormFilter; import com.qlangtech.tis.plugin.IdentityName; import com.qlangtech.tis.plugin.datax.SelectedTab; import com.qlangtech.tis.runtime.module.misc.impl.DefaultFieldErrorHandler; @@ -49,6 +50,10 @@ public abstract class BaseSubFormProperties extends PluginFormProperties impleme public Class instClazz; public final Descriptor subFormFieldsDescriptor; + @Override + public boolean isCollectionType() { + throw new UnsupportedOperationException(); + } /** * @param subFormField @@ -107,7 +112,7 @@ public JSON getInstancePropsJson(Object instance) { // } } - public abstract DescriptorsJSON.IPropGetter getSubFormIdListGetter(IPropertyType.SubFormFilter filter); + public abstract DescriptorsJSON.IPropGetter getSubFormIdListGetter(SubFormFilter filter); public Collection getSubFormPropVal(Object instance) { Class fieldType = subFormField.getType(); diff --git a/tis-plugin/src/main/java/com/qlangtech/tis/extension/impl/IncrSourceExtendSelected.java b/tis-plugin/src/main/java/com/qlangtech/tis/extension/impl/IncrSourceExtendSelected.java index e45a91ca6..32e48853b 100644 --- a/tis-plugin/src/main/java/com/qlangtech/tis/extension/impl/IncrSourceExtendSelected.java +++ b/tis-plugin/src/main/java/com/qlangtech/tis/extension/impl/IncrSourceExtendSelected.java @@ -22,7 +22,7 @@ import com.qlangtech.tis.TIS; import com.qlangtech.tis.extension.Describable; import com.qlangtech.tis.extension.Descriptor; -import com.qlangtech.tis.extension.IPropertyType; +import com.qlangtech.tis.extension.SubFormFilter; import com.qlangtech.tis.plugin.IdentityName; import com.qlangtech.tis.plugin.datax.SelectedTabExtend; import com.qlangtech.tis.plugin.datax.SelectedTab; @@ -79,7 +79,7 @@ public boolean atLeastOne() { } @Override - public DescriptorsJSON.IPropGetter getSubFormIdListGetter(IPropertyType.SubFormFilter filter) { + public DescriptorsJSON.IPropGetter getSubFormIdListGetter(SubFormFilter filter) { // return (filter) -> { // IPluginStore readerSubFieldStore // = HeteroEnum.getDataXReaderAndWriterStore(filter.uploadPluginMeta.getPluginContext(), diff --git a/tis-plugin/src/main/java/com/qlangtech/tis/extension/impl/PropertyType.java b/tis-plugin/src/main/java/com/qlangtech/tis/extension/impl/PropertyType.java index 71b20a97f..76af00e6f 100644 --- a/tis-plugin/src/main/java/com/qlangtech/tis/extension/impl/PropertyType.java +++ b/tis-plugin/src/main/java/com/qlangtech/tis/extension/impl/PropertyType.java @@ -39,7 +39,7 @@ import com.qlangtech.tis.plugin.ds.DataTypeMeta; import com.qlangtech.tis.plugin.ds.ElementCreatorFactory; import com.qlangtech.tis.runtime.module.misc.IMessageHandler; -import com.qlangtech.tis.trigger.util.JsonUtil; +import com.qlangtech.tis.trigger.util.UnCacheString; import org.apache.commons.beanutils.ConvertUtilsBean; import org.apache.commons.beanutils.Converter; import org.apache.commons.lang.StringUtils; @@ -71,8 +71,8 @@ public class PropertyType implements IPropertyType { convertUtils.register(new Converter() { @Override public T convert(Class type, Object value) { - if (value instanceof JsonUtil.UnCacheString) { - return (T) ((JsonUtil.UnCacheString) value).getValue(); + if (value instanceof UnCacheString) { + return (T) ((UnCacheString) value).getValue(); } else if (value instanceof JSONArray) { JSONArray array = (JSONArray) value; List convert = array.toJavaList(String.class); @@ -119,7 +119,11 @@ public PropertyType(Field f, FormField formField) { this.formField = formField; } - + @Override + public boolean isCollectionType() { + // PropertyType pt = (PropertyType) propertyType; + return List.class.isAssignableFrom(this.clazz); + } public static Map filterFieldProp(Map props) { diff --git a/tis-plugin/src/main/java/com/qlangtech/tis/extension/impl/SuFormProperties.java b/tis-plugin/src/main/java/com/qlangtech/tis/extension/impl/SuFormProperties.java index 98c3be8ee..390e66ec6 100644 --- a/tis-plugin/src/main/java/com/qlangtech/tis/extension/impl/SuFormProperties.java +++ b/tis-plugin/src/main/java/com/qlangtech/tis/extension/impl/SuFormProperties.java @@ -21,7 +21,7 @@ import com.qlangtech.tis.TIS; import com.qlangtech.tis.extension.Describable; import com.qlangtech.tis.extension.Descriptor; -import com.qlangtech.tis.extension.IPropertyType; +import com.qlangtech.tis.extension.SubFormFilter; import com.qlangtech.tis.extension.util.GroovyShellEvaluate; import com.qlangtech.tis.extension.util.MultiItemsViewType; import com.qlangtech.tis.extension.util.PluginExtraProps; @@ -59,7 +59,7 @@ public static SuFormGetterContext setSuFormGetterContext(Describable plugin, Upl } SuFormProperties.SuFormGetterContext subFormContext = subFormGetterProcessThreadLocal.get(); subFormContext.plugin = plugin; - pluginMeta.putExtraParams(IPropertyType.SubFormFilter.PLUGIN_META_SUBFORM_DETAIL_ID_VALUE, subFormDetailId); + pluginMeta.putExtraParams(SubFormFilter.PLUGIN_META_SUBFORM_DETAIL_ID_VALUE, subFormDetailId); subFormContext.param = pluginMeta; return subFormContext; } @@ -221,7 +221,7 @@ public SuFormProperties overWriteInstClazz(Class instClazz) { @Override - public DescriptorsJSON.IPropGetter getSubFormIdListGetter(IPropertyType.SubFormFilter filter) { + public DescriptorsJSON.IPropGetter getSubFormIdListGetter(SubFormFilter filter) { if (filter.isIncrProcessExtend()) { return (f) -> { @@ -246,10 +246,11 @@ public DescriptorsJSON.IPropGetter getSubFormIdListGetter(IPropertyType.SubFormF + "import com.qlangtech.tis.coredefine.module.action.DataxAction; " // + "import com.qlangtech.tis.util.DescriptorsJSON.IPropGetter; " // + "import com.qlangtech.tis.extension.IPropertyType; " // + + "import com.qlangtech.tis.extension.SubFormFilter;" + "class " + className + " implements IPropGetter {" // + " " // + "@Override" // - + " public Object build(IPropertyType.SubFormFilter filter)" + " " // + + " public Object build(SubFormFilter filter)" + " " // + "{" + this.getIdListGetScript() + " }" + "}"; // //this.getIdListGetScript() // loader.loadMyClass(className, script); @@ -324,9 +325,9 @@ public T getContextAttr(String key, Function func) { } public String getSubFormIdentityField() { - String id = param.getExtraParam(IPropertyType.SubFormFilter.PLUGIN_META_SUBFORM_DETAIL_ID_VALUE); + String id = param.getExtraParam(SubFormFilter.PLUGIN_META_SUBFORM_DETAIL_ID_VALUE); if (StringUtils.isEmpty(id)) { - throw new IllegalStateException(IPropertyType.SubFormFilter.PLUGIN_META_SUBFORM_DETAIL_ID_VALUE + " " + "can not be empty"); + throw new IllegalStateException(SubFormFilter.PLUGIN_META_SUBFORM_DETAIL_ID_VALUE + " " + "can not be empty"); } return id; } diff --git a/tis-plugin/src/main/java/com/qlangtech/tis/extension/model/UpdateSite.java b/tis-plugin/src/main/java/com/qlangtech/tis/extension/model/UpdateSite.java index 45f6d51bc..5bd7648bd 100644 --- a/tis-plugin/src/main/java/com/qlangtech/tis/extension/model/UpdateSite.java +++ b/tis-plugin/src/main/java/com/qlangtech/tis/extension/model/UpdateSite.java @@ -498,13 +498,13 @@ public Plugin(String sourceId, JSONObject o) { if (pluginTags == null) { throw new IllegalStateException(JsonUtil.toString(o) + " lack relevant property pluginTags"); } - this.pluginTags = ((Stream) pluginTags.stream()) + this.pluginTags = ((Stream) pluginTags.stream()) .map((t) -> IPluginTaggable.PluginTag.parse((String) t)).collect(Collectors.toSet()); JSONArray endTypes = o.getJSONArray("endTypes"); if (endTypes == null) { throw new IllegalStateException(JsonUtil.toString(o) + " lack relevant property endTypes"); } - this.endTypes = ((Stream) endTypes.stream()).map((e) -> (String) e).collect(Collectors.toSet()); + this.endTypes = ((Stream) endTypes.stream()).map((e) -> (String) e).collect(Collectors.toSet()); JSONObject extendPoints = o.getJSONObject("extendPoints"); this.extendPoints = Maps.newHashMap(); @@ -1184,7 +1184,7 @@ public final class Data { supportMultiClassifier = pluginMeta.getBooleanValue("supportMultiClassifier"); JSONArray classifierInfo = pluginMeta.getJSONArray(PluginManager.PACAKGE_CLASSIFIER); if (supportMultiClassifier && classifierInfo != null && classifierInfo.size() > 0) { - classifiers = ((Stream) classifierInfo.stream()) + classifiers = ((Stream) classifierInfo.stream()) .map((i) -> { return (new DftCoord(e.getKey(), (JSONObject) i)); }).collect(Collectors.toList()); diff --git a/tis-plugin/src/main/java/com/qlangtech/tis/extension/util/GroovyShellEvaluate.java b/tis-plugin/src/main/java/com/qlangtech/tis/extension/util/GroovyShellEvaluate.java index c00ee9224..aeb78fcd4 100644 --- a/tis-plugin/src/main/java/com/qlangtech/tis/extension/util/GroovyShellEvaluate.java +++ b/tis-plugin/src/main/java/com/qlangtech/tis/extension/util/GroovyShellEvaluate.java @@ -17,11 +17,10 @@ */ package com.qlangtech.tis.extension.util; -import com.qlangtech.tis.trigger.util.JsonUtil; +import com.qlangtech.tis.trigger.util.UnCacheString; import com.qlangtech.tis.util.UploadPluginMeta; import groovy.lang.Script; -import java.util.ServiceLoader; import java.util.concurrent.Callable; import java.util.function.Function; @@ -91,7 +90,7 @@ public static Object scriptEval(String script, Function... proce } return eval(meta.getName()); }; - return unCache ? new JsonUtil.UnCacheString(valGetter) : valGetter.call(); + return unCache ? new UnCacheString(valGetter) : valGetter.call(); } catch (Exception e) { throw new RuntimeException("script:" + script, e); } diff --git a/tis-plugin/src/main/java/com/qlangtech/tis/extension/util/MultiItemsViewType.java b/tis-plugin/src/main/java/com/qlangtech/tis/extension/util/MultiItemsViewType.java index 8685a3f4c..91aba49fc 100644 --- a/tis-plugin/src/main/java/com/qlangtech/tis/extension/util/MultiItemsViewType.java +++ b/tis-plugin/src/main/java/com/qlangtech/tis/extension/util/MultiItemsViewType.java @@ -104,7 +104,8 @@ public List getPostSelectedItems(PropertyType attrDe @Override public void appendExternalJsonProp(JSONObject biz) { - this.tupleFactory.appendExternalJsonProp(biz); + + this.tupleFactory.appendExternalJsonProp(this.propertyType, biz); } @Override @@ -192,7 +193,7 @@ public enum ViewFormatType { // String keyColsMeta = StringUtils.EMPTY; JSONArray mcols = eprops.getJSONObject(Descriptor.KEY_ENUM_PROP).getJSONArray("_mcols"); - CMeta.ParsePostMCols parsePostMCols = elementCreator.parsePostMCols(msgHandler, context, keyColsMeta, mcols); + CMeta.ParsePostMCols parsePostMCols = elementCreator.parsePostMCols(attrDesc,msgHandler, context, attrDesc.f.getName(), mcols); if (parsePostMCols.validateFaild) { return Collections.emptyList(); } diff --git a/tis-plugin/src/main/java/com/qlangtech/tis/plugin/annotation/FormFieldType.java b/tis-plugin/src/main/java/com/qlangtech/tis/plugin/annotation/FormFieldType.java index baedaac60..1af19f819 100644 --- a/tis-plugin/src/main/java/com/qlangtech/tis/plugin/annotation/FormFieldType.java +++ b/tis-plugin/src/main/java/com/qlangtech/tis/plugin/annotation/FormFieldType.java @@ -19,12 +19,10 @@ import com.alibaba.citrus.turbine.Context; import com.qlangtech.tis.datax.TimeFormat; -import com.qlangtech.tis.extension.IPropertyType; +import com.qlangtech.tis.extension.SubFormFilter; import com.qlangtech.tis.extension.impl.PropertyType; import com.qlangtech.tis.extension.impl.PropertyType.PropVal; import com.qlangtech.tis.manage.common.Option; -import com.qlangtech.tis.plugin.datax.SelectedTab; -import com.qlangtech.tis.plugin.ds.CMeta; import com.qlangtech.tis.plugin.ds.TypeBase; import com.qlangtech.tis.runtime.module.misc.IFieldErrorHandler; import org.apache.commons.lang.StringUtils; @@ -183,7 +181,7 @@ public interface IMultiSelectValidator { * @param items 多选条目列表 * @return */ - public default boolean validate(IFieldErrorHandler msgHandler, Optional subFormFilter, + public default boolean validate(IFieldErrorHandler msgHandler, Optional subFormFilter, Context context, String fieldName, List items) { int selectCount = 0; diff --git a/tis-plugin/src/main/java/com/qlangtech/tis/plugin/annotation/SubForm.java b/tis-plugin/src/main/java/com/qlangtech/tis/plugin/annotation/SubForm.java index 13691b33c..0fbcb5ea5 100644 --- a/tis-plugin/src/main/java/com/qlangtech/tis/plugin/annotation/SubForm.java +++ b/tis-plugin/src/main/java/com/qlangtech/tis/plugin/annotation/SubForm.java @@ -19,7 +19,7 @@ import com.alibaba.citrus.turbine.Context; import com.qlangtech.tis.extension.Describable; -import com.qlangtech.tis.extension.IPropertyType; +import com.qlangtech.tis.extension.SubFormFilter; import com.qlangtech.tis.extension.impl.BaseSubFormProperties; import com.qlangtech.tis.plugin.datax.SelectedTab; import com.qlangtech.tis.runtime.module.misc.IControlMsgHandler; @@ -70,7 +70,7 @@ interface ISubFormItemValidate { * @return */ boolean validateSubFormItems(IControlMsgHandler msgHandler, Context context, BaseSubFormProperties props, - IPropertyType.SubFormFilter subFormFilter, AttrVals formData); + SubFormFilter subFormFilter, AttrVals formData); /** * 校验单个selected表单 diff --git a/tis-plugin/src/main/java/com/qlangtech/tis/plugin/datax/SelectedTab.java b/tis-plugin/src/main/java/com/qlangtech/tis/plugin/datax/SelectedTab.java index 9bac276f3..f07f761fe 100644 --- a/tis-plugin/src/main/java/com/qlangtech/tis/plugin/datax/SelectedTab.java +++ b/tis-plugin/src/main/java/com/qlangtech/tis/plugin/datax/SelectedTab.java @@ -25,8 +25,8 @@ import com.qlangtech.tis.datax.impl.ESTableAlias; import com.qlangtech.tis.extension.Describable; import com.qlangtech.tis.extension.Descriptor; -import com.qlangtech.tis.extension.IPropertyType; import com.qlangtech.tis.extension.PluginFormProperties; +import com.qlangtech.tis.extension.SubFormFilter; import com.qlangtech.tis.extension.TISExtension; import com.qlangtech.tis.extension.impl.BaseSubFormProperties; import com.qlangtech.tis.extension.impl.EnumFieldMode; @@ -282,7 +282,7 @@ protected final boolean verify(IControlMsgHandler msgHandler, Context context, P } @Override - public boolean validate(IFieldErrorHandler msgHandler, Optional subFormFilter, + public boolean validate(IFieldErrorHandler msgHandler, Optional subFormFilter, Context context, String fieldName, List items) { if (SelectedTab.KEY_FIELD_COLS.equals(fieldName)) { @@ -315,7 +315,7 @@ protected final boolean validateAll(IControlMsgHandler msgHandler, Context conte @Override public boolean validateSubFormItems(IControlMsgHandler msgHandler, Context context, - BaseSubFormProperties props, IPropertyType.SubFormFilter filter, + BaseSubFormProperties props, SubFormFilter filter, AttrVals formData) { formData.vistAttrValMap((tab, item) -> { @@ -344,7 +344,7 @@ public boolean validateSubFormItems(IControlMsgHandler msgHandler, Context conte return true; } - public PluginFormProperties getPluginFormPropertyTypes(Optional subFormFilter) { + public PluginFormProperties getPluginFormPropertyTypes(Optional subFormFilter) { return super.getPluginFormPropertyTypes(subFormFilter); } diff --git a/tis-plugin/src/main/java/com/qlangtech/tis/plugin/datax/SelectedTabExtend.java b/tis-plugin/src/main/java/com/qlangtech/tis/plugin/datax/SelectedTabExtend.java index f38225a8f..7dc6f435c 100644 --- a/tis-plugin/src/main/java/com/qlangtech/tis/plugin/datax/SelectedTabExtend.java +++ b/tis-plugin/src/main/java/com/qlangtech/tis/plugin/datax/SelectedTabExtend.java @@ -32,7 +32,6 @@ import com.qlangtech.tis.plugin.incr.TISSinkFactory; import com.qlangtech.tis.util.*; import com.qlangtech.tis.datax.impl.DataxReader; -import org.apache.commons.collections.CollectionUtils; import org.apache.commons.collections.MapUtils; import java.util.*; @@ -376,7 +375,7 @@ public Descriptor getDescriptor() { protected static abstract class BaseDescriptor extends Descriptor { @Override - public PluginFormProperties getPluginFormPropertyTypes(Optional subFormFilter) { + public PluginFormProperties getPluginFormPropertyTypes(Optional subFormFilter) { return super.getPluginFormPropertyTypes(Optional.empty()); } } diff --git a/tis-plugin/src/main/java/com/qlangtech/tis/plugin/datax/transformer/PluginImpl.java b/tis-plugin/src/main/java/com/qlangtech/tis/plugin/datax/transformer/PluginImpl.java new file mode 100644 index 000000000..03f84597d --- /dev/null +++ b/tis-plugin/src/main/java/com/qlangtech/tis/plugin/datax/transformer/PluginImpl.java @@ -0,0 +1,28 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.qlangtech.tis.plugin.datax.transformer; + +/** + * + * @author: 百岁(baisui@qlangtech.com) + * @create: 2024-06-12 13:59 + **/ +public interface PluginImpl { + public String getImpl(); +} diff --git a/tis-plugin/src/main/java/com/qlangtech/tis/plugin/datax/transformer/PluginLiteria.java b/tis-plugin/src/main/java/com/qlangtech/tis/plugin/datax/transformer/PluginLiteria.java new file mode 100644 index 000000000..dc02a638d --- /dev/null +++ b/tis-plugin/src/main/java/com/qlangtech/tis/plugin/datax/transformer/PluginLiteria.java @@ -0,0 +1,35 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.qlangtech.tis.plugin.datax.transformer; + +import java.util.List; + +/** + * + * @author: 百岁(baisui@qlangtech.com) + * @create: 2024-06-12 14:00 + **/ +public interface PluginLiteria { + /** + * 用于粗略生成Transformer 的描述 + * + * @return + */ + public List getLiteria(); +} diff --git a/tis-plugin/src/main/java/com/qlangtech/tis/plugin/datax/transformer/PluginLiteriaDesc.java b/tis-plugin/src/main/java/com/qlangtech/tis/plugin/datax/transformer/PluginLiteriaDesc.java index 419ab71a5..574bf4033 100644 --- a/tis-plugin/src/main/java/com/qlangtech/tis/plugin/datax/transformer/PluginLiteriaDesc.java +++ b/tis-plugin/src/main/java/com/qlangtech/tis/plugin/datax/transformer/PluginLiteriaDesc.java @@ -18,18 +18,11 @@ package com.qlangtech.tis.plugin.datax.transformer; -import java.util.List; - /** - * * @author: 百岁(baisui@qlangtech.com) * @create: 2024-06-09 17:05 **/ -public interface PluginLiteriaDesc { - /** - * 用于粗略生成Transformer 的描述 - * - * @return - */ - public List getLiteria(); +public interface PluginLiteriaDesc extends PluginLiteria, PluginImpl { + + } diff --git a/tis-plugin/src/main/java/com/qlangtech/tis/plugin/datax/transformer/RecordTransformer.java b/tis-plugin/src/main/java/com/qlangtech/tis/plugin/datax/transformer/RecordTransformer.java index ebf90896f..5989aa477 100644 --- a/tis-plugin/src/main/java/com/qlangtech/tis/plugin/datax/transformer/RecordTransformer.java +++ b/tis-plugin/src/main/java/com/qlangtech/tis/plugin/datax/transformer/RecordTransformer.java @@ -27,14 +27,15 @@ public class RecordTransformer extends TypeBase { public final String getName() { - return target; + // return target; + return null; } /** * 自定义规则 */ private UDFDefinition udf; - private String target; + // private String target; public UDFDefinition getUdf() { return udf; @@ -44,11 +45,11 @@ public void setUdf(UDFDefinition udf) { this.udf = udf; } - public String getTarget() { - return target; - } - - public void setTarget(String target) { - this.target = target; - } +// public String getTarget() { +// return target; +// } +// +// public void setTarget(String target) { +// this.target = target; +// } } diff --git a/tis-plugin/src/main/java/com/qlangtech/tis/plugin/datax/transformer/TargetColumn.java b/tis-plugin/src/main/java/com/qlangtech/tis/plugin/datax/transformer/TargetColumn.java index d80ebf1cc..76f69515a 100644 --- a/tis-plugin/src/main/java/com/qlangtech/tis/plugin/datax/transformer/TargetColumn.java +++ b/tis-plugin/src/main/java/com/qlangtech/tis/plugin/datax/transformer/TargetColumn.java @@ -23,6 +23,7 @@ import com.qlangtech.tis.extension.Describable; import com.qlangtech.tis.extension.Descriptor; import com.qlangtech.tis.plugin.IPluginStore.AfterPluginSaved; +import com.qlangtech.tis.plugin.IdentityName; import com.qlangtech.tis.util.IPluginContext; import java.util.Optional; @@ -31,12 +32,18 @@ * @author: 百岁(baisui@qlangtech.com) * @create: 2024-06-09 13:43 **/ -public abstract class TargetColumn implements Describable, AfterPluginSaved, PluginLiteriaDesc { +public abstract class TargetColumn implements Describable, AfterPluginSaved, PluginLiteriaDesc, IdentityName { public abstract String getName(); + @JSONField(serialize = false) + @Override + public Class getDescribleClass() { + return IdentityName.super.getDescribleClass(); + } + @Override public final String getImpl() { return this.getClass().getName(); } diff --git a/tis-plugin/src/main/java/com/qlangtech/tis/plugin/datax/transformer/UDFDesc.java b/tis-plugin/src/main/java/com/qlangtech/tis/plugin/datax/transformer/UDFDesc.java new file mode 100644 index 000000000..23cec6913 --- /dev/null +++ b/tis-plugin/src/main/java/com/qlangtech/tis/plugin/datax/transformer/UDFDesc.java @@ -0,0 +1,46 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.qlangtech.tis.plugin.datax.transformer; + +import com.google.common.collect.Lists; +import com.qlangtech.tis.manage.common.Option; + +import java.util.List; + +/** + * 每个Transformer udf 对应的转化器 + * + * @author: 百岁(baisui@qlangtech.com) + * @create: 2024-06-12 14:06 + **/ +public class UDFDesc { + private List

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.qlangtech.tis.plugin.datax.transformer.impl; + +import com.google.common.collect.Lists; +import com.qlangtech.tis.extension.Descriptor; +import com.qlangtech.tis.extension.TISExtension; +import com.qlangtech.tis.plugin.annotation.FormField; +import com.qlangtech.tis.plugin.annotation.FormFieldType; +import com.qlangtech.tis.plugin.annotation.Validator; +import com.qlangtech.tis.plugin.datax.transformer.UDFDefinition; +import com.qlangtech.tis.plugin.datax.transformer.UDFDesc; +import com.qlangtech.tis.plugin.datax.transformer.jdbcprop.TargetColType; + +import java.util.List; + +/** + * 将某列JSON内容拆解成多个字段 + * + * @author: 百岁(baisui@qlangtech.com) + * @create: 2024-06-11 14:38 + **/ +public class JSONSplitterUDF extends AbstractFromColumnUDFDefinition { + + @FormField(ordinal = 2, type = FormFieldType.MULTI_SELECTABLE, validate = {Validator.require}) + public List to; + + @Override + public List getLiteria() { + List result = Lists.newArrayList(super.getLiteria()); + + List toDesc = Lists.newArrayList(); + this.to.forEach((colType) -> { + toDesc.addAll(colType.getLiteria()); + }); + result.add(new UDFDesc("to", toDesc)); + return result; + } + + public static List getCols() { + return Lists.newArrayList(); + } + + @TISExtension + public static final class DefaultDescriptor extends Descriptor { + public DefaultDescriptor() { + super(); + } + + @Override + public String getDisplayName() { + return "JSON Splitter"; + } + } + +} diff --git a/tis-plugin/src/main/java/com/qlangtech/tis/plugin/datax/transformer/impl/TransformerRuleElementCreatorFactory.java b/tis-plugin/src/main/java/com/qlangtech/tis/plugin/datax/transformer/impl/TransformerRuleElementCreatorFactory.java index b8c2a29a3..5ba9ddc5f 100644 --- a/tis-plugin/src/main/java/com/qlangtech/tis/plugin/datax/transformer/impl/TransformerRuleElementCreatorFactory.java +++ b/tis-plugin/src/main/java/com/qlangtech/tis/plugin/datax/transformer/impl/TransformerRuleElementCreatorFactory.java @@ -24,6 +24,7 @@ import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.qlangtech.tis.extension.Descriptor.ParseDescribable; +import com.qlangtech.tis.extension.IPropertyType; import com.qlangtech.tis.plugin.ValidatorCommons; import com.qlangtech.tis.plugin.annotation.Validator; import com.qlangtech.tis.plugin.datax.SelectedTab; @@ -59,10 +60,10 @@ public RecordTransformer createDefault() { } @Override - public void appendExternalJsonProp(JSONObject biz) { + public void appendExternalJsonProp(IPropertyType propertyType, JSONObject biz) { // 源表可选的列 - List colsCandidate = SelectedTab.getColsCandidate(); - biz.put("sourceTabCols", colsCandidate); +// List colsCandidate = SelectedTab.getColsCandidate(); +// biz.put("sourceTabCols", colsCandidate); } @Override @@ -72,8 +73,8 @@ public RecordTransformer create(JSONObject targetCol, BiConsumer @Override - public ParsePostMCols parsePostMCols( - IFieldErrorHandler msgHandler, Context context, String _keyColsMeta, JSONArray targetCols) { + public ParsePostMCols parsePostMCols(IPropertyType propertyType, + IFieldErrorHandler msgHandler, Context context, String _keyColsMeta, JSONArray targetCols) { final String keyColsMeta = "rules"; final String keyTarget = "target"; final String keyUdf = "udf"; @@ -81,40 +82,40 @@ public ParsePostMCols parsePostMCols( RecordTransformer transformerRule = null; postMCols.validateFaild = false; - DataType dataType = null; + // DataType dataType = null; UDFDefinition udf = null; JSONObject udfObj = null; AttrValMap valsMap = null; - String targetColName = null; + // String targetColName = null; // 校验是否有相同的列的名称 Map> duplicateCols = Maps.newHashMap(); for (int i = 0; i < targetCols.size(); i++) { final int index = i; JSONObject targetCol = targetCols.getJSONObject(index); udfObj = targetCol.getJSONObject(keyUdf); - targetColName = targetCol.getString(keyTarget); + // targetColName = targetCol.getString(keyTarget); - if (!Validator.require.validate(msgHandler, context - , IFieldErrorHandler.joinField(keyColsMeta, Collections.singletonList(index), keyTarget), targetColName)) { - postMCols.validateFaild = true; - } else { - List cols = duplicateCols.get(targetColName); - if (cols == null) { - cols = Lists.newArrayList(); - duplicateCols.put(targetColName, cols); - } - cols.add(index); - } +// if (!Validator.require.validate(msgHandler, context +// , IFieldErrorHandler.joinField(keyColsMeta, Collections.singletonList(index), keyTarget), targetColName)) { +// postMCols.validateFaild = true; +// } else { +// List cols = duplicateCols.get(targetColName); +// if (cols == null) { +// cols = Lists.newArrayList(); +// duplicateCols.put(targetColName, cols); +// } +// cols.add(index); +// } transformerRule = new RecordTransformer(); - dataType = CMeta.parseType(targetCol, (propKey, errMsg) -> { - msgHandler.addFieldError(context - , IFieldErrorHandler.joinField(keyColsMeta, Collections.singletonList(index), propKey) - , errMsg); - postMCols.validateFaild = true; - }); +// dataType = CMeta.parseType(targetCol, (propKey, errMsg) -> { +// msgHandler.addFieldError(context +// , IFieldErrorHandler.joinField(keyColsMeta, Collections.singletonList(index), propKey) +// , errMsg); +// postMCols.validateFaild = true; +// }); if (udfObj == null) { msgHandler.addFieldError(context @@ -122,7 +123,7 @@ public ParsePostMCols parsePostMCols( , ValidatorCommons.MSG_EMPTY_INPUT_ERROR); postMCols.validateFaild = true; } - if (dataType == null || udfObj == null) { + if (udfObj == null) { continue; } @@ -143,8 +144,8 @@ public ParsePostMCols parsePostMCols( ParseDescribable describable = valsMap.createDescribable(null); udf = (UDFDefinition) describable.getInstance(); transformerRule.setUdf(udf); - transformerRule.setType(dataType); - transformerRule.setTarget(targetColName); + // transformerRule.setType(dataType); + // transformerRule.setTarget(targetColName); postMCols.writerCols.add(transformerRule); } // postMCols.validateFaild = false; diff --git a/tis-plugin/src/main/java/com/qlangtech/tis/plugin/datax/transformer/impl/VirtualTargetColumn.java b/tis-plugin/src/main/java/com/qlangtech/tis/plugin/datax/transformer/impl/VirtualTargetColumn.java index 2c9402e0a..2fe397296 100644 --- a/tis-plugin/src/main/java/com/qlangtech/tis/plugin/datax/transformer/impl/VirtualTargetColumn.java +++ b/tis-plugin/src/main/java/com/qlangtech/tis/plugin/datax/transformer/impl/VirtualTargetColumn.java @@ -24,6 +24,7 @@ import com.qlangtech.tis.plugin.annotation.FormFieldType; import com.qlangtech.tis.plugin.annotation.Validator; import com.qlangtech.tis.plugin.datax.transformer.TargetColumn; +import com.qlangtech.tis.plugin.datax.transformer.UDFDesc; import java.util.Collections; import java.util.List; @@ -34,7 +35,7 @@ **/ public class VirtualTargetColumn extends TargetColumn { - @FormField(ordinal = 1, type = FormFieldType.INPUTTEXT, validate = {Validator.require, Validator.db_col_name}) + @FormField(ordinal = 1, identity = true, type = FormFieldType.INPUTTEXT, validate = {Validator.require, Validator.db_col_name}) public String name; @Override @@ -43,8 +44,13 @@ public String getName() { } @Override - public List getLiteria() { - return Collections.singletonList("virtual col:" + this.name); + public String identityValue() { + return this.getName(); + } + + @Override + public List getLiteria() { + return Collections.singletonList(new UDFDesc("virtual col", this.name)); } @TISExtension diff --git a/tis-plugin/src/main/java/com/qlangtech/tis/plugin/datax/transformer/jdbcprop/JdbcPropertyElementCreatorFactory.java b/tis-plugin/src/main/java/com/qlangtech/tis/plugin/datax/transformer/jdbcprop/JdbcPropertyElementCreatorFactory.java index 065aebb51..23d7c8c09 100644 --- a/tis-plugin/src/main/java/com/qlangtech/tis/plugin/datax/transformer/jdbcprop/JdbcPropertyElementCreatorFactory.java +++ b/tis-plugin/src/main/java/com/qlangtech/tis/plugin/datax/transformer/jdbcprop/JdbcPropertyElementCreatorFactory.java @@ -1,19 +1,19 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package com.qlangtech.tis.plugin.datax.transformer.jdbcprop; @@ -21,12 +21,32 @@ import com.alibaba.citrus.turbine.Context; import com.alibaba.fastjson.JSONArray; import com.alibaba.fastjson.JSONObject; +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import com.qlangtech.tis.extension.Descriptor.ParseDescribable; +import com.qlangtech.tis.extension.IPropertyType; +import com.qlangtech.tis.extension.impl.PropertyType; +import com.qlangtech.tis.plugin.IdentityName; +import com.qlangtech.tis.plugin.ValidatorCommons; +import com.qlangtech.tis.plugin.datax.SelectedTab; +import com.qlangtech.tis.plugin.datax.transformer.TargetColumn; +import com.qlangtech.tis.plugin.datax.transformer.UDFDefinition; +import com.qlangtech.tis.plugin.datax.transformer.impl.VirtualTargetColumn; +import com.qlangtech.tis.plugin.ds.CMeta; import com.qlangtech.tis.plugin.ds.CMeta.ParsePostMCols; +import com.qlangtech.tis.plugin.ds.DataType; import com.qlangtech.tis.plugin.ds.ElementCreatorFactory; import com.qlangtech.tis.plugin.ds.TypeBase; import com.qlangtech.tis.plugin.ds.ViewContent; import com.qlangtech.tis.runtime.module.misc.IFieldErrorHandler; +import com.qlangtech.tis.util.AttrValMap; +import org.apache.commons.lang3.StringUtils; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Optional; import java.util.function.BiConsumer; /** @@ -35,22 +55,113 @@ **/ public class JdbcPropertyElementCreatorFactory implements ElementCreatorFactory { + private static final String KEY_TARGET_COL = "name"; + @Override public ViewContent getViewContentType() { return ViewContent.JdbcTypeProps; } @Override - public ParsePostMCols parsePostMCols( - IFieldErrorHandler msgHandler, Context context, String keyColsMeta, JSONArray targetCols) { - return null; + public void appendExternalJsonProp(IPropertyType propertyType, JSONObject biz) { + biz.put("isList", propertyType.isCollectionType()); + List colsCandidate = SelectedTab.getColsCandidate(); + biz.put("sourceTabCols", colsCandidate); + biz.put("dftStrType", DataType.createVarChar(32)); + // biz.put(); + // ElementCreatorFactory.super.appendExternalJsonProp(propertyType, biz); + } + + + @Override + public ParsePostMCols parsePostMCols(IPropertyType propertyType, + IFieldErrorHandler msgHandler, Context context, String keyColsMeta, JSONArray targetCols) { + boolean isCollection = propertyType.isCollectionType(); + + ParsePostMCols result = new ParsePostMCols<>(); + if (isCollection) { + final int[] index = new int[1]; + Map> duplicateCols = Maps.newHashMap(); + for (Object e : targetCols) { + try { + JSONObject element = (JSONObject) e; + String target = element.getString(KEY_TARGET_COL); + if (StringUtils.isEmpty(target)) { + msgHandler.addFieldError(context // + , IFieldErrorHandler.joinField(keyColsMeta, Collections.singletonList(index[0]), KEY_TARGET_COL) // + , ValidatorCommons.MSG_EMPTY_INPUT_ERROR); + result.validateFaild = true; + } + + Collection sameKeys = duplicateCols.get(target); + if (sameKeys == null) { + sameKeys = Lists.newArrayList(); + duplicateCols.put(target, sameKeys); + } + sameKeys.add(index[0]); + DataType type = CMeta.parseType(element, (propKey, errMsg) -> { + msgHandler.addFieldError(context + , IFieldErrorHandler.joinField(keyColsMeta, Collections.singletonList(index[0]), propKey) + , errMsg); + result.validateFaild = true; + }); + TargetColType targetColType = new TargetColType(); + PainTargetColumn vcol = new PainTargetColumn(target); + targetColType.setTarget(vcol); + targetColType.setType(type); + result.writerCols.add(targetColType); + } finally { + index[0]++; + } + } + + for (Map.Entry> entry : duplicateCols.entrySet()) { + if (entry.getValue().size() > 1) { + for (Integer duplicateIndex : entry.getValue()) { + msgHandler.addFieldError(context // + , IFieldErrorHandler.joinField(keyColsMeta, Collections.singletonList(duplicateIndex), KEY_TARGET_COL) // + , IdentityName.MSG_ERROR_NAME_DUPLICATE); + result.validateFaild = true; + } + } + } + } else { + this.parseSinglePojo(targetCols, result); + } + + return result; + } + + private void parseSinglePojo(JSONArray targetCols, ParsePostMCols result) { + for (Object e : targetCols) { + JSONObject element = (JSONObject) e; + // JSONObject type = element.getJSONObject("type"); + JSONObject target = element.getJSONObject(KEY_TARGET_COL); + + DataType type = CMeta.parseType(element, (propKey, errMsg) -> { +// msgHandler.addFieldError(context +// , IFieldErrorHandler.joinField(keyColsMeta, Collections.singletonList(index), propKey) +// , errMsg); + result.validateFaild = true; + }); + + AttrValMap valsMap = AttrValMap.parseDescribableMap(Optional.empty(), target); + ParseDescribable describable = valsMap.createDescribable(null); + TargetColumn targetCol = (TargetColumn) describable.getInstance(); + TargetColType targetColType = new TargetColType(); + targetColType.setTarget(targetCol); + targetColType.setType(type); + result.writerCols.add(targetColType); + break; + } } /** * 根据目标属性是否是List类型,创建 + * + * @return * @see com.qlangtech.tis.plugin.datax.transformer.jdbcprop.TargetColType * @see com.qlangtech.tis.plugin.datax.transformer.jdbcprop.VirtualColType - * @return */ @Override public TypeBase createDefault() { diff --git a/tis-plugin/src/main/java/com/qlangtech/tis/plugin/datax/transformer/jdbcprop/PainTargetColumn.java b/tis-plugin/src/main/java/com/qlangtech/tis/plugin/datax/transformer/jdbcprop/PainTargetColumn.java new file mode 100644 index 000000000..ac735a967 --- /dev/null +++ b/tis-plugin/src/main/java/com/qlangtech/tis/plugin/datax/transformer/jdbcprop/PainTargetColumn.java @@ -0,0 +1,54 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.qlangtech.tis.plugin.datax.transformer.jdbcprop; + +import com.alibaba.fastjson.annotation.JSONType; +import com.google.common.collect.Lists; +import com.qlangtech.tis.plugin.datax.transformer.TargetColumn; +import com.qlangtech.tis.plugin.datax.transformer.UDFDesc; + +import java.util.List; + +/** + * @author: 百岁(baisui@qlangtech.com) + * @create: 2024-06-12 16:56 + **/ +@JSONType(serializer = PainTargetColumnSerializer.class) +public class PainTargetColumn extends TargetColumn { + private final String colName; + + public PainTargetColumn(String colName) { + this.colName = colName; + } + + @Override + public String getName() { + return this.colName; + } + + @Override + public String identityValue() { + throw new UnsupportedOperationException(); + } + + @Override + public List getLiteria() { + return Lists.newArrayList(new UDFDesc("column", this.colName)); + } +} diff --git a/tis-plugin/src/main/java/com/qlangtech/tis/plugin/datax/transformer/jdbcprop/PainTargetColumnSerializer.java b/tis-plugin/src/main/java/com/qlangtech/tis/plugin/datax/transformer/jdbcprop/PainTargetColumnSerializer.java new file mode 100644 index 000000000..95c05e270 --- /dev/null +++ b/tis-plugin/src/main/java/com/qlangtech/tis/plugin/datax/transformer/jdbcprop/PainTargetColumnSerializer.java @@ -0,0 +1,44 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.qlangtech.tis.plugin.datax.transformer.jdbcprop; + +import com.alibaba.fastjson.serializer.JSONSerializer; +import com.alibaba.fastjson.serializer.ObjectSerializer; +import com.alibaba.fastjson2.JSONWriter; + +import java.io.IOException; +import java.io.UnsupportedEncodingException; +import java.lang.reflect.Type; + +/** + * @author: 百岁(baisui@qlangtech.com) + * @create: 2024-06-12 16:56 + **/ +public class PainTargetColumnSerializer implements ObjectSerializer { + @Override + public void write(JSONWriter jsonWriter, Object object, Object fieldName, Type fieldType, long features) { + PainTargetColumn targetColumn = (PainTargetColumn) object; + jsonWriter.writeRaw("\"" + targetColumn.getName() + "\""); + } + + @Override + public void write(JSONSerializer jsonSerializer, Object o, Object o1, Type type, int i) throws IOException { + throw new UnsupportedEncodingException(); + } +} diff --git a/tis-plugin/src/main/java/com/qlangtech/tis/plugin/datax/transformer/jdbcprop/TargetColType.java b/tis-plugin/src/main/java/com/qlangtech/tis/plugin/datax/transformer/jdbcprop/TargetColType.java index f974c694b..44777ddb0 100644 --- a/tis-plugin/src/main/java/com/qlangtech/tis/plugin/datax/transformer/jdbcprop/TargetColType.java +++ b/tis-plugin/src/main/java/com/qlangtech/tis/plugin/datax/transformer/jdbcprop/TargetColType.java @@ -18,17 +18,35 @@ package com.qlangtech.tis.plugin.datax.transformer.jdbcprop; +import com.alibaba.fastjson.annotation.JSONType; +import com.google.common.collect.Lists; +import com.qlangtech.tis.plugin.datax.transformer.PluginLiteria; import com.qlangtech.tis.plugin.datax.transformer.TargetColumn; +import com.qlangtech.tis.plugin.datax.transformer.UDFDesc; import com.qlangtech.tis.plugin.ds.TypeBase; +import java.util.List; + /** + * https://github.com/alibaba/fastjson/wiki/JSONType_serializer * * @author: 百岁(baisui@qlangtech.com) * @create: 2024-06-10 10:10 **/ -public class TargetColType extends TypeBase { +@JSONType(serializer = TargetColTypeSerializer.class) +public class TargetColType extends TypeBase implements PluginLiteria { + + TargetColumn target; - private TargetColumn target; + @Override + public List getLiteria() { + + List result = Lists.newArrayList(target.getLiteria()); + result.forEach((desc) -> { + desc.addPair("type", this.getType().getTypeDesc()); + }); + return result; + } @Override public String getName() { diff --git a/tis-plugin/src/main/java/com/qlangtech/tis/plugin/datax/transformer/jdbcprop/TargetColTypeSerializer.java b/tis-plugin/src/main/java/com/qlangtech/tis/plugin/datax/transformer/jdbcprop/TargetColTypeSerializer.java new file mode 100644 index 000000000..932298b64 --- /dev/null +++ b/tis-plugin/src/main/java/com/qlangtech/tis/plugin/datax/transformer/jdbcprop/TargetColTypeSerializer.java @@ -0,0 +1,47 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.qlangtech.tis.plugin.datax.transformer.jdbcprop; + +import com.alibaba.fastjson.serializer.JSONSerializer; +import com.alibaba.fastjson.serializer.ObjectSerializer; +import com.google.common.collect.Maps; + +import java.io.IOException; +import java.lang.reflect.Type; +import java.util.Map; +import java.util.Objects; + +/** + * @author: 百岁(baisui@qlangtech.com) + * @create: 2024-06-12 11:43 + **/ +public class TargetColTypeSerializer implements ObjectSerializer { + @Override + public void write(JSONSerializer serializer, Object object, Object fieldName, Type fieldType, int features) throws IOException { + serializer.write(serialize((TargetColType) object)); + } + + + private Map serialize(TargetColType colType) { + Map result = Maps.newHashMap(); + result.put("name", Objects.requireNonNull(colType.target, "target can not be null")); + result.put("type", colType.getType()); + return result; + } +} diff --git a/tis-plugin/src/main/java/com/qlangtech/tis/util/AttrValMap.java b/tis-plugin/src/main/java/com/qlangtech/tis/util/AttrValMap.java index cfed88aad..2fc972907 100644 --- a/tis-plugin/src/main/java/com/qlangtech/tis/util/AttrValMap.java +++ b/tis-plugin/src/main/java/com/qlangtech/tis/util/AttrValMap.java @@ -24,10 +24,8 @@ import com.qlangtech.tis.TIS; import com.qlangtech.tis.extension.Descriptor; import com.qlangtech.tis.extension.INotebookable; -import com.qlangtech.tis.extension.IPropertyType; import com.qlangtech.tis.extension.PluginFormProperties; -import com.qlangtech.tis.extension.impl.RootFormProperties; -import com.qlangtech.tis.plugin.IRepositoryTargetFile; +import com.qlangtech.tis.extension.SubFormFilter; import com.qlangtech.tis.runtime.module.misc.IControlMsgHandler; import com.qlangtech.tis.util.impl.AttrVals; @@ -64,10 +62,10 @@ public static void removeCurrentRootPluginValidator() { public final Descriptor descriptor; //private IControlMsgHandler msgHandler; - private final Optional subFormFilter; + private final Optional subFormFilter; public static List describableAttrValMapList(JSONArray itemsArray, - Optional subFormFilter) { + Optional subFormFilter) { List describableAttrValMapList = Lists.newArrayList(); AttrValMap describableAttrValMap = null; JSONObject itemObj = null; @@ -79,7 +77,7 @@ public static List describableAttrValMapList(JSONArray itemsArray, return describableAttrValMapList; } - public static AttrValMap parseDescribableMap(Optional subFormFilter, + public static AttrValMap parseDescribableMap(Optional subFormFilter, com.alibaba.fastjson.JSONObject jsonObject) { String impl = null; Descriptor descriptor; @@ -94,7 +92,7 @@ public static AttrValMap parseDescribableMap(Optional subFormFilter, Descriptor descriptor) { + AttrVals attrValMap, Optional subFormFilter, Descriptor descriptor) { this.attrValMap = attrValMap; this.descriptor = descriptor; // this.msgHandler = msgHandler; diff --git a/tis-plugin/src/main/java/com/qlangtech/tis/util/DescribableJSON.java b/tis-plugin/src/main/java/com/qlangtech/tis/util/DescribableJSON.java index 5af9a8a28..f2fc523b3 100644 --- a/tis-plugin/src/main/java/com/qlangtech/tis/util/DescribableJSON.java +++ b/tis-plugin/src/main/java/com/qlangtech/tis/util/DescribableJSON.java @@ -21,11 +21,10 @@ import com.alibaba.fastjson.JSONObject; import com.qlangtech.tis.extension.Describable; import com.qlangtech.tis.extension.Descriptor; -import com.qlangtech.tis.extension.IPropertyType; import com.qlangtech.tis.extension.PluginFormProperties; +import com.qlangtech.tis.extension.SubFormFilter; import com.qlangtech.tis.extension.impl.BaseSubFormProperties; import com.qlangtech.tis.extension.impl.RootFormProperties; -import com.qlangtech.tis.plugin.CompanionPluginFactory; import com.qlangtech.tis.plugin.IdentityName; import java.util.Objects; @@ -55,7 +54,7 @@ public JSONObject getItemJson() throws Exception { return this.getItemJson(Optional.empty()); } - public JSONObject getItemJson(Optional subFormFilter) throws Exception { + public JSONObject getItemJson(Optional subFormFilter) throws Exception { PluginFormProperties pluginFormPropertyTypes = descriptor.getPluginFormPropertyTypes(subFormFilter); return getItemJson(pluginFormPropertyTypes); } diff --git a/tis-plugin/src/main/java/com/qlangtech/tis/util/DescriptorsJSON.java b/tis-plugin/src/main/java/com/qlangtech/tis/util/DescriptorsJSON.java index 68ad461d7..df63e6488 100644 --- a/tis-plugin/src/main/java/com/qlangtech/tis/util/DescriptorsJSON.java +++ b/tis-plugin/src/main/java/com/qlangtech/tis/util/DescriptorsJSON.java @@ -23,8 +23,8 @@ import com.qlangtech.tis.TIS; import com.qlangtech.tis.extension.Describable; import com.qlangtech.tis.extension.Descriptor; -import com.qlangtech.tis.extension.IPropertyType; import com.qlangtech.tis.extension.PluginFormProperties; +import com.qlangtech.tis.extension.SubFormFilter; import com.qlangtech.tis.extension.TISExtensible; import com.qlangtech.tis.extension.impl.BaseSubFormProperties; import com.qlangtech.tis.extension.impl.PropertyType; @@ -98,9 +98,9 @@ protected Object clone() throws CloneNotSupportedException { public static abstract class SubFormFieldVisitor implements PluginFormProperties.IVisitor { - final Optional subFormFilter; + final Optional subFormFilter; - public SubFormFieldVisitor(Optional subFormFilter) { + public SubFormFieldVisitor(Optional subFormFilter) { this.subFormFilter = subFormFilter; } @@ -111,7 +111,7 @@ public DescriptorsJSONResult getDescriptorsJSON() { } - public DescriptorsJSONResult getDescriptorsJSON(Optional subFormFilter) { + public DescriptorsJSONResult getDescriptorsJSON(Optional subFormFilter) { JSONArray attrs; String key; PropertyType val; @@ -145,7 +145,7 @@ public Descriptor visit(BaseSubFormProperties props) { subForm.put("fieldName", props.getSubFormFieldName()); if (subFormFilter.isPresent()) { - IPropertyType.SubFormFilter filter = subFormFilter.get(); + SubFormFilter filter = subFormFilter.get(); if (!filter.subformDetailView) { desJson.put("subForm", true); subForm.put("idList", props.getSubFormIdListGetter(filter).build(filter)); @@ -237,7 +237,7 @@ public Descriptor visit(BaseSubFormProperties props) { return descriptors; } - private List> getAcceptDescs(Optional subFormFilter) { + private List> getAcceptDescs(Optional subFormFilter) { PluginFormProperties pluginFormPropertyTypes = null; List> acceptDescs = Lists.newArrayList(this.descriptors); for (Descriptor dd : this.descriptors) { @@ -279,6 +279,6 @@ public static List getSelectOptions(Descriptor descript } public interface IPropGetter { - public Object build(IPropertyType.SubFormFilter filter); + public Object build(SubFormFilter filter); } } diff --git a/tis-plugin/src/main/java/com/qlangtech/tis/util/HeteroEnum.java b/tis-plugin/src/main/java/com/qlangtech/tis/util/HeteroEnum.java index 4ba79b2cb..7a9276b04 100644 --- a/tis-plugin/src/main/java/com/qlangtech/tis/util/HeteroEnum.java +++ b/tis-plugin/src/main/java/com/qlangtech/tis/util/HeteroEnum.java @@ -29,8 +29,8 @@ import com.qlangtech.tis.extension.Describable; import com.qlangtech.tis.extension.Descriptor; import com.qlangtech.tis.extension.ExtensionList; -import com.qlangtech.tis.extension.IPropertyType; import com.qlangtech.tis.extension.PluginFormProperties; +import com.qlangtech.tis.extension.SubFormFilter; import com.qlangtech.tis.extension.TISExtension; import com.qlangtech.tis.extension.impl.BaseSubFormProperties; import com.qlangtech.tis.extension.impl.SuFormProperties; @@ -55,7 +55,6 @@ import com.qlangtech.tis.plugin.k8s.K8sImage; import com.qlangtech.tis.plugin.k8s.K8sImage.ImageCategory; import com.qlangtech.tis.plugin.trigger.JobTrigger; -import com.qlangtech.tis.runtime.module.misc.IControlMsgHandler; import org.apache.commons.lang.StringUtils; import java.io.File; @@ -415,11 +414,11 @@ public static IPluginStore getDataXReaderAndWriterStore(IPluginContext plugin public static IPluginStore getDataXReaderAndWriterStore(IPluginContext pluginContext, boolean getReader, UploadPluginMeta pluginMeta, - Optional subFormFilter) { + Optional subFormFilter) { IPluginStore store = null; if (subFormFilter.isPresent()) { - IPropertyType.SubFormFilter filter = subFormFilter.get(); + SubFormFilter filter = subFormFilter.get(); Descriptor targetDescriptor = filter.getTargetDescriptor(); final Class clazz = targetDescriptor.getT(); // Optional firstDesc = heteroEnum.descriptors().stream() diff --git a/tis-plugin/src/main/java/com/qlangtech/tis/util/HeteroList.java b/tis-plugin/src/main/java/com/qlangtech/tis/util/HeteroList.java index 11ce6d132..499269e96 100644 --- a/tis-plugin/src/main/java/com/qlangtech/tis/util/HeteroList.java +++ b/tis-plugin/src/main/java/com/qlangtech/tis/util/HeteroList.java @@ -21,7 +21,7 @@ import com.alibaba.fastjson.JSONObject; import com.qlangtech.tis.extension.Describable; import com.qlangtech.tis.extension.Descriptor; -import com.qlangtech.tis.extension.IPropertyType; +import com.qlangtech.tis.extension.SubFormFilter; import com.qlangtech.tis.extension.util.GroovyShellUtil; import com.qlangtech.tis.manage.common.Config; import com.qlangtech.tis.manage.common.TisUTF8; @@ -118,7 +118,7 @@ public JSONObject toJSON() throws Exception { Config.TIS_PUB_PLUGINS_DOC_URL + URLEncoder.encode(StringUtils.lowerCase(StringUtils.remove(this.extensionPoint.getName(), ".")), TisUTF8.getName())); - Optional subFormFilter = pluginMeta.getSubFormFilter(); + Optional subFormFilter = pluginMeta.getSubFormFilter(); DescriptorsJSON desc2Json = new DescriptorsJSON(this.descriptors, true); @@ -139,7 +139,7 @@ public JSONObject toJSON() throws Exception { private static > JSONArray createItemsJSONArray( UploadPluginMeta pluginMeta, List items, - Optional subFormFilter) throws Exception { + Optional subFormFilter) throws Exception { JSONArray result = new JSONArray(); JSONObject item = null; diff --git a/tis-plugin/src/main/java/com/qlangtech/tis/util/UploadPluginMeta.java b/tis-plugin/src/main/java/com/qlangtech/tis/util/UploadPluginMeta.java index 489506c58..cb7032bc3 100644 --- a/tis-plugin/src/main/java/com/qlangtech/tis/util/UploadPluginMeta.java +++ b/tis-plugin/src/main/java/com/qlangtech/tis/util/UploadPluginMeta.java @@ -25,7 +25,7 @@ import com.qlangtech.tis.datax.impl.DataxReader; import com.qlangtech.tis.extension.Describable; import com.qlangtech.tis.extension.Descriptor; -import com.qlangtech.tis.extension.IPropertyType; +import com.qlangtech.tis.extension.SubFormFilter; import com.qlangtech.tis.extension.impl.IncrSourceExtendSelected; import com.qlangtech.tis.offline.DataxUtils; import com.qlangtech.tis.plugin.IPluginStore; @@ -36,7 +36,6 @@ import org.apache.commons.lang.StringUtils; import java.util.*; -import java.util.concurrent.atomic.AtomicBoolean; import java.util.regex.Matcher; import java.util.regex.Pattern; import java.util.stream.Collectors; @@ -223,11 +222,11 @@ public List getDataxReaders(IPluginContext pluginContext) { public IPluginEnum getHeteroEnum() { - Optional subFormFilter = null; + Optional subFormFilter = null; subFormFilter = this.getSubFormFilter(); if (subFormFilter.isPresent()) { - IPropertyType.SubFormFilter subFilter = subFormFilter.get(); + SubFormFilter subFilter = subFormFilter.get(); final boolean[] incrExtend = new boolean[1]; if ((incrExtend[0] = subFilter.isIncrProcessExtend()) || subFilter.isBatchSourceProcessExtend()) { @@ -298,12 +297,12 @@ public void setRequired(boolean required) { } - public Optional getSubFormFilter() { + public Optional getSubFormFilter() { TargetDesc targetDesc = this.getTargetDesc(); - String subFormField = this.getExtraParam(IPropertyType.SubFormFilter.PLUGIN_META_SUB_FORM_FIELD); + String subFormField = this.getExtraParam(SubFormFilter.PLUGIN_META_SUB_FORM_FIELD); if (StringUtils.isNotEmpty(targetDesc.descDisplayName) && StringUtils.isNotEmpty(subFormField)) { - return Optional.of(new IPropertyType.SubFormFilter(this, targetDesc //, targetDescImpl + return Optional.of(new SubFormFilter(this, targetDesc //, targetDescImpl , subFormField)); } return Optional.empty(); @@ -341,7 +340,7 @@ public boolean shallMatchTargetDesc() { } public boolean isNameMatch(String displayName) { - return IPropertyType.SubFormFilter.KEY_INCR_PROCESS_EXTEND.equals(matchTargetPluginDescName) // + return SubFormFilter.KEY_INCR_PROCESS_EXTEND.equals(matchTargetPluginDescName) // || StringUtils.equals(displayName, this.matchTargetPluginDescName); } diff --git a/tis-plugin/src/main/resources/com/qlangtech/tis/plugin/datax/transformer/impl/JSONSplitterUDF.json b/tis-plugin/src/main/resources/com/qlangtech/tis/plugin/datax/transformer/impl/JSONSplitterUDF.json new file mode 100644 index 000000000..c4d7230ee --- /dev/null +++ b/tis-plugin/src/main/resources/com/qlangtech/tis/plugin/datax/transformer/impl/JSONSplitterUDF.json @@ -0,0 +1,7 @@ +{ + "to": { + "elementCreator": "com.qlangtech.tis.plugin.datax.transformer.jdbcprop.JdbcPropertyElementCreatorFactory", + "enum": "com.qlangtech.tis.plugin.datax.transformer.impl.JSONSplitterUDF.getCols():uncache_true", + "viewtype": "tuplelist" + } +} \ No newline at end of file diff --git a/tis-plugin/src/test/java/TestAll.java b/tis-plugin/src/test/java/TestAll.java index ba72b50eb..cdff43281 100644 --- a/tis-plugin/src/test/java/TestAll.java +++ b/tis-plugin/src/test/java/TestAll.java @@ -38,7 +38,10 @@ import com.qlangtech.tis.plugin.TestPluginStore; import com.qlangtech.tis.plugin.annotation.TestValidator; import com.qlangtech.tis.plugin.datax.TestSelectedTab; +import com.qlangtech.tis.plugin.datax.transformer.impl.TestCopyValUDF; import com.qlangtech.tis.plugin.datax.transformer.impl.TestTransformerRuleElementCreatorFactory; +import com.qlangtech.tis.plugin.datax.transformer.jdbcprop.TestPainTargetColumn; +import com.qlangtech.tis.plugin.datax.transformer.jdbcprop.TestTargetColType; import com.qlangtech.tis.plugin.ds.TestDataSourceFactoryPluginStore; import com.qlangtech.tis.plugin.ds.TestTableInDB; import com.qlangtech.tis.plugin.k8s.TestK8sImage; @@ -89,6 +92,9 @@ public static Test suite() { suite.addTestSuite(TestK8sImage.class); suite.addTestSuite(TestServerLaunchToken.class); suite.addTestSuite(TestTransformerRuleElementCreatorFactory.class); + suite.addTestSuite(TestCopyValUDF.class); + suite.addTestSuite(TestTargetColType.class); + suite.addTestSuite(TestPainTargetColumn.class); return suite; } diff --git a/tis-plugin/src/test/java/com/qlangtech/tis/coredefine/module/action/DataxAction.java b/tis-plugin/src/test/java/com/qlangtech/tis/coredefine/module/action/DataxAction.java index 60d7509bc..814a5cef2 100644 --- a/tis-plugin/src/test/java/com/qlangtech/tis/coredefine/module/action/DataxAction.java +++ b/tis-plugin/src/test/java/com/qlangtech/tis/coredefine/module/action/DataxAction.java @@ -19,7 +19,7 @@ import com.google.common.collect.Lists; import com.qlangtech.tis.common.utils.Assert; -import com.qlangtech.tis.extension.IPropertyType; +import com.qlangtech.tis.extension.SubFormFilter; import com.qlangtech.tis.manage.common.Option; import com.qlangtech.tis.offline.DataxUtils; import com.qlangtech.tis.util.TestHeteroList; @@ -37,7 +37,7 @@ public static List

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.qlangtech.tis.plugin.datax.transformer.jdbcprop; + +import com.qlangtech.tis.common.utils.Assert; +import com.qlangtech.tis.manage.common.Option; +import com.qlangtech.tis.plugin.datax.transformer.UDFDesc; +import com.qlangtech.tis.trigger.util.JsonUtil; +import junit.framework.TestCase; +import org.junit.Test; + +import java.util.List; + +/** + * @author: 百岁(baisui@qlangtech.com) + * @create: 2024-06-12 17:01 + **/ +public class TestPainTargetColumn extends TestCase { + + @Test + public void testSerialize() { + String colName = "user_name"; + PainTargetColumn targetColumn = new PainTargetColumn(colName); + Assert.assertEquals("\"" + colName + "\"", JsonUtil.toString(targetColumn, true)); + + List descs = targetColumn.getLiteria(); + Assert.assertEquals(1, descs.size()); + for (UDFDesc desc : descs) { + Assert.assertEquals(1, desc.getPairs().size()); + for (Option opt : desc.getPairs()) { + Assert.assertEquals("column", opt.getName()); + Assert.assertEquals(colName, opt.getValue()); + } + } + } +} diff --git a/tis-plugin/src/test/java/com/qlangtech/tis/plugin/datax/transformer/jdbcprop/TestTargetColType.java b/tis-plugin/src/test/java/com/qlangtech/tis/plugin/datax/transformer/jdbcprop/TestTargetColType.java new file mode 100644 index 000000000..eda605e46 --- /dev/null +++ b/tis-plugin/src/test/java/com/qlangtech/tis/plugin/datax/transformer/jdbcprop/TestTargetColType.java @@ -0,0 +1,51 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.qlangtech.tis.plugin.datax.transformer.jdbcprop; + +import com.qlangtech.tis.common.utils.Assert; +import com.qlangtech.tis.plugin.datax.transformer.impl.ExistTargetCoumn; +import com.qlangtech.tis.plugin.ds.DataType; +import com.qlangtech.tis.trigger.util.JsonUtil; +import junit.framework.TestCase; +import org.junit.Test; + +/** + * @author: 百岁(baisui@qlangtech.com) + * @create: 2024-06-12 11:18 + **/ +public class TestTargetColType extends TestCase { + + @Test + public void testSerialize() { + TargetColType colType = new TargetColType(); + colType.setType(DataType.createVarChar(32)); + ExistTargetCoumn existTargetCoumn = new ExistTargetCoumn(); + existTargetCoumn.name = "user_name"; + colType.setTarget(existTargetCoumn); + + + + String assertFileName = "target-col-type.json"; + JsonUtil.assertJSONEqual(TargetColType.class, assertFileName + , JsonUtil.toString(colType, true) // + , (message, expected, actual) -> { + Assert.assertEquals(message, expected, actual); + }); + } +} diff --git a/tis-plugin/src/test/java/com/qlangtech/tis/util/TestHeteroList.java b/tis-plugin/src/test/java/com/qlangtech/tis/util/TestHeteroList.java index 563f86607..3fc4e0d83 100644 --- a/tis-plugin/src/test/java/com/qlangtech/tis/util/TestHeteroList.java +++ b/tis-plugin/src/test/java/com/qlangtech/tis/util/TestHeteroList.java @@ -21,7 +21,7 @@ import com.qlangtech.tis.TIS; import com.qlangtech.tis.datax.impl.DataxReader; import com.qlangtech.tis.extension.ExtensionFinder; -import com.qlangtech.tis.extension.IPropertyType; +import com.qlangtech.tis.extension.SubFormFilter; import com.qlangtech.tis.extension.impl.ClassicPluginStrategy; import com.qlangtech.tis.manage.common.CenterResource; import com.qlangtech.tis.manage.common.Config; @@ -80,7 +80,7 @@ public void testSubFormFieldToJson() throws Exception { HeteroEnum dataxReader = HeteroEnum.DATAX_READER; String pluginMeta = dataxReader.identity + ":require," + UploadPluginMeta.PLUGIN_META_TARGET_DESCRIPTOR_NAME - + "_MySQL," + IPropertyType.SubFormFilter.PLUGIN_META_SUB_FORM_FIELD + "_selectedTabs," + DataxUtils.DATAX_NAME + "_" + DATAX_INSTANCE_NAME; + + "_MySQL," + SubFormFilter.PLUGIN_META_SUB_FORM_FIELD + "_selectedTabs," + DataxUtils.DATAX_NAME + "_" + DATAX_INSTANCE_NAME; UploadPluginMeta meta = UploadPluginMeta.parse(pluginMeta); IPluginContext pluginContext = EasyMock.createMock("pluginContext", IPluginContext.class); diff --git a/tis-plugin/src/test/java/com/qlangtech/tis/util/TestUploadPluginMeta.java b/tis-plugin/src/test/java/com/qlangtech/tis/util/TestUploadPluginMeta.java index 8da91251a..d656df63f 100644 --- a/tis-plugin/src/test/java/com/qlangtech/tis/util/TestUploadPluginMeta.java +++ b/tis-plugin/src/test/java/com/qlangtech/tis/util/TestUploadPluginMeta.java @@ -18,7 +18,7 @@ package com.qlangtech.tis.util; import com.qlangtech.tis.extension.Descriptor; -import com.qlangtech.tis.extension.IPropertyType; +import com.qlangtech.tis.extension.SubFormFilter; import junit.framework.TestCase; import org.easymock.EasyMock; @@ -98,16 +98,16 @@ public void testPluginMetaParse() { plugins = new String[]{pluginName + ":" + UploadPluginMeta.PLUGIN_META_TARGET_DESCRIPTOR_NAME + "_" + targetDescriptor + "," + UploadPluginMeta.PLUGIN_META_TARGET_DESCRIPTOR_IMPLEMENTION + "_" + targetDescriptorImpl - + "," + IPropertyType.SubFormFilter.PLUGIN_META_SUB_FORM_FIELD + "_" + subFieldName + ",require"}; + + "," + SubFormFilter.PLUGIN_META_SUB_FORM_FIELD + "_" + subFieldName + ",require"}; pluginMetas = UploadPluginMeta.parse(plugins); assertEquals(1, pluginMetas.size()); meta = pluginMetas.get(0); assertTrue(meta.isRequired()); - Optional subFormFilter = meta.getSubFormFilter(); + Optional subFormFilter = meta.getSubFormFilter(); assertTrue(subFormFilter.isPresent()); - IPropertyType.SubFormFilter filter = subFormFilter.get(); + SubFormFilter filter = subFormFilter.get(); assertEquals(targetDescriptor, filter.targetDesc.descDisplayName); assertEquals(targetDescriptorImpl, filter.targetDesc.impl); diff --git a/tis-plugin/src/test/resources/com/qlangtech/tis/plugin/datax/transformer/impl/transformer-rule-sample.json b/tis-plugin/src/test/resources/com/qlangtech/tis/plugin/datax/transformer/impl/transformer-rule-sample.json index 0967080e9..f628baaad 100644 --- a/tis-plugin/src/test/resources/com/qlangtech/tis/plugin/datax/transformer/impl/transformer-rule-sample.json +++ b/tis-plugin/src/test/resources/com/qlangtech/tis/plugin/datax/transformer/impl/transformer-rule-sample.json @@ -1,12 +1,4 @@ [{ - "target": "sort_num", - "type": { - "columnSize": 19, - "decimalDigits": 0, - "type": -5, - "typeDesc": "bigint", - "typeName": "BIGINT" - }, "udf": { "updateModel": false, "impl": "com.qlangtech.tis.plugin.datax.transformer.impl.CopyValUDF", diff --git a/tis-plugin/src/test/resources/com/qlangtech/tis/plugin/datax/transformer/jdbcprop/pain-target-column.json b/tis-plugin/src/test/resources/com/qlangtech/tis/plugin/datax/transformer/jdbcprop/pain-target-column.json new file mode 100644 index 000000000..e69de29bb diff --git a/tis-plugin/src/test/resources/com/qlangtech/tis/plugin/datax/transformer/jdbcprop/target-col-type.json b/tis-plugin/src/test/resources/com/qlangtech/tis/plugin/datax/transformer/jdbcprop/target-col-type.json new file mode 100644 index 000000000..f70e5291e --- /dev/null +++ b/tis-plugin/src/test/resources/com/qlangtech/tis/plugin/datax/transformer/jdbcprop/target-col-type.json @@ -0,0 +1,16 @@ +{ + "name": { + "impl": "com.qlangtech.tis.plugin.datax.transformer.impl.ExistTargetCoumn", + "literia": [ + "replace col:user_name" + ], + "name": "user_name" + }, + "type": { + "columnSize": 32, + "decimalDigits": -1, + "type": 12, + "typeDesc": "varchar(32)", + "typeName": "VARCHAR" + } +} diff --git a/tis-plugin/src/test/resources/com/qlangtech/tis/plugin/datax/transformer/record-transformer-rules-descriptor.json b/tis-plugin/src/test/resources/com/qlangtech/tis/plugin/datax/transformer/record-transformer-rules-descriptor.json index 9e26dfeeb..c85391c53 100644 --- a/tis-plugin/src/test/resources/com/qlangtech/tis/plugin/datax/transformer/record-transformer-rules-descriptor.json +++ b/tis-plugin/src/test/resources/com/qlangtech/tis/plugin/datax/transformer/record-transformer-rules-descriptor.json @@ -1 +1,345 @@ -{} \ No newline at end of file +{ + "com.qlangtech.tis.plugin.datax.transformer.impl.CopyValUDF":{ + "attrs":[ + { + "describable":false, + "eprops":{ + "enum":[ + + ] + }, + "key":"from", + "ord":1, + "pk":false, + "required":true, + "type":5 + }, + { + "describable":false, + "eprops":{ + "elementCreator":"com.qlangtech.tis.plugin.datax.transformer.jdbcprop.JdbcPropertyElementCreatorFactory", + "enum":{ + "colMetas":[ + { + "containColSize":false, + "containDecimalRange":false, + "type":{ + "columnSize":-1, + "decimalDigits":-1, + "type":-6, + "typeDesc":"tinyint", + "typeName":"TINYINT" + } + }, + { + "containColSize":false, + "containDecimalRange":false, + "type":{ + "columnSize":-1, + "decimalDigits":-1, + "type":6, + "typeDesc":"float", + "typeName":"FLOAT" + } + }, + { + "containColSize":false, + "containDecimalRange":false, + "type":{ + "columnSize":-1, + "decimalDigits":-1, + "type":7, + "typeDesc":"float", + "typeName":"REAL" + } + }, + { + "colsSizeRange":{ + "max":2000, + "min":1 + }, + "containColSize":true, + "containDecimalRange":false, + "type":{ + "columnSize":32, + "decimalDigits":-1, + "type":12, + "typeDesc":"varchar(32)", + "typeName":"VARCHAR" + } + }, + { + "colsSizeRange":{ + "max":46, + "min":1 + }, + "containColSize":true, + "containDecimalRange":true, + "decimalRange":{ + "max":20, + "min":1 + }, + "type":{ + "columnSize":20, + "decimalDigits":0, + "type":3, + "typeDesc":"decimal(20,0)", + "typeName":"DECIMAL" + } + }, + { + "containColSize":false, + "containDecimalRange":false, + "type":{ + "columnSize":-1, + "decimalDigits":-1, + "type":-7, + "typeDesc":"bit", + "typeName":"BIT" + } + }, + { + "containColSize":false, + "containDecimalRange":false, + "type":{ + "columnSize":-1, + "decimalDigits":-1, + "type":-16, + "typeDesc":"varchar(-1)", + "typeName":"LONGNVARCHAR" + } + }, + { + "containColSize":false, + "containDecimalRange":false, + "type":{ + "columnSize":-1, + "decimalDigits":-1, + "type":92, + "typeDesc":"time", + "typeName":"TIME" + } + }, + { + "colsSizeRange":{ + "max":2000, + "min":1 + }, + "containColSize":true, + "containDecimalRange":false, + "type":{ + "columnSize":1000, + "decimalDigits":-1, + "type":-4, + "typeDesc":"blob(1000)", + "typeName":"LONGVARBINARY" + } + }, + { + "containColSize":false, + "containDecimalRange":false, + "type":{ + "columnSize":-1, + "decimalDigits":-1, + "type":-1, + "typeDesc":"varchar(-1)", + "typeName":"LONGVARCHAR" + } + }, + { + "containColSize":false, + "containDecimalRange":false, + "type":{ + "columnSize":-1, + "decimalDigits":-1, + "type":4, + "typeDesc":"int", + "typeName":"INTEGER" + } + }, + { + "containColSize":false, + "containDecimalRange":false, + "type":{ + "columnSize":-1, + "decimalDigits":-1, + "type":1, + "typeDesc":"varchar(-1)", + "typeName":"CHAR" + } + }, + { + "colsSizeRange":{ + "max":2000, + "min":1 + }, + "containColSize":true, + "containDecimalRange":false, + "type":{ + "columnSize":32, + "decimalDigits":-1, + "type":1111, + "typeDesc":"varchar(32)", + "typeName":"OTHER" + } + }, + { + "containColSize":false, + "containDecimalRange":false, + "type":{ + "columnSize":-1, + "decimalDigits":-1, + "type":-5, + "typeDesc":"bigint", + "typeName":"BIGINT" + } + }, + { + "colsSizeRange":{ + "max":4000, + "min":1 + }, + "containColSize":true, + "containDecimalRange":false, + "type":{ + "columnSize":1000, + "decimalDigits":-1, + "type":-3, + "typeDesc":"blob(1000)", + "typeName":"VARBINARY" + } + }, + { + "containColSize":false, + "containDecimalRange":false, + "type":{ + "columnSize":-1, + "decimalDigits":-1, + "type":93, + "typeDesc":"timestamp", + "typeName":"TIMESTAMP" + } + }, + { + "containColSize":false, + "containDecimalRange":false, + "type":{ + "columnSize":-1, + "decimalDigits":-1, + "type":16, + "typeDesc":"bool", + "typeName":"BOOLEAN" + } + }, + { + "containColSize":false, + "containDecimalRange":false, + "type":{ + "columnSize":-1, + "decimalDigits":-1, + "type":91, + "typeDesc":"date", + "typeName":"DATE" + } + }, + { + "colsSizeRange":{ + "max":2000, + "min":1 + }, + "containColSize":true, + "containDecimalRange":false, + "type":{ + "columnSize":1000, + "decimalDigits":-1, + "type":-2, + "typeDesc":"blob(1000)", + "typeName":"BINARY" + } + }, + { + "colsSizeRange":{ + "max":2000, + "min":1 + }, + "containColSize":true, + "containDecimalRange":false, + "type":{ + "columnSize":1000, + "decimalDigits":-1, + "type":2004, + "typeDesc":"blob(1000)", + "typeName":"BLOB" + } + }, + { + "containColSize":false, + "containDecimalRange":false, + "type":{ + "columnSize":-1, + "decimalDigits":-1, + "type":5, + "typeDesc":"smallint", + "typeName":"SMALLINT" + } + }, + { + "containColSize":false, + "containDecimalRange":false, + "type":{ + "columnSize":-1, + "decimalDigits":-1, + "type":8, + "typeDesc":"double", + "typeName":"DOUBLE" + } + }, + { + "colsSizeRange":{ + "max":46, + "min":1 + }, + "containColSize":true, + "containDecimalRange":true, + "decimalRange":{ + "max":20, + "min":1 + }, + "type":{ + "columnSize":20, + "decimalDigits":0, + "type":2, + "typeDesc":"decimal(20,0)", + "typeName":"NUMERIC" + } + } + ], + "isList":false, + "tabMapper":[ + + ], + "viewContentType":"jdbcTypeProps" + }, + "viewtype":"tuplelist" + }, + "key":"to", + "ord":2, + "pk":false, + "required":true, + "type":8 + } + ], + "containAdvance":false, + "displayName":"CopyVal", + "extendPoint":"com.qlangtech.tis.plugin.datax.transformer.UDFDefinition", + "extractProps":{ + "notebook":{ + "ability":false, + "activate":false + } + }, + "impl":"com.qlangtech.tis.plugin.datax.transformer.impl.CopyValUDF", + "implUrl":"http://tis.pub/docs/plugin/plugins/#comqlangtechtisplugindataxtransformerimplcopyvaludf", + "veriflable":false + } +} \ No newline at end of file