add transformer for TIS adapter in reltine flink module

datax-config/src/main/java/com/alibaba/datax/common/element/ColumnAwareRecord.java

@@ -0,0 +1,53 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http:https://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.alibaba.datax.common.element;
+
+import java.util.Map;
+
+/**
+ *
+ * @author: 百岁（[email protected]）
+ * @create: 2024-06-18 17:40
+ **/
+public interface ColumnAwareRecord<ColValType> {
+
+ /**
+ * 设置列名称到列所在index的位置
+ *
+ * @param mapper
+ */
+ public void setCol2Index(Map<String, Integer> mapper);
+
+ /**
+ * @param field 字段名称
+ * @param colVal
+ */
+ public void setColumn(String field, final ColValType colVal);
+
+
+ public void setString(String field, final String val);
+
+ /**
+ * @param field 字段名称
+ * @return
+ */
+ public ColValType getColumn(String field);
+
+ public String getString(String field);
+}

datax-config/src/main/java/com/alibaba/datax/common/element/Record.java

@@ -1,42 +1,51 @@
 /**
- *  Licensed to the Apache Software Foundation (ASF) under one
- *  or more contributor license agreements. See the NOTICE file
- *  distributed with this work for additional information
- *  regarding copyright ownership. The ASF licenses this file
- *  to you under the Apache License, Version 2.0 (the
- *  "License"); you may not use this file except in compliance
- *  with the License. You may obtain a copy of the License at
- *
- *  http:https://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http:https://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  */
 
 package com.alibaba.datax.common.element;
 
-import com.alibaba.datax.common.element.Column;
+import java.util.Map;
+
 /**
  * Created by jingxing on 14-8-24.
  */
 
-public interface Record {
+public interface Record extends ColumnAwareRecord<Column> {
+
+
+ public void addColumn(Column column);
+
+ public void setColumn(int i, final Column column);
 
- public void addColumn(Column column);
 
- public void setColumn(int i, final Column column);
+ public Column getColumn(int i);
 
- public Column getColumn(int i);
+ @Override
+ default String getString(String field) {
+ Column colVal = getColumn(field);
+ return colVal != null ? colVal.asString() : null;
+ }
 
- public String toString();
+ public String toString();
 
- public int getColumnNumber();
+ public int getColumnNumber();
 
- public int getByteSize();
+ public int getByteSize();
 
- public int getMemorySize();
+ public int getMemorySize();
 
 }

datax-config/src/main/java/com/alibaba/datax/core/job/IJobContainerContext.java

@@ -3,10 +3,12 @@
 import com.qlangtech.tis.datax.IDataXNameAware;
 import com.qlangtech.tis.datax.IDataXTaskRelevant;
 
+import java.util.Optional;
+
 /**
  * @author: 百岁（[email protected]）
  * @create: 2023-02-23 10:06
  **/
 public interface IJobContainerContext extends IDataXTaskRelevant, IDataXNameAware {
-
+ <T extends ITransformerBuildInfo> Optional<T> getTransformerBuildCfg();
 }

tis-plugin/src/main/java/com/qlangtech/tis/plugin/datax/transformer/jdbcprop/VirtualColType.java → datax-config/src/main/java/com/alibaba/datax/core/job/ITransformerBuildInfo.java

@@ -16,24 +16,15 @@
  * limitations under the License.
  */
 
-package com.qlangtech.tis.plugin.datax.transformer.jdbcprop;
+package com.alibaba.datax.core.job;
 
-import com.qlangtech.tis.plugin.ds.TypeBase;
+import java.util.List;
 
 /**
+ *
  * @author: 百岁（[email protected]）
- * @create: 2024-06-10 10:14
+ * @create: 2024-06-15 12:34
  **/
-public class VirtualColType extends TypeBase {
-
- private String name;
-
- @Override
- public String getName() {
- return this.name;
- }
-
- public void setName(String name) {
- this.name = name;
- }
+public interface ITransformerBuildInfo {
+ List<String> relevantOutterColKeys();
 }

datax-config/src/main/java/com/alibaba/datax/core/util/container/TransformerConstant.java

@@ -0,0 +1,11 @@
+package com.alibaba.datax.core.util.container;
+
+/**
+ * @author: 百岁（[email protected]）
+ * @create: 2024-06-15 08:22
+ **/
+public interface TransformerConstant {
+ String JOB_TRANSFORMER = "transformer";
+ String JOB_TRANSFORMER_NAME = "name";
+ String JOB_TRANSFORMER_RELEVANT_KEYS = "cols";
+}

datax-config/src/main/java/com/qlangtech/tis/datax/IDataXNameAware.java

@@ -18,16 +18,30 @@
  * limitations under the License.
  */
 
+import org.apache.commons.lang3.StringUtils;
+
 /**
  * @author: 百岁（[email protected]）
  * @create: 2023-02-23 10:35
  **/
 public interface IDataXNameAware {
+ /**
+ * 是否在索引
+ *
+ * @return
+ */
+ default boolean isCollectionAware() {
+ return StringUtils.isNotEmpty(getCollectionName());
+ }
+
+ String getCollectionName();
 
  /**
  * 取得DataX名称
  *
  * @return
  */
- public String getTISDataXName();
+ public default String getTISDataXName() {
+ return this.getCollectionName();
+ }
 }

tis-base-test/src/main/java/com/qlangtech/tis/test/TISEasyMock.java

@@ -17,15 +17,13 @@
  */
 package com.qlangtech.tis.test;
 
-import org.junit.Before;
-
 /**
  * @author 百岁（[email protected]）
  * @date 2021-03-04 12:41
  */
 public interface TISEasyMock {
 
- @Before
+
  default void clearMocks() {
  EasyMockUtil.clearMocks();
  }

tis-builder-api/src/main/java/com/qlangtech/tis/realtime/transfer/UnderlineUtils.java

@@ -1,19 +1,19 @@
 /**
- *  Licensed to the Apache Software Foundation (ASF) under one
- *  or more contributor license agreements. See the NOTICE file
- *  distributed with this work for additional information
- *  regarding copyright ownership. The ASF licenses this file
- *  to you under the Apache License, Version 2.0 (the
- *  "License"); you may not use this file except in compliance
- *  with the License. You may obtain a copy of the License at
- *
- *  http:https://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http:https://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  */
 package com.qlangtech.tis.realtime.transfer;
 
@@ -48,13 +48,28 @@ public static StringBuffer removeUnderline(String value) {
  }
 
  public static StringBuffer addUnderline(String value) {
+ //return UnderlineUtils.addUnderline(value);
  StringBuffer parsedName = new StringBuffer();
  char[] nameAry = value.toCharArray();
+ boolean firstAppend = true;
+ boolean previousUnderline = false;
  for (int i = 0; i < nameAry.length; i++) {
  if (Character.isUpperCase(nameAry[i])) {
- parsedName.append('_').append(Character.toLowerCase(nameAry[i]));
+ if (firstAppend) {
+ parsedName.append(Character.toLowerCase(nameAry[i]));
+ firstAppend = false;
+ } else {
+ if (!previousUnderline) {
+ parsedName.append('_');
+ }
+ parsedName.append(Character.toLowerCase(nameAry[i]));
+ previousUnderline = true;
+ }
  } else {
  parsedName.append(nameAry[i]);
+ firstAppend = false;
+ previousUnderline = false;
+ // .append(Character.toLowerCase());
  }
  }
  return parsedName;

tis-builder-api/src/test/java/com/qlangtech/tis/TestAll.java

@@ -20,6 +20,7 @@
 
 import com.qlangtech.tis.fs.TestIPath;
 import com.qlangtech.tis.manage.common.TestConfig;
+import com.qlangtech.tis.realtime.transfer.TestUnderlineUtils;
 import com.qlangtech.tis.utils.TestTisMetaProps;
 import com.qlangtech.tis.utils.TestUtils;
 import junit.framework.Test;
@@ -41,6 +42,7 @@ public static Test suite() {
  suite.addTestSuite(TestUtils.class);
  suite.addTestSuite(TestTisMetaProps.class);
  suite.addTestSuite(TestIPath.class);
+ suite.addTestSuite(TestUnderlineUtils.class);
 
  return suite;
  }

tis-builder-api/src/test/java/com/qlangtech/tis/realtime/transfer/TestUnderlineUtils.java

@@ -0,0 +1,34 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http:https://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.qlangtech.tis.realtime.transfer;
+
+import junit.framework.TestCase;
+import org.junit.Assert;
+
+/**
+ * @author: 百岁（[email protected]）
+ * @create: 2024-06-18 10:18
+ **/
+public class TestUnderlineUtils extends TestCase {
+
+ public void testAddUnderline() {
+ StringBuffer copyValUDF = UnderlineUtils.addUnderline("CopyValUDF");
+ Assert.assertEquals("copy_val_udf", copyValUDF.toString());
+ }
+}