modify dftUsingClusterIP default value setter

datavane · Mar 28, 2024 · eb73caa · eb73caa
1 parent 04167e2
commit eb73caa
Show file tree

Hide file tree

Showing 4 changed files with 58 additions and 24 deletions.
diff --git a/tis-console.yaml b/tis-console.yaml
@@ -1,3 +1,4 @@
+# export KUBECONFIG=~/.kube/aliyun
 apiVersion: v1
 kind: Service
 metadata:
@@ -152,7 +153,7 @@ spec:
  mountPath: /opt/app/tis-uber/tis-assemble/conf/tis-web-config/
  - name: tis-console-pvc
  mountPath: "/opt/data"
- image: registry.cn-hangzhou.aliyuncs.com/tis/tis-console:4.0.0.2
+ image: registry.cn-hangzhou.aliyuncs.com/tis/tis-console:4.0.0.5
  # command: [ "/bin/sh", "-c", "sleep 1000000" ]
  ports:
  - name: tis-8080

diff --git a/tis-console/src/main/java/com/qlangtech/tis/runtime/module/action/SysInitializeAction.java b/tis-console/src/main/java/com/qlangtech/tis/runtime/module/action/SysInitializeAction.java
@@ -88,7 +88,6 @@ public class SysInitializeAction extends BasicModule {
  private static Boolean _isSysInitialized;
 
 
-
  /**
  * 当使用TIS作为docker容器启动，本地data目录作为容器卷，初始状态是空的，需要将空的卷初始化
  *
@@ -347,11 +346,16 @@ public Void p(int status, InputStream stream, Map<String, List<String>> headerFi
  for (String c : tmp.list()) {
  File child = new File(tmp, c);
  File dest = new File(dataDir, c);
+ if (dest.exists()) {
+ logger.info("dest :{} is already exist ,skip it", dest.getAbsolutePath());
+ continue;
+ }
  if (child.isFile()) {
  FileUtils.moveFile(child, dest);
  } else {
  FileUtils.moveDirectory(child, dest);
  }
+ logger.info("successful move {} to dest :{}", child.getAbsolutePath(), dest.getAbsolutePath());
  }
 
  }

diff --git a/tis-plugin/src/main/java/com/qlangtech/tis/datax/impl/DataXCfgGenerator.java b/tis-plugin/src/main/java/com/qlangtech/tis/datax/impl/DataXCfgGenerator.java
@@ -25,7 +25,16 @@
 import com.google.common.collect.Lists;
 import com.google.common.collect.Maps;
 import com.google.common.collect.Sets;
-import com.qlangtech.tis.datax.*;
+import com.qlangtech.tis.datax.DataXJobInfo;
+import com.qlangtech.tis.datax.IDataXPluginMeta;
+import com.qlangtech.tis.datax.IDataxContext;
+import com.qlangtech.tis.datax.IDataxGlobalCfg;
+import com.qlangtech.tis.datax.IDataxProcessor;
+import com.qlangtech.tis.datax.IDataxReader;
+import com.qlangtech.tis.datax.IDataxReaderContext;
+import com.qlangtech.tis.datax.IDataxWriter;
+import com.qlangtech.tis.datax.IGroupChildTaskIterator;
+import com.qlangtech.tis.datax.TableAliasMapper;
 import com.qlangtech.tis.manage.common.TisUTF8;
 import com.qlangtech.tis.offline.DataxUtils;
 import com.qlangtech.tis.plugin.datax.CreateTableSqlBuilder;
@@ -43,7 +52,13 @@
 import java.io.File;
 import java.io.IOException;
 import java.io.StringWriter;
-import java.util.*;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Properties;
+import java.util.Set;
 import java.util.concurrent.atomic.AtomicReference;
 import java.util.stream.Collectors;
 
@@ -524,25 +539,10 @@ public void setGenTime(long genTime) {
  }
  }
 
- private IDataxProcessor.TableMap createTableMap(TableAliasMapper tabAlias, Map<String, ISelectedTab> selectedTabs
+ private IDataxProcessor.TableMap createTableMap(
+ TableAliasMapper tabAlias, Map<String, ISelectedTab> selectedTabs
  , IDataxReaderContext readerContext) {
-
  return readerContext.createTableMap(tabAlias, selectedTabs);
-
-// TableAlias tableAlias = tabAlias.get(readerContext.getSourceTableName());
-// if (tableAlias == null) {
-// throw new IllegalStateException("sourceTable:" + readerContext.getSourceTableName() + " can not find " +
-// "relevant 'tableAlias' keys:[" + tabAlias.getFromTabDesc() + "]");
-// }
-// ISelectedTab selectedTab = selectedTabs.get(readerContext.getSourceTableName());
-// if (selectedTab == null) {
-// throw new IllegalStateException("sourceTable:" + readerContext.getSourceTableName() + " can not find " +
-// "relevant '" + ISelectedTab.class.getSimpleName() + "' keys:[" + String.join(",",
-// selectedTabs.keySet()) + "]");
-// }
-// IDataxProcessor.TableMap tableMap = new IDataxProcessor.TableMap(tableAlias, selectedTab);
-//
-// return tableMap;
  }
 
  /**
@@ -551,8 +551,9 @@ private IDataxProcessor.TableMap createTableMap(TableAliasMapper tabAlias, Map<S
  * @return 生成的配置文件内容
  * @throws IOException
  */
- public String generateDataxConfig(IDataxReaderContext readerContext, IDataxWriter writer, IDataxReader reader,
- Optional<IDataxProcessor.TableMap> tableMap) throws IOException {
+ public String generateDataxConfig(
+ IDataxReaderContext readerContext, IDataxWriter writer, IDataxReader reader,
+ Optional<IDataxProcessor.TableMap> tableMap) throws IOException {
  Objects.requireNonNull(writer, "writer can not be null");
  StringWriter writerContent = null;
  final String tpl = getTemplateContent(reader, writer);
@@ -570,7 +571,7 @@ public String generateDataxConfig(IDataxReaderContext readerContext, IDataxWrite
  try {
  JSONObject cfg = JSON.parseObject(content);
  validatePluginName(writer, reader, cfg);
- return JsonUtil.toString(cfg);
+ return JsonUtil.toString(cfg, true);
  } catch (Exception e) {
  throw new RuntimeException(content, e);
  }

diff --git a/tis-test-mysql.yaml b/tis-test-mysql.yaml
@@ -0,0 +1,28 @@
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+ name: mysql
+spec:
+ replicas: 1
+ selector:
+ matchLabels:
+ app: mysql
+ serviceName: mysql-svc
+ template:
+ spec:
+ containers:
+ - env:
+ - name: MYSQL_ROOT_PASSWORD
+ value: "123456"
+ image: mysql:5.7
+ imagePullPolicy: IfNotPresent
+ name: mysql
+ ports:
+ - containerPort: 3306
+ name: mysql-3306
+ protocol: TCP
+ resources:
+ requests:
+ cpu: "2"
+ memory: 4Gi
+ terminationGracePeriodSeconds: 3