optimize systetm initialization process

tis-builder-api/src/main/java/com/qlangtech/tis/extension/model/UpdateCenterResource.java

@@ -49,11 +49,10 @@ public static URL getTISReleaseRes(String ver, String subPath, String res) {
  /**
  * 取得 TIS的预置包
  *
- * @param ver
  * @param pkgName
  * @return
  */
- public static URL getTISTarPkg(String ver, String pkgName) {
- return getTISReleaseRes(ver, "/tis", pkgName);
+ public static URL getTISTarPkg(String pkgName) {
+ return getTISReleaseRes(TisMetaProps.getInstance().getVersion(), "/tis", pkgName);
  }
 }

tis-console/src/main/java/com/qlangtech/tis/runtime/module/action/SysInitializeAction.java

@@ -17,44 +17,58 @@
  */
 package com.qlangtech.tis.runtime.module.action;
 
+import com.alibaba.citrus.turbine.Context;
 import com.google.common.collect.Lists;
-import com.qlangtech.tis.manage.biz.dal.dao.*;
-import com.qlangtech.tis.manage.biz.dal.pojo.*;
-import com.qlangtech.tis.manage.common.*;
+import com.qlangtech.tis.extension.model.UpdateCenterResource;
+import com.qlangtech.tis.manage.biz.dal.pojo.Application;
+import com.qlangtech.tis.manage.biz.dal.pojo.Department;
+import com.qlangtech.tis.manage.biz.dal.pojo.ServerGroupCriteria;
+import com.qlangtech.tis.manage.biz.dal.pojo.Snapshot;
+import com.qlangtech.tis.manage.biz.dal.pojo.SnapshotCriteria;
+import com.qlangtech.tis.manage.biz.dal.pojo.UsrDptRelationCriteria;
+import com.qlangtech.tis.manage.common.Config;
+import com.qlangtech.tis.manage.common.ConfigFileContext.StreamProcess;
+import com.qlangtech.tis.manage.common.ConfigFileReader;
+import com.qlangtech.tis.manage.common.HttpUtils;
+import com.qlangtech.tis.manage.common.MockContext;
+import com.qlangtech.tis.manage.common.TisUTF8;
 import com.qlangtech.tis.manage.spring.TISDataSourceFactory;
 import com.qlangtech.tis.pubhook.common.RunEnvironment;
 import com.qlangtech.tis.runtime.module.action.UploadJarAction.ConfigContentGetter;
 import org.apache.commons.io.FileUtils;
 import org.apache.commons.io.IOUtils;
 import org.apache.commons.io.LineIterator;
 import org.apache.commons.lang.StringUtils;
+import org.apache.tools.tar.TarEntry;
+import org.apache.tools.tar.TarInputStream;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
-import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.beans.factory.config.AutowireCapableBeanFactory;
 import org.springframework.context.ApplicationContext;
 import org.springframework.context.support.ClassPathXmlApplicationContext;
 
 import java.io.File;
 import java.io.IOException;
 import java.io.InputStream;
+import java.net.URL;
 import java.sql.Connection;
 import java.sql.SQLException;
 import java.sql.Statement;
 import java.util.Date;
 import java.util.List;
+import java.util.Map;
 import java.util.Objects;
 import java.util.regex.Pattern;
 import java.util.stream.Collectors;
+import java.util.zip.GZIPInputStream;
 
 /**
  * 系统初始化
  *
  * @author 百岁（[email protected]）
  * @date 2019年1月17日
  */
-public class SysInitializeAction //extends BasicModule
-{
+public class SysInitializeAction extends BasicModule {
 
  private static final long serialVersionUID = 1L;
  private static final Logger logger = LoggerFactory.getLogger(SysInitializeAction.class);
@@ -164,7 +178,7 @@ public static void systemDataInitialize() throws Exception {
  "classpath:/tis.application.context.xml", "classpath:/tis.application.mockable.context.xml");
  appContext.getAutowireCapableBeanFactory().autowireBeanProperties(
  initAction, AutowireCapableBeanFactory.AUTOWIRE_BY_NAME, false);
- initAction.doInit();
+ initAction.doInit(MockContext.instance);
  }
 
  /**
@@ -203,20 +217,22 @@ private static List<String> convert2BatchSql(TISDataSourceFactory.SystemDBInit d
  return batchs;
  }
 
- public void doInit() throws Exception {
+ public void doInit(Context context) throws Exception {
 
  // final File sysInitializedToken = new File(Config.getDataDir(), "system_initialized_token");
  if (isSysInitialized()) {
  throw new IllegalStateException("tis has initialized:" + getSysInitializedTokenFile().getAbsolutePath());
  }
 
-// logger.info("needZkInit:{}", needZkInit);
-//
-// if (needZkInit && !initializeZkPath(Config.getZKHost())) {
-// // 初始化ZK失败
-// logger.warn("ZkInit falid,zkAddress:{}", Config.getZKHost());
-// return;
-// }
+ /**
+ * 下载data目录
+ */
+ copyDataTarToLocal();
+
+ /**
+ * 取得SQL并且初始化
+ */
+
 
  UsrDptRelationCriteria c = new UsrDptRelationCriteria();
  c.createCriteria().andUsrIdEqualTo(ADMIN_ID);
@@ -228,17 +244,53 @@ public void doInit() throws Exception {
  }
  // 添加一个系统管理员
  this.getUsrDptRelationDAO().addAdminUser();
-
  this.initializeDepartment();
-
  this.initializeAppAndSchema();
+
  touchSysInitializedToken();
  }
 
- private IUsrDptRelationDAO getUsrDptRelationDAO() {
- return this.getDaoContext().getUsrDptRelationDAO();
+ void copyDataTarToLocal() throws IOException {
+ final String dataPkgName = "tis-data.tar.gz";
+ URL dataPkg = UpdateCenterResource.getTISTarPkg(dataPkgName);
+ File tmpDataDir = FileUtils.getTempDirectory();// Config.getDataDir();
+ HttpUtils.get(dataPkg, new StreamProcess<Void>() {
+ @Override
+ public Void p(int status, InputStream stream, Map<String, List<String>> headerFields) throws IOException {
+ try (GZIPInputStream gis = new GZIPInputStream(stream);
+ TarInputStream tis = new TarInputStream(gis)) {
+ TarEntry entry = null;
+ while ((entry = tis.getNextEntry()) != null) {
+ if (entry.isDirectory()) {
+ File dir = new File(tmpDataDir, entry.getName());
+ if (!dir.exists()) {
+ if (!dir.mkdirs()) {
+ throw new IOException("Failed to create directory: " + dir.getAbsolutePath());
+ }
+ }
+ } else {
+ File file = new File(tmpDataDir, entry.getName());
+ FileUtils.copyToFile(tis, file);
+ logger.info("write file:{}", file.getAbsolutePath());
+ }
+ }
+ return null;
+ }
+ }
+ }
+ );
+ File tmp = new File(tmpDataDir, "data");
+ final File dataDir = Config.getDataDir();
+ logger.info("move to:{} from:{}", dataDir.getAbsolutePath(), tmp);
+ FileUtils.deleteDirectory(dataDir);
+ FileUtils.moveDirectory(tmp, dataDir);
+
  }
 
+// private IUsrDptRelationDAO getUsrDptRelationDAO() {
+// return this.getDaoContext().getUsrDptRelationDAO();
+// }
+
  public void initializeAppAndSchema() throws IOException {
  this.getApplicationDAO().deleteByPrimaryKey(TEMPLATE_APPLICATION_DEFAULT_ID);
  SnapshotCriteria snapshotQuery = new SnapshotCriteria();
@@ -498,34 +550,9 @@ void initializeDepartment() {
  }
  }
 
- public IApplicationDAO getApplicationDAO() {
- return getDaoContext().getApplicationDAO();
- }
-
- public IServerGroupDAO getServerGroupDAO() {
- return getDaoContext().getServerGroupDAO();
- }
-
- public ISnapshotDAO getSnapshotDAO() {
- return getDaoContext().getSnapshotDAO();
- }
-
- public IUploadResourceDAO getUploadResourceDAO() {
- return getDaoContext().getUploadResourceDAO();
- }
-
- public IDepartmentDAO getDepartmentDAO() {
- return getDaoContext().getDepartmentDAO();
- }
-
- private RunContextGetter daoContextGetter;
+// private RunContext getDaoContext() {
+// return daoContextGetter.get();
+// }
 
- private RunContext getDaoContext() {
- return daoContextGetter.get();
- }
 
- @Autowired
- public final void setRunContextGetter(RunContextGetter daoContextGetter) {
- this.daoContextGetter = daoContextGetter;
- }
 }

tis-console/src/main/java/com/qlangtech/tis/runtime/module/action/SysInitializeZookeeper.java

@@ -1,25 +1,23 @@
 /**
- *  Licensed to the Apache Software Foundation (ASF) under one
- *  or more contributor license agreements. See the NOTICE file
- *  distributed with this work for additional information
- *  regarding copyright ownership. The ASF licenses this file
- *  to you under the Apache License, Version 2.0 (the
- *  "License"); you may not use this file except in compliance
- *  with the License. You may obtain a copy of the License at
- *
- *  http:https://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http:https://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  */
 
 package com.qlangtech.tis.runtime.module.action;
 
-import com.qlangtech.tis.manage.common.Config;
-
 /**
  * @author: 百岁（[email protected]）
  * @create: 2021-07-25 15:19

tis-console/src/test/java/com/qlangtech/tis/runtime/module/action/TestSysInitializeAction.java

@@ -1,19 +1,19 @@
 /**
- *  Licensed to the Apache Software Foundation (ASF) under one
- *  or more contributor license agreements. See the NOTICE file
- *  distributed with this work for additional information
- *  regarding copyright ownership. The ASF licenses this file
- *  to you under the Apache License, Version 2.0 (the
- *  "License"); you may not use this file except in compliance
- *  with the License. You may obtain a copy of the License at
- *
- *  http:https://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http:https://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  */
 package com.qlangtech.tis.runtime.module.action;
 
@@ -28,6 +28,7 @@
 import org.easymock.EasyMock;
 
 import java.io.File;
+import java.io.IOException;
 import java.util.concurrent.atomic.AtomicBoolean;
 
 /**
@@ -54,6 +55,12 @@ public void setUp() throws Exception {
 //
 // }
 
+ public void testCopyDataTarToLocal() throws IOException {
+ Config.setTestDataDir();
+ SysInitializeAction action = new SysInitializeAction();
+ action.copyDataTarToLocal();
+ }
+
  /**
  * 测试系统数据库初始化
  */
@@ -65,7 +72,7 @@ public void testSystemDBInitializWithDerby() throws Exception {
  Config config = this.mock("config", Config.class);
 
 
- // EasyMock.expect(config.getZkHost()).andReturn("192.168.28.200:2181/tis/cloud");
+  // EasyMock.expect(config.getZkHost()).andReturn("192.168.28.200:2181/tis/cloud");
  EasyMock.expect(config.getRuntime()).andReturn(RunEnvironment.DAILY.getKeyName()).anyTimes();
  Config.TisDbConfig mockDBType = new Config.TisDbConfig();
  mockDBType.dbtype = Config.DB_TYPE_DERBY;

tis-manage-pojo/src/main/java/com/qlangtech/tis/manage/common/ConfigFileContext.java

@@ -227,7 +227,7 @@ public T p(HttpURLConnection conn, InputStream stream) throws IOException {
  * @param
  * @return
  */
- public abstract T p(int status, InputStream stream, Map<String, List<String>> headerFields);
+ public abstract T p(int status, InputStream stream, Map<String, List<String>> headerFields) throws IOException;
 
  public void error(int status, InputStream errstream, IOException e) throws Exception {
  logger.error("error code:" + status + "\n" + IOUtils.toString(errstream, TisUTF8.get()));

tis-plugin/src/main/java/com/qlangtech/tis/extension/Descriptor.java

@@ -37,6 +37,7 @@
 import com.qlangtech.tis.extension.util.PluginExtraProps;
 import com.qlangtech.tis.manage.common.Config;
 import com.qlangtech.tis.manage.common.Option;
+import com.qlangtech.tis.plugin.IDataXEndTypeGetter;
 import com.qlangtech.tis.plugin.IEndTypeGetter;
 import com.qlangtech.tis.plugin.IEndTypeGetter.EndType;
 import com.qlangtech.tis.plugin.IEndTypeGetter.IEndType;
@@ -1239,14 +1240,23 @@ protected final void registerSelectOptions(String fieldName, Callable<List<? ext
  public final List<SelectOption> getSelectOptions(String name) {
  Callable<List<? extends IdentityName>> opsCallable = selectOptsRegister.get(name);
  if (opsCallable == null) {
- throw new IllegalStateException("fieldName:" + name + " is select options has not been register,class:" + this.getClass().getName() + ",has registed:" + selectOptsRegister.keySet().stream().collect(Collectors.joining(",")));
+ throw new IllegalStateException("fieldName:" + name + " is select options has not been register,class:"
+ + this.getClass().getName()
+ + ",has registed:" + selectOptsRegister.keySet().stream().collect(Collectors.joining(",")));
  }
  try {
  List<? extends IdentityName> opts = opsCallable.call();
  if (opts == null) {
  return Collections.emptyList();
  }
- return opts.stream().map((r) -> new SelectOption(r.identityValue(), r.getDescribleClass())).collect(Collectors.toList());
+ return opts.stream().map((r) -> {
+ Descriptor desc = null;
+ EndType endType = null;
+ if ((desc = ((Describable) r).getDescriptor()) instanceof IEndTypeGetter) {
+ endType = ((IEndTypeGetter) desc).getEndType();
+ }
+ return new SelectOption(r.identityValue(), r.getDescribleClass(), endType);
+ }).collect(Collectors.toList());
  } catch (Exception e) {
  throw new RuntimeException("field name:" + name + ",class:" + this.getClass().getName(), e);
  }
@@ -1257,10 +1267,19 @@ public static class SelectOption {
  private final String name;
 
  private final Class<?> implClass;
+ private final IDataXEndTypeGetter.EndType endType;
 
- public SelectOption(String name, Class<?> implClass) {
+ public SelectOption(String name, Class<?> implClass, IDataXEndTypeGetter.EndType endType) {
  this.name = name;
  this.implClass = implClass;
+ this.endType = endType;
+ }
+
+ public String getEndType() {
+ if (this.endType != null) {
+ return this.endType.getVal();
+ }
+ return null;
  }
 
  public String getName() {