commit make elastic search useable

tis-console/src/main/java/com/qlangtech/tis/coredefine/module/action/DataxAction.java

@@ -55,6 +55,7 @@
 import com.qlangtech.tis.runtime.module.misc.IControlMsgHandler;
 import com.qlangtech.tis.runtime.module.misc.IFieldErrorHandler;
 import com.qlangtech.tis.runtime.module.misc.impl.DelegateControl4JsonPostMsgHandler;
+import com.qlangtech.tis.solrdao.ISchema;
 import com.qlangtech.tis.util.*;
 import com.qlangtech.tis.workflow.pojo.WorkFlowBuildHistory;
 import com.qlangtech.tis.workflow.pojo.WorkFlowBuildHistoryCriteria;
@@ -805,16 +806,30 @@ public void doGotoEsAppCreateConfirm(Context context) throws Exception {
  // 这里只做schema的校验
  CreateIndexConfirmModel confiemModel = parseJsonPost(CreateIndexConfirmModel.class);
  String schemaContent = null;
+ ISchema schema = null;
+ ISearchEngineTypeTransfer typeTransfer = ISearchEngineTypeTransfer.load(this, confiemModel.getDataxName());
  if (confiemModel.isExpertModel()) {
- schemaContent = confiemModel.getExpert().getXml();
+ CreateIndexConfirmModel.ExpertEditorModel expect = confiemModel.getExpert();
+ schemaContent = expect.getXml();
+ schema = typeTransfer.projectionFromExpertModel(expect.asJson());
  } else {
-
- ISearchEngineTypeTransfer typeTransfer = ISearchEngineTypeTransfer.load(this, confiemModel.getDataxName());
- schemaContent = typeTransfer.mergeFromStupidModel(confiemModel.getStupid().getModel()
+ schema = confiemModel.getStupid().getModel();
+ schemaContent = typeTransfer.mergeFromStupidModel(schema
  , ISearchEngineTypeTransfer.getOriginExpertSchema(null)).toJSONString();
  }
 
+ if (!schema.isValid()) {
+ for (String err : schema.getErrors()) {
+ this.addErrorMessage(context, err);
+ }
+ return;
+ }
+
+ DataxProcessor.DataXCreateProcessMeta processMeta = DataxProcessor.getDataXCreateProcessMeta(this, confiemModel.getDataxName());
+ List<ISelectedTab> selectedTabs = processMeta.getReader().getSelectedTabs();
  ESTableAlias esTableAlias = new ESTableAlias();
+ esTableAlias.setFrom(selectedTabs.stream().findFirst().get().getName());
+ esTableAlias.setTo(((ISearchEngineTypeTransfer) processMeta.getWriter()).getIndexName());
  esTableAlias.setSchemaContent(schemaContent);
 
  this.saveTableMapper(this, confiemModel.getDataxName(), Collections.singletonList(esTableAlias));

tis-console/src/main/java/com/qlangtech/tis/runtime/module/action/CreateIndexConfirmModel.java

@@ -1,24 +1,26 @@
 /**
- *  Licensed to the Apache Software Foundation (ASF) under one
- *  or more contributor license agreements. See the NOTICE file
- *  distributed with this work for additional information
- *  regarding copyright ownership. The ASF licenses this file
- *  to you under the Apache License, Version 2.0 (the
- *  "License"); you may not use this file except in compliance
- *  with the License. You may obtain a copy of the License at
- *
- *  https://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * https://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  */
 package com.qlangtech.tis.runtime.module.action;
 
+import com.alibaba.fastjson.JSONObject;
 import com.qlangtech.tis.coredefine.module.control.SelectableServer;
 import com.qlangtech.tis.coredefine.module.control.SelectableServer.CoreNode;
+import com.qlangtech.tis.datax.ISearchEngineTypeTransfer;
 import com.qlangtech.tis.fullbuild.indexbuild.LuceneVersion;
 import com.qlangtech.tis.runtime.module.action.AddAppAction.ExtendApp;
 import com.qlangtech.tis.runtime.module.action.SchemaAction.UploadSchemaWithRawContentForm;
@@ -150,6 +152,10 @@ public String getXml() {
  return this.xml;
  }
 
+ public JSONObject asJson() {
+ return ISearchEngineTypeTransfer.getOriginExpertSchema(getXml());
+ }
+
  public void setXml(String xml) {
  this.xml = xml;
  }

tis-console/src/main/java/com/qlangtech/tis/runtime/module/action/SchemaAction.java

@@ -91,11 +91,8 @@ public class SchemaAction extends BasicModule {
  protected static final String FIELD_Recept = "recept";
  protected static final String FIELD_DptId = "dptId";
  private static final long serialVersionUID = 1L;
-
  protected static final String INDEX_PREFIX = "search4";
 
- //private static Logger log = LoggerFactory.getLogger(SchemaAction.class);
-
  /**
  * DataX 创建流程中取得es的默认字段
  *
@@ -112,31 +109,45 @@ public void doGetEsTplFields(Context context) throws Exception {
 
  ISearchEngineTypeTransfer typeTransfer = ISearchEngineTypeTransfer.load(this, dataxName);
  DataxReader dataxReader = DataxReader.load(this, dataxName);
+ ISelectedTab esTab = null;
+ for (ISelectedTab tab : dataxReader.getSelectedTabs()) {
+ esTab = tab;
+ break;
+ }
+ Objects.requireNonNull(esTab, "esTab can not be null");
  // ESField field = null;
  if (stepType.update) {
  DataxProcessor dataxProcessor = DataxProcessor.load(this, dataxName);
 
  Optional<TableAlias> f = dataxProcessor.getTabAlias().findFirst();
  if (f.isPresent()) {
- writerStructFields(context, f.get(), typeTransfer);
- return;
+ TableAlias tabMapper = f.get();
+ if (StringUtils.equals(tabMapper.getFrom(), esTab.getName())) {
+ writerStructFields(context, tabMapper, typeTransfer);
+ return;
+ } else {
+ // 更新时，源表改变了重新初始化
+ writeStructFields(context, typeTransfer, esTab);
+ return;
+ }
  }
-// for (Map.Entry<String, TableAlias> e : dataxProcessor.getTabAlias().entrySet()) {
-// writerStructFields(context, e.getValue(), typeTransfer);
-// return;
-// }
+
  } else {
- for (ISelectedTab tab : dataxReader.getSelectedTabs()) {
- // ESSchema parseResult = new ESSchema();
- SchemaMetaContent tplSchema = typeTransfer.initSchemaMetaContent(tab);
- this.setBizResult(context, tplSchema.toJSON());
- return;
- }
+// for (ISelectedTab tab : dataxReader.getSelectedTabs()) {
+ // ESSchema parseResult = new ESSchema();
+ writeStructFields(context, typeTransfer, esTab);
+ return;
+ //}
  }
 
  throw new IllegalStateException("have not find any tab in DataXReader");
  }
 
+ private void writeStructFields(Context context, ISearchEngineTypeTransfer typeTransfer, ISelectedTab esTab) {
+ SchemaMetaContent tplSchema = typeTransfer.initSchemaMetaContent(esTab);
+ this.setBizResult(context, tplSchema.toJSON());
+ }
+
  /**
  * 创建新索引流程取得通过workflow反射Schema 生成索引
  *
@@ -369,51 +380,39 @@ private boolean validateStupidContent(Context context, UploadSchemaWithRawConten
  this.addErrorMessage(context, "字段名称不能为空");
  hasBlankError = true;
  }
- FieldErrorInfo err = fieldsErrors.getFieldErrorInfo(f.getId());// new FieldErrorInfo(f.getId());
+ FieldErrorInfo err = fieldsErrors.getFieldErrorInfo(f.getId());
  err.setFieldNameError(true);
  } else {
  if (!duplicate.add(f.getName())) {
- // if (!hasDuplicateError) {
  this.addErrorMessage(context, "字段名‘" + f.getName() + "’不能重复");
- // }
  // 有重复
  hasDuplicateError = true;
  FieldErrorInfo err = fieldsErrors.getFieldErrorInfo(f.getId());
  err.setFieldNameError(true);
- // fieldsErrors.add(err);
  } else {
  if (!PATTERN_FIELD.matcher(f.getName()).matches() && !f.isDynamic()) {
- // if (!hasNamePatternError) {
  this.addErrorMessage(context, "字段名‘" + f.getName() + "’开通必须以[a-z]作为开头且中间不能有除数字、下划线、大小写字母以外的字符出现");
- // }
- // hasNamePatternError = true;
  FieldErrorInfo err = fieldsErrors.getFieldErrorInfo(f.getId());
  err.setFieldNameError(true);
-
  } else if (StringUtils.isBlank(f.getFieldtype())) {
  this.addErrorMessage(context, "请为字段‘" + f.getName() + "’选择类型");
  hasFieldTypeError = true;
  FieldErrorInfo err = fieldsErrors.getFieldErrorInfo(f.getId());
  err.setFieldTypeError(true);
- //fieldsErrors.add(err);
  } else if ("string".equalsIgnoreCase(f.getFieldtype()) //
  && (StringUtils.isBlank(f.getTokenizerType())
  || "-1".equalsIgnoreCase(f.getTokenizerType()))) {
- // if (!hasFieldTypeError) {
  this.addErrorMessage(context, "请为字段‘" + f.getName() + "’选择分词类型");
  hasFieldTypeError = true;
- // }
  FieldErrorInfo err = fieldsErrors.getFieldErrorInfo(f.getId());
  err.setFieldTypeError(true);
- // fieldsErrors.add(err);
  }
  }
 
  if (!f.getSortable() && !f.isIndexed() && !f.isStored()) {
- this.addErrorMessage(context, SolrFieldsParser.getFieldPropRequiredErr(f.getName()));
+ this.addErrorMessage(context, ISchema.getFieldPropRequiredErr(f.getName()));
  FieldErrorInfo err = fieldsErrors.getFieldErrorInfo(f.getId());
  err.setFieldPropRequiredError(true);
- // fieldsErrors.add(err);
  }
 
  }
@@ -444,23 +443,17 @@ public void doToggleEsStupidModel(Context context) throws Exception {
  // 整段xml文本
  com.alibaba.fastjson.JSONObject body = this.parseJsonPost();
 
- ISearchEngineTypeTransfer typeTransfer = ISearchEngineTypeTransfer.load(this, body.getString(DataxUtils.DATAX_NAME));
-
- // final String content = body.getString("content");
- //byte[] schemaContent = body.getString("content").getBytes(TisUTF8.get());
- //JSONArray fields = JSON.parseArray(content);
+ ISearchEngineTypeTransfer typeTransfer
+ = ISearchEngineTypeTransfer.load(this, body.getString(DataxUtils.DATAX_NAME));
  writerStructFields(context, body, typeTransfer);
  }
 
  private void writerStructFields(Context context, TableAlias tableAlias, ISearchEngineTypeTransfer typeTransfer) {
 
  SchemaMetaContent schemaContent = new SchemaMetaContent();
- //schemaContent.content = content.getBytes(TisUTF8.get());
  schemaContent.parseResult = typeTransfer.projectionFromExpertModel(tableAlias, (content) -> {
  schemaContent.content = content;
  });
- // this.getStructSchema(context, ISchemaPluginContext.NULL, schemaContent);
-
  this.setBizResult(context, schemaContent.toJSON());
  }
 
@@ -469,8 +462,6 @@ private void writerStructFields(Context context, JSONObject body, ISearchEngineT
  SchemaMetaContent schemaContent = new SchemaMetaContent();
  schemaContent.content = content.getBytes(TisUTF8.get());
  schemaContent.parseResult = typeTransfer.projectionFromExpertModel(body);
- // this.getStructSchema(context, ISchemaPluginContext.NULL, schemaContent);
-
  this.setBizResult(context, schemaContent.toJSON());
  }
 

tis-console/src/main/java/com/qlangtech/tis/runtime/module/misc/SchemaField.java

@@ -48,8 +48,6 @@ public class SchemaField implements ISchemaField {
 
  private boolean multiValue = false;
 
- //private boolean sortable;
-
  // 如果选择了string分词
  private String textAnalysis;
 
@@ -58,8 +56,6 @@ public class SchemaField implements ISchemaField {
 
  // 是否开通docvalue
  private boolean docval;
-
-
  public int getId() {
  return id;
  }
@@ -73,7 +69,6 @@ public boolean isMultiValue() {
  return this.multiValue;
  }
 
-
  @Override
  public boolean isDynamic() {
  return StringUtils.indexOf(this.name, "*") > -1;

tis-console/src/main/java/com/qlangtech/tis/runtime/module/misc/UploadSchemaForm.java

@@ -1,27 +1,29 @@
 /**
- *  Licensed to the Apache Software Foundation (ASF) under one
- *  or more contributor license agreements. See the NOTICE file
- *  distributed with this work for additional information
- *  regarding copyright ownership. The ASF licenses this file
- *  to you under the Apache License, Version 2.0 (the
- *  "License"); you may not use this file except in compliance
- *  with the License. You may obtain a copy of the License at
- *
- *  https://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * https://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  */
 package com.qlangtech.tis.runtime.module.misc;
 
 import com.alibaba.fastjson.JSONArray;
 import com.qlangtech.tis.solrdao.ISchema;
+import org.apache.commons.collections.CollectionUtils;
 import org.apache.commons.lang.StringUtils;
 
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.List;
 
 /**
@@ -31,21 +33,15 @@
  * @date 2017年5月8日
  */
 public class UploadSchemaForm implements ISchema {
-
  private final List<SchemaField> fields = new ArrayList<SchemaField>();
 
- // private final Set<String> fieldKeys = new HashSet<String>();
+ @Override
  public List<SchemaField> getSchemaFields() {
- return fields;
-// List<ISchemaField> result = new ArrayList<>();
-// for (SchemaField f : fields) {
-// result.add(f);
-// }
-// return result;
+ return this.fields;
  }
 
  public List<SchemaField> getFields() {
- return this.fields;
+ return this.getSchemaFields();
  }
 
  public boolean containsField(String name) {
@@ -103,4 +99,22 @@ public JSONArray serialTypes() {
  public void clearFields() {
  this.fields.clear();
  }
+
+
+ private List<String> errlist = new ArrayList<String>();
+
+ @Override
+ public boolean isValid() {
+ if (!this.errlist.isEmpty()) {
+ return false;
+ }
+ return CollectionUtils.isEmpty(this.errlist = ISchema.validateSchema(this.fields));
+ }
+
+ @Override
+ public List<String> getErrors() {
+ return Collections.unmodifiableList(this.errlist);
+ }
+
+
 }