Upgrade iceberg to 0.6.3. Add column comments for iceberg tables.

Netflix · Feb 26, 2019 · 5ff06fe · 5ff06fe
1 parent 4b4d32f
commit 5ff06fe
Show file tree

Hide file tree

Showing 5 changed files with 15 additions and 9 deletions.
diff --git a/gradle.properties b/gradle.properties
@@ -39,4 +39,4 @@ hive_version=1.2.1
 org.gradle.parallel=false
 org.gradle.daemon=false
 
-iceberg_version=0.6.2
+iceberg_version=0.6.3
diff --git a/...r-hive/src/main/java/com/netflix/metacat/connector/hive/converters/HiveTypeConverter.java b/...r-hive/src/main/java/com/netflix/metacat/connector/hive/converters/HiveTypeConverter.java
@@ -118,13 +118,12 @@ public List<FieldInfo> icebergeSchemaTofieldDtos(final Schema schema,
  partitionFields.stream()
  .map(PartitionField::name).collect(Collectors.toList());
 
- for (int i = 0; i < schema.columns().size(); i++) {
- final Types.NestedField field = schema.columns().get(i);
+ for (Types.NestedField field : schema.columns()) {
  final FieldInfo fieldInfo = new FieldInfo();
  fieldInfo.setName(field.name());
  fieldInfo.setType(toMetacatType(fromIcebergToHiveType(field.type())));
  fieldInfo.setIsNullable(field.isOptional());
-
+ fieldInfo.setComment(field.doc());
  fieldInfo.setPartitionKey(partitionNames.contains(field.name()));
  fields.add(fieldInfo);
  }

diff --git a/...roovy/com/netflix/metacat/connector/hive/converters/HiveConnectorInfoConvertorSpec.groovy b/...roovy/com/netflix/metacat/connector/hive/converters/HiveConnectorInfoConvertorSpec.groovy
@@ -515,11 +515,13 @@ class HiveConnectorInfoConvertorSpec extends Specification{
  1 * partSpec.fields() >> [ field]
  1 * icebergTable.schema() >> schema
  1 * field.name() >> "fieldName"
- 5 * schema.columns() >> [nestedField, nestedField2]
+ 1 * schema.columns() >> [nestedField, nestedField2]
  2 * nestedField.name() >> "fieldName"
+ 1 * nestedField.doc() >> "fieldName doc"
  1 * nestedField.type() >> type
  1 * nestedField.isOptional() >> true
  2 * nestedField2.name() >> "fieldName2"
+ 1 * nestedField2.doc() >> "fieldName2 doc"
  1 * nestedField2.type() >> type
  1 * nestedField2.isOptional() >> true
  2 * type.typeId() >> Type.TypeID.BOOLEAN
@@ -528,6 +530,7 @@ class HiveConnectorInfoConvertorSpec extends Specification{
  tableInfo.getMetadata().get(DirectSqlTable.PARAM_METADATA_LOCATION).equals("/tmp/test")
  tableInfo.getFields().size() == 2
  tableInfo.getFields().get(0).isPartitionKey() != tableInfo.getFields().get(1).isPartitionKey()
+ tableInfo.getFields().get(0).getComment() == 'fieldName doc'
 
  }
 }
diff --git a/...at-test-cluster/etc-metacat/data/00088-5641e8bf-06b8-46b3-a0fc-5c867f5bca58.metadata.json b/...at-test-cluster/etc-metacat/data/00088-5641e8bf-06b8-46b3-a0fc-5c867f5bca58.metadata.json
@@ -9,17 +9,20 @@
  "id" : 1,
  "name" : "id",
  "required" : false,
- "type" : "long"
+ "type" : "long",
+ "doc" : "1st field"
  }, {
  "id" : 2,
  "name" : "data",
  "required" : false,
- "type" : "string"
+ "type" : "string",
+ "doc" : "2nd field"
  }, {
  "id" : 3,
  "name" : "dateint",
  "required" : false,
- "type" : "int"
+ "type" : "int",
+ "doc" : "3rd field"
  } ]
  },
  "partition-spec" : [ {

diff --git a/...at-functional-tests/src/functionalTest/groovy/com/netflix/metacat/MetacatSmokeSpec.groovy b/...at-functional-tests/src/functionalTest/groovy/com/netflix/metacat/MetacatSmokeSpec.groovy
@@ -509,6 +509,7 @@ class MetacatSmokeSpec extends Specification {
  noExceptionThrown()
  tableDTO.metadata.get("metadata_location").equals(metadataLocation)
  tableDTO.getFields().size() == 3
+ tableDTO.getFields().get(0).getComment() != null
  parts.size() == 2
  parts.get(0).dataMetadata != null
  partkeys.size() == 2
@@ -564,7 +565,7 @@ class MetacatSmokeSpec extends Specification {
 
 
  }
- 
+
  @Unroll
  def "Test delete table #catalogName/#databaseName/#tableName"() {
  given: