Fixing the functionalTest and addressing comments

gradle.properties

@@ -37,3 +37,4 @@ hive_version=1.2.1
 ## Gradle Properties
 
 org.gradle.parallel=false
+org.gradle.daemon=false

metacat-client/src/main/java/com/netflix/metacat/client/api/TagV1.java

@@ -18,6 +18,8 @@
 package com.netflix.metacat.client.api;
 
 import com.netflix.metacat.common.QualifiedName;
+import com.netflix.metacat.common.dto.TagCreateRequestDto;
+import com.netflix.metacat.common.dto.TagRemoveRequestDto;
 
 import javax.ws.rs.Consumes;
 import javax.ws.rs.DELETE;
@@ -126,6 +128,32 @@ Set<String> setTableTags(
  Set<String> tags
  );
 
+ /**
+ * Sets the tags on the given qualified name.
+ *
+ * @param tagCreateRequestDto tag create request dto
+ * @return set of tags
+ */
+ @POST
+ @Consumes(MediaType.APPLICATION_JSON)
+ @Produces(MediaType.APPLICATION_JSON)
+ Set<String> setTags(
+ TagCreateRequestDto tagCreateRequestDto
+ );
+
+
+ /**
+ * Remove the tags on the given qualified name.
+ *
+ * @param tagRemoveRequestDto tag remove request dto
+ */
+ @DELETE
+ @Consumes(MediaType.APPLICATION_JSON)
+ @Produces(MediaType.APPLICATION_JSON)
+ void removeTags(
+ TagRemoveRequestDto tagRemoveRequestDto
+ );
+
  /**
  * Remove the tags from the given table.
  *
@@ -149,6 +177,6 @@ void removeTableTags(
  @DefaultValue("false")
  @QueryParam("all")
  Boolean deleteAll,
-  Set<String> tags
+ Set<String> tags
  );
 }

metacat-common-server/src/main/java/com/netflix/metacat/common/server/connectors/exception/AlreadyExistsException.java

@@ -45,7 +45,7 @@ protected AlreadyExistsException(
  final boolean enableSuppression,
  final boolean writableStackTrace
  ) {
- this(name, String.format("%s '%s' already exists.", name.getType().name(), name.toString()),
+ this(name, String.format("%s '%s' already exists.", name.getType(), name.toString()),
  cause, enableSuppression, writableStackTrace);
  }
 

metacat-common-server/src/main/java/com/netflix/metacat/common/server/usermetadata/TagService.java

@@ -95,11 +95,11 @@ default Set<String> setTags(
  }
 
  /**
- * Removes the tags from the given table.
+ * Removes the tags from the given qualified name.
  *
  * @param qualifiedName table name
- * @param deleteAll if true, will delete all tags associated with the given table
- * @param tags list of tags to be removed for the given table
+ * @param deleteAll if true, will delete all tags associated with the given qualified name
+ * @param tags list of tags to be removed for the given qualified name
  * @param updateUserMetadata if true, updates the tags in the user metadata
  */
  default void removeTags(
@@ -113,18 +113,18 @@ default void removeTags(
  /**
  * Delete the tag item along with its associated tags.
  *
- * @param name table name
+ * @param name qualified name
  * @param updateUserMetadata if true, updates the tags in the user metadata
  */
  default void delete(final QualifiedName name, final boolean updateUserMetadata) {
  }
 
  /**
  * Renames the tag item name with the new table name.
- *
+ * Can only be used in table rename
  * @param name table qualified name
  * @param newTableName new table name
  */
- default void rename(final QualifiedName name, final String newTableName) {
+ default void renameTableTags(final QualifiedName name, final String newTableName) {
  }
 }

metacat-common/src/main/java/com/netflix/metacat/common/QualifiedName.java

@@ -182,6 +182,7 @@ public static QualifiedName fromString(@NonNull final String s, final boolean is
 
  /**
  * Returns a copy of this qualified name with the database/table/view names in upper case.
+ *
  * @return QualifiedName
  */
  public QualifiedName cloneWithUpperCase() {

metacat-common/src/main/java/com/netflix/metacat/common/dto/TagCreateRequestDto.java

@@ -19,19 +19,25 @@
 
 import com.netflix.metacat.common.QualifiedName;
 import io.swagger.annotations.ApiModelProperty;
+import lombok.AllArgsConstructor;
+import lombok.Builder;
 import lombok.Data;
 import lombok.EqualsAndHashCode;
+import lombok.NoArgsConstructor;
 
 import java.util.List;
 
 /**
  * Tag Create Request Dto.
  */
 @Data
+@Builder
+@NoArgsConstructor
+@AllArgsConstructor
 @EqualsAndHashCode(callSuper = false)
 public class TagCreateRequestDto extends BaseDto {
  private static final long serialVersionUID = -990374882621118670L;
- @ApiModelProperty(value = "the name of this entity", required = true)
+ @ApiModelProperty(value = "The qualified name", required = true)
  private QualifiedName name;
 
  @ApiModelProperty(value = "Tags to insert")

metacat-common/src/main/java/com/netflix/metacat/common/dto/TagRemoveRequestDto.java

@@ -19,24 +19,30 @@
 
 import com.netflix.metacat.common.QualifiedName;
 import io.swagger.annotations.ApiModelProperty;
+import lombok.AllArgsConstructor;
+import lombok.Builder;
 import lombok.Data;
 import lombok.EqualsAndHashCode;
+import lombok.NoArgsConstructor;
 
 import java.util.List;
 
 /**
  * Tag Remove Request Dto.
  */
 @Data
+@Builder
+@NoArgsConstructor
+@AllArgsConstructor
 @EqualsAndHashCode(callSuper = false)
 public class TagRemoveRequestDto extends BaseDto {
  private static final long serialVersionUID = 8698531483258796673L;
- @ApiModelProperty(value = "the name of this entity", required = true)
+ @ApiModelProperty(value = "The qualified name", required = true)
  private QualifiedName name;
 
- @ApiModelProperty(value = "flag to delete all tags")
+ @ApiModelProperty(value = "True to delete all tags")
  private Boolean deleteAll;
 
- @ApiModelProperty(value = "Tags to insert")
+ @ApiModelProperty(value = "Tags to remove")
  private List<String> tags;
 }

metacat-common/src/test/groovy/com/netflix/metacat/common/dto/DtoVerificationSpec.groovy

@@ -68,7 +68,6 @@ class DtoVerificationSpec extends Specification {
  if (writeMethod) {
  Class<?> type = descriptor.propertyType
  Field field = clazz.declaredFields.find { it.name == descriptor.name }
- print "checking ${field}"
  if (field) {
  def randomValue = getRandomValue(type, field)
  writeMethod.invoke(dto, randomValue)

metacat-functional-tests/metacat-test-cluster/docker-compose.yml

@@ -52,12 +52,14 @@ services:
  -Dmetacat.mysqlmetadataservice.enabled=true
  -Dmetacat.type.converter=com.netflix.metacat.connector.pig.converters.PigTypeConverter
  -Dmetacat.definition.metadata.delete.enableForTable=false
- -Dmetacat.definition.metadata.delete.enableDeleteForQualifiedNames=hive-metastore/hsmoke_ddb,hive-metastore/hsmoke_ddb1/test_create_table1,embedded-hive-metastore,embedded-fast-hive-metastore/fsmoke_db1,embedded-fast-hive-metastore/fsmoke_ddb1,embedded-fast-hive-metastore/shard,s3-mysql-db,mysql-56-db
+ -Dmetacat.definition.metadata.delete.enableDeleteForQualifiedNames=hive-metastore/hsmoke_ddb,hive-metastore/hsmoke_ddb1/test_create_table1,embedded-hive-metastore,embedded-fast-hive-metastore/fsmoke_db1,embedded-fast-hive-metastore/fsmoke_ddb1,embedded-fast-hive-metastore/shard,embedded-fast-hive-metastore/fsmoke_db4,s3-mysql-db,mysql-56-db
  -Dmetacat.hive.metastore.batchSize=10
  -Dmetacat.usermetadata.config.location=/etc/metacat/usermetadata.properties'
  labels:
  - "com.netflix.metacat.oss.test"
  - "com.netflix.metacat.oss.test.war"
+ ##REMOVED cassandra and druid container to reduce the memory demand (orig git hash 43890dd)
+ ##TODO: We need to add them back with light images
  hive-metastore-db:
  image: mysql:5.6
  volumes:

metacat-functional-tests/src/functionalTest/groovy/com/netflix/metacat/MetacatFunctionalSpec.groovy

@@ -19,11 +19,11 @@ package com.netflix.metacat
 import com.fasterxml.jackson.databind.node.ObjectNode
 import com.google.common.base.Throwables
 import com.netflix.metacat.client.Client
-import com.netflix.metacat.client.api.TagV1
-import com.netflix.metacat.common.QualifiedName
 import com.netflix.metacat.client.api.MetacatV1
 import com.netflix.metacat.client.api.PartitionV1
 import com.netflix.metacat.client.api.ResolverV1
+import com.netflix.metacat.client.api.TagV1
+import com.netflix.metacat.common.QualifiedName
 import com.netflix.metacat.common.dto.*
 import com.netflix.metacat.common.exception.MetacatAlreadyExistsException
 import com.netflix.metacat.common.exception.MetacatBadRequestException
@@ -35,7 +35,6 @@ import feign.RetryableException
 import org.apache.hadoop.hive.metastore.Warehouse
 import org.joda.time.Instant
 import org.skyscreamer.jsonassert.JSONAssert
-import spock.lang.Ignore
 import spock.lang.Specification
 import spock.lang.Stepwise
 import spock.lang.Unroll
@@ -140,22 +139,26 @@ class MetacatFunctionalSpec extends Specification {
  given:
  def now = new Date().time
  def metadata = metacatJson.parseJsonObject("""{
- "test_time": ${now},
- "batch_id": ${BATCH_ID}
+ "updated_part": {
+ "test_time": ${now},
+ "batch_id": ${BATCH_ID}
+ }
 }""")
-
  when:
- def catResponse = api.getCatalog(catalog.name)
-
- then:
- !catResponse.definitionMetadata
+ def catResponse1 = api.getCatalog(catalog.name)
 
- when:
  api.updateCatalog(catalog.name, new CreateCatalogDto(type: catalog.type, definitionMetadata: metadata))
- catResponse = api.getCatalog(catalog.name)
+ def catResponse2 = api.getCatalog(catalog.name)
 
  then:
- catResponse.definitionMetadata == metadata
+ def originMetadata = catResponse1.definitionMetadata
+ if ( originMetadata == null ) {
+ originMetadata = metacatJson.parseJsonObject("{}");
+ }
+ def ret = metacatJson.mergeIntoPrimary(originMetadata, metadata)
+
+ catResponse2.definitionMetadata.get("updated_part") == metadata.get("updated_part")
+ catResponse2.definitionMetadata == originMetadata
 
  where:
  catalog << TestCatalogs.ALL

metacat-functional-tests/src/functionalTest/groovy/com/netflix/metacat/MetacatSmokeSpec.groovy

@@ -1083,6 +1083,107 @@ class MetacatSmokeSpec extends Specification {
  's3-mysql-db' | 'smoke_db6' | 'part' | ['test', 'unused'] as Set<String> | false
  }
 
+ @Unroll
+ def "Test tags for catalogName, databaseName and tableName"() {
+ when:
+ createTable(catalogName, databaseName, tableName)
+ def catalog = QualifiedName.ofCatalog(catalogName)
+ tagApi.setTags(TagCreateRequestDto.builder().name(catalog).tags(tags).build())
+ if (repeat) {
+ tagApi.setTags(TagCreateRequestDto.builder().name(catalog).tags(tags).build())
+ }
+
+ def database = QualifiedName.ofDatabase(catalogName, databaseName)
+ tagApi.setTags(TagCreateRequestDto.builder().name(database).tags(tags).build())
+
+ def table = QualifiedName.ofTable(catalogName, databaseName, tableName)
+ tagApi.setTags(TagCreateRequestDto.builder().name(table).tags(tags).build())
+
+ def catalogDto = api.getCatalog(catalogName)
+ def databaseDto = api.getDatabase(catalogName, databaseName, true, false)
+ def tableDto = api.getTable(catalogName, databaseName, tableName, true, true, false)
+
+ def ret = tagApi.search('test_tag', null, null, null)
+ def ret2 = tagApi.list(['test_tag'] as Set<String>, null, null, null, null)
+ def ret3 = tagApi.list(['never_tag'] as Set<String>, tags as Set<String>, null, null, null)
+
+ tagApi.removeTags(TagRemoveRequestDto.builder().name(catalog).tags([]).deleteAll(true).build())
+ tagApi.removeTags(TagRemoveRequestDto.builder().name(database).tags([]).deleteAll(true).build())
+ tagApi.removeTags(TagRemoveRequestDto.builder().name(table).tags([]).deleteAll(true).build())
+
+ def ret_new = tagApi.search('test_tag', null, null, null)
+ def ret2_new = tagApi.list(['test_tag'] as Set<String>, null, null, null, null)
+
+ then:
+ metacatJson.convertValue(catalogDto.getDefinitionMetadata().get('tags'), Set.class) == tags as Set<String>
+ metacatJson.convertValue(databaseDto.getDefinitionMetadata().get('tags'), Set.class) == tags as Set<String>
+ metacatJson.convertValue(tableDto.getDefinitionMetadata().get('tags'), Set.class) == tags as Set<String>
+
+ assert ret.size() == 3
+ assert ret2.size() == 3
+ assert ret3.size() == 0
+
+ assert ret_new.size() == 0
+ assert ret2_new.size() == 0
+
+ cleanup:
+ api.deleteTable(catalogName, databaseName, tableName)
+
+ where:
+ catalogName | databaseName | tableName | tags | repeat
+ 'embedded-hive-metastore' | 'smoke_db6' | 'part' | ['test_tag'] as List<String> | true
+ 'embedded-hive-metastore' | 'smoke_db6' | 'part' | ['test_tag', 'unused'] as List<String> | false
+ 'embedded-fast-hive-metastore' | 'fsmoke_db6' | 'part' | ['test_tag'] as List<String> | true
+ 'embedded-fast-hive-metastore' | 'fsmoke_db6' | 'part' | ['test_tag', 'unused'] as List<String> | false
+ 'embedded-fast-hive-metastore' | 'shard' | 'part' | ['test_tag'] as List<String> | true
+ 'embedded-fast-hive-metastore' | 'shard' | 'part' | ['test_tag', 'unused'] as List<String> | false
+ 'hive-metastore' | 'hsmoke_db6' | 'part' | ['test_tag'] as List<String> | true
+ 'hive-metastore' | 'hsmoke_db6' | 'part' | ['test_tag', 'unused'] as List<String> | false
+ 's3-mysql-db' | 'smoke_db6' | 'part' | ['test_tag'] as List<String> | true
+ 's3-mysql-db' | 'smoke_db6' | 'part' | ['test_tag', 'unused'] as List<String> | false
+ }
+
+ @Unroll
+ def "Test tags for #catalogName/#databaseName/#tableName/#viewName"() {
+ when:
+ try {
+ api.createDatabase(catalogName, 'franklinviews', new DatabaseCreateRequestDto())
+ } catch (Exception ignored) {
+ }
+ createTable(catalogName, databaseName, tableName)
+ api.createMView(catalogName, databaseName, tableName, viewName, true, null)
+
+ def view = QualifiedName.ofView(catalogName, databaseName, tableName, viewName)
+ tagApi.setTags(TagCreateRequestDto.builder().name(view).tags(tags).build())
+
+ def ret = tagApi.search('test_tag', null, null, null)
+ def ret2 = tagApi.list(['test_tag'] as Set<String>, null, null, null, null)
+ def ret3 = tagApi.list(['never_tag'] as Set<String>, tags as Set<String>, null, null, null)
+
+ tagApi.removeTags(TagRemoveRequestDto.builder().name(view).tags([]).deleteAll(true).build())
+
+ def ret_new = tagApi.search('test_tag', null, null, null)
+ def ret2_new = tagApi.list(['test_tag'] as Set<String>, null, null, null, null)
+
+ then:
+ assert ret.size() == 1
+ assert ret2.size() == 1
+ assert ret3.size() == 0
+
+ assert ret_new.size() == 0
+ assert ret2_new.size() == 0
+
+ cleanup:
+ api.deleteMView(catalogName, databaseName, tableName, viewName)
+ where:
+ catalogName | databaseName | tableName | viewName |tags
+ 'embedded-hive-metastore' | 'smoke_db4' | 'part' | 'part_view' | ['test_tag'] as List<String>
+ 'embedded-fast-hive-metastore' | 'fsmoke_db4' | 'part' | 'part_view' | ['test_tag'] as List<String>
+ 'embedded-fast-hive-metastore' | 'shard' | 'part' | 'part_view' | ['test_tag'] as List<String>
+ 'hive-metastore' | 'hsmoke_db4' | 'part' | 'part_view' | ['test_tag'] as List<String>
+ 's3-mysql-db' | 'smoke_db4' | 'part' | 'part_view' | ['test_tag'] as List<String>
+ }
+
  @Unroll
  def "Delete invalid partition for #catalogName/#databaseName/#tableName"() {
  when: