Upgrade to Spring boot 2.7.6 (#542)

* Add unit tests runs

* Fix functional tests

* Add a workflow for functional tests

* Fix smoke tests

* Set owner for all tables in functional tests

.github/workflows/nebula-pr-functional-tests.yml

@@ -0,0 +1,44 @@
+name: "Run functional tests on pull requests"
+on:
+ pull_request:
+
+jobs:
+ build:
+ runs-on: ubuntu-latest
+ strategy:
+ matrix:
+ # test against JDK 8
+ java: [ 8 ]
+ name: Functional tests with Java ${{ matrix.java }}
+ steps:
+ - uses: actions/checkout@v1
+ - name: Setup jdk
+ uses: actions/setup-java@v1
+ with:
+ java-version: ${{ matrix.java }}
+ - uses: actions/cache@v1
+ id: gradle-cache
+ with:
+ path: ~/.gradle/caches
+ key: ${{ runner.os }}-gradle-${{ hashFiles('**/gradle/dependency-locks/*.lockfile') }}
+ restore-keys: |
+ - ${{ runner.os }}-gradle-
+ - uses: actions/cache@v1
+ id: gradle-wrapper-cache
+ with:
+ path: ~/.gradle/wrapper
+ key: ${{ runner.os }}-gradlewrapper-${{ hashFiles('gradle/wrapper/*') }}
+ restore-keys: |
+ - ${{ runner.os }}-gradlewrapper-
+ - name: Build with Gradle
+ run: ./gradlew --info --stacktrace functionalTest
+ env:
+ CI_NAME: github_actions
+ CI_BUILD_NUMBER: ${{ github.sha }}
+ CI_BUILD_URL: 'https://github.com/${{ github.repository }}'
+ CI_BRANCH: ${{ github.ref }}
+ COVERALLS_REPO_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+ if: ${{ failure() }} |
+ - name=$(date +"%Y-%m-%d") && cat ./metacat-functional-tests/build/logs/metacat/catalina.$name.log
+ - name=$(date +"%Y-%m-%d") && cat ./metacat-functional-tests/build/logs/metacat/localhost.$name.log
+ - cat ./metacat-functional-tests/build/logs/metacat/spring.log

build.gradle

@@ -145,6 +145,7 @@ configure(javaProjects) {
  dependency("com.esotericsoftware.kryo:kryo:2.22")
  dependency("org.apache.iceberg:iceberg-spark-runtime:${iceberg_version}")
  dependency("com.datastax.cassandra:cassandra-driver-core:3.7.2")
+ dependency("mysql:mysql-connector-java:8.0.27")
  }
  }
 
@@ -184,8 +185,17 @@ configure(javaProjects) {
  testCompile("org.spockframework:spock-core")
  testCompile("org.spockframework:spock-guice")
  testCompile("org.spockframework:spock-spring")
- testCompile("org.springframework.boot:spring-boot-starter-test")
+ testCompile("org.springframework.boot:spring-boot-starter-test") {
+ exclude group: 'com.vaadin.external.google', module: 'android-json'
+ }
  testRuntime("org.hsqldb:hsqldb")
+ testRuntimeOnly 'org.junit.vintage:junit-vintage-engine:5.3.1'
+ testCompile("org.junit.jupiter:junit-jupiter-api:5.4.2")
+ testImplementation("org.junit.jupiter:junit-jupiter-api:5.4.2")
+ testRuntimeOnly("org.junit.jupiter:junit-jupiter-engine:5.4.2")
+ testRuntimeOnly "org.junit.platform:junit-platform-commons:1.7.0"
+ testImplementation 'org.junit.platform:junit-platform-launcher:1.8.1'
+ testImplementation 'org.junit.platform:junit-platform-runner:1.8.1'
  }
 
  compileJava {
@@ -229,8 +239,10 @@ configure(javaProjects) {
 
  // Print out full stack traces when our tests fail to assist debugging (e.g., when scanning Jenkins console output)
  tasks.withType(Test) {
+ useJUnitPlatform()
  testLogging {
  exceptionFormat = "full"
+ events "PASSED", "FAILED", "SKIPPED"
  }
  }
 

gradle.properties

@@ -19,10 +19,10 @@
 ## Spring Dependency Versions
 
 # Used in documentation and for including the Gradle plugin
-spring_boot_version=2.3.3.RELEASE
+spring_boot_version=2.7.6
 spring_cloud_version=Hoxton.SR8
 google_guice_version=4.1.0
-spock_version=1.3-groovy-2.5
+spock_version=2.0-groovy-3.0
 
 ## Override Spring Boot versions
 

metacat-app/build.gradle

@@ -40,7 +40,7 @@ jar {
 }
 
 springBoot {
- mainClassName = "com.netflix.metacat.MetacatApplication"
+ mainClass = "com.netflix.metacat.MetacatApplication"
 }
 
 def env = project.hasProperty("env") ? project.getProperty("env") : "dev"

metacat-connector-cassandra/src/test/groovy/com/netflix/metacat/connector/cassandra/CassandraTypeConverterSpec.groovy

@@ -34,7 +34,7 @@ class CassandraTypeConverterSpec extends Specification {
  def converter = new CassandraTypeConverter()
 
  @Unroll
- "Can convert Cassandra string: #type to Metacat Type: #signature"() {
+ "Can convert Cassandra string: #type to Metacat Type: #metacatType"() {
 
  expect:
  this.converter.toMetacatType(type) == metacatType

metacat-connector-mysql/src/test/groovy/com/netflix/metacat/connector/mysql/MySqlTypeConverterSpec.groovy

@@ -33,7 +33,7 @@ class MySqlTypeConverterSpec extends Specification {
  def converter = new MySqlTypeConverter()
 
  @Unroll
- "Can convert MySQL string: #type to Metacat Type: #signature"() {
+ "Can convert MySQL string: #type to Metacat Type: #metacatType"() {
 
  expect:
  this.converter.toMetacatType(type) == metacatType

metacat-connector-polaris/build.gradle

@@ -56,7 +56,9 @@ dependencies {
  * Test Dependencies
  *******************************/
  testCompile('org.springframework:spring-test')
- testImplementation('org.springframework.boot:spring-boot-starter-test')
+ testImplementation('org.springframework.boot:spring-boot-starter-test') {
+ exclude group: 'com.vaadin.external.google', module: 'android-json'
+ }
 
  testRuntimeOnly 'org.junit.vintage:junit-vintage-engine:5.3.1'
  testCompile("org.junit.jupiter:junit-jupiter-api:5.4.2")

metacat-connector-polaris/src/functionalTest/resources/application-polaris_functional_test.yml

@@ -1,11 +1,8 @@
 spring:
  datasource:
- platform: h2db
  url: jdbc:postgresql:https://127.0.0.1:26257/defaultdb?sslmode=disable
  username: admin
  password:
- schema: classpath:schema.sql
- initialization-mode: always
  driverClassName: org.postgresql.Driver
  hikari:
  connection-timeout: 5000 # 5 seconds
@@ -22,8 +19,13 @@ spring:
  hibernate:
  dialect: org.hibernate.dialect.PostgreSQLDialect
  show_sql: true
+ sql:
+ init:
+ schema-locations: classpath:schema.sql
+ mode: always
+ platform: h2db
 
 logging:
  level:
  org.hibernate.SQL: DEBUG
- org.hibernate.type: TRACE
+ org.hibernate.type: TRACE

metacat-connector-polaris/src/main/java/com/netflix/metacat/connector/polaris/store/entities/PolarisDatabaseEntity.java

@@ -6,6 +6,7 @@
 import lombok.Getter;
 import lombok.NoArgsConstructor;
 import lombok.ToString;
+import org.hibernate.annotations.GenericGenerator;
 import org.springframework.data.jpa.domain.support.AuditingEntityListener;
 
 import javax.persistence.Basic;
@@ -14,7 +15,6 @@
 import javax.persistence.Entity;
 import javax.persistence.EntityListeners;
 import javax.persistence.GeneratedValue;
-import javax.persistence.GenerationType;
 import javax.persistence.Id;
 import javax.persistence.Table;
 import javax.persistence.Version;
@@ -38,7 +38,8 @@ public class PolarisDatabaseEntity {
 
  @Basic
  @Id
- @GeneratedValue(strategy = GenerationType.IDENTITY)
+ @GeneratedValue(generator = "uuid")
+ @GenericGenerator(name = "uuid", strategy = "uuid2")
  @Column(name = "id", nullable = false, unique = true, updatable = false)
  private String dbId;
 

metacat-connector-polaris/src/main/java/com/netflix/metacat/connector/polaris/store/entities/PolarisTableEntity.java

@@ -7,6 +7,7 @@
 import lombok.NoArgsConstructor;
 import lombok.Setter;
 import lombok.ToString;
+import org.hibernate.annotations.GenericGenerator;
 import org.springframework.data.jpa.domain.support.AuditingEntityListener;
 
 import javax.persistence.Basic;
@@ -15,7 +16,6 @@
 import javax.persistence.Entity;
 import javax.persistence.EntityListeners;
 import javax.persistence.GeneratedValue;
-import javax.persistence.GenerationType;
 import javax.persistence.Id;
 import javax.persistence.Table;
 import javax.persistence.Version;
@@ -39,7 +39,8 @@ public class PolarisTableEntity {
 
  @Basic
  @Id
- @GeneratedValue(strategy = GenerationType.IDENTITY)
+ @GeneratedValue(generator = "uuid")
+ @GenericGenerator(name = "uuid", strategy = "uuid2")
  @Column(name = "id", nullable = false, unique = true, updatable = false)
  private String tblId;
 

metacat-connector-polaris/src/test/resources/application-polaristest.yml

@@ -1,11 +1,8 @@
 spring:
  datasource:
- platform: h2db
  url: jdbc:h2:mem:testdb
  username: sa
  password:
- schema: classpath:/h2db/schema.sql
- initialization-mode: always
  driverClassName: org.h2.Driver
  hikari:
  connection-timeout: 5000 # 5 seconds
@@ -22,3 +19,18 @@ spring:
  hibernate:
  dialect: org.hibernate.dialect.H2Dialect
  show_sql: true
+ format_sql: true
+ show-sql: true
+ sql:
+ init:
+ platform: h2db
+ schema-locations: classpath:/h2db/schema.sql
+ mode: always
+logging:
+ level:
+ root: debug
+ org:
+ hibernate:
+ type: trace
+ descriptor:
+ sql: trace

metacat-connector-polaris/src/test/resources/h2db/schema.sql

@@ -1,9 +1,9 @@
-drop table TBLS if exists;
-drop table DBS if exists;
+drop table if exists TBLS;
+drop table if exists DBS;
 
 create table DBS (
  version bigint not null,
- id IDENTITY not null primary key,
+ id varchar(255) default random_uuid() not null primary key,
  name varchar(255) not null unique,
  location varchar(1024),
  created_by varchar(255),
@@ -14,7 +14,7 @@ create table DBS (
 
 create table TBLS (
  version bigint not null,
- id IDENTITY not null primary key,
+ id varchar(255) default random_uuid() not null primary key,
  db_name varchar(255) not null,
  tbl_name varchar(255) not null,
  previous_metadata_location varchar(1024),

metacat-connector-postgresql/src/test/groovy/com/netflix/metacat/connector/postgresql/PostgreSqlTypeConverterSpec.groovy

@@ -33,7 +33,7 @@ class PostgreSqlTypeConverterSpec extends Specification {
  def converter = new PostgreSqlTypeConverter()
 
  @Unroll
- "Can convert PostgreSQL string: #type to Metacat Type: #signature"() {
+ "Can convert PostgreSQL string: #type to Metacat Type: #metacatType"() {
 
  expect:
  this.converter.toMetacatType(type) == metacatType

metacat-connector-redshift/src/test/groovy/com/netflix/metacat/connector/redshift/RedshiftTypeConverterSpec.groovy

@@ -33,7 +33,7 @@ class RedshiftTypeConverterSpec extends Specification {
  def converter = new RedshiftTypeConverter()
 
  @Unroll
- "Can convert Redshift string: #type to Metacat Type: #signature"() {
+ "Can convert Redshift string: #type to Metacat Type: #metacatType"() {
 
  expect:
  this.converter.toMetacatType(type) == metacatType

metacat-connector-snowflake/src/test/groovy/com/netflix/metacat/connector/snowflake/SnowflakeTypeConverterSpec.groovy

@@ -33,7 +33,7 @@ class SnowflakeTypeConverterSpec extends Specification {
  def converter = new SnowflakeTypeConverter()
 
  @Unroll
- "Can convert Snowflake string: #type to Metacat Type: #signature"() {
+ "Can convert Snowflake string: #type to Metacat Type: #metacatType"() {
 
  expect:
  this.converter.toMetacatType(type) == metacatType

metacat-functional-tests/build.gradle

@@ -128,13 +128,13 @@ task metacatPorts {
  ext.hive_debug_port = get_docker_port('com.netflix.metacat.oss.test.hive', 8005)
 
  println '++ Container Exposed ports ++'
- println '+++ Metacat Web Port:' + ext.http_port
- println '+++ Metacat Web Debug Port:' + ext.metacat_http_debug_port
- println '+++ Metacat Hive Thrift Port:' + ext.metacat_hive_thrift_port
- println '+++ Metacat Embedded Hive Thrift Port:' + ext.metacat_embedded_hive_thrift_port
- println '+++ Metacat Embedded Fast Hive Thrift Port:' + ext.metacat_embedded_fast_hive_thrift_port
- println '+++ Metacat Hive Metastore Port:' + ext.hive_thrift_port
- println '+++ Metacat Hive Metastore Debug Port:' + ext.hive_debug_port
+ println '+++ Metacat Web Port: [' + ext.http_port + ']'
+ println '+++ Metacat Web Debug Port: [' + ext.metacat_http_debug_port + ']'
+ println '+++ Metacat Hive Thrift Port: [' + ext.metacat_hive_thrift_port + ']'
+ println '+++ Metacat Embedded Hive Thrift Port: [' + ext.metacat_embedded_hive_thrift_port + ']'
+ println '+++ Metacat Embedded Fast Hive Thrift Port: [' + ext.metacat_embedded_fast_hive_thrift_port + ']'
+ println '+++ Metacat Hive Metastore Port: [' + ext.hive_thrift_port + ']'
+ println '+++ Metacat Hive Metastore Debug Port: [' + ext.hive_debug_port + ']'
  }
 }
 

metacat-functional-tests/src/functionalTest/groovy/com/netflix/metacat/MetacatFunctionalSpec.groovy

@@ -654,7 +654,7 @@ class MetacatFunctionalSpec extends Specification {
  catalog << TestCatalogs.getCanCreateTable(TestCatalogs.ALL)
  }
 
- def 'updateTable: #name'() {
+ def 'updateTable'() {
  given:
  def now = new Date()
 
@@ -1007,7 +1007,10 @@ class MetacatFunctionalSpec extends Specification {
  type: 'chararray',
  partition_key: false
  ),
- ]
+ ],
+ serde: new StorageDto(
+ owner: 'ssarma'
+ )
  )
 
  when:

metacat-functional-tests/src/functionalTest/groovy/com/netflix/metacat/MetacatSmokeSpec.groovy

@@ -123,6 +123,14 @@ class MetacatSmokeSpec extends Specification {
  if (metadata != null) {
  newTable.getMetadata() == null? newTable.setMetadata(metadata): newTable.getMetadata().putAll(metadata)
  }
+
+ if (newTable.getSerde() == null) {
+ newTable.setSerde(new StorageDto(owner: owner))
+ }
+
+ if (newTable.getSerde().getOwner() == null) {
+ newTable.getSerde().setOwner(owner)
+ }
  api.createTable(catalogName, databaseName, tableName, newTable)
  }
  }
@@ -861,7 +869,7 @@ class MetacatSmokeSpec extends Specification {
  then:
  thrown(RetryableException)
  when:
- tagApi.removeTableTags(catalogName, databaseName, tableNameTagNoRename, false, ['do_not_rename'] as Set)
+ tagApi.removeTableTags(catalogName, databaseName, tableNameTagNoRename, false, ['iceberg_migration_do_not_modify'] as Set)
  api.renameTable(catalogName, databaseName, tableNameTagNoRename, tableNameTagNoRenamed)
  tagApi.removeTableTags(catalogName, databaseName, tableNameTagNoRenamed, true, [] as Set)
  then:
@@ -1265,7 +1273,10 @@ class MetacatSmokeSpec extends Specification {
  then:
  thrown(MetacatNotFoundException)
  when:
- api.createTable(catalogName, 'invalid', 'invalid', new TableDto(name: QualifiedName.ofTable('invalid', 'invalid', 'invalid')))
+ api.createTable(catalogName, 'invalid', 'invalid', new TableDto(
+ name: QualifiedName.ofTable('invalid', 'invalid', 'invalid'),
+ serde: new StorageDto(owner: 'ssarma')
+ ))
  then:
  thrown(MetacatNotFoundException)
  when:
@@ -1336,7 +1347,10 @@ class MetacatSmokeSpec extends Specification {
  thrown(MetacatNotFoundException)
  when:
  createTable(catalogName, 'invalid', 'invalid')
- api.createTable(catalogName, 'invalid', 'invalid', new TableDto(name: QualifiedName.ofTable('invalid', 'invalid', 'invalid')))
+ api.createTable(catalogName, 'invalid', 'invalid', new TableDto(
+ name: QualifiedName.ofTable('invalid', 'invalid', 'invalid'),
+ serde: new StorageDto(owner: 'ssarma')
+ ))
  then:
  thrown(MetacatAlreadyExistsException)
  when: