Adding internal hive connector (#68)

* package name refactor

* package name refactor

* adding embedded client

* fixing the functionTest

* fixing check style

* removing the port setting in function test

* fixing the thrif functional test with embedded hive store

* fixing checkstyple

* removing commented code in functionalTest

* fixing the hardcode string to constants

* removing unused property

* removing the guava-retrying in dependencies

gradle.properties

@@ -10,10 +10,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
-
 # Temporarily change release scope to major
 release.scope=major
-
 amazon_sns_version=1.11.60
 airlift_version=0.116
 cassandra_driver_version=3.1.4

metacat-common-server/src/main/java/com/netflix/metacat/common/server/connectors/ConnectorFactory.java

@@ -1,3 +1,21 @@
+/*
+ *
+ * Copyright 2017 Netflix, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http:https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
 package com.netflix.metacat.common.server.connectors;
 
 /**

metacat-common-server/src/main/java/com/netflix/metacat/common/server/connectors/ConnectorInfoConverter.java

@@ -1,3 +1,21 @@
+/*
+ *
+ * Copyright 2017 Netflix, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http:https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
 package com.netflix.metacat.common.server.connectors;
 
 import com.netflix.metacat.common.QualifiedName;

metacat-common-server/src/main/java/com/netflix/metacat/common/server/connectors/ConnectorPlugin.java

@@ -1,3 +1,21 @@
+/*
+ *
+ * Copyright 2017 Netflix, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http:https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
 package com.netflix.metacat.common.server.connectors;
 
 import javax.annotation.Nonnull;

metacat-common-server/src/main/java/com/netflix/metacat/common/server/connectors/model/AuditInfo.java

@@ -1,3 +1,21 @@
+/*
+ *
+ * Copyright 2017 Netflix, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http:https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
 package com.netflix.metacat.common.server.connectors.model;
 
 import lombok.AllArgsConstructor;

metacat-common-server/src/main/java/com/netflix/metacat/common/server/converter/ConverterUtil.java

@@ -1,3 +1,21 @@
+/*
+ *
+ * Copyright 2017 Netflix, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http:https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
 package com.netflix.metacat.common.server.converter;
 
 import com.google.common.collect.Maps;

metacat-common-server/src/main/java/com/netflix/metacat/common/server/converter/DozerTypeConverter.java

@@ -1,3 +1,21 @@
+/*
+ *
+ * Copyright 2017 Netflix, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http:https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
 package com.netflix.metacat.common.server.converter;
 
 import com.netflix.metacat.common.server.connectors.ConnectorTypeConverter;

metacat-connector-hive/build.gradle

@@ -20,5 +20,6 @@ dependencies {
  compile "com.google.inject:guice:${guice_version}"
  compile "com.google.inject.extensions:guice-persist:${guice_version}"
  compile "com.google.inject.extensions:guice-multibindings:${guice_version}"
+ runtime "mysql:mysql-connector-java:${mysql_connector_version}"
  testCompile project(':metacat-testdata-provider').sourceSets.test.output
 }

metacat-connector-hive/src/main/java/com/netflix/metacat/connector/hive/HiveConnectorDatabaseService.java

@@ -30,8 +30,10 @@
 import com.netflix.metacat.common.server.exception.DatabaseNotFoundException;
 import com.netflix.metacat.common.server.exception.InvalidMetaException;
 import com.netflix.metacat.connector.hive.converters.HiveConnectorInfoConverter;
+import lombok.NonNull;
 import org.apache.hadoop.hive.metastore.api.AlreadyExistsException;
 import org.apache.hadoop.hive.metastore.api.Database;
+import org.apache.hadoop.hive.metastore.api.InvalidObjectException;
 import org.apache.hadoop.hive.metastore.api.InvalidOperationException;
 import org.apache.hadoop.hive.metastore.api.MetaException;
 import org.apache.hadoop.hive.metastore.api.NoSuchObjectException;
@@ -47,6 +49,7 @@
  * HiveConnectorDatabaseService.
  *
  * @author zhenl
+ * @since 1.0.0
  */
 public class HiveConnectorDatabaseService implements ConnectorDatabaseService {
  private final IMetacatHiveClient metacatHiveClient;
@@ -62,8 +65,8 @@ public class HiveConnectorDatabaseService implements ConnectorDatabaseService {
  */
  @Inject
  public HiveConnectorDatabaseService(@Named("catalogName") final String catalogName,
- @Nonnull final IMetacatHiveClient metacatHiveClient,
- @Nonnull final HiveConnectorInfoConverter hiveMetacatConverters) {
+ @Nonnull @NonNull final IMetacatHiveClient metacatHiveClient,
+ @Nonnull @NonNull final HiveConnectorInfoConverter hiveMetacatConverters) {
  this.metacatHiveClient = metacatHiveClient;
  this.hiveMetacatConverters = hiveMetacatConverters;
  this.catalogName = catalogName;
@@ -73,24 +76,28 @@ public HiveConnectorDatabaseService(@Named("catalogName") final String catalogNa
  * {@inheritDoc}.
  */
  @Override
- public void create(@Nonnull final ConnectorContext requestContext, @Nonnull final DatabaseInfo databaseInfo) {
+ public void create(@Nonnull @NonNull final ConnectorContext requestContext,
+ @Nonnull @NonNull final DatabaseInfo databaseInfo) {
+ final QualifiedName databaseName = databaseInfo.getName();
  try {
  this.metacatHiveClient.createDatabase(hiveMetacatConverters.fromDatabaseInfo(databaseInfo));
  } catch (AlreadyExistsException exception) {
- throw new DatabaseAlreadyExistsException(databaseInfo.getName(), exception);
- } catch (MetaException exception) {
- throw new InvalidMetaException(databaseInfo.getName(), exception);
+ throw new DatabaseAlreadyExistsException(databaseName, exception);
+ } catch (MetaException | InvalidObjectException exception) {
+ throw new InvalidMetaException(databaseName, exception);
  } catch (TException exception) {
  throw new ConnectorException(
- String.format("Failed creating hive database %s", databaseInfo.getName()), exception);
+ String.format("Failed creating hive database %s", databaseName), exception);
  }
  }
 
  /**
  * {@inheritDoc}.
  */
  @Override
- public void delete(@Nonnull final ConnectorContext requestContext, @Nonnull final QualifiedName name) {
+ public void delete(@Nonnull @NonNull final ConnectorContext requestContext,
+ @Nonnull @NonNull final QualifiedName name) {
+
  try {
  this.metacatHiveClient.dropDatabase(name.getDatabaseName());
  } catch (NoSuchObjectException exception) {
@@ -108,7 +115,28 @@ public void delete(@Nonnull final ConnectorContext requestContext, @Nonnull fina
  * {@inheritDoc}.
  */
  @Override
- public DatabaseInfo get(@Nonnull final ConnectorContext requestContext, @Nonnull final QualifiedName name) {
+ public void update(@Nonnull @NonNull final ConnectorContext context,
+ @Nonnull @NonNull final DatabaseInfo databaseInfo) {
+ final QualifiedName databaseName = databaseInfo.getName();
+ try {
+ this.metacatHiveClient.alterDatabase(databaseName.getDatabaseName(),
+ hiveMetacatConverters.fromDatabaseInfo(databaseInfo));
+ } catch (AlreadyExistsException exception) {
+ throw new DatabaseAlreadyExistsException(databaseName, exception);
+ } catch (MetaException | InvalidObjectException exception) {
+ throw new InvalidMetaException(databaseName, exception);
+ } catch (TException exception) {
+ throw new ConnectorException(
+ String.format("Failed updating hive database %s", databaseName), exception);
+ }
+ }
+
+ /**
+ * {@inheritDoc}.
+ */
+ @Override
+ public DatabaseInfo get(@Nonnull @NonNull final ConnectorContext requestContext,
+ @Nonnull @NonNull final QualifiedName name) {
  try {
  final Database database = metacatHiveClient.getDatabase(name.getDatabaseName());
  return hiveMetacatConverters.toDatabaseInfo(name, database);
@@ -125,7 +153,8 @@ public DatabaseInfo get(@Nonnull final ConnectorContext requestContext, @Nonnull
  * {@inheritDoc}.
  */
  @Override
- public boolean exists(@Nonnull final ConnectorContext requestContext, @Nonnull final QualifiedName name) {
+ public boolean exists(@Nonnull @NonNull final ConnectorContext requestContext,
+ @Nonnull @NonNull final QualifiedName name) {
  try {
  final Database database = this.metacatHiveClient.getDatabase(name.getDatabaseName());
  return database != null;
@@ -139,8 +168,8 @@ public boolean exists(@Nonnull final ConnectorContext requestContext, @Nonnull f
  */
  @Override
  public List<QualifiedName> listNames(
- @Nonnull final ConnectorContext requestContext,
- @Nonnull final QualifiedName name,
+ @Nonnull @NonNull final ConnectorContext requestContext,
+ @Nonnull @NonNull final QualifiedName name,
  @Nullable final QualifiedName prefix,
  @Nullable final Sort sort,
  @Nullable final Pageable pageable
@@ -173,8 +202,8 @@ public List<QualifiedName> listNames(
  */
  @Override
  public List<DatabaseInfo> list(
- @Nonnull final ConnectorContext requestContext,
- @Nonnull final QualifiedName name,
+ @Nonnull @NonNull final ConnectorContext requestContext,
+ @Nonnull @NonNull final QualifiedName name,
  @Nullable final QualifiedName prefix,
  @Nullable final Sort sort,
  @Nullable final Pageable pageable