-
Notifications
You must be signed in to change notification settings - Fork 279
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Fixing hive partition getQualifiedName, adding resolver ending point #111
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,90 @@ | ||
/* | ||
* Copyright 2017 Netflix, Inc. | ||
* | ||
* Licensed under the Apache License, Version 2.0 (the "License"); | ||
* you may not use this file except in compliance with the License. | ||
* You may obtain a copy of the License at | ||
* | ||
* https://www.apache.org/licenses/LICENSE-2.0 | ||
* | ||
* Unless required by applicable law or agreed to in writing, software | ||
* distributed under the License is distributed on an "AS IS" BASIS, | ||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
* See the License for the specific language governing permissions and | ||
* limitations under the License. | ||
*/ | ||
|
||
|
||
package com.netflix.metacat.common.api; | ||
|
||
import com.netflix.metacat.common.dto.ResolveByUriRequestDto; | ||
import com.netflix.metacat.common.dto.ResolveByUriResponseDto; | ||
import com.wordnik.swagger.annotations.Api; | ||
import com.wordnik.swagger.annotations.ApiOperation; | ||
import com.wordnik.swagger.annotations.ApiParam; | ||
|
||
import javax.ws.rs.Consumes; | ||
import javax.ws.rs.DefaultValue; | ||
import javax.ws.rs.POST; | ||
import javax.ws.rs.Path; | ||
import javax.ws.rs.Produces; | ||
import javax.ws.rs.QueryParam; | ||
import javax.ws.rs.core.MediaType; | ||
import javax.ws.rs.core.Response; | ||
|
||
|
||
/** | ||
* ResolverV1. | ||
* | ||
* @author zhenl | ||
* @since 1.0.0 | ||
*/ | ||
@Path("mds/v1/resolver") | ||
@Api(value = "ResolverV1", | ||
description = "Metadata resolver operations", | ||
produces = MediaType.APPLICATION_JSON, | ||
consumes = MediaType.APPLICATION_JSON) | ||
@Consumes(MediaType.APPLICATION_JSON) | ||
@Produces(MediaType.APPLICATION_JSON) | ||
public interface ResolverV1 { | ||
/** | ||
* resolveByUri. | ||
* | ||
* @param prefixSearch use prefix search | ||
* @param resolveByUriRequestDto resolveByUriRequest | ||
* @return response from uri search | ||
*/ | ||
@POST | ||
@ApiOperation( | ||
position = 0, | ||
value = "Returns the list of qualified names of tables and partitions containing the given URI path", | ||
notes = "Returns the list of qualified names of tables and partitions containing the given URI path") | ||
ResolveByUriResponseDto resolveByUri( | ||
@ApiParam(value = "do prefix search for URI", required = false) | ||
@DefaultValue("false") | ||
@QueryParam("prefixSearch") | ||
Boolean prefixSearch, | ||
ResolveByUriRequestDto resolveByUriRequestDto); | ||
|
||
/** | ||
* isUriUsedMoreThanOnce. | ||
* | ||
* @param prefixSearch use prefix search | ||
* @param resolveByUriRequestDto resolveByUriRequest | ||
* @return response of check if a uri used more than once | ||
*/ | ||
@POST | ||
@Path("isUriUsedMoreThanOnce") | ||
@ApiOperation( | ||
position = 1, | ||
value = "Returns status 200 if the given URI is being referred more than once." | ||
+ " Returns status 404 if the given URI is not found or not being referred more than once.", | ||
notes = "Returns status 200 if the given URI is being referred more than once." | ||
+ " Returns status 404 if the given URI is not found or not being referred more than once.") | ||
Response isUriUsedMoreThanOnce( | ||
@ApiParam(value = "do prefix search for URI", required = false) | ||
@DefaultValue("false") | ||
@QueryParam("prefixSearch") | ||
Boolean prefixSearch, | ||
ResolveByUriRequestDto resolveByUriRequestDto); | ||
} |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,17 @@ | ||
package com.netflix.metacat.common.dto; | ||
|
||
import lombok.Data; | ||
import lombok.EqualsAndHashCode; | ||
|
||
/** | ||
* ResolveByUriRequestDto. | ||
* | ||
* @author zhenl | ||
* @since 1.0.0 | ||
*/ | ||
@Data | ||
@EqualsAndHashCode(callSuper = false) | ||
public class ResolveByUriRequestDto extends BaseDto { | ||
private static final long serialVersionUID = -2649784382533439526L; | ||
private String uri; | ||
} |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,21 @@ | ||
package com.netflix.metacat.common.dto; | ||
|
||
import com.netflix.metacat.common.QualifiedName; | ||
import lombok.Data; | ||
import lombok.EqualsAndHashCode; | ||
|
||
import java.util.List; | ||
|
||
/** | ||
* ResolveByUriResponseDto. | ||
* | ||
* @author zhenl | ||
* @since 1.0.0 | ||
*/ | ||
@Data | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. if this is a DTO please make the variables final. That way lombok won't create setters. |
||
@EqualsAndHashCode(callSuper = false) | ||
public class ResolveByUriResponseDto extends BaseDto { | ||
private static final long serialVersionUID = -4505346090786555046L; | ||
private List<QualifiedName> tables; | ||
private List<QualifiedName> partitions; | ||
} |
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -23,6 +23,7 @@ import com.netflix.metacat.client.Client | |
import com.netflix.metacat.common.QualifiedName | ||
import com.netflix.metacat.common.api.MetacatV1 | ||
import com.netflix.metacat.common.api.PartitionV1 | ||
import com.netflix.metacat.common.api.ResolverV1 | ||
import com.netflix.metacat.common.dto.* | ||
import com.netflix.metacat.common.exception.MetacatAlreadyExistsException | ||
import com.netflix.metacat.common.exception.MetacatBadRequestException | ||
|
@@ -47,14 +48,14 @@ import static TestUtilities.dateCloseEnough | |
class MetacatFunctionalSpec extends Specification { | ||
public static MetacatV1 api | ||
public static PartitionV1 partitionApi | ||
public static ResolverV1 resolverApi | ||
public static final MetacatJson metacatJson = MetacatJsonLocator.INSTANCE | ||
public static final long BATCH_ID = System.currentTimeSeconds() | ||
public static final int timediff = 24 * 3600 | ||
|
||
def setupSpec() { | ||
String httpPort = System.properties['metacat_http_port']?.toString()?.trim() | ||
assert httpPort, 'Required system property "metacat_http_port" is not set' | ||
|
||
def client = Client.builder() | ||
.withHost("https://localhost:$httpPort") | ||
.withDataTypeContext('pig') | ||
|
@@ -64,9 +65,8 @@ class MetacatFunctionalSpec extends Specification { | |
.build() | ||
api = client.api | ||
partitionApi = client.partitionApi | ||
|
||
resolverApi = client.resolverApi | ||
TestCatalogs.resetAll() | ||
|
||
} | ||
|
||
|
||
|
@@ -518,6 +518,17 @@ class MetacatFunctionalSpec extends Specification { | |
then: | ||
database.tables.contains(tableName) | ||
|
||
when: | ||
def resovlerRep = resolverApi.resolveByUri(false, new ResolveByUriRequestDto(uri: dataUri)) | ||
then: | ||
!resovlerRep.tables.empty | ||
|
||
when: | ||
resovlerRep = resolverApi.resolveByUri(true, | ||
new ResolveByUriRequestDto(uri: "file:/tmp/${catalog.name}/${databaseName}/".toString())) | ||
then: | ||
!resovlerRep.tables.empty | ||
|
||
when: | ||
def table = api.getTable(catalog.name, databaseName, tableName, true, true, true) | ||
def name = QualifiedName.ofTable(catalog.name, databaseName, tableName) | ||
|
@@ -773,6 +784,8 @@ class MetacatFunctionalSpec extends Specification { | |
] | ||
) | ||
|
||
def resolveByUridto = new ResolveByUriRequestDto(uri: dataUri) | ||
|
||
when: | ||
def keys = partitionApi.getPartitionKeys(name.catalogName, name.databaseName, name.tableName, null, null, null, null, null) | ||
|
||
|
@@ -787,6 +800,22 @@ class MetacatFunctionalSpec extends Specification { | |
response.added.contains(escapedName.partitionName) | ||
response.updated.empty | ||
|
||
when: | ||
def resovlerRep = resolverApi.resolveByUri(false, resolveByUridto) | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Same here. Please add a test case with prefixSearch=true There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Added the test. |
||
then: | ||
!resovlerRep.partitions.empty | ||
|
||
when: | ||
resovlerRep = resolverApi.resolveByUri(true, | ||
new ResolveByUriRequestDto(uri: "file:/tmp/${name.catalogName}/${name.databaseName}/${name.tableName}".toString())) | ||
then: | ||
!resovlerRep.partitions.empty | ||
|
||
when: | ||
def usedMoreThanOnce = resolverApi.isUriUsedMoreThanOnce(false, resolveByUridto) | ||
then: | ||
thrown(MetacatNotFoundException) | ||
|
||
when: | ||
keys = partitionApi.getPartitionKeys(name.catalogName, name.databaseName, name.tableName, null, null, null, null, null) | ||
|
||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,69 @@ | ||
package com.netflix.metacat.main.api; | ||
|
||
import com.google.inject.Inject; | ||
import com.netflix.metacat.common.api.ResolverV1; | ||
import com.netflix.metacat.common.dto.ResolveByUriRequestDto; | ||
import com.netflix.metacat.common.dto.ResolveByUriResponseDto; | ||
import com.netflix.metacat.main.services.PartitionService; | ||
import com.netflix.metacat.main.services.TableService; | ||
|
||
import javax.ws.rs.core.Response; | ||
|
||
/** | ||
* ResovlerServiceImpl. | ||
* | ||
* @author zhenl | ||
* @since 1.0.0 | ||
*/ | ||
public class ResovlerV1Resource implements ResolverV1 { | ||
private TableService tableService; | ||
private PartitionService partitionService; | ||
|
||
/** | ||
* ResovlerServiceImpl. | ||
* @param tableService table service | ||
* @param partitionService partition service | ||
*/ | ||
@Inject | ||
ResovlerV1Resource(final TableService tableService, final PartitionService partitionService) { | ||
this.tableService = tableService; | ||
this.partitionService = partitionService; | ||
} | ||
|
||
/** | ||
* Gets the qualified name by uri. | ||
* | ||
* @param resolveByUriRequestDto resolveByUriRequestDto | ||
* @param prefixSearch search by prefix flag | ||
* @return the qualified name of uri | ||
*/ | ||
@Override | ||
public ResolveByUriResponseDto resolveByUri(final Boolean prefixSearch, | ||
final ResolveByUriRequestDto resolveByUriRequestDto) { | ||
final ResolveByUriResponseDto result = new ResolveByUriResponseDto(); | ||
result.setTables(tableService.getQualifiedNames(resolveByUriRequestDto.getUri(), prefixSearch)); | ||
result.setPartitions(partitionService.getQualifiedNames(resolveByUriRequestDto.getUri(), prefixSearch)); | ||
return result; | ||
} | ||
|
||
/** | ||
* Check if the uri used more than once. | ||
* | ||
* @param prefixSearch search by prefix flag | ||
* @param resolveByUriRequestDto resolveByUriRequestDto | ||
* @return true if the uri used more than once | ||
*/ | ||
@Override | ||
public Response isUriUsedMoreThanOnce(final Boolean prefixSearch, | ||
final ResolveByUriRequestDto resolveByUriRequestDto) { | ||
int size = tableService.getQualifiedNames(resolveByUriRequestDto.getUri(), prefixSearch).size(); | ||
if (size < 2) { | ||
size += partitionService.getQualifiedNames(resolveByUriRequestDto.getUri(), prefixSearch).size(); | ||
} | ||
if (size > 1) { | ||
return Response.ok().build(); | ||
} else { | ||
return Response.status(Response.Status.NOT_FOUND).build(); | ||
} | ||
} | ||
} |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
does this work? cool
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Yes, it works!