Add namespace, before, and after to search endpoint (MarquezPro…

…ject#2556) * Add namespace and after query params Signed-off-by: wslulciuc <[email protected]> * continued: Add namespace and after query params Signed-off-by: wslulciuc <[email protected]> * continued: Add namespace and after query params Signed-off-by: wslulciuc <[email protected]> * Add before query param to search Signed-off-by: wslulciuc <[email protected]> * Add `namespace`, `before`, and `after` to searchAPI (MarquezProject#2553) Signed-off-by: wslulciuc <[email protected]> * Add namespace, before and after to SearchDao (MarquezProject#2555) Add support for the following parameters in the SearchDao: * `namespace` - match jobs or datasets within the given namespace. * `before` - match jobs or datasets before YYYY-MM-DD. * `after` - match jobs or datasets after YYYY-MM-DD. Relates to issue: MarquezProject#2550 Signed-off-by: Tatiana Al-Chueyr <[email protected]> Co-authored-by: Willy Lulciuc <[email protected]> * Enforce `YYYY-MM-DD` for query params Signed-off-by: wslulciuc <[email protected]> * Use `YYYY-MM-DD` as `LocalDate` Signed-off-by: wslulciuc <[email protected]> --------- Signed-off-by: wslulciuc <[email protected]> Signed-off-by: Tatiana Al-Chueyr <[email protected]> Co-authored-by: Tatiana Al-Chueyr <[email protected]>
ubermuse · Sep 14, 2023 · 3bbcd80 · 3bbcd80
1 parent d0d35fa
commit 3bbcd80
Show file tree

Hide file tree

Showing 5 changed files with 175 additions and 41 deletions.
diff --git a/api/src/main/java/marquez/api/SearchResource.java b/api/src/main/java/marquez/api/SearchResource.java
@@ -6,15 +6,18 @@
 package marquez.api;
 
 import static javax.ws.rs.core.MediaType.APPLICATION_JSON;
+import static marquez.common.Utils.toLocateDateOrNull;
 
 import com.codahale.metrics.annotation.ExceptionMetered;
 import com.codahale.metrics.annotation.ResponseMetered;
 import com.codahale.metrics.annotation.Timed;
 import com.fasterxml.jackson.annotation.JsonCreator;
 import java.util.List;
 import javax.annotation.Nullable;
+import javax.validation.Valid;
 import javax.validation.constraints.Min;
-import javax.validation.constraints.NotNull;
+import javax.validation.constraints.NotBlank;
+import javax.validation.constraints.Pattern;
 import javax.ws.rs.DefaultValue;
 import javax.ws.rs.GET;
 import javax.ws.rs.Path;
@@ -33,6 +36,7 @@
 @Slf4j
 @Path("/api/v1/search")
 public class SearchResource {
+ private static final String YYYY_MM_DD = "^\\d{4}-\\d{2}-\\d{2}$";
  private static final String DEFAULT_SORT = "name";
  private static final String DEFAULT_LIMIT = "10";
  private static final int MIN_LIMIT = 0;
@@ -49,25 +53,23 @@ public SearchResource(@NonNull final SearchDao searchDao) {
  @GET
  @Produces(APPLICATION_JSON)
  public Response search(
- @QueryParam("q") @NotNull String query,
+ @QueryParam("q") @NotBlank String query,
  @QueryParam("filter") @Nullable SearchFilter filter,
  @QueryParam("sort") @DefaultValue(DEFAULT_SORT) SearchSort sort,
- @QueryParam("limit") @DefaultValue(DEFAULT_LIMIT) @Min(MIN_LIMIT) int limit) {
- return Response.ok(
- isQueryBlank(query)
- ? SearchResults.EMPTY
- : searchWithNonBlankQuery(query, filter, sort, limit))
- .build();
- }
-
- private static boolean isQueryBlank(@NonNull String query) {
- return query.trim().isEmpty();
- }
-
- private SearchResults searchWithNonBlankQuery(
- String query, SearchFilter filter, SearchSort sort, int limit) {
- final List<SearchResult> results = searchDao.search(query, filter, sort, limit);
- return new SearchResults(results);
+ @QueryParam("limit") @DefaultValue(DEFAULT_LIMIT) @Min(MIN_LIMIT) int limit,
+ @QueryParam("namespace") @Nullable String namespace,
+ @QueryParam("before") @Valid @Pattern(regexp = YYYY_MM_DD) @Nullable String before,
+ @QueryParam("after") @Valid @Pattern(regexp = YYYY_MM_DD) @Nullable String after) {
+ final List<SearchResult> searchResults =
+ searchDao.search(
+ query,
+ filter,
+ sort,
+ limit,
+ namespace,
+ toLocateDateOrNull(before),
+ toLocateDateOrNull(after));
+ return Response.ok(new SearchResults(searchResults)).build();
  }
 
  /** Wrapper for {@link SearchResult}s which also contains a {@code total count}. */
@@ -81,7 +83,5 @@ public SearchResults(@NonNull final List<SearchResult> results) {
  this.totalCount = results.size();
  this.results = results;
  }
-
- static final SearchResults EMPTY = new SearchResults(List.of());
  }
 }
diff --git a/api/src/main/java/marquez/common/Utils.java b/api/src/main/java/marquez/common/Utils.java
@@ -31,9 +31,11 @@
 import java.nio.ByteBuffer;
 import java.nio.charset.StandardCharsets;
 import java.time.Instant;
+import java.time.LocalDate;
 import java.time.ZonedDateTime;
 import java.util.List;
 import java.util.Map;
+import java.util.Optional;
 import java.util.Set;
 import java.util.TreeSet;
 import java.util.UUID;
@@ -105,6 +107,10 @@ public static void addZonedDateTimeMixin(ObjectMapper mapper) {
  @JsonDeserialize(using = FlexibleDateTimeDeserializer.class)
  static final class ZonedDateTimeMixin {}
 
+ public static LocalDate toLocateDateOrNull(@Nullable final String timeAsString) {
+ return Optional.ofNullable(timeAsString).map(LocalDate::parse).orElse(null);
+ }
+
  public static String toJson(@NonNull final Object value) {
  try {
  return MAPPER.writeValueAsString(value);

diff --git a/api/src/main/java/marquez/db/SearchDao.java b/api/src/main/java/marquez/db/SearchDao.java
@@ -5,17 +5,45 @@
 
 package marquez.db;
 
+import java.time.LocalDate;
 import java.util.List;
+import javax.annotation.Nullable;
 import marquez.api.models.SearchFilter;
 import marquez.api.models.SearchResult;
 import marquez.api.models.SearchSort;
 import marquez.db.mappers.SearchResultMapper;
+import org.jdbi.v3.sqlobject.SqlObject;
 import org.jdbi.v3.sqlobject.config.RegisterRowMapper;
 import org.jdbi.v3.sqlobject.statement.SqlQuery;
 
 /** The DAO for {@link SearchResult}. */
 @RegisterRowMapper(SearchResultMapper.class)
-public interface SearchDao {
+public interface SearchDao extends SqlObject {
+
+ default List<SearchResult> search(String query, SearchFilter filter, SearchSort sort, int limit) {
+ return search(query, filter, sort, limit, null, null, null);
+ }
+
+ default List<SearchResult> search(
+ String query, SearchFilter filter, SearchSort sort, int limit, String namespace) {
+ return search(query, filter, sort, limit, namespace, null, null);
+ }
+
+ default List<SearchResult> search(
+ String query, SearchFilter filter, SearchSort sort, int limit, LocalDate before) {
+ return search(query, filter, sort, limit, null, before, null);
+ }
+
+ default List<SearchResult> search(
+ String query,
+ SearchFilter filter,
+ SearchSort sort,
+ int limit,
+ LocalDate before,
+ LocalDate after) {
+ return search(query, filter, sort, limit, null, before, after);
+ }
+
  /**
  * Returns all datasets and jobs that match the provided query; matching of datasets and jobs are
  * string based and case-insensitive.
@@ -24,26 +52,41 @@ public interface SearchDao {
  * @param filter The filter to apply to the query result.
  * @param sort The sort to apply to the query result.
  * @param limit The limit to apply to the query result.
+ * @param namespace Match jobs or datasets within the given namespace.
+ * @param before Match jobs or datasets before YYYY-MM-DD.
+ * @param after Match jobs or datasets after YYYY-MM-DD.
  * @return A {@link SearchResult} object.
  */
  @SqlQuery(
  """
- SELECT type, name, updated_at, namespace_name
- FROM (
- SELECT 'DATASET' AS type, d.name, d.updated_at, d.namespace_name
- FROM datasets_view AS d
- WHERE d.name ilike '%' || :query || '%'
- UNION
- SELECT DISTINCT ON (j.namespace_name, j.name)\s
- 'JOB' AS type, j.name, j.updated_at, j.namespace_name
- FROM (SELECT namespace_name, name, unnest(COALESCE(aliases, Array[NULL]::varchar[])) AS alias, updated_at\s
- FROM jobs_view WHERE symlink_target_uuid IS NULL
- ORDER BY updated_at DESC) AS j
- WHERE j.name ilike '%' || :query || '%'
- OR j.alias ilike '%' || :query || '%'
- ) AS results
- WHERE type = :filter OR CAST(:filter AS TEXT) IS NULL
- ORDER BY :sort
- LIMIT :limit""")
- List<SearchResult> search(String query, SearchFilter filter, SearchSort sort, int limit);
+ SELECT type, name, updated_at, namespace_name
+ FROM (
+ SELECT 'DATASET' AS type, d.name, d.updated_at, d.namespace_name
+ FROM datasets_view AS d
+ WHERE (d.namespace_name = :namespace OR CAST(:namespace AS TEXT) IS NULL)
+ AND (d.updated_at < :before OR CAST(:before AS TEXT) IS NULL)
+ AND (d.updated_at > :after OR CAST(:after AS TEXT) IS NULL)
+ AND (d.name ILIKE '%' || :query || '%')
+ UNION
+ SELECT DISTINCT ON (j.namespace_name, j.name)
+ 'JOB' AS type, j.name, j.updated_at, j.namespace_name
+ FROM (SELECT namespace_name, name, UNNEST(COALESCE(aliases, Array[NULL]::varchar[])) AS alias, updated_at
+ FROM jobs_view WHERE symlink_target_uuid IS NULL
+ ORDER BY updated_at DESC) AS j
+ WHERE (j.namespace_name = :namespace OR CAST(:namespace AS TEXT) IS NULL)
+ AND (j.updated_at < :before OR CAST(:before AS TEXT) IS NULL)
+ AND (j.updated_at > :after OR CAST(:after AS TEXT) IS NULL)
+ AND (j.name ILIKE '%' || :query || '%' OR j.alias ILIKE '%' || :query || '%')
+ ) AS results
+ WHERE type = :filter OR CAST(:filter AS TEXT) IS NULL
+ ORDER BY :sort
+ LIMIT :limit""")
+ List<SearchResult> search(
+ String query,
+ SearchFilter filter,
+ SearchSort sort,
+ int limit,
+ @Nullable String namespace,
+ @Nullable LocalDate before,
+ @Nullable LocalDate after);
 }
diff --git a/api/src/test/java/marquez/db/SearchDaoTest.java b/api/src/test/java/marquez/db/SearchDaoTest.java
@@ -11,6 +11,9 @@
 import java.net.URL;
 import java.sql.SQLException;
 import java.time.Instant;
+import java.time.LocalDate;
+import java.time.ZoneOffset;
+import java.time.temporal.ChronoUnit;
 import java.util.List;
 import java.util.Map;
 import java.util.stream.Collectors;
@@ -36,6 +39,10 @@
 @Tag("DataAccessTests")
 @ExtendWith(MarquezJdbiExternalPostgresExtension.class)
 public class SearchDaoTest {
+ private static final LocalDate BEFORE =
+ LocalDate.ofInstant(Instant.now().plus(1, ChronoUnit.DAYS), ZoneOffset.systemDefault());
+ private static final LocalDate AFTER =
+ LocalDate.ofInstant(Instant.now().minus(1, ChronoUnit.DAYS), ZoneOffset.systemDefault());
 
  static final int LIMIT = 25;
  static final int NUM_OF_JOBS = 2;
@@ -62,6 +69,11 @@ public static void setUpOnce(final Jdbi jdbi) throws SQLException {
  DbTestUtils.newDataset(jdbi, "time_ordering_1");
  DbTestUtils.newDataset(jdbi, "time_ordering_2");
 
+ DbTestUtils.newDataset(jdbi, "namespace1", "datasetA");
+ DbTestUtils.newDataset(jdbi, "namespace1", "datasetB");
+ DbTestUtils.newDataset(jdbi, "namespace111", "excludeMe");
+ DbTestUtils.newDataset(jdbi, "namespace2", "datasetC");
+
  ImmutableSet<JobRow> jobRows = DbTestUtils.newJobs(jdbi, NUM_OF_JOBS);
 
  // add a symlinked job - validate that the number of results is the same in the below unit test
@@ -107,6 +119,52 @@ public static void setUpOnce(final Jdbi jdbi) throws SQLException {
  });
  }
 
+ @Test
+ public void testSearch_filterByNamespace() {
+ final String query = "dataset";
+ final String namespace = "namespace1";
+ final List<SearchResult> resultsWithSort =
+ searchDao.search(query, SearchFilter.DATASET, SearchSort.UPDATE_AT, LIMIT, namespace);
+
+ // Ensure sorted search results contain N datasets.
+ assertThat(resultsWithSort).hasSize(2);
+ assertThat(resultsWithSort).extracting("name").contains("datasetA", "datasetB");
+ }
+
+ @Test
+ public void testSearch_filterByNamespaceAndAfter() {
+ final String query = "dataset";
+ final String namespace = "namespace2";
+ final List<SearchResult> resultsWithSort =
+ searchDao.search(
+ query, SearchFilter.DATASET, SearchSort.UPDATE_AT, LIMIT, namespace, null, AFTER);
+
+ // Ensure sorted search results contain N datasets.
+ assertThat(resultsWithSort).hasSize(1);
+ assertThat(resultsWithSort).extracting("name").contains("datasetC");
+ }
+
+ @Test
+ public void testSearch_filterByNamespaceBeforeFuture() {
+ final String query = "dataset";
+ final List<SearchResult> resultsWithSort =
+ searchDao.search(query, SearchFilter.DATASET, SearchSort.UPDATE_AT, LIMIT, BEFORE);
+
+ // Ensure sorted search results contain N datasets.
+ assertThat(resultsWithSort).hasSize(15);
+ assertThat(resultsWithSort).extracting("name").contains("datasetA", "datasetB");
+ }
+
+ @Test
+ public void testSearch_filterByNamespaceBeforePast() {
+ final String query = "dataset";
+ final List<SearchResult> resultsWithSort =
+ searchDao.search(query, SearchFilter.DATASET, SearchSort.UPDATE_AT, LIMIT, AFTER);
+
+ // Ensure sorted search results contain N datasets.
+ assertThat(resultsWithSort).hasSize(0);
+ }
+
  @Test
  public void testSearch(Jdbi jdbi) {
  jdbi.withHandle(

diff --git a/spec/openapi.yml b/spec/openapi.yml
@@ -669,6 +669,33 @@ paths:
  - $ref: '#/components/parameters/filter'
  - $ref: '#/components/parameters/sort'
  - $ref: '#/components/parameters/limit'
+ - name: namespace
+ in: query
+ description: Match jobs or datasets within the given namespace.
+ required: false
+ schema:
+ type: string
+ maxLength: 1024
+ example: my-namespace
+ - before:
+ name: before
+ in: query
+ description: Match jobs or datasets **before** `YYYY-MM-DD`.
+ required: false
+ schema:
+ type: string
+ pattern: YYYY-MM-DD
+ example: "2022-09-15"
+ - after:
+ name: after
+ in: query
+ description: Match jobs or datasets **after** `YYYY-MM-DD`.
+ required: false
+ schema:
+ type: string
+ pattern: YYYY-MM-DD
+ example: "2022-09-15"
+
  summary: Query all datasets and jobs
  description: Returns one or more datasets and jobs of your query.
  tags:
@@ -773,7 +800,7 @@ components:
  limit:
  name: limit
  in: query
- description: The number of results to return from offset
+ description: The number of results to return from offset.
  required: false
  schema:
  type: integer
@@ -783,7 +810,7 @@ components:
  offset:
  name: offset
  in: query
- description: The initial position from which to return results
+ description: The initial position from which to return results.
  required: false
  schema:
  type: integer