diff --git a/README.md b/README.md index 2aa0c68f..27a2c7b2 100644 --- a/README.md +++ b/README.md @@ -229,6 +229,9 @@ Supported Features * ThetaSketchEstimate * ThetaSketchSetOp +#### Virtual Columns +* Expression + #### Granularity * Duration * Period diff --git a/src/main/java/in/zapr/druid/druidry/dimension/enums/OutputType.java b/src/main/java/in/zapr/druid/druidry/dimension/enums/OutputType.java index 66c51bc4..9c710484 100644 --- a/src/main/java/in/zapr/druid/druidry/dimension/enums/OutputType.java +++ b/src/main/java/in/zapr/druid/druidry/dimension/enums/OutputType.java @@ -19,7 +19,7 @@ import com.fasterxml.jackson.annotation.JsonValue; public enum OutputType { - STRING, LONG, FLOAT; + STRING, LONG, FLOAT, DOUBLE; @JsonValue public String getName() { diff --git a/src/main/java/in/zapr/druid/druidry/query/aggregation/DruidAggregationQuery.java b/src/main/java/in/zapr/druid/druidry/query/aggregation/DruidAggregationQuery.java index bff05361..09219278 100644 --- a/src/main/java/in/zapr/druid/druidry/query/aggregation/DruidAggregationQuery.java +++ b/src/main/java/in/zapr/druid/druidry/query/aggregation/DruidAggregationQuery.java @@ -26,6 +26,7 @@ import in.zapr.druid.druidry.granularity.Granularity; import in.zapr.druid.druidry.postAggregator.DruidPostAggregator; import in.zapr.druid.druidry.query.DruidQuery; +import in.zapr.druid.druidry.virtualColumn.DruidVirtualColumn; import lombok.EqualsAndHashCode; import lombok.Getter; @@ -35,6 +36,7 @@ public abstract class DruidAggregationQuery extends DruidQuery { protected List intervals; protected Granularity granularity; + protected List virtualColumns; protected DruidFilter filter; protected List aggregations; protected List postAggregations; diff --git a/src/main/java/in/zapr/druid/druidry/query/aggregation/DruidGroupByQuery.java b/src/main/java/in/zapr/druid/druidry/query/aggregation/DruidGroupByQuery.java index 94a2c786..5d65f133 100644 --- a/src/main/java/in/zapr/druid/druidry/query/aggregation/DruidGroupByQuery.java +++ b/src/main/java/in/zapr/druid/druidry/query/aggregation/DruidGroupByQuery.java @@ -29,6 +29,7 @@ import in.zapr.druid.druidry.limitSpec.DefaultLimitSpec; import in.zapr.druid.druidry.postAggregator.DruidPostAggregator; import in.zapr.druid.druidry.query.QueryType; +import in.zapr.druid.druidry.virtualColumn.DruidVirtualColumn; import lombok.Builder; import lombok.EqualsAndHashCode; import lombok.Getter; @@ -50,6 +51,7 @@ private DruidGroupByQuery(@NonNull String dataSource, @NonNull List dimensions, DefaultLimitSpec limitSpec, @NonNull Granularity granularity, + List virtualColumns, DruidFilter filter, List aggregators, List postAggregators, @@ -61,10 +63,11 @@ private DruidGroupByQuery(@NonNull String dataSource, this.dimensions = dimensions; this.limitSpec = limitSpec; this.granularity = granularity; + this.virtualColumns = virtualColumns; this.filter = filter; this.aggregations = aggregators; this.postAggregations = postAggregators; this.intervals = intervals; this.context = context; } -} \ No newline at end of file +} diff --git a/src/main/java/in/zapr/druid/druidry/query/aggregation/DruidTimeSeriesQuery.java b/src/main/java/in/zapr/druid/druidry/query/aggregation/DruidTimeSeriesQuery.java index c3c52bb2..76de8a97 100644 --- a/src/main/java/in/zapr/druid/druidry/query/aggregation/DruidTimeSeriesQuery.java +++ b/src/main/java/in/zapr/druid/druidry/query/aggregation/DruidTimeSeriesQuery.java @@ -27,6 +27,7 @@ import in.zapr.druid.druidry.granularity.Granularity; import in.zapr.druid.druidry.postAggregator.DruidPostAggregator; import in.zapr.druid.druidry.query.QueryType; +import in.zapr.druid.druidry.virtualColumn.DruidVirtualColumn; import lombok.Builder; import lombok.EqualsAndHashCode; import lombok.Getter; @@ -42,6 +43,7 @@ public class DruidTimeSeriesQuery extends DruidAggregationQuery { @Builder private DruidTimeSeriesQuery(@NonNull String dataSource, Boolean descending, @NonNull List intervals, @NonNull Granularity granularity, + List virtualColumns, DruidFilter filter, List aggregators, List postAggregators, Context context) { @@ -50,6 +52,7 @@ private DruidTimeSeriesQuery(@NonNull String dataSource, Boolean descending, this.descending = descending; this.intervals = intervals; this.granularity = granularity; + this.virtualColumns = virtualColumns; this.filter = filter; this.aggregations = aggregators; this.postAggregations = postAggregators; diff --git a/src/main/java/in/zapr/druid/druidry/query/aggregation/DruidTopNQuery.java b/src/main/java/in/zapr/druid/druidry/query/aggregation/DruidTopNQuery.java index ba12711e..0455664c 100644 --- a/src/main/java/in/zapr/druid/druidry/query/aggregation/DruidTopNQuery.java +++ b/src/main/java/in/zapr/druid/druidry/query/aggregation/DruidTopNQuery.java @@ -31,6 +31,7 @@ import in.zapr.druid.druidry.postAggregator.DruidPostAggregator; import in.zapr.druid.druidry.query.QueryType; import in.zapr.druid.druidry.topNMetric.TopNMetric; +import in.zapr.druid.druidry.virtualColumn.DruidVirtualColumn; import lombok.Builder; import lombok.EqualsAndHashCode; import lombok.Getter; @@ -49,6 +50,7 @@ public class DruidTopNQuery extends DruidAggregationQuery { private DruidTopNQuery(@NonNull String dataSource, @NonNull List intervals, @NonNull Granularity granularity, + List virtualColumns, DruidFilter filter, List aggregators, List postAggregators, @@ -61,6 +63,7 @@ private DruidTopNQuery(@NonNull String dataSource, this.dataSource = dataSource; this.intervals = intervals; this.granularity = granularity; + this.virtualColumns = virtualColumns; this.filter = filter; this.aggregations = aggregators; this.postAggregations = postAggregators; diff --git a/src/main/java/in/zapr/druid/druidry/query/scan/DruidScanQuery.java b/src/main/java/in/zapr/druid/druidry/query/scan/DruidScanQuery.java index 67530cb1..a20dcca9 100644 --- a/src/main/java/in/zapr/druid/druidry/query/scan/DruidScanQuery.java +++ b/src/main/java/in/zapr/druid/druidry/query/scan/DruidScanQuery.java @@ -27,6 +27,7 @@ import in.zapr.druid.druidry.filter.DruidFilter; import in.zapr.druid.druidry.query.DruidQuery; import in.zapr.druid.druidry.query.QueryType; +import in.zapr.druid.druidry.virtualColumn.DruidVirtualColumn; import lombok.Builder; import lombok.EqualsAndHashCode; import lombok.Getter; @@ -36,7 +37,7 @@ @JsonInclude(JsonInclude.Include.NON_NULL) @EqualsAndHashCode(callSuper = true) public class DruidScanQuery extends DruidQuery { - + private List virtualColumns; private DruidFilter filter; private Integer batchSize; private List intervals; @@ -46,7 +47,10 @@ public class DruidScanQuery extends DruidQuery { private Boolean legacy; @Builder - private DruidScanQuery(@NonNull String dataSource, DruidFilter filter, Integer batchSize, @NonNull List intervals, List columns, ResultFormat resultFormat, Long limit, Boolean legacy, Context context) { + private DruidScanQuery(@NonNull String dataSource, List virtualColumns, DruidFilter filter, + Integer batchSize, @NonNull List intervals, List columns, ResultFormat resultFormat, + Long limit, Boolean legacy, Context context) { + this.virtualColumns = virtualColumns; this.filter = filter; this.intervals = intervals; this.columns = columns; diff --git a/src/main/java/in/zapr/druid/druidry/query/search/DruidSearchQuery.java b/src/main/java/in/zapr/druid/druidry/query/search/DruidSearchQuery.java index 87287c55..57d5a414 100644 --- a/src/main/java/in/zapr/druid/druidry/query/search/DruidSearchQuery.java +++ b/src/main/java/in/zapr/druid/druidry/query/search/DruidSearchQuery.java @@ -16,7 +16,6 @@ package in.zapr.druid.druidry.query.search; - import com.fasterxml.jackson.annotation.JsonInclude; import java.util.List; @@ -30,6 +29,7 @@ import in.zapr.druid.druidry.granularity.Granularity; import in.zapr.druid.druidry.query.DruidQuery; import in.zapr.druid.druidry.query.QueryType; +import in.zapr.druid.druidry.virtualColumn.DruidVirtualColumn; import lombok.Builder; import lombok.EqualsAndHashCode; import lombok.Getter; @@ -41,6 +41,7 @@ public class DruidSearchQuery extends DruidQuery { private Granularity granularity; + private List virtualColumns; private DruidFilter filter; private Integer limit; private List intervals; @@ -51,6 +52,7 @@ public class DruidSearchQuery extends DruidQuery { @Builder private DruidSearchQuery(@NonNull String dataSource, @NonNull Granularity granularity, + List virtualColumns, DruidFilter filter, Integer limit, @NonNull List intervals, @@ -62,6 +64,7 @@ private DruidSearchQuery(@NonNull String dataSource, this.queryType = QueryType.SEARCH; this.dataSource = dataSource; this.granularity = granularity; + this.virtualColumns = virtualColumns; this.filter = filter; this.limit = limit; this.intervals = intervals; diff --git a/src/main/java/in/zapr/druid/druidry/query/select/DruidSelectQuery.java b/src/main/java/in/zapr/druid/druidry/query/select/DruidSelectQuery.java index d99a26f4..eb83627d 100644 --- a/src/main/java/in/zapr/druid/druidry/query/select/DruidSelectQuery.java +++ b/src/main/java/in/zapr/druid/druidry/query/select/DruidSelectQuery.java @@ -26,6 +26,7 @@ import in.zapr.druid.druidry.granularity.Granularity; import in.zapr.druid.druidry.query.DruidQuery; import in.zapr.druid.druidry.query.QueryType; +import in.zapr.druid.druidry.virtualColumn.DruidVirtualColumn; import lombok.Builder; import lombok.EqualsAndHashCode; import lombok.Getter; @@ -39,6 +40,7 @@ @EqualsAndHashCode(callSuper = true) public class DruidSelectQuery extends DruidQuery { private List intervals; + private List virtualColumns; private DruidFilter filter; private Boolean descending; private Granularity granularity; @@ -49,6 +51,7 @@ public class DruidSelectQuery extends DruidQuery { @Builder public DruidSelectQuery( @NonNull String dataSource, + List virtualColumns, DruidFilter filter, Boolean descending, Granularity granularity, @@ -60,6 +63,7 @@ public DruidSelectQuery( this.queryType = QueryType.SELECT; this.context = context; this.dataSource = dataSource; + this.virtualColumns = virtualColumns; this.filter = filter; this.descending = descending; this.granularity = granularity; diff --git a/src/main/java/in/zapr/druid/druidry/virtualColumn/DruidVirtualColumn.java b/src/main/java/in/zapr/druid/druidry/virtualColumn/DruidVirtualColumn.java new file mode 100644 index 00000000..deba1a75 --- /dev/null +++ b/src/main/java/in/zapr/druid/druidry/virtualColumn/DruidVirtualColumn.java @@ -0,0 +1,21 @@ +package in.zapr.druid.druidry.virtualColumn; + +import com.fasterxml.jackson.annotation.JsonInclude; + +import in.zapr.druid.druidry.dimension.enums.OutputType; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.NonNull; + +@Getter +@EqualsAndHashCode +@JsonInclude(JsonInclude.Include.NON_NULL) +public abstract class DruidVirtualColumn { + @NonNull + protected String type; + + @NonNull + protected String name; + + protected OutputType outputType; +} diff --git a/src/main/java/in/zapr/druid/druidry/virtualColumn/ExpressionVirtualColumn.java b/src/main/java/in/zapr/druid/druidry/virtualColumn/ExpressionVirtualColumn.java new file mode 100644 index 00000000..af186fc0 --- /dev/null +++ b/src/main/java/in/zapr/druid/druidry/virtualColumn/ExpressionVirtualColumn.java @@ -0,0 +1,27 @@ +package in.zapr.druid.druidry.virtualColumn; + +import com.fasterxml.jackson.annotation.JsonInclude; + +import in.zapr.druid.druidry.dimension.enums.OutputType; +import lombok.Builder; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.NonNull; + +@Getter +@EqualsAndHashCode(callSuper = true) +@JsonInclude(JsonInclude.Include.NON_NULL) +public class ExpressionVirtualColumn extends DruidVirtualColumn { + private static final String EXPRESSION_VIRTUAL_COLUMN = "expression"; + + @NonNull + private String expression; + + @Builder + public ExpressionVirtualColumn(@NonNull String name, @NonNull String expression, OutputType outputType) { + this.type = EXPRESSION_VIRTUAL_COLUMN; + this.name = name; + this.outputType = outputType; + this.expression = expression; + } +} diff --git a/src/test/java/in/zapr/druid/druidry/query/scan/DruidScanQueryTest.java b/src/test/java/in/zapr/druid/druidry/query/scan/DruidScanQueryTest.java index cb9b6319..ea737da2 100644 --- a/src/test/java/in/zapr/druid/druidry/query/scan/DruidScanQueryTest.java +++ b/src/test/java/in/zapr/druid/druidry/query/scan/DruidScanQueryTest.java @@ -32,8 +32,10 @@ import java.util.List; import in.zapr.druid.druidry.Interval; +import in.zapr.druid.druidry.dimension.enums.OutputType; import in.zapr.druid.druidry.filter.DruidFilter; import in.zapr.druid.druidry.filter.SelectorFilter; +import in.zapr.druid.druidry.virtualColumn.ExpressionVirtualColumn; public class DruidScanQueryTest { private static ObjectMapper objectMapper; @@ -62,6 +64,7 @@ public void testSampleQuery() throws JsonProcessingException, JSONException { DruidScanQuery query = DruidScanQuery.builder() .dataSource("sample_datasource") .columns(searchDimensions) + .virtualColumns(Collections.singletonList(new ExpressionVirtualColumn("dim3", "dim1 + dim2", OutputType.FLOAT))) .filter(filter) .resultFormat(ResultFormat.LIST) .intervals(Collections.singletonList(interval)) @@ -77,6 +80,12 @@ public void testSampleQuery() throws JsonProcessingException, JSONException { " \"dim1\",\n" + " \"dim2\"\n" + " ],\n" + + " \"virtualColumns\": [{\n" + + " \"type\": \"expression\",\n" + + " \"name\": \"dim3\",\n" + + " \"outputType\": \"FLOAT\",\n" + + " \"expression\": \"dim1 + dim2\"\n" + + " }],\n" + " \"filter\": {\n" + " \"type\": \"selector\",\n" + " \"dimension\": \"dim1\",\n" + @@ -92,7 +101,7 @@ public void testSampleQuery() throws JsonProcessingException, JSONException { "}"; String actualJson = objectMapper.writeValueAsString(query); - JSONAssert.assertEquals(actualJson, expectedJsonAsString, JSONCompareMode.NON_EXTENSIBLE); + JSONAssert.assertEquals(expectedJsonAsString, actualJson, JSONCompareMode.NON_EXTENSIBLE); } @@ -207,4 +216,3 @@ public void testSampleQueryWithEmptyLines() throws JsonProcessingException, JSON } } - diff --git a/src/test/java/in/zapr/druid/druidry/query/search/DruidSearchQueryTest.java b/src/test/java/in/zapr/druid/druidry/query/search/DruidSearchQueryTest.java index 83884b48..94242eee 100644 --- a/src/test/java/in/zapr/druid/druidry/query/search/DruidSearchQueryTest.java +++ b/src/test/java/in/zapr/druid/druidry/query/search/DruidSearchQueryTest.java @@ -36,12 +36,14 @@ import in.zapr.druid.druidry.SortingOrder; import in.zapr.druid.druidry.dimension.DruidDimension; import in.zapr.druid.druidry.dimension.SimpleDimension; +import in.zapr.druid.druidry.dimension.enums.OutputType; import in.zapr.druid.druidry.filter.DruidFilter; import in.zapr.druid.druidry.filter.SelectorFilter; import in.zapr.druid.druidry.filter.searchQuerySpec.InsensitiveContainsSearchQuerySpec; import in.zapr.druid.druidry.filter.searchQuerySpec.SearchQuerySpec; import in.zapr.druid.druidry.granularity.PredefinedGranularity; import in.zapr.druid.druidry.granularity.SimpleGranularity; +import in.zapr.druid.druidry.virtualColumn.ExpressionVirtualColumn; public class DruidSearchQueryTest { private static ObjectMapper objectMapper; @@ -68,6 +70,7 @@ public void testSampleQuery() throws JsonProcessingException, JSONException { DruidSearchQuery query = DruidSearchQuery.builder() .dataSource("sample_datasource") .granularity(new SimpleGranularity(PredefinedGranularity.DAY)) + .virtualColumns(Collections.singletonList(new ExpressionVirtualColumn("dim3", "dim1 + dim2", OutputType.FLOAT))) .searchDimensions(searchDimensions) .query(searchQuerySpec) .sort(SortingOrder.LEXICOGRAPHIC) @@ -78,6 +81,12 @@ public void testSampleQuery() throws JsonProcessingException, JSONException { " \"queryType\": \"search\",\n" + " \"dataSource\": \"sample_datasource\",\n" + " \"granularity\": \"day\",\n" + + " \"virtualColumns\": [{\n" + + " \"type\": \"expression\",\n" + + " \"name\": \"dim3\",\n" + + " \"outputType\": \"FLOAT\",\n" + + " \"expression\": \"dim1 + dim2\"\n" + + " }],\n" + " \"searchDimensions\": [\n" + " \"dim1\",\n" + " \"dim2\"\n" + @@ -156,6 +165,7 @@ public void testAllFields() throws JsonProcessingException, JSONException { DruidSearchQuery query = DruidSearchQuery.builder() .dataSource("sample_datasource") .granularity(new SimpleGranularity(PredefinedGranularity.DAY)) + .virtualColumns(Collections.singletonList(new ExpressionVirtualColumn("dim3", "dim1 + dim2", OutputType.FLOAT))) .filter(druidFilter) .limit(16) .searchDimensions(searchDimensions) @@ -169,6 +179,12 @@ public void testAllFields() throws JsonProcessingException, JSONException { " \"queryType\": \"search\",\n" + " \"dataSource\": \"sample_datasource\",\n" + " \"granularity\": \"day\",\n" + + " \"virtualColumns\": [{\n" + + " \"type\": \"expression\",\n" + + " \"name\": \"dim3\",\n" + + " \"outputType\": \"FLOAT\",\n" + + " \"expression\": \"dim1 + dim2\"\n" + + " }],\n" + " \"filter\": {\n" + " \"type\": \"selector\",\n" + " \"dimension\": \"Dim\",\n" + diff --git a/src/test/java/in/zapr/druid/druidry/query/select/DruidSelectQueryTest.java b/src/test/java/in/zapr/druid/druidry/query/select/DruidSelectQueryTest.java index f6abe385..51f538c0 100644 --- a/src/test/java/in/zapr/druid/druidry/query/select/DruidSelectQueryTest.java +++ b/src/test/java/in/zapr/druid/druidry/query/select/DruidSelectQueryTest.java @@ -31,9 +31,11 @@ import java.util.HashMap; import in.zapr.druid.druidry.Interval; +import in.zapr.druid.druidry.dimension.enums.OutputType; import in.zapr.druid.druidry.granularity.Granularity; import in.zapr.druid.druidry.granularity.PredefinedGranularity; import in.zapr.druid.druidry.granularity.SimpleGranularity; +import in.zapr.druid.druidry.virtualColumn.ExpressionVirtualColumn; public class DruidSelectQueryTest { private static ObjectMapper objectMapper; @@ -60,6 +62,7 @@ public void testSampleQuery() throws JsonProcessingException, JSONException { .dataSource("wikipedia") .descending(false) .granularity(granularity) + .virtualColumns(Collections.singletonList(new ExpressionVirtualColumn("dim3", "dim1 + dim2", OutputType.FLOAT))) .intervals(Collections.singletonList(interval)) .pagingSpec(pagingSpec) .build(); @@ -72,6 +75,12 @@ public void testSampleQuery() throws JsonProcessingException, JSONException { " ]," + " \"descending\": false,\n" + " \"granularity\": \"all\",\n" + + " \"virtualColumns\": [{\n" + + " \"type\": \"expression\",\n" + + " \"name\": \"dim3\",\n" + + " \"outputType\": \"FLOAT\",\n" + + " \"expression\": \"dim1 + dim2\"\n" + + " }],\n" + " \"pagingSpec\": {\n" + " \"threshold\": 5,\n" + " \"pagingIdentifiers\": {}\n" + @@ -177,4 +186,3 @@ public void testNullablePagingSpec() { .build(); } } - diff --git a/src/test/java/in/zapr/druid/druidry/virtualColumn/ExpressionVirtualColumnTest.java b/src/test/java/in/zapr/druid/druidry/virtualColumn/ExpressionVirtualColumnTest.java new file mode 100644 index 00000000..d4aa6535 --- /dev/null +++ b/src/test/java/in/zapr/druid/druidry/virtualColumn/ExpressionVirtualColumnTest.java @@ -0,0 +1,57 @@ +package in.zapr.druid.druidry.virtualColumn; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; + +import org.json.JSONException; +import org.json.JSONObject; +import org.skyscreamer.jsonassert.JSONAssert; +import org.skyscreamer.jsonassert.JSONCompareMode; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.Test; + +import in.zapr.druid.druidry.dimension.enums.OutputType; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public class ExpressionVirtualColumnTest { + private static ObjectMapper objectMapper; + + @BeforeClass + public void init() { + objectMapper = new ObjectMapper(); + } + + @Test + public void testAllFields() throws JsonProcessingException, JSONException { + ExpressionVirtualColumn column = new ExpressionVirtualColumn("foo", "a + b", OutputType.LONG); + JSONObject expected = new JSONObject(); + expected.put("type", "expression"); + expected.put("name", "foo"); + expected.put("outputType", "LONG"); + expected.put("expression", "a + b"); + String actualJSON = objectMapper.writeValueAsString(column); + JSONAssert.assertEquals(expected.toString(), actualJSON, JSONCompareMode.NON_EXTENSIBLE); + } + + @Test + public void testRequiredFields() throws JsonProcessingException, JSONException { + ExpressionVirtualColumn column = new ExpressionVirtualColumn("foo", "a + b", null); + JSONObject expected = new JSONObject(); + expected.put("type", "expression"); + expected.put("name", "foo"); + expected.put("expression", "a + b"); + String actualJSON = objectMapper.writeValueAsString(column); + JSONAssert.assertEquals(expected.toString(), actualJSON, JSONCompareMode.NON_EXTENSIBLE); + } + + @Test(expectedExceptions = NullPointerException.class) + public void testNameMissingFields() { + new ExpressionVirtualColumn(null, "a + b", null); + } + + @Test(expectedExceptions = NullPointerException.class) + public void testExpressionMissingFields() { + new ExpressionVirtualColumn("foo", null, null); + } +}