Add combine action (#1506)

* Add combine action

* Address review comments

processor/metricstransformprocessor/README.md

@@ -14,21 +14,23 @@ a batch of metrics**. It does not do any aggregation across batches, so it is
 not suitable for aggregating metrics from multiple sources (e.g. multiple nodes
 or clients).
 
-| Operation  | Example (based on metric `system.cpu.usage`) |
-|-------------------------------------------------------------------|-------------------------------------------------------------------------------------------------|
-| Rename metrics  | Rename to `system.cpu.usage_time` |
-| Add labels  | Add new label `identifier` with value `1` to all points |
-| Rename label keys  | Rename label `state` to `cpu_state` |
-| Rename label values  | For label `state`, rename value `idle` to `-` |
-| Delete data points based on label values  | Delete all points where label `state` has value `idle` |
-| Toggle the data type of scalar metrics between `int` and `double` | Change from `int` data points to `double` data points |
-| Aggregate across label sets by sum, mean, min, or max | Retain only the label `state`, sum all points with the same value for this label  |
-| Aggregate across label values by sum, mean, min, or max | For label `state`, sum points where the value is `user` or `system` into `used = user + system` |
+| Operation | Example (based on metric `system.cpu.usage`) |
+|-------------------------------|-------------------------------------------------------------------------------------------------|
+| Rename metrics | Rename to `system.cpu.usage_time` |
+| Add labels | Add new label `identifier` with value `1` to all points |
+| Rename label keys | Rename label `state` to `cpu_state` |
+| Rename label values | For label `state`, rename value `idle` to `-` |
+| Delete data points | Delete all points where label `state` has value `idle` |
+| Toggle data type   | Change from `int` data points to `double` data points |
+| Aggregate across label sets  | Retain only the label `state`, average all points with the same value for this label |
+| Aggregate across label values | For label `state`, sum points where the value is `user` or `system` into `used = user + system` |
 
 In addition to the above:
 
 - Operations can be applied to one or more metrics using a `strict` or `regexp`
  filter
+- Metrics can be updated in place or on an inserted copy
+- A set of matching metrics can be combined into a single metric
 - When renaming metrics, capturing groups from the `regexp` filter will be
  expanded
 - When adding or updating a label value, `{{version}}` will be replaced with
@@ -48,10 +50,12 @@ transforms:
  - include: <metric_name>
  # match_type specifies whether the include name should be used as a strict match or regexp match, default = strict
  match_type: {strict, regexp}
- # action specifies if the operations are performed on the metric in place, or on an inserted clone
- action: {update, insert}
+ # action specifies if the operations are performed on a metric in place, on an inserted clone, or if the matched metrics should be combined
+ action: {update, insert, combine}
  # new_name specifies the updated name of the metric; if action is insert, new_name is required
  new_name: <new_metric_name_inserted>
+ # aggregation_type defines how combined data points will be aggregated; if action is combine, aggregation_type is required
+ aggregation_type: {sum, mean, min, max}
  # operations contain a list of operations that will be performed on the selected metrics
  operations:
  # action defines the type of operation that will be performed, see examples below for more details
@@ -68,7 +72,7 @@ transforms:
  label_value: <label_value>
  # label_set contains a list of labels that will remain after aggregation; if action is aggregate_labels, label_set is required
  label_set: [labels...]
- # aggregation_type defines how excluded labels will be aggregated; if action is aggregate_labels or aggregate_label_values, aggregation_type is required
+ # aggregation_type defines how data points will be aggregated; if action is aggregate_labels or aggregate_label_values, aggregation_type is required
  aggregation_type: {sum, mean, min, max}
  # value_actions contain a list of operations that will be performed on the selected label
  value_actions:
@@ -210,3 +214,18 @@ operations:
  new_value: slab 
  aggregation_type: sum
 ```
+
+### Combine metrics
+```yaml
+# convert a set of metrics for each http_method into a single metric with an http_method label, i.e.
+#
+# Web Service (*)/Total DELETE Requests iis.requests{http_method=DELETE}
+# Web Service (*)/Total GET Requests > iis.requests{http_method=GET}
+# Web Service (*)/Total POST Requests iis.requests{http_method=POST}
+metric_names: ^Web Service \(\*\)/Total (?P<http_method>.*) Requests$
+match_type: regexp
+action: combine
+new_name: iis.requests
+operations:
+ ...
+```

processor/metricstransformprocessor/config.go

@@ -34,6 +34,9 @@ const (
  // NewNameFieldName is the mapstructure field name for NewName field
  NewNameFieldName = "new_name"
 
+ // AggregationTypeFieldName is the mapstructure field name for AggregationType field
+ AggregationTypeFieldName = "aggregation_type"
+
  // LabelFieldName is the mapstructure field name for Label field
  LabelFieldName = "label"
 
@@ -44,16 +47,6 @@ const (
  NewValueFieldName = "new_value"
 )
 
-const (
- // StrictMatchType is the FilterType for filtering by exact string matches.
- StrictMatchType = "strict"
-
- // RegexpMatchType is the FilterType for filtering by regexp string matches.
- RegexpMatchType = "regexp"
-)
-
-var MatchTypes = []string{StrictMatchType, RegexpMatchType}
-
 // Config defines configuration for Resource processor.
 type Config struct {
  configmodels.ProcessorSettings `mapstructure:",squash"`
@@ -80,6 +73,10 @@ type Transform struct {
  // REQUIRED only if Action is INSERT.
  NewName string `mapstructure:"new_name"`
 
+ // AggregationType specifies how to aggregate.
+ // REQUIRED only if Action is COMBINE.
+ AggregationType AggregationType `mapstructure:"aggregation_type"`
+
  // Operations contains a list of operations that will be performed on the selected metric.
  Operations []Operation `mapstructure:"operations"`
 }
@@ -89,7 +86,7 @@ type FilterConfig struct {
  Include string `mapstructure:"include"`
 
  // MatchType determines how the Include string is matched: <strict|regexp>.
- MatchType string `mapstructure:"match_type"`
+ MatchType MatchType `mapstructure:"match_type"`
 }
 
 // Operation defines the specific operation performed on the selected metrics.
@@ -135,22 +132,33 @@ type ValueAction struct {
 // ConfigAction is the enum to capture the two types of actions to perform on a metric.
 type ConfigAction string
 
-// OperationAction is the enum to capture the thress types of actions to perform for an operation.
-type OperationAction string
-
-// AggregationType os the enum to capture the three types of aggregation for the aggregation operation.
-type AggregationType string
-
 const (
  // Insert adds a new metric to the batch with a new name.
  Insert ConfigAction = "insert"
 
  // Update updates an existing metric.
  Update ConfigAction = "update"
 
- // ToggleScalarDataType changes the data type from int64 to double, or vice-versa
- ToggleScalarDataType OperationAction = "toggle_scalar_data_type"
+ // Combine combines multiple metrics into a single metric.
+ Combine ConfigAction = "combine"
+)
+
+var Actions = []ConfigAction{Insert, Update, Combine}
 
+func (ca ConfigAction) isValid() bool {
+ for _, configAction := range Actions {
+ if ca == configAction {
+ return true
+ }
+ }
+
+ return false
+}
+
+// OperationAction is the enum to capture the thress types of actions to perform for an operation.
+type OperationAction string
+
+const (
  // AddLabel adds a new label to an existing metric.
  AddLabel OperationAction = "add_label"
 
@@ -160,23 +168,78 @@ const (
  // DeleteLabelValue deletes a label value by also removing all the points associated with this label value
  DeleteLabelValue OperationAction = "delete_label_value"
 
+ // ToggleScalarDataType changes the data type from int64 to double, or vice-versa
+ ToggleScalarDataType OperationAction = "toggle_scalar_data_type"
+
  // AggregateLabels aggregates away all labels other than the ones in Operation.LabelSet
  // by the method indicated by Operation.AggregationType.
  AggregateLabels OperationAction = "aggregate_labels"
 
  // AggregateLabelValues aggregates away the values in Operation.AggregatedValues
  // by the method indicated by Operation.AggregationType.
  AggregateLabelValues OperationAction = "aggregate_label_values"
+)
 
- // Mean indicates taking the mean of the aggregated data.
- Mean AggregationType = "mean"
+var OperationActions = []OperationAction{AddLabel, UpdateLabel, DeleteLabelValue, ToggleScalarDataType, AggregateLabels, AggregateLabelValues}
 
- // Max indicates taking the max of the aggregated data.
- Max AggregationType = "max"
+func (oa OperationAction) isValid() bool {
+ for _, operationAction := range OperationActions {
+ if oa == operationAction {
+ return true
+ }
+ }
+
+ return false
+}
+
+// AggregationType is the enum to capture the three types of aggregation for the aggregation operation.
+type AggregationType string
 
+const (
  // Sum indicates taking the sum of the aggregated data.
  Sum AggregationType = "sum"
 
+ // Mean indicates taking the mean of the aggregated data.
+ Mean AggregationType = "mean"
+
  // Min indicates taking the minimum of the aggregated data.
  Min AggregationType = "min"
+
+ // Max indicates taking the max of the aggregated data.
+ Max AggregationType = "max"
 )
+
+var AggregationTypes = []AggregationType{Sum, Mean, Min, Max}
+
+func (at AggregationType) isValid() bool {
+ for _, aggregationType := range AggregationTypes {
+ if at == aggregationType {
+ return true
+ }
+ }
+
+ return false
+}
+
+// MatchType is the enum to capture the two types of matching metric(s) that should have operations applied to them.
+type MatchType string
+
+const (
+ // StrictMatchType is the FilterType for filtering by exact string matches.
+ StrictMatchType MatchType = "strict"
+
+ // RegexpMatchType is the FilterType for filtering by regexp string matches.
+ RegexpMatchType MatchType = "regexp"
+)
+
+var MatchTypes = []MatchType{StrictMatchType, RegexpMatchType}
+
+func (mt MatchType) isValid() bool {
+ for _, matchType := range MatchTypes {
+ if mt == matchType {
+ return true
+ }
+ }
+
+ return false
+}