Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[exporter/awsemfexporter] add tags feature for Cloudwatch Log Group #19406

Merged
merged 29 commits into from
May 11, 2023
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
29 commits
Select commit Hold shift + click to select a range
52e29ba
Add set tags to config
humivo Mar 3, 2023
639d0c1
Add config to cloudwatchlogsexporter too
humivo Mar 7, 2023
e25c61f
Add unit testing to aws emf
humivo Mar 8, 2023
08215ec
Add unit testing for other code changes and update README
humivo Mar 8, 2023
fb8c6c7
Add chlog entry
humivo Mar 8, 2023
e5c93d7
Merge branch 'main' into SetTagsLogGroup
humivo Mar 8, 2023
9792497
Fix issue number
humivo Mar 8, 2023
d086990
Change naming to accountID
humivo Mar 8, 2023
a56ab0c
Add sts endpoint and fix validate func
humivo Mar 16, 2023
80c0914
Fix import
humivo Mar 16, 2023
1af294c
Merge branch 'main' into SetTagsLogGroup
humivo Mar 16, 2023
6a72fec
Add comments to client code
humivo Mar 16, 2023
0e32886
Move functions to utils.go
humivo Mar 28, 2023
2ab601c
Remove line
humivo Mar 28, 2023
37403f4
Change tags call
humivo Apr 26, 2023
aa69c3d
Merge branch 'main' into SetTagsLogGroup
humivo Apr 26, 2023
57be341
Fix code after merge
humivo Apr 26, 2023
2ac02ce
Fix spelling error
humivo Apr 26, 2023
3576ddb
gci the file
humivo Apr 26, 2023
c3acf58
gci the exporter files
humivo Apr 26, 2023
71eb45c
Add gci option prefix
humivo Apr 26, 2023
27a640a
Add comment to cwlog_client
humivo Apr 27, 2023
ed6562f
Address comments on PR
humivo Apr 27, 2023
3701061
Improve testing
humivo May 9, 2023
18dacb3
Run gci command
humivo May 9, 2023
5a64e2d
Merge branch 'main' into SetTagsLogGroup
humivo May 9, 2023
5e99c16
Fix license header
humivo May 9, 2023
71e81c3
Simplify return statement
humivo May 11, 2023
26df4c3
Run gci command on code
humivo May 11, 2023
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Address comments on PR
  • Loading branch information
humivo committed Apr 27, 2023
commit ed6562fc5388f6825409c9183799123f6c1e8d69
2 changes: 1 addition & 1 deletion exporter/awscloudwatchlogsexporter/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ The following settings can be optionally configured:
- `region`: The AWS region where the log stream is in.
- `endpoint`: The CloudWatch Logs service endpoint which the requests are forwarded to. [See the CloudWatch Logs endpoints](https://docs.aws.amazon.com/general/latest/gr/cwl_region.html) for a list.
- `log_retention`: LogRetention is the option to set the log retention policy for only newly created CloudWatch Log Groups. Defaults to Never Expire if not specified or set to 0. Possible values for retention in days are 1, 3, 5, 7, 14, 30, 60, 90, 120, 150, 180, 365, 400, 545, 731, 1827, 2192, 2557, 2922, 3288, or 3653.
- `tags`: Tags is the option to set tags for the CloudWatch Log Group. If specified, please add at most 50 tags. Input is a string to string map like so: { 'key': 'value' }. Keys must be between 1-128 characters and follow the regex pattern: `^([\p{L}\p{Z}\p{N}_.:/=+\-@]+)$`. Values must be between 1-256 characters and follow the regex pattern: `^([\p{L}\p{Z}\p{N}_.:/=+\-@]*)$`.
- `tags`: Tags is the option to set tags for the CloudWatch Log Group. If specified, please add at most 50 tags. Input is a string to string map like so: { 'key': 'value' }. Keys must be between 1-128 characters and follow the regex pattern: `^([\p{L}\p{Z}\p{N}_.:/=+\-@]+)$`(alphanumerics, whitespace, and _.:/=+-!). Values must be between 1-256 characters and follow the regex pattern: `^([\p{L}\p{Z}\p{N}_.:/=+\-@]*)$`(alphanumerics, whitespace, and _.:/=+-!). [Link to tagging restrictions](https://docs.aws.amazon.com/AmazonCloudWatchLogs/latest/APIReference/API_CreateLogGroup.html#:~:text=Required%3A%20Yes-,tags,-The%20key%2Dvalue)
Aneurysm9 marked this conversation as resolved.
Show resolved Hide resolved
- `role_arn`: The AWS IAM role to upload segments to a same/different account
- `raw_log`: Boolean default false. If you want to export only the log message to cw logs. This is required for emf logs.

Expand Down
2 changes: 1 addition & 1 deletion exporter/awscloudwatchlogsexporter/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -83,7 +83,7 @@ func (config *Config) Validate() error {
if config.QueueSettings.QueueSize < 1 {
return errors.New("'sending_queue.queue_size' must be 1 or greater")
}
if retErr := cwlogs.IsValidRetentionValue(config.LogRetention); retErr != nil {
if retErr := cwlogs.ValidateRetentionValue(config.LogRetention); retErr != nil {
return retErr
}
tagInputErr := cwlogs.ValidateTagsInput(config.Tags)
Expand Down
30 changes: 21 additions & 9 deletions exporter/awscloudwatchlogsexporter/config_test.go
Aneurysm9 marked this conversation as resolved.
Show resolved Hide resolved
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ package awscloudwatchlogsexporter
import (
"path/filepath"
"strconv"
"strings"
"testing"

"github.com/cenkalti/backoff/v4"
Expand Down Expand Up @@ -165,6 +166,23 @@ func TestTagsValidateCorrect(t *testing.T) {

}

func TestTagsValidateTooLittleTags(t *testing.T) {
m := make(map[string]*string)
defaultRetrySettings := exporterhelper.NewDefaultRetrySettings()
wrongcfg := &Config{
RetrySettings: defaultRetrySettings,
LogGroupName: "test-1",
LogStreamName: "testing",
Endpoint: "",
Tags: m,
AWSSessionSettings: awsutil.CreateDefaultSessionConfig(),
QueueSettings: QueueSettings{
QueueSize: exporterhelper.NewDefaultQueueSettings().QueueSize,
},
}
assert.EqualError(t, component.ValidateConfig(wrongcfg), "invalid amount of items. Please input at least 1 tag or remove the tag field")
}

func TestTagsValidateTooManyTags(t *testing.T) {
m := make(map[string]*string)
avalue := "avalue"
Expand Down Expand Up @@ -239,17 +257,14 @@ func TestTagsValidateKeyTooShort(t *testing.T) {

func TestTagsValidateKeyTooLong(t *testing.T) {
avalue := "avalue"
akey := ""
for i := 0; i < 129; i++ {
akey += "a"
}
akey := strings.Repeat("a", 129)
defaultRetrySettings := exporterhelper.NewDefaultRetrySettings()
wrongcfg := &Config{
RetrySettings: defaultRetrySettings,
LogGroupName: "test-1",
LogStreamName: "testing",
Endpoint: "",
Tags: map[string]*string{"": &avalue},
Tags: map[string]*string{akey: &avalue},
AWSSessionSettings: awsutil.CreateDefaultSessionConfig(),
QueueSettings: QueueSettings{
QueueSize: exporterhelper.NewDefaultQueueSettings().QueueSize,
Expand All @@ -276,10 +291,7 @@ func TestTagsValidateValueTooShort(t *testing.T) {
}

func TestTagsValidateValueTooLong(t *testing.T) {
avalue := ""
for i := 0; i < 257; i++ {
avalue += "a"
}
avalue := strings.Repeat("a", 257)
defaultRetrySettings := exporterhelper.NewDefaultRetrySettings()
wrongcfg := &Config{
RetrySettings: defaultRetrySettings,
Expand Down
2 changes: 1 addition & 1 deletion exporter/awsemfexporter/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ The following exporter configuration parameters are supported.
| `log_group_name` | Customized log group name which supports `{ClusterName}` and `{TaskId}` placeholders. One valid example is `/aws/metrics/{ClusterName}`. It will search for `ClusterName` (or `aws.ecs.cluster.name`) resource attribute in the metrics data and replace with the actual cluster name. If none of them are found in the resource attribute map, `{ClusterName}` will be replaced by `undefined`. Similar way, for the `{TaskId}`, it searches for `TaskId` (or `aws.ecs.task.id`) key in the resource attribute map. For `{NodeName}`, it searches for `NodeName` (or `k8s.node.name`) |"/metrics/default"|
| `log_stream_name` | Customized log stream name which supports `{TaskId}`, `{ClusterName}`, `{NodeName}`, `{ContainerInstanceId}`, and `{TaskDefinitionFamily}` placeholders. One valid example is `{TaskId}`. It will search for `TaskId` (or `aws.ecs.task.id`) resource attribute in the metrics data and replace with the actual task id. If none of them are found in the resource attribute map, `{TaskId}` will be replaced by `undefined`. Similarly, for the `{TaskDefinitionFamily}`, it searches for `TaskDefinitionFamily` (or `aws.ecs.task.family`). For the `{ClusterName}`, it searches for `ClusterName` (or `aws.ecs.cluster.name`). For `{NodeName}`, it searches for `NodeName` (or `k8s.node.name`). For `{ContainerInstanceId}`, it searches for `ContainerInstanceId` (or `aws.ecs.container.instance.id`). (Note: ContainerInstanceId (or `aws.ecs.container.instance.id`) only works for AWS ECS EC2 launch type. |"otel-stream"|
| `log_retention` | LogRetention is the option to set the log retention policy for only newly created CloudWatch Log Groups. Defaults to Never Expire if not specified or set to 0. Possible values for retention in days are 1, 3, 5, 7, 14, 30, 60, 90, 120, 150, 180, 365, 400, 545, 731, 1827, 2192, 2557, 2922, 3288, or 3653. |"Never Expire"|
| `tags` | Tags is the option to set tags for the CloudWatch Log Group. If specified, please add at most 50 tags. Input is a string to string map like so: { 'key': 'value' }. Keys must be between 1-128 characters and follow the regex pattern: `^([\p{L}\p{Z}\p{N}_.:/=+\-@]+)$`. Values must be between 1-256 characters and follow the regex pattern: `^([\p{L}\p{Z}\p{N}_.:/=+\-@]*)$` | No tags set |
| `tags` | Tags is the option to set tags for the CloudWatch Log Group. If specified, please add at most 50 tags. Input is a string to string map like so: { 'key': 'value' }. Keys must be between 1-128 characters and follow the regex pattern: `^([\p{L}\p{Z}\p{N}_.:/=+\-@]+)$`(alphanumerics, whitespace, and _.:/=+-!). Values must be between 1-256 characters and follow the regex pattern: `^([\p{L}\p{Z}\p{N}_.:/=+\-@]*)$`(alphanumerics, whitespace, and _.:/=+-!). [Link to tagging restrictions](https://docs.aws.amazon.com/AmazonCloudWatchLogs/latest/APIReference/API_CreateLogGroup.html#:~:text=Required%3A%20Yes-,tags,-The%20key%2Dvalue) | No tags set |
| `namespace` | Customized CloudWatch metrics namespace | "default" |
| `endpoint` | Optionally override the default CloudWatch service endpoint. | |
| `no_verify_ssl` | Enable or disable TLS certificate verification. | false |
Expand Down
2 changes: 1 addition & 1 deletion exporter/awsemfexporter/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -142,7 +142,7 @@ func (config *Config) Validate() error {
}
config.MetricDescriptors = validDescriptors

if retErr := cwlogs.IsValidRetentionValue(config.LogRetention); retErr != nil {
if retErr := cwlogs.ValidateRetentionValue(config.LogRetention); retErr != nil {
return retErr
}

Expand Down
26 changes: 18 additions & 8 deletions exporter/awsemfexporter/config_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ package awsemfexporter
import (
"path/filepath"
"strconv"
"strings"
"testing"

"github.com/stretchr/testify/assert"
Expand Down Expand Up @@ -201,6 +202,21 @@ func TestTagsValidateCorrect(t *testing.T) {

}

func TestTagsValidateTooLittleTags(t *testing.T) {
m := make(map[string]*string)
wrongcfg := &Config{
AWSSessionSettings: awsutil.AWSSessionSettings{
RequestTimeoutSeconds: 30,
MaxRetries: 1,
},
DimensionRollupOption: "ZeroAndSingleDimensionRollup",
Tags: m,
ResourceToTelemetrySettings: resourcetotelemetry.Settings{Enabled: true},
logger: zap.NewNop(),
}
assert.EqualError(t, component.ValidateConfig(wrongcfg), "invalid amount of items. Please input at least 1 tag or remove the tag field")
}

func TestTagsValidateTooManyTags(t *testing.T) {
m := make(map[string]*string)
avalue := "avalue"
Expand Down Expand Up @@ -267,10 +283,7 @@ func TestTagsValidateKeyTooShort(t *testing.T) {

func TestTagsValidateKeyTooLong(t *testing.T) {
avalue := "avalue"
akey := ""
for i := 0; i < 129; i++ {
akey += "a"
}
akey := strings.Repeat("a", 129)
wrongcfg := &Config{
AWSSessionSettings: awsutil.AWSSessionSettings{
RequestTimeoutSeconds: 30,
Expand Down Expand Up @@ -300,10 +313,7 @@ func TestTagsValidateValueTooShort(t *testing.T) {
}

func TestTagsValidateValueTooLong(t *testing.T) {
avalue := ""
for i := 0; i < 257; i++ {
avalue += "a"
}
avalue := strings.Repeat("a", 257)
wrongcfg := &Config{
AWSSessionSettings: awsutil.AWSSessionSettings{
RequestTimeoutSeconds: 30,
Expand Down
14 changes: 4 additions & 10 deletions internal/aws/cwlogs/cwlog_client.go
Original file line number Diff line number Diff line change
Expand Up @@ -157,16 +157,10 @@ func (client *Client) CreateStream(logGroup, streamName *string) (token string,
var awsErr awserr.Error
if errors.As(err, &awsErr) && awsErr.Code() == cloudwatchlogs.ErrCodeResourceNotFoundException {
// Create Log Group with tags if they exist and were specified in the config
if client.tags != nil && len(client.tags) > 0 {
_, err = client.svc.CreateLogGroup(&cloudwatchlogs.CreateLogGroupInput{
LogGroupName: logGroup,
Tags: client.tags,
})
} else {
_, err = client.svc.CreateLogGroup(&cloudwatchlogs.CreateLogGroupInput{
LogGroupName: logGroup,
})
}
_, err = client.svc.CreateLogGroup(&cloudwatchlogs.CreateLogGroupInput{
LogGroupName: logGroup,
Tags: client.tags,
})
if err == nil {
// For newly created log groups, set the log retention polic if specified or non-zero. Otheriwse, set to Never Expire
if client.logRetention != 0 {
Expand Down
6 changes: 5 additions & 1 deletion internal/aws/cwlogs/utils.go
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ import (
)

// Added function to check if value is an accepted number of log retention days
func IsValidRetentionValue(input int64) error {
func ValidateRetentionValue(input int64) error {
switch input {
case
0,
Expand Down Expand Up @@ -53,9 +53,13 @@ func IsValidRetentionValue(input int64) error {

// Check if the tags input is valid
func ValidateTagsInput(input map[string]*string) error {
if input != nil && len(input) < 1 {
return fmt.Errorf("invalid amount of items. Please input at least 1 tag or remove the tag field")
}
if len(input) > 50 {
return fmt.Errorf("invalid amount of items. Please input at most 50 tags")
}
// The regex for the Key and Value requires "alphanumerics, whitespace, and _.:/=+-!" as noted here: https://docs.aws.amazon.com/AmazonCloudWatchLogs/latest/APIReference/API_CreateLogGroup.html#:~:text=%5E(%5B%5Cp%7BL%7D%5Cp%7BZ%7D%5Cp%7BN%7D_.%3A/%3D%2B%5C%2D%40%5D%2B)%24
validKeyPattern := regexp.MustCompile(`^([\p{L}\p{Z}\p{N}_.:/=+\-@]+)$`)
validValuePattern := regexp.MustCompile(`^([\p{L}\p{Z}\p{N}_.:/=+\-@]*)$`)
for key, value := range input {
Expand Down