Skip to content

Commit

Permalink
Reduce Retry Time
Browse files Browse the repository at this point in the history 
Add A Zap Logger To Imds Retryer And Reduce Retry Time
  • Loading branch information
@sethAmazon
sethAmazon committed Jul 27, 2023
1 parent d272937 commit 3cf5b47
Show file tree
Hide file tree
Showing 21 changed files with 67 additions and 32 deletions.
2 changes: 1 addition & 1 deletion cmd/configschema/go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -277,7 +277,7 @@ require (
github.com/alecthomas/participle/v2 v2.0.0 // indirect
github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 // indirect
github.com/aliyun/aliyun-log-go-sdk v0.1.44 // indirect
github.com/amazon-contributing/opentelemetry-collector-contrib/override/aws v0.0.0-20230719154634-7dfcb28377c2 // indirect
github.com/amazon-contributing/opentelemetry-collector-contrib/override/aws v0.0.0-20230727225902-317222bcea83 // indirect
github.com/andybalholm/brotli v1.0.5 // indirect
github.com/antonmedv/expr v1.12.5 // indirect
github.com/apache/arrow/go/arrow v0.0.0-20211112161151-bc219186db40 // indirect
Expand Down
2 changes: 1 addition & 1 deletion cmd/otelcontribcol/go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ module github.com/open-telemetry/opentelemetry-collector-contrib/cmd/otelcontrib
go 1.19

require (
github.com/amazon-contributing/opentelemetry-collector-contrib/override/aws v0.0.0-20230719154634-7dfcb28377c2
github.com/amazon-contributing/opentelemetry-collector-contrib/override/aws v0.0.0-20230727225902-317222bcea83
github.com/open-telemetry/opentelemetry-collector-contrib/connector/countconnector v0.77.0
github.com/open-telemetry/opentelemetry-collector-contrib/connector/servicegraphconnector v0.77.0
github.com/open-telemetry/opentelemetry-collector-contrib/connector/spanmetricsconnector v0.77.0
Expand Down
2 changes: 1 addition & 1 deletion exporter/awscloudwatchlogsexporter/go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ require (
)

require (
github.com/amazon-contributing/opentelemetry-collector-contrib/override/aws v0.0.0-20230719154634-7dfcb28377c2 // indirect
github.com/amazon-contributing/opentelemetry-collector-contrib/override/aws v0.0.0-20230727225902-317222bcea83 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/gogo/protobuf v1.3.2 // indirect
github.com/golang/protobuf v1.5.3 // indirect
Expand Down
2 changes: 1 addition & 1 deletion exporter/awsemfexporter/go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ require (
)

require (
github.com/amazon-contributing/opentelemetry-collector-contrib/override/aws v0.0.0-20230719154634-7dfcb28377c2 // indirect
github.com/amazon-contributing/opentelemetry-collector-contrib/override/aws v0.0.0-20230727225902-317222bcea83 // indirect
github.com/cenkalti/backoff/v4 v4.2.1 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/gogo/protobuf v1.3.2 // indirect
Expand Down
2 changes: 1 addition & 1 deletion exporter/awsxrayexporter/go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ require (
)

require (
github.com/amazon-contributing/opentelemetry-collector-contrib/override/aws v0.0.0-20230719154634-7dfcb28377c2 // indirect
github.com/amazon-contributing/opentelemetry-collector-contrib/override/aws v0.0.0-20230727225902-317222bcea83 // indirect
github.com/cenkalti/backoff/v4 v4.2.1 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/gogo/protobuf v1.3.2 // indirect
Expand Down
2 changes: 1 addition & 1 deletion exporter/datadogexporter/go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ require (
github.com/DataDog/opentelemetry-mapping-go/pkg/quantile v0.2.0
github.com/DataDog/sketches-go v1.4.2
github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.13.1
github.com/amazon-contributing/opentelemetry-collector-contrib/override/aws v0.0.0-20230719154634-7dfcb28377c2
github.com/amazon-contributing/opentelemetry-collector-contrib/override/aws v0.0.0-20230727225902-317222bcea83
github.com/aws/aws-sdk-go v1.44.293
github.com/cenkalti/backoff/v4 v4.2.1
github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/ecsutil v0.77.0
Expand Down
2 changes: 1 addition & 1 deletion extension/awsproxy/go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ require (
)

require (
github.com/amazon-contributing/opentelemetry-collector-contrib/override/aws v0.0.0-20230719154634-7dfcb28377c2 // indirect
github.com/amazon-contributing/opentelemetry-collector-contrib/override/aws v0.0.0-20230727225902-317222bcea83 // indirect
github.com/aws/aws-sdk-go v1.44.293 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/fsnotify/fsnotify v1.6.0 // indirect
Expand Down
4 changes: 2 additions & 2 deletions extension/awsproxy/go.sum
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -257,7 +257,7 @@ require (
github.com/alecthomas/participle/v2 v2.0.0 // indirect
github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 // indirect
github.com/aliyun/aliyun-log-go-sdk v0.1.44 // indirect
github.com/amazon-contributing/opentelemetry-collector-contrib/override/aws v0.0.0-20230719154634-7dfcb28377c2 // indirect
github.com/amazon-contributing/opentelemetry-collector-contrib/override/aws v0.0.0-20230727225902-317222bcea83 // indirect
github.com/andybalholm/brotli v1.0.5 // indirect
github.com/antonmedv/expr v1.12.5 // indirect
github.com/apache/arrow/go/arrow v0.0.0-20211112161151-bc219186db40 // indirect
Expand Down
2 changes: 1 addition & 1 deletion internal/aws/awsutil/go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ module github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/aw
go 1.19

require (
github.com/amazon-contributing/opentelemetry-collector-contrib/override/aws v0.0.0-20230719154634-7dfcb28377c2
github.com/amazon-contributing/opentelemetry-collector-contrib/override/aws v0.0.0-20230727225902-317222bcea83
github.com/aws/aws-sdk-go v1.44.293
github.com/stretchr/testify v1.8.2
go.uber.org/zap v1.24.0
Expand Down
2 changes: 1 addition & 1 deletion internal/aws/proxy/go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ go 1.19
replace github.com/amazon-contributing/opentelemetry-collector-contrib/override/aws => ../../../override/aws

require (
github.com/amazon-contributing/opentelemetry-collector-contrib/override/aws v0.0.0-20230719154634-7dfcb28377c2
github.com/amazon-contributing/opentelemetry-collector-contrib/override/aws v0.0.0-20230727225902-317222bcea83
github.com/aws/aws-sdk-go v1.44.293
github.com/open-telemetry/opentelemetry-collector-contrib/internal/common v0.77.0
github.com/stretchr/testify v1.8.2
Expand Down
2 changes: 1 addition & 1 deletion internal/aws/xray/go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ module github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/xr
go 1.19

require (
github.com/amazon-contributing/opentelemetry-collector-contrib/override/aws v0.0.0-20230719154634-7dfcb28377c2
github.com/amazon-contributing/opentelemetry-collector-contrib/override/aws v0.0.0-20230727225902-317222bcea83
github.com/aws/aws-sdk-go v1.44.293
github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/awsutil v0.77.0
github.com/stretchr/testify v1.8.2
Expand Down
5 changes: 4 additions & 1 deletion internal/metadataproviders/go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ replace github.com/amazon-contributing/opentelemetry-collector-contrib/override/

require (
github.com/Showmax/go-fqdn v1.0.0
github.com/amazon-contributing/opentelemetry-collector-contrib/override/aws v0.0.0-20230719154634-7dfcb28377c2
github.com/amazon-contributing/opentelemetry-collector-contrib/override/aws v0.0.0-20230727225902-317222bcea83
github.com/aws/aws-sdk-go v1.44.293
github.com/docker/docker v23.0.5+incompatible
github.com/hashicorp/consul/api v1.20.0
Expand Down Expand Up @@ -41,6 +41,9 @@ require (
github.com/pkg/errors v0.9.1 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/stretchr/objx v0.5.0 // indirect
go.uber.org/atomic v1.7.0 // indirect
go.uber.org/multierr v1.6.0 // indirect
go.uber.org/zap v1.24.0 // indirect
golang.org/x/net v0.9.0 // indirect
golang.org/x/sys v0.7.0 // indirect
golang.org/x/time v0.3.0 // indirect
Expand Down
8 changes: 8 additions & 0 deletions internal/metadataproviders/go.sum
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

11 changes: 9 additions & 2 deletions override/aws/go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,13 @@ module github.com/amazon-contributing/opentelemetry-collector-contrib/override/a

go 1.19

require github.com/aws/aws-sdk-go v1.44.293
require (
github.com/aws/aws-sdk-go v1.44.293
go.uber.org/zap v1.24.0
)

require github.com/jmespath/go-jmespath v0.4.0 // indirect
require (
github.com/jmespath/go-jmespath v0.4.0 // indirect
go.uber.org/atomic v1.7.0 // indirect
go.uber.org/multierr v1.6.0 // indirect
)
15 changes: 14 additions & 1 deletion override/aws/go.sum
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

24 changes: 14 additions & 10 deletions override/aws/imdsretryer.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ import (
"github.com/aws/aws-sdk-go/aws/awserr"
"github.com/aws/aws-sdk-go/aws/client"
"github.com/aws/aws-sdk-go/aws/request"
"go.uber.org/zap"
)

// IMDSRetryer this must implement request.Retryer
Expand All @@ -33,28 +34,28 @@ import (
var IMDSRetryer request.Retryer = newIMDSRetryer()

const (
TimePerCall = 2 * time.Minute
TimePerCall = 30 * time.Second
)

type iMDSRetryer struct {
client.DefaultRetryer
logger *zap.Logger
}

// newIMDSRetryer allows us to retry imds errors
// 2 imds calls 1 for hostname 1 for doc
// 2 calls 1 for imdsv2 1 for imdsv1
// 2 seconds 4 seconds 8 seconds 16 seconds 32 seconds = 1 minute 2 seconds
// total is 4 minutes 8 seconds
// random jitter is applied of half the retry time
// max retry total time is 6 minutes 12 seconds
// min retry total time is 2 minutes 4 seconds
// .5 seconds 1 seconds 2 seconds 4 seconds 8 seconds = 15.5 seconds
func newIMDSRetryer() iMDSRetryer {
return iMDSRetryer{
imdsRetryer := iMDSRetryer{
DefaultRetryer: client.DefaultRetryer{
NumMaxRetries: 5,
MinRetryDelay: 2 * time.Second,
MinRetryDelay: time.Second / 2,
},
}
logger, err := zap.NewDevelopment()
if err == nil {
imdsRetryer.logger = logger
}
return imdsRetryer
}

func (r iMDSRetryer) ShouldRetry(req *request.Request) bool {
Expand All @@ -65,5 +66,8 @@ func (r iMDSRetryer) ShouldRetry(req *request.Request) bool {
if awsError, ok := req.Error.(awserr.Error); r.DefaultRetryer.ShouldRetry(req) || (ok && awsError != nil && awsError.Code() == "EC2MetadataError") {
shouldRetry = true
}
if r.logger != nil {
r.logger.Debug("imds error : ", zap.Bool("shouldRetry", shouldRetry), zap.Error(req.Error))
}
return shouldRetry
}
2 changes: 1 addition & 1 deletion processor/resourcedetectionprocessor/go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ require (
github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect
github.com/Microsoft/go-winio v0.5.2 // indirect
github.com/Showmax/go-fqdn v1.0.0 // indirect
github.com/amazon-contributing/opentelemetry-collector-contrib/override/aws v0.0.0-20230719154634-7dfcb28377c2 // indirect
github.com/amazon-contributing/opentelemetry-collector-contrib/override/aws v0.0.0-20230727225902-317222bcea83 // indirect
github.com/armon/go-metrics v0.3.10 // indirect
github.com/benbjohnson/clock v1.3.0 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
Expand Down
4 changes: 2 additions & 2 deletions processor/resourcedetectionprocessor/go.sum
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion receiver/awscontainerinsightreceiver/go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ module github.com/open-telemetry/opentelemetry-collector-contrib/receiver/awscon
go 1.19

require (
github.com/amazon-contributing/opentelemetry-collector-contrib/override/aws v0.0.0-20230719154634-7dfcb28377c2
github.com/amazon-contributing/opentelemetry-collector-contrib/override/aws v0.0.0-20230727225902-317222bcea83
github.com/aws/aws-sdk-go v1.44.293
github.com/go-kit/log v0.2.1
github.com/google/cadvisor v0.47.1
Expand Down
2 changes: 1 addition & 1 deletion receiver/awsxrayreceiver/go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ require (

require (
contrib.go.opencensus.io/exporter/prometheus v0.4.2 // indirect
github.com/amazon-contributing/opentelemetry-collector-contrib/override/aws v0.0.0-20230719154634-7dfcb28377c2 // indirect
github.com/amazon-contributing/opentelemetry-collector-contrib/override/aws v0.0.0-20230727225902-317222bcea83 // indirect
github.com/beorn7/perks v1.0.1 // indirect
github.com/cespare/xxhash/v2 v2.2.0 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
Expand Down

0 comments on commit 3cf5b47

Please sign in to comment.