fix(executor): Retry kubectl on internal transient error (#8092)

* fix(executor): Retry kubectl on internal transient error fix(executor): Retry kubectl on internal transient error Signed-off-by: wujayway <[email protected]> * add ut add ut Signed-off-by: wujayway <[email protected]> * add ut add ut Signed-off-by: wujayway <[email protected]>
argoproj · Mar 18, 2022 · 4c61c8d · 4c61c8d
1 parent 47b78d4
commit 4c61c8d
Show file tree

Hide file tree

Showing 2 changed files with 17 additions and 5 deletions.
diff --git a/util/errors/errors.go b/util/errors/errors.go
@@ -38,7 +38,7 @@ func matchTransientErrPattern(err error) bool {
  if pattern == "" {
  return false
  }
- match, _ := regexp.MatchString(pattern, err.Error())
+ match, _ := regexp.MatchString(pattern, generateErrorString(err))
  return match
 }
 
@@ -63,10 +63,7 @@ func isTransientNetworkErr(err error) bool {
  }
  }
 
- errorString := err.Error()
- if exitErr, ok := err.(*exec.ExitError); ok {
- errorString = fmt.Sprintf("%s %s", errorString, exitErr.Stderr)
- }
+ errorString := generateErrorString(err)
  if strings.Contains(errorString, "net/http: TLS handshake timeout") {
  // If error is - tlsHandshakeTimeoutError, retry.
  return true
@@ -80,3 +77,11 @@ func isTransientNetworkErr(err error) bool {
 
  return false
 }
+
+func generateErrorString(err error) string {
+ errorString := err.Error()
+ if exitErr, ok := err.(*exec.ExitError); ok {
+ errorString = fmt.Sprintf("%s %s", errorString, exitErr.Stderr)
+ }
+ return errorString
+}
diff --git a/util/errors/errors_test.go b/util/errors/errors_test.go
@@ -5,6 +5,7 @@ import (
  "net"
  "net/url"
  "os"
+ "os/exec"
  "testing"
 
  "github.com/stretchr/testify/assert"
@@ -23,6 +24,10 @@ var (
  ioTimeoutErr net.Error = netError("i/o timeout")
  connectionTimedout net.Error = netError("connection timed out")
  transientErr net.Error = netError("this error is transient")
+ transientExitErr = exec.ExitError{
+ ProcessState: &os.ProcessState{},
+ Stderr: []byte("this error is transient"),
+ }
 )
 
 const transientEnvVarKey = "TRANSIENT_ERROR_PATTERN"
@@ -67,9 +72,11 @@ func TestIsTransientErr(t *testing.T) {
  t.Run("TransientErrorPattern", func(t *testing.T) {
  _ = os.Setenv(transientEnvVarKey, "this error is transient")
  assert.True(t, IsTransientErr(transientErr))
+ assert.True(t, IsTransientErr(&transientExitErr))
 
  _ = os.Setenv(transientEnvVarKey, "this error is not transient")
  assert.False(t, IsTransientErr(transientErr))
+ assert.False(t, IsTransientErr(&transientExitErr))
 
  _ = os.Setenv(transientEnvVarKey, "")
  assert.False(t, IsTransientErr(transientErr))