Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix: Check child node status before backoff in retry #2407

Merged
merged 3 commits into from
Mar 12, 2020
Merged
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
added test
  • Loading branch information
simster7 committed Mar 11, 2020
commit 5e092edec7165ea10c9f83b6e2c070d5e16767d5
55 changes: 55 additions & 0 deletions workflow/controller/operator_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -254,6 +254,61 @@ func TestProcessNodesWithRetriesOnErrors(t *testing.T) {
assert.Equal(t, n.Phase, wfv1.NodeError)
}

func TestProcessNodesWithRetriesWithBackoff(t *testing.T) {
controller := newController()
assert.NotNil(t, controller)
wf := unmarshalWF(helloWorldWf)
assert.NotNil(t, wf)
woc := newWorkflowOperationCtx(wf, controller)
assert.NotNil(t, woc)

// Verify that there are no nodes in the wf status.
assert.Zero(t, len(woc.wf.Status.Nodes))

// Add the parent node for retries.
nodeName := "test-node"
nodeID := woc.wf.NodeID(nodeName)
node := woc.initializeNode(nodeName, wfv1.NodeTypeRetry, "", &wfv1.Template{}, "", wfv1.NodeRunning)
retries := wfv1.RetryStrategy{}
retryLimit := int32(2)
retries.Limit = &retryLimit
retries.Backoff = &wfv1.Backoff{
Duration: "10s",
Factor: 2,
MaxDuration: "10m",
}
retries.RetryPolicy = wfv1.RetryPolicyAlways
woc.wf.Status.Nodes[nodeID] = *node

assert.Equal(t, node.Phase, wfv1.NodeRunning)

// Ensure there are no child nodes yet.
lastChild, err := woc.getLastChildNode(node)
assert.Nil(t, err)
assert.Nil(t, lastChild)

woc.initializeNode("child-node-1", wfv1.NodeTypePod, "", &wfv1.Template{}, "", wfv1.NodeRunning)
woc.addChildNode(nodeName, "child-node-1")

n := woc.getNodeByName(nodeName)
lastChild, err = woc.getLastChildNode(n)
assert.Nil(t, err)
assert.NotNil(t, lastChild)

// Last child is still running. processNodesWithRetries() should return false since
// there should be no retries at this point.
n, _, err = woc.processNodeRetries(n, retries)
assert.Nil(t, err)
assert.Equal(t, n.Phase, wfv1.NodeRunning)

// Mark lastChild as successful.
woc.markNodePhase(lastChild.Name, wfv1.NodeSucceeded)
n, _, err = woc.processNodeRetries(n, retries)
assert.Nil(t, err)
// The parent node also gets marked as Succeeded.
assert.Equal(t, n.Phase, wfv1.NodeSucceeded)
}

// TestProcessNodesWithRetries tests retrying when RetryOn.Error is disabled
func TestProcessNodesNoRetryWithError(t *testing.T) {
controller := newController()
Expand Down