perf: Optimize time-based filtering on large number of workflows (#3340)

🚀
argoproj · Jul 1, 2020 · 1db93c0 · 1db93c0
1 parent 2ab9495
commit 1db93c0
Show file tree

Hide file tree

Showing 4 changed files with 47 additions and 8 deletions.
diff --git a/cmd/argo/commands/list.go b/cmd/argo/commands/list.go
@@ -114,15 +114,23 @@ func listWorkflows(ctx context.Context, serviceClient workflowpkg.WorkflowServic
  Filter(func(wf wfv1.Workflow) bool {
  return strings.HasPrefix(wf.ObjectMeta.Name, flags.prefix)
  })
- if flags.createdSince != "" {
- t, err := argotime.ParseSince(flags.createdSince)
+ if flags.createdSince != "" && flags.finishedAfter != "" {
+ startTime, err := argotime.ParseSince(flags.createdSince)
  errors.CheckError(err)
- workflows = workflows.Filter(wfv1.WorkflowCreatedAfter(*t))
- }
- if flags.finishedAfter != "" {
- t, err := argotime.ParseSince(flags.finishedAfter)
+ endTime, err := argotime.ParseSince(flags.finishedAfter)
  errors.CheckError(err)
- workflows = workflows.Filter(wfv1.WorkflowFinishedBefore(*t))
+ workflows = workflows.Filter(wfv1.WorkflowRanBetween(*startTime, *endTime))
+ } else {
+ if flags.createdSince != "" {
+ t, err := argotime.ParseSince(flags.createdSince)
+ errors.CheckError(err)
+ workflows = workflows.Filter(wfv1.WorkflowCreatedAfter(*t))
+ }
+ if flags.finishedAfter != "" {
+ t, err := argotime.ParseSince(flags.finishedAfter)
+ errors.CheckError(err)
+ workflows = workflows.Filter(wfv1.WorkflowFinishedBefore(*t))
+ }
  }
  sort.Sort(workflows)
  return workflows, nil

diff --git a/pkg/apis/workflow/v1alpha1/item_test.go b/pkg/apis/workflow/v1alpha1/item_test.go
@@ -64,4 +64,4 @@ func TestItem_GetStrVal(t *testing.T) {
  assert.NoError(t, err)
  val := item.GetStrVal()
  assert.Equal(t, "foo", val)
-}
+}
diff --git a/pkg/apis/workflow/v1alpha1/workflow_types.go b/pkg/apis/workflow/v1alpha1/workflow_types.go
@@ -129,6 +129,11 @@ var (
  return !wf.Status.FinishedAt.IsZero() && wf.Status.FinishedAt.Time.Before(t)
  }
  }
+ WorkflowRanBetween = func(startTime time.Time, endTime time.Time) WorkflowPredicate {
+ return func(wf Workflow) bool {
+ return wf.ObjectMeta.CreationTimestamp.After(startTime) && !wf.Status.FinishedAt.IsZero() && wf.Status.FinishedAt.Time.Before(endTime)
+ }
+ }
 )
 
 // WorkflowList is list of Workflow resources

diff --git a/pkg/apis/workflow/v1alpha1/workflow_types_test.go b/pkg/apis/workflow/v1alpha1/workflow_types_test.go
@@ -47,6 +47,32 @@ func TestWorkflowFinishedBefore(t *testing.T) {
  assert.True(t, WorkflowFinishedBefore(t1)(Workflow{Status: WorkflowStatus{FinishedAt: metav1.Time{Time: t0}}}))
 }
 
+func TestWorkflowHappenedBetween(t *testing.T) {
+ t0 := time.Time{}
+ t1 := t0.Add(time.Second)
+ t2 := t1.Add(time.Second)
+ t3 := t2.Add(time.Second)
+ assert.False(t, WorkflowRanBetween(t0, t3)(Workflow{}))
+ assert.False(t, WorkflowRanBetween(t0, t1)(Workflow{
+ ObjectMeta: metav1.ObjectMeta{CreationTimestamp: metav1.Time{Time: t0}},
+ Status: WorkflowStatus{FinishedAt: metav1.Time{Time: t1}}}))
+ assert.False(t, WorkflowRanBetween(t1, t2)(Workflow{
+ ObjectMeta: metav1.ObjectMeta{CreationTimestamp: metav1.Time{Time: t0}},
+ Status: WorkflowStatus{FinishedAt: metav1.Time{Time: t1}}}))
+ assert.False(t, WorkflowRanBetween(t2, t3)(Workflow{
+ ObjectMeta: metav1.ObjectMeta{CreationTimestamp: metav1.Time{Time: t0}},
+ Status: WorkflowStatus{FinishedAt: metav1.Time{Time: t1}}}))
+ assert.False(t, WorkflowRanBetween(t0, t1)(Workflow{
+ ObjectMeta: metav1.ObjectMeta{CreationTimestamp: metav1.Time{Time: t1}},
+ Status: WorkflowStatus{FinishedAt: metav1.Time{Time: t2}}}))
+ assert.False(t, WorkflowRanBetween(t2, t3)(Workflow{
+ ObjectMeta: metav1.ObjectMeta{CreationTimestamp: metav1.Time{Time: t1}},
+ Status: WorkflowStatus{FinishedAt: metav1.Time{Time: t2}}}))
+ assert.True(t, WorkflowRanBetween(t0, t3)(Workflow{
+ ObjectMeta: metav1.ObjectMeta{CreationTimestamp: metav1.Time{Time: t1}},
+ Status: WorkflowStatus{FinishedAt: metav1.Time{Time: t2}}}))
+}
+
 func TestArtifactLocation_HasLocation(t *testing.T) {
  assert.False(t, (&ArtifactLocation{}).HasLocation())
  assert.False(t, (&ArtifactLocation{ArchiveLogs: pointer.BoolPtr(true)}).HasLocation())