Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

perf: Optimize time-based filtering on large number of workflows #3340

Merged
merged 3 commits into from
Jul 1, 2020
Merged

perf: Optimize time-based filtering on large number of workflows #3340

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
7dfc57f
perf: Optimize argo list time-based filtering
terrytangyuan Jun 29, 2020
a31b209
Format
terrytangyuan Jun 29, 2020
1ad922c
address comment
terrytangyuan Jun 29, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 15 additions & 7 deletions cmd/argo/commands/list.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -114,15 +114,23 @@ func listWorkflows(ctx context.Context, serviceClient workflowpkg.WorkflowServic
Filter(func(wf wfv1.Workflow) bool {
return strings.HasPrefix(wf.ObjectMeta.Name, flags.prefix)
})
if flags.createdSince != "" {
t, err := argotime.ParseSince(flags.createdSince)
if flags.createdSince != "" && flags.finishedAfter != "" {
startTime, err := argotime.ParseSince(flags.createdSince)
errors.CheckError(err)
workflows = workflows.Filter(wfv1.WorkflowCreatedAfter(*t))
}
if flags.finishedAfter != "" {
t, err := argotime.ParseSince(flags.finishedAfter)
endTime, err := argotime.ParseSince(flags.finishedAfter)
errors.CheckError(err)
workflows = workflows.Filter(wfv1.WorkflowFinishedBefore(*t))
workflows = workflows.Filter(wfv1.WorkflowRanBetween(*startTime, *endTime))
} else {
if flags.createdSince != "" {
t, err := argotime.ParseSince(flags.createdSince)
errors.CheckError(err)
workflows = workflows.Filter(wfv1.WorkflowCreatedAfter(*t))
}
if flags.finishedAfter != "" {
t, err := argotime.ParseSince(flags.finishedAfter)
errors.CheckError(err)
workflows = workflows.Filter(wfv1.WorkflowFinishedBefore(*t))
}
}
sort.Sort(workflows)
return workflows, nil
Expand Down
2 changes: 1 addition & 1 deletion pkg/apis/workflow/v1alpha1/item_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -64,4 +64,4 @@ func TestItem_GetStrVal(t *testing.T) {
assert.NoError(t, err)
val := item.GetStrVal()
assert.Equal(t, "foo", val)
}
}
5 changes: 5 additions & 0 deletions pkg/apis/workflow/v1alpha1/workflow_types.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -125,6 +125,11 @@ var (
return !wf.Status.FinishedAt.IsZero() && wf.Status.FinishedAt.Time.Before(t)
}
}
WorkflowRanBetween = func(startTime time.Time, endTime time.Time) WorkflowPredicate {
Copy link
Contributor

@alexec alexec Jul 1, 2020 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

interesting, I can think of other interpretation of this:

  • the workflow started between times A and B - e.g. WorkflowStartedBetween
  • the workflow finished between times A and. B - e.g. WorkflowFinishedBetween
  • the workflow was running between the two times - i.e. start time is before B and (not finished OR finished after time A) - e.g. WorkflowRanBetween?
  • the workflow was ran and finished between the two times - i.e. started after A and finished before B - e.g. WorkflowWasBetween?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes. Should we discuss further on a better design in a separate issue? The original purpose of this PR is to optimize the performance of the existing functionality. We can definitely break this down and see if we can come up something more comprehensive and useful in practice.

return func(wf Workflow) bool {
return wf.ObjectMeta.CreationTimestamp.After(startTime) && !wf.Status.FinishedAt.IsZero() && wf.Status.FinishedAt.Time.Before(endTime)
}
}
)

// WorkflowList is list of Workflow resources
Expand Down
26 changes: 26 additions & 0 deletions pkg/apis/workflow/v1alpha1/workflow_types_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,32 @@ func TestWorkflowFinishedBefore(t *testing.T) {
assert.True(t, WorkflowFinishedBefore(t1)(Workflow{Status: WorkflowStatus{FinishedAt: metav1.Time{Time: t0}}}))
}

func TestWorkflowHappenedBetween(t *testing.T) {
t0 := time.Time{}
t1 := t0.Add(time.Second)
t2 := t1.Add(time.Second)
t3 := t2.Add(time.Second)
assert.False(t, WorkflowRanBetween(t0, t3)(Workflow{}))
assert.False(t, WorkflowRanBetween(t0, t1)(Workflow{
ObjectMeta: metav1.ObjectMeta{CreationTimestamp: metav1.Time{Time: t0}},
Status: WorkflowStatus{FinishedAt: metav1.Time{Time: t1}}}))
assert.False(t, WorkflowRanBetween(t1, t2)(Workflow{
ObjectMeta: metav1.ObjectMeta{CreationTimestamp: metav1.Time{Time: t0}},
Status: WorkflowStatus{FinishedAt: metav1.Time{Time: t1}}}))
assert.False(t, WorkflowRanBetween(t2, t3)(Workflow{
ObjectMeta: metav1.ObjectMeta{CreationTimestamp: metav1.Time{Time: t0}},
Status: WorkflowStatus{FinishedAt: metav1.Time{Time: t1}}}))
assert.False(t, WorkflowRanBetween(t0, t1)(Workflow{
ObjectMeta: metav1.ObjectMeta{CreationTimestamp: metav1.Time{Time: t1}},
Status: WorkflowStatus{FinishedAt: metav1.Time{Time: t2}}}))
assert.False(t, WorkflowRanBetween(t2, t3)(Workflow{
ObjectMeta: metav1.ObjectMeta{CreationTimestamp: metav1.Time{Time: t1}},
Status: WorkflowStatus{FinishedAt: metav1.Time{Time: t2}}}))
assert.True(t, WorkflowRanBetween(t0, t3)(Workflow{
ObjectMeta: metav1.ObjectMeta{CreationTimestamp: metav1.Time{Time: t1}},
Status: WorkflowStatus{FinishedAt: metav1.Time{Time: t2}}}))
}

func TestArtifactLocation_HasLocation(t *testing.T) {
assert.False(t, (&ArtifactLocation{}).HasLocation())
assert.False(t, (&ArtifactLocation{ArchiveLogs: pointer.BoolPtr(true)}).HasLocation())
Expand Down