API add/generalize pagination (#9452)

* paginate results * fixed deadlock * prevented breaking change * updated swagger * go fmt * fixed find topic * go mod tidy * go mod vendor with go1.13.5 * fixed repo find topics * fixed unit test * added Limit method to Engine struct; use engine variable when provided; fixed gitignore * use ItemsPerPage for default pagesize; fix GetWatchers, getOrgUsersByOrgID and GetStargazers; fix GetAllCommits headers; reverted some changed behaviors * set Page value on Home route * improved memory allocations * fixed response headers * removed logfiles * fixed import order * import order * improved swagger * added function to get models.ListOptions from context * removed pagesize diff on unit test * fixed imports * removed unnecessary struct field * fixed go fmt * scoped PR * code improvements * code improvements * go mod tidy * fixed import order * fixed commit statuses session * fixed files headers * fixed headers; added pagination for notifications * go mod tidy * go fmt * removed Private from user search options; added setting.UI.IssuePagingNum as default valeu on repo's issues list * Apply suggestions from code review Co-Authored-By: 6543 <[email protected]> Co-Authored-By: zeripath <[email protected]> * fixed build error * CI.restart() * fixed merge conflicts resolve * fixed conflicts resolve * improved FindTrackedTimesOptions.ToOptions() method * added backwards compatibility on ListReleases request; fixed issue tracked time ToSession * fixed build error; fixed swagger template * fixed swagger template * fixed ListReleases backwards compatibility * added page to user search route Co-authored-by: techknowlogick <[email protected]> Co-authored-by: 6543 <[email protected]> Co-authored-by: zeripath <[email protected]>
go-gitea · Jan 24, 2020 · 1f01f53 · 1f01f53
1 parent 333401e
commit 1f01f53
Show file tree

Hide file tree

Showing 113 changed files with 1,890 additions and 569 deletions.
diff --git a/cmd/admin.go b/cmd/admin.go
@@ -351,9 +351,11 @@ func runRepoSyncReleases(c *cli.Context) error {
  log.Trace("Synchronizing repository releases (this may take a while)")
  for page := 1; ; page++ {
  repos, count, err := models.SearchRepositoryByName(&models.SearchRepoOptions{
- Page: page,
- PageSize: models.RepositoryListDefaultPageSize,
- Private: true,
+ ListOptions: models.ListOptions{
+ PageSize: models.RepositoryListDefaultPageSize,
+ Page: page,
+ },
+ Private: true,
  })
  if err != nil {
  return fmt.Errorf("SearchRepositoryByName: %v", err)

diff --git a/models/access.go b/models/access.go
@@ -215,7 +215,7 @@ func (repo *Repository) refreshAccesses(e Engine, accessMap map[int64]*userAcces
 
 // refreshCollaboratorAccesses retrieves repository collaborations with their access modes.
 func (repo *Repository) refreshCollaboratorAccesses(e Engine, accessMap map[int64]*userAccess) error {
- collaborators, err := repo.getCollaborators(e)
+ collaborators, err := repo.getCollaborators(e, ListOptions{})
  if err != nil {
  return fmt.Errorf("getCollaborations: %v", err)
  }

diff --git a/models/commit_status.go b/models/commit_status.go
@@ -104,7 +104,7 @@ func CalcCommitStatus(statuses []*CommitStatus) *CommitStatus {
 
 // CommitStatusOptions holds the options for query commit statuses
 type CommitStatusOptions struct {
- Page int
+ ListOptions
  State string
  SortType string
 }
@@ -114,18 +114,22 @@ func GetCommitStatuses(repo *Repository, sha string, opts *CommitStatusOptions)
  if opts.Page <= 0 {
  opts.Page = 1
  }
+ if opts.PageSize <= 0 {
+ opts.Page = ItemsPerPage
+ }
 
  countSession := listCommitStatusesStatement(repo, sha, opts)
+ countSession = opts.setSessionPagination(countSession)
  maxResults, err := countSession.Count(new(CommitStatus))
  if err != nil {
  log.Error("Count PRs: %v", err)
  return nil, maxResults, err
  }
 
- statuses := make([]*CommitStatus, 0, ItemsPerPage)
+ statuses := make([]*CommitStatus, 0, opts.PageSize)
  findSession := listCommitStatusesStatement(repo, sha, opts)
+ findSession = opts.setSessionPagination(findSession)
  sortCommitStatusesSession(findSession, opts.SortType)
- findSession.Limit(ItemsPerPage, (opts.Page-1)*ItemsPerPage)
  return statuses, maxResults, findSession.Find(&statuses)
 }
 

diff --git a/models/gpg_key.go b/models/gpg_key.go
@@ -64,9 +64,14 @@ func (key *GPGKey) AfterLoad(session *xorm.Session) {
 }
 
 // ListGPGKeys returns a list of public keys belongs to given user.
-func ListGPGKeys(uid int64) ([]*GPGKey, error) {
- keys := make([]*GPGKey, 0, 5)
- return keys, x.Where("owner_id=? AND primary_key_id=''", uid).Find(&keys)
+func ListGPGKeys(uid int64, listOptions ListOptions) ([]*GPGKey, error) {
+ sess := x.Where("owner_id=? AND primary_key_id=''", uid)
+ if listOptions.Page != 0 {
+ sess = listOptions.setSessionPagination(sess)
+ }
+
+ keys := make([]*GPGKey, 0, 2)
+ return keys, sess.Find(&keys)
 }
 
 // GetGPGKeyByID returns public key by given ID.
@@ -628,7 +633,7 @@ func ParseCommitWithSignature(c *git.Commit) *CommitVerification {
 
  // Now try to associate the signature with the committer, if present
  if committer.ID != 0 {
- keys, err := ListGPGKeys(committer.ID)
+ keys, err := ListGPGKeys(committer.ID, ListOptions{})
  if err != nil { //Skipping failed to get gpg keys of user
  log.Error("ListGPGKeys: %v", err)
  return &CommitVerification{

diff --git a/models/issue.go b/models/issue.go
@@ -1103,13 +1103,12 @@ func GetIssuesByIDs(issueIDs []int64) ([]*Issue, error) {
 
 // IssuesOptions represents options of an issue.
 type IssuesOptions struct {
+ ListOptions
  RepoIDs []int64 // include all repos if empty
  AssigneeID int64
  PosterID int64
  MentionedID int64
  MilestoneID int64
- Page int
- PageSize int
  IsClosed util.OptionalBool
  IsPull util.OptionalBool
  LabelIDs []int64

diff --git a/models/issue_comment.go b/models/issue_comment.go
@@ -777,6 +777,7 @@ func GetCommentByID(id int64) (*Comment, error) {
 
 // FindCommentsOptions describes the conditions to Find comments
 type FindCommentsOptions struct {
+ ListOptions
  RepoID int64
  IssueID int64
  ReviewID int64
@@ -814,6 +815,11 @@ func findComments(e Engine, opts FindCommentsOptions) ([]*Comment, error) {
  if opts.RepoID > 0 {
  sess.Join("INNER", "issue", "issue.id = comment.issue_id")
  }
+
+ if opts.Page != 0 {
+ sess = opts.setSessionPagination(sess)
+ }
+
  return comments, sess.
  Asc("comment.created_unix").
  Asc("comment.id").

diff --git a/models/issue_label.go b/models/issue_label.go
@@ -1,4 +1,5 @@
 // Copyright 2016 The Gogs Authors. All rights reserved.
+// Copyright 2020 The Gitea Authors.
 // Use of this source code is governed by a MIT-style
 // license that can be found in the LICENSE file.
 
@@ -298,7 +299,7 @@ func GetLabelsInRepoByIDs(repoID int64, labelIDs []int64) ([]*Label, error) {
  Find(&labels)
 }
 
-func getLabelsByRepoID(e Engine, repoID int64, sortType string) ([]*Label, error) {
+func getLabelsByRepoID(e Engine, repoID int64, sortType string, listOptions ListOptions) ([]*Label, error) {
  labels := make([]*Label, 0, 10)
  sess := e.Where("repo_id = ?", repoID)
 
@@ -313,12 +314,16 @@ func getLabelsByRepoID(e Engine, repoID int64, sortType string) ([]*Label, error
  sess.Asc("name")
  }
 
+ if listOptions.Page != 0 {
+ sess = listOptions.setSessionPagination(sess)
+ }
+
  return labels, sess.Find(&labels)
 }
 
 // GetLabelsByRepoID returns all labels that belong to given repository by ID.
-func GetLabelsByRepoID(repoID int64, sortType string) ([]*Label, error) {
- return getLabelsByRepoID(x, repoID, sortType)
+func GetLabelsByRepoID(repoID int64, sortType string, listOptions ListOptions) ([]*Label, error) {
+ return getLabelsByRepoID(x, repoID, sortType, listOptions)
 }
 
 func getLabelsByIssueID(e Engine, issueID int64) ([]*Label, error) {

diff --git a/models/issue_label_test.go b/models/issue_label_test.go
@@ -131,7 +131,7 @@ func TestGetLabelsInRepoByIDs(t *testing.T) {
 func TestGetLabelsByRepoID(t *testing.T) {
  assert.NoError(t, PrepareTestDatabase())
  testSuccess := func(repoID int64, sortType string, expectedIssueIDs []int64) {
- labels, err := GetLabelsByRepoID(repoID, sortType)
+ labels, err := GetLabelsByRepoID(repoID, sortType, ListOptions{})
  assert.NoError(t, err)
  assert.Len(t, labels, len(expectedIssueIDs))
  for i, label := range labels {

diff --git a/models/issue_milestone.go b/models/issue_milestone.go
@@ -221,7 +221,7 @@ func (milestones MilestoneList) getMilestoneIDs() []int64 {
 }
 
 // GetMilestonesByRepoID returns all opened milestones of a repository.
-func GetMilestonesByRepoID(repoID int64, state api.StateType) (MilestoneList, error) {
+func GetMilestonesByRepoID(repoID int64, state api.StateType, listOptions ListOptions) (MilestoneList, error) {
  sess := x.Where("repo_id = ?", repoID)
 
  switch state {
@@ -238,7 +238,11 @@ func GetMilestonesByRepoID(repoID int64, state api.StateType) (MilestoneList, er
  sess = sess.And("is_closed = ?", false)
  }
 
- miles := make([]*Milestone, 0, 10)
+ if listOptions.Page != 0 {
+ sess = listOptions.setSessionPagination(sess)
+ }
+
+ miles := make([]*Milestone, 0, listOptions.PageSize)
  return miles, sess.Asc("deadline_unix").Asc("id").Find(&miles)
 }
 

diff --git a/models/issue_milestone_test.go b/models/issue_milestone_test.go
@@ -71,7 +71,7 @@ func TestGetMilestonesByRepoID(t *testing.T) {
  assert.NoError(t, PrepareTestDatabase())
  test := func(repoID int64, state api.StateType) {
  repo := AssertExistsAndLoadBean(t, &Repository{ID: repoID}).(*Repository)
- milestones, err := GetMilestonesByRepoID(repo.ID, state)
+ milestones, err := GetMilestonesByRepoID(repo.ID, state, ListOptions{})
  assert.NoError(t, err)
 
  var n int
@@ -105,7 +105,7 @@ func TestGetMilestonesByRepoID(t *testing.T) {
  test(3, api.StateClosed)
  test(3, api.StateAll)
 
- milestones, err := GetMilestonesByRepoID(NonexistentID, api.StateOpen)
+ milestones, err := GetMilestonesByRepoID(NonexistentID, api.StateOpen, ListOptions{})
  assert.NoError(t, err)
  assert.Len(t, milestones, 0)
 }

diff --git a/models/issue_reaction.go b/models/issue_reaction.go
@@ -30,6 +30,7 @@ type Reaction struct {
 
 // FindReactionsOptions describes the conditions to Find reactions
 type FindReactionsOptions struct {
+ ListOptions
  IssueID int64
  CommentID int64
  UserID int64
@@ -71,20 +72,28 @@ func FindCommentReactions(comment *Comment) (ReactionList, error) {
 }
 
 // FindIssueReactions returns a ReactionList of all reactions from an issue
-func FindIssueReactions(issue *Issue) (ReactionList, error) {
+func FindIssueReactions(issue *Issue, listOptions ListOptions) (ReactionList, error) {
  return findReactions(x, FindReactionsOptions{
- IssueID: issue.ID,
- CommentID: -1,
+ ListOptions: listOptions,
+ IssueID: issue.ID,
+ CommentID: -1,
  })
 }
 
 func findReactions(e Engine, opts FindReactionsOptions) ([]*Reaction, error) {
- reactions := make([]*Reaction, 0, 10)
- sess := e.Where(opts.toConds())
- return reactions, sess.
+ e = e.
+ Where(opts.toConds()).
  In("reaction.`type`", setting.UI.Reactions).
- Asc("reaction.issue_id", "reaction.comment_id", "reaction.created_unix", "reaction.id").
- Find(&reactions)
+ Asc("reaction.issue_id", "reaction.comment_id", "reaction.created_unix", "reaction.id")
+ if opts.Page != 0 {
+ e = opts.setEnginePagination(e)
+
+ reactions := make([]*Reaction, 0, opts.PageSize)
+ return reactions, e.Find(&reactions)
+ }
+
+ reactions := make([]*Reaction, 0, 10)
+ return reactions, e.Find(&reactions)
 }
 
 func createReaction(e *xorm.Session, opts *ReactionOptions) (*Reaction, error) {

diff --git a/models/issue_stopwatch.go b/models/issue_stopwatch.go
@@ -33,9 +33,14 @@ func getStopwatch(e Engine, userID, issueID int64) (sw *Stopwatch, exists bool,
 }
 
 // GetUserStopwatches return list of all stopwatches of a user
-func GetUserStopwatches(userID int64) (sws *Stopwatches, err error) {
- sws = new(Stopwatches)
- err = x.Where("stopwatch.user_id = ?", userID).Find(sws)
+func GetUserStopwatches(userID int64, listOptions ListOptions) (*Stopwatches, error) {
+ sws := new(Stopwatches)
+ sess := x.Where("stopwatch.user_id = ?", userID)
+ if listOptions.Page != 0 {
+ sess = listOptions.setSessionPagination(sess)
+ }
+
+ err := sess.Find(sws)
  if err != nil {
  return nil, err
  }

diff --git a/models/issue_test.go b/models/issue_test.go
@@ -139,24 +139,30 @@ func TestIssues(t *testing.T) {
  IssuesOptions{
  RepoIDs: []int64{1, 3},
  SortType: "oldest",
- Page: 1,
- PageSize: 4,
+ ListOptions: ListOptions{
+ Page: 1,
+ PageSize: 4,
+ },
  },
  []int64{1, 2, 3, 5},
  },
  {
  IssuesOptions{
  LabelIDs: []int64{1},
- Page: 1,
- PageSize: 4,
+ ListOptions: ListOptions{
+ Page: 1,
+ PageSize: 4,
+ },
  },
  []int64{2, 1},
  },
  {
  IssuesOptions{
  LabelIDs: []int64{1, 2},
- Page: 1,
- PageSize: 4,
+ ListOptions: ListOptions{
+ Page: 1,
+ PageSize: 4,
+ },
  },
  []int64{}, // issues with **both** label 1 and 2, none of these issues matches, TODO: add more tests
  },

diff --git a/models/issue_tracked_time.go b/models/issue_tracked_time.go
@@ -10,7 +10,6 @@ import (
  "code.gitea.io/gitea/modules/setting"
 
  "xorm.io/builder"
- "xorm.io/xorm"
 )
 
 // TrackedTime represents a time that was spent for a specific issue.
@@ -71,6 +70,7 @@ func (tl TrackedTimeList) LoadAttributes() (err error) {
 
 // FindTrackedTimesOptions represent the filters for tracked times. If an ID is 0 it will be ignored.
 type FindTrackedTimesOptions struct {
+ ListOptions
  IssueID int64
  UserID int64
  RepositoryID int64
@@ -104,11 +104,19 @@ func (opts *FindTrackedTimesOptions) ToCond() builder.Cond {
 }
 
 // ToSession will convert the given options to a xorm Session by using the conditions from ToCond and joining with issue table if required
-func (opts *FindTrackedTimesOptions) ToSession(e Engine) *xorm.Session {
+func (opts *FindTrackedTimesOptions) ToSession(e Engine) Engine {
+ sess := e
  if opts.RepositoryID > 0 || opts.MilestoneID > 0 {
- return e.Join("INNER", "issue", "issue.id = tracked_time.issue_id").Where(opts.ToCond())
+ sess = e.Join("INNER", "issue", "issue.id = tracked_time.issue_id")
  }
- return e.Where(opts.ToCond())
+
+ sess = sess.Where(opts.ToCond())
+
+ if opts.Page != 0 {
+ sess = opts.setEnginePagination(sess)
+ }
+
+ return sess
 }
 
 func getTrackedTimes(e Engine, options FindTrackedTimesOptions) (trackedTimes TrackedTimeList, err error) {

diff --git a/models/issue_watch.go b/models/issue_watch.go
@@ -77,18 +77,22 @@ func GetIssueWatchersIDs(issueID int64) ([]int64, error) {
 }
 
 // GetIssueWatchers returns watchers/unwatchers of a given issue
-func GetIssueWatchers(issueID int64) (IssueWatchList, error) {
- return getIssueWatchers(x, issueID)
+func GetIssueWatchers(issueID int64, listOptions ListOptions) (IssueWatchList, error) {
+ return getIssueWatchers(x, issueID, listOptions)
 }
 
-func getIssueWatchers(e Engine, issueID int64) (watches IssueWatchList, err error) {
- err = e.
+func getIssueWatchers(e Engine, issueID int64, listOptions ListOptions) (watches IssueWatchList, err error) {
+ sess := e.
  Where("`issue_watch`.issue_id = ?", issueID).
  And("`issue_watch`.is_watching = ?", true).
  And("`user`.is_active = ?", true).
  And("`user`.prohibit_login = ?", false).
- Join("INNER", "`user`", "`user`.id = `issue_watch`.user_id").
- Find(&watches)
+ Join("INNER", "`user`", "`user`.id = `issue_watch`.user_id")
+
+ if listOptions.Page == 0 {
+ sess = listOptions.setSessionPagination(sess)
+ }
+ err = sess.Find(&watches)
  return
 }
 

diff --git a/models/issue_watch_test.go b/models/issue_watch_test.go
@@ -42,22 +42,22 @@ func TestGetIssueWatch(t *testing.T) {
 func TestGetIssueWatchers(t *testing.T) {
  assert.NoError(t, PrepareTestDatabase())
 
- iws, err := GetIssueWatchers(1)
+ iws, err := GetIssueWatchers(1, ListOptions{})
  assert.NoError(t, err)
  // Watcher is inactive, thus 0
  assert.Len(t, iws, 0)
 
- iws, err = GetIssueWatchers(2)
+ iws, err = GetIssueWatchers(2, ListOptions{})
  assert.NoError(t, err)
  // Watcher is explicit not watching
  assert.Len(t, iws, 0)
 
- iws, err = GetIssueWatchers(5)
+ iws, err = GetIssueWatchers(5, ListOptions{})
  assert.NoError(t, err)
  // Issue has no Watchers
  assert.Len(t, iws, 0)
 
- iws, err = GetIssueWatchers(7)
+ iws, err = GetIssueWatchers(7, ListOptions{})
  assert.NoError(t, err)
  // Issue has one watcher
  assert.Len(t, iws, 1)