adding paging to get repositories from the database

store/datastore/repos.go

@@ -21,21 +21,37 @@ func (db *datastore) GetRepoName(name string) (*model.Repo, error) {
 
 func (db *datastore) GetRepoListOf(listof []*model.RepoLite) ([]*model.Repo, error) {
  var (
- repos []*model.Repo
- args  []interface{}
- stmt string
- err error
+ err error
+ repos = []*model.Repo{}
+ toListRepoLite func([]*model.RepoLite) (string, []interface{})
+ total = len(listof)
  )
+
+ if total == 0 {
+ return repos, nil
+ }
+
  switch meddler.Default {
  case meddler.PostgreSQL:
- stmt, args = toListPosgres(listof)
+ toListRepoLite = toListPosgres
  default:
- stmt, args = toList(listof)
+ toListRepoLite = toList
  }
- if len(args) > 0 {
- err = meddler.QueryAll(db, &repos, fmt.Sprintf(repoListOfQuery, stmt), args...)
+
+ pages := calculatePagination(total, maxRepoPage)
+
+ for i := 0; i < pages; i++ {
+ stmt, args := toListRepoLite(resizeList(listof, i, maxRepoPage))
+
+ var tmpRepos []*model.Repo
+ err = meddler.QueryAll(db, &tmpRepos, fmt.Sprintf(repoListOfQuery, stmt), args...)
+ if err != nil {
+ return nil, err
+ }
+
+ repos = append(repos, tmpRepos...)
  }
- return repos, err
+ return repos, nil
 }
 
 func (db *datastore) GetRepoCount() (int, error) {

store/datastore/repos_test.go

@@ -1,6 +1,7 @@
 package datastore
 
 import (
+ "fmt"
  "testing"
 
  "github.com/drone/drone/model"
@@ -138,6 +139,52 @@ func TestRepos(t *testing.T) {
  g.Assert(count).Equal(2)
  })
 
+ g.It("Should Get a Repo List Paginated", func() {
+ _repos := []*model.RepoLite{
+ {FullName: "bradrydzewski/drone"},
+ {FullName: "drone/drone"},
+ }
+ _repo := []*model.Repo{
+ {
+ UserID: 1,
+ Owner: "bradrydzewski",
+ Name: "drone",
+ FullName: "bradrydzewski/drone",
+ },
+ {
+ UserID: 2,
+ Owner: "drone",
+ Name: "drone",
+ FullName: "drone/drone",
+ },
+ }
+ s.CreateRepo(_repo[0])
+ s.CreateRepo(_repo[1])
+
+ for i := 0; i < 3000; i++ {
+ // find by 3000 repos
+ _repos = append(_repos, &model.RepoLite{FullName: "octocat/hello-world" + fmt.Sprint(i)})
+ // but create only 2000 repos
+ if i >= 1999 && i < 2999 {
+ continue
+ }
+ repo := &model.Repo{
+ UserID: 2,
+ Owner: "octocat",
+ Name: "hello-world" + fmt.Sprint(i),
+ FullName: "octocat/hello-world" + fmt.Sprint(i),
+ }
+ _repo = append(_repo, repo)
+ s.CreateRepo(repo)
+ }
+ repos, err := s.GetRepoListOf(_repos)
+ g.Assert(err == nil).IsTrue()
+ g.Assert(len(repos)).Equal(2002)
+ g.Assert(repos[0].ID).Equal(_repo[0].ID)
+ g.Assert(repos[1].ID).Equal(_repo[1].ID)
+ g.Assert(repos[2001].ID).Equal(_repo[2001].ID)
+ })
+
  g.It("Should Delete a Repo", func() {
  repo := model.Repo{
  UserID: 1,

store/datastore/users.go

@@ -2,6 +2,7 @@ package datastore
 
 import (
  "fmt"
+ "sort"
 
  "github.com/drone/drone/model"
  "github.com/russross/meddler"
@@ -27,42 +28,76 @@ func (db *datastore) GetUserList() ([]*model.User, error) {
 
 func (db *datastore) GetUserFeed(listof []*model.RepoLite) ([]*model.Feed, error) {
  var (
- args []interface{}
- stmt string
- err error
-
- feed = []*model.Feed{}
+ err error
+ feed = feedHelper{}
+ total = len(listof)
+ toListRepoLite func([]*model.RepoLite) (string, []interface{})
  )
+
+ if total == 0 {
+ return feed, nil
+ }
+
  switch meddler.Default {
  case meddler.PostgreSQL:
- stmt, args = toListPosgres(listof)
+ toListRepoLite = toListPosgres
  default:
- stmt, args = toList(listof)
+ toListRepoLite = toList
  }
- if len(args) > 0 {
- err = meddler.QueryAll(db, &feed, fmt.Sprintf(userFeedQuery, stmt), args...)
+
+ pages := calculatePagination(total, maxRepoPage)
+ for i := 0; i < pages; i++ {
+ stmt, args := toListRepoLite(resizeList(listof, i, maxRepoPage))
+
+ var tmpFeed []*model.Feed
+ err = meddler.QueryAll(db, &tmpFeed, fmt.Sprintf(userFeedQuery, stmt), args...)
+ if err != nil {
+ return nil, err
+ }
+
+ feed = append(feed, tmpFeed...)
+ }
+
+ if len(feed) <= 50 {
+ return feed, nil
  }
- return feed, err
+
+ sort.Sort(sort.Reverse(feed))
+
+ return feed[:50], nil
 }
 
 func (db *datastore) GetUserFeedLatest(listof []*model.RepoLite) ([]*model.Feed, error) {
  var (
- args []interface{}
- stmt string
- err error
-
+ err error
  feed = []*model.Feed{}
+ toListRepoLite func([]*model.RepoLite) (string, []interface{})
+ total = len(listof)
  )
+
+ if total == 0 {
+ return feed, nil
+ }
+
  switch meddler.Default {
  case meddler.PostgreSQL:
- stmt, args = toListPosgres(listof)
+ toListRepoLite = toListPosgres
  default:
- stmt, args = toList(listof)
+ toListRepoLite = toList
  }
- if len(args) > 0 {
- err = meddler.QueryAll(db, &feed, fmt.Sprintf(userFeedLatest, stmt), args...)
+
+ pages := calculatePagination(total, maxRepoPage)
+ for i := 0; i < pages; i++ {
+ stmt, args := toListRepoLite(resizeList(listof, i, maxRepoPage))
+ var tmpFeed []*model.Feed
+ err = meddler.QueryAll(db, &tmpFeed, fmt.Sprintf(userFeedLatest, stmt), args...)
+ if err != nil {
+ return nil, err
+ }
+
+ feed = append(feed, tmpFeed...)
  }
- return feed, err
+ return feed, nil
 }
 
 func (db *datastore) GetUserCount() (int, error) {
@@ -170,4 +205,4 @@ FROM repos LEFT OUTER JOIN builds ON build_id = (
  LIMIT 1
 )
 WHERE repo_full_name IN (%s)
-`
+`

store/datastore/utils.go

@@ -8,6 +8,24 @@ import (
  "github.com/russross/meddler"
 )
 
+// indicate max of each page when paginated repo list
+const maxRepoPage = 999
+
+// helper type that sort Feed List
+type feedHelper []*model.Feed
+
+func (slice feedHelper) Len() int {
+ return len(slice)
+}
+
+func (slice feedHelper) Less(i, j int) bool {
+ return slice[i].Created < slice[j].Created;
+}
+
+func (slice feedHelper) Swap(i, j int) {
+ slice[i], slice[j] = slice[j], slice[i]
+}
+
 // rebind is a helper function that changes the sql
 // bind type from ? to $ for postgres queries.
 func rebind(query string) string {
@@ -36,14 +54,32 @@ func rebind(query string) string {
  return string(rqb)
 }
 
+// helper function that calculate pagination
+func calculatePagination(total, limit int) (pages int) {
+ pages = total / limit
+ if total % limit != 0 {
+ pages++
+ }
+ return
+}
+
+// helper function that resize list of repo to pagination
+func resizeList(listof []*model.RepoLite, page, limit int) []*model.RepoLite {
+ var total = len(listof)
+ var end = (page * limit) + limit
+ if end > total{
+ end = total
+ }
+ if total > limit{
+ return listof[page * limit:end]
+ }
+ return listof
+}
+
 // helper function that converts a simple repsitory list
 // to a sql IN statment.
 func toList(listof []*model.RepoLite) (string, []interface{}) {
  var size = len(listof)
- if size > 999 {
- size = 999
- listof = listof[:999]
- }
  var qs = make([]string, size, size)
  var in = make([]interface{}, size, size)
  for i, repo := range listof {
@@ -57,10 +93,6 @@ func toList(listof []*model.RepoLite) (string, []interface{}) {
 // to a sql IN statement compatible with postgres.
 func toListPosgres(listof []*model.RepoLite) (string, []interface{}) {
  var size = len(listof)
- if size > 999 {
- size = 999
- listof = listof[:999]
- }
  var qs = make([]string, size, size)
  var in = make([]interface{}, size, size)
  for i, repo := range listof {

store/datastore/utils_test.go

@@ -0,0 +1,47 @@
+package datastore
+
+import (
+ "testing"
+
+ "github.com/drone/drone/model"
+ "github.com/franela/goblin"
+)
+
+func TestUtils(t *testing.T) {
+ g := goblin.Goblin(t)
+ g.Describe("Utils", func() {
+
+ g.It("Should Calculate Pagination", func() {
+ g.Assert(calculatePagination(10, 100)).Equal(1)
+ g.Assert(calculatePagination(10, 5)).Equal(2)
+ g.Assert(calculatePagination(10, 3)).Equal(4)
+ })
+
+ g.It("Should Resize a RepoLite List", func() {
+ repoLiteList := []*model.RepoLite{
+ {FullName: "bradrydzewski/drone"},
+ {FullName: "drone/drone"},
+ {FullName: "octocat/hello-world-1"},
+ {FullName: "octocat/hello-world-2"},
+ {FullName: "octocat/hello-world-3"},
+ }
+ notResized := resizeList(repoLiteList, 0, 10)
+ page1 := resizeList(repoLiteList, 0, 2)
+ page2 := resizeList(repoLiteList, 1, 2)
+ page3 := resizeList(repoLiteList, 2, 2)
+
+ g.Assert(len(notResized)).Equal(5)
+
+ g.Assert(len(page1)).Equal(2)
+ g.Assert(page1[0].FullName).Equal("bradrydzewski/drone")
+ g.Assert(page1[1].FullName).Equal("drone/drone")
+
+ g.Assert(len(page2)).Equal(2)
+ g.Assert(page2[0].FullName).Equal("octocat/hello-world-1")
+ g.Assert(page2[1].FullName).Equal("octocat/hello-world-2")
+
+ g.Assert(len(page3)).Equal(1)
+ g.Assert(page3[0].FullName).Equal("octocat/hello-world-3")
+ })
+ })
+}