Index code and stats only for non-empty repositories

Fix test and switch to unique queue

Fix MySQL support when deleting old statistics

models/fixtures/repository.yml

@@ -4,6 +4,7 @@
  owner_name: user2
  lower_name: repo1
  name: repo1
+ is_empty: false
  is_private: false
  num_issues: 2
  num_closed_issues: 1

models/repo_indexer.go

@@ -34,6 +34,8 @@ func GetUnindexedRepos(indexerType RepoIndexerType, maxRepoID int64, page, pageS
  ids := make([]int64, 0, 50)
  cond := builder.Cond(builder.IsNull{
  "repo_indexer_status.id",
+ }).And(builder.Eq{
+ "repository.is_empty": false,
  })
  sess := x.Table("repository").Join("LEFT OUTER", "repo_indexer_status", "repository.id = repo_indexer_status.repo_id AND repo_indexer_status.indexer_type = ?", indexerType)
  if maxRepoID > 0 {
@@ -66,11 +68,11 @@ func (repo *Repository) getIndexerStatus(e Engine, indexerType RepoIndexerType)
  return repo.StatsIndexerStatus, nil
  }
  }
- status := &RepoIndexerStatus{RepoID: repo.ID, IndexerType: indexerType}
- has, err := e.Get(status)
- if err != nil {
+ status := &RepoIndexerStatus{RepoID: repo.ID}
+ if has, err := e.Where("`indexer_type` = ?", indexerType).Get(status); err != nil {
  return nil, err
  } else if !has {
+ status.IndexerType = indexerType
  status.CommitSha = ""
  }
  switch indexerType {

models/repo_language_stats.go

@@ -125,10 +125,19 @@ func (repo *Repository) UpdateLanguageStats(commitID string, stats map[string]fl
  }
  }
  // Delete old languages
- if _, err := sess.Where("`id` IN (SELECT `id` FROM `language_stat` WHERE `repo_id` = ? AND `commit_id` != ?)", repo.ID, commitID).Delete(&LanguageStat{}); err != nil {
- return err
+ statsToDelete := make([]int64, 0, len(oldstats))
+ for _, s := range oldstats {
+ if s.CommitID != commitID {
+ statsToDelete = append(statsToDelete, s.ID)
+ }
+ }
+ if len(statsToDelete) > 0 {
+ if _, err := sess.In("`id`", statsToDelete).Delete(&LanguageStat{}); err != nil {
+ return err
+ }
  }
 
+ // Update indexer status
  if err = repo.updateIndexerStatus(sess, RepoIndexerTypeStats, commitID); err != nil {
  return err
  }

modules/indexer/stats/queue.go

@@ -14,7 +14,7 @@ import (
 )
 
 // statsQueue represents a queue to handle repository stats updates
-var statsQueue queue.Queue
+var statsQueue queue.UniqueQueue
 
 // handle passed PR IDs and test the PRs
 func handle(data ...queue.Data) {
@@ -27,7 +27,7 @@ func handle(data ...queue.Data) {
 }
 
 func initStatsQueue() error {
- statsQueue = queue.CreateQueue("repo_stats_update", handle, int64(0)).(queue.Queue)
+ statsQueue = queue.CreateUniqueQueue("repo_stats_update", handle, int64(0)).(queue.UniqueQueue)
  if statsQueue == nil {
  return fmt.Errorf("Unable to create repo_stats_update Queue")
  }