From 4120aa1e427744a228b9c06031bab649d3bb70f5 Mon Sep 17 00:00:00 2001 From: 6543 <6543@obermui.de> Date: Fri, 17 Apr 2020 16:29:57 +0200 Subject: [PATCH 01/27] rename check-db to check-db-version to have multible db checks --- cmd/doctor.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/doctor.go b/cmd/doctor.go index f469496cfd24..140a41ce6c9c 100644 --- a/cmd/doctor.go +++ b/cmd/doctor.go @@ -85,7 +85,7 @@ var checklist = []check{ }, { title: "Check Database Version", - name: "check-db", + name: "check-db-version", isDefault: true, f: runDoctorCheckDBVersion, abortIfFailed: true, From 2627b81d0c51cc07758e6ff338cbf63ce7d60d8f Mon Sep 17 00:00:00 2001 From: 6543 <6543@obermui.de> Date: Fri, 17 Apr 2020 17:23:51 +0200 Subject: [PATCH 02/27] add runDoctorCheckDBConsistency --- cmd/doctor.go | 67 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 67 insertions(+) diff --git a/cmd/doctor.go b/cmd/doctor.go index 140a41ce6c9c..714a4b091f5a 100644 --- a/cmd/doctor.go +++ b/cmd/doctor.go @@ -114,6 +114,12 @@ var checklist = []check{ isDefault: false, f: runDoctorPRMergeBase, }, + { + title: "Check consistency of database", + name: "check-db-consistency", + isDefault: false, + f: runDoctorCheckDBConsistency, + }, // more checks please append here } @@ -495,3 +501,64 @@ func runDoctorScriptType(ctx *cli.Context) ([]string, error) { } return []string{fmt.Sprintf("ScriptType %s is on the current PATH at %s", setting.ScriptType, path)}, nil } + +func runDoctorCheckDBConsistency(ctx *cli.Context) ([]string, error) { + _, committer, err := models.TxDBContext() + if err != nil { + return nil, err + } + sess := committer.(models.Engine) + defer committer.Close() + var results []string + + //find issues without existing repository + count, err := sess.Table("issue"). + Join("LEFT", "repository", "issue.repo_id=repository.id"). + Where("repository.id is NULL"). + Count("id") + if err != nil { + return nil, err + } + if count > 0 { + if ctx.Bool("fix") { + if _, err = sess.In("id", builder.Select("issue.id"). + From("issue"). + Join("LEFT", "repository", "issue.repo_id=repository.id"). + Where(builder.IsNull{"repository.id"})). + Delete(models.Issue{}); err != nil { + return nil, err + } + results = append(results, fmt.Sprintf("%d issues without existing repository deleted", count)) + } else { + results = append(results, fmt.Sprintf("%d issues without existing repository", count)) + } + } + + //find tracked times without existing issues/pulls + count, err = sess.Table("tracked_time"). + Join("LEFT", "issue", "tracked_time.issue_id=issue.id"). + Where("issue.id is NULL"). + Count("id") + if err != nil { + return nil, err + } + if count > 0 { + if ctx.Bool("fix") { + if _, err = sess.In("id", builder.Select("tracked_time.id"). + From("tracked_time"). + Join("LEFT", "issue", "tracked_time.issue_id=issue.id"). + Where(builder.IsNull{"issue.id"})). + Delete(models.TrackedTime{}); err != nil { + return nil, err + } + results = append(results, fmt.Sprintf("%d tracked times without existing issue deleted", count)) + } else { + results = append(results, fmt.Sprintf("%d tracked times without existing issue", count)) + } + } + + if ctx.Bool("fix") { + return results, committer.Commit() + } + return results, nil +} From fe480547444c43ff43b41cb63d2e82fbaa6df72d Mon Sep 17 00:00:00 2001 From: 6543 <6543@obermui.de> Date: Fri, 17 Apr 2020 17:29:22 +0200 Subject: [PATCH 03/27] add find pulls without existing issues --- cmd/doctor.go | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/cmd/doctor.go b/cmd/doctor.go index 714a4b091f5a..ac0c36da0c31 100644 --- a/cmd/doctor.go +++ b/cmd/doctor.go @@ -534,6 +534,29 @@ func runDoctorCheckDBConsistency(ctx *cli.Context) ([]string, error) { } } + //find pulls without existing issues + count, err = sess.Table("pull_request"). + Join("LEFT", "issue", "pull_request.issue_id=issue.id"). + Where("issue.id is NULL"). + Count("id") + if err != nil { + return nil, err + } + if count > 0 { + if ctx.Bool("fix") { + if _, err = sess.In("id", builder.Select("pull_request.id"). + From("pull_request"). + Join("LEFT", "issue", "pull_request.issue_id=issue.id"). + Where(builder.IsNull{"issue.id"})). + Delete(models.PullRequest{}); err != nil { + return nil, err + } + results = append(results, fmt.Sprintf("%d pull requests without existing issue deleted", count)) + } else { + results = append(results, fmt.Sprintf("%d pull requests without existing issue", count)) + } + } + //find tracked times without existing issues/pulls count, err = sess.Table("tracked_time"). Join("LEFT", "issue", "tracked_time.issue_id=issue.id"). From 5cfa0d41be1cd9dd3f08d14c47f36961e518cee4 Mon Sep 17 00:00:00 2001 From: 6543 <6543@obermui.de> Date: Sun, 19 Apr 2020 02:32:25 +0200 Subject: [PATCH 04/27] move Issue Deletion to issue.go --- models/issue.go | 93 +++++++++++++++++++++++++++++++++++++++++++++++++ models/repo.go | 65 ++-------------------------------- 2 files changed, 96 insertions(+), 62 deletions(-) diff --git a/models/issue.go b/models/issue.go index 17ec0a6888d3..8bdac90dedb0 100644 --- a/models/issue.go +++ b/models/issue.go @@ -1874,3 +1874,96 @@ func UpdateReactionsMigrationsByType(gitServiceType structs.GitServiceType, orig }) return err } + +// DeleteIssuesByIDs and Objects who depend on them +func DeleteIssuesByIDs(ids []int64) (err error) { + + sess := x.NewSession() + defer sess.Close() + if err = sess.Begin(); err != nil { + return err + } + + deleteCond := builder.Select("id").From("issue").Where(builder.In("id", ids)) + var attachmentPaths []string + if attachmentPaths, err = deleteIssuesByBuilder(sess, deleteCond); err != nil { + return err + } + + if err = sess.Commit(); err != nil { + return err + } + + // Remove issue attachment files. + for i := range attachmentPaths { + removeAllWithNotice(x, "Delete issue attachment", attachmentPaths[i]) + } + + return nil +} + +func deleteIssuesByBuilder(sess Engine, issueIDBuilder *builder.Builder) (attachmentPaths []string, err error) { + + // Delete comments and attachments + if _, err = sess.In("issue_id", issueIDBuilder). + Delete(&Comment{}); err != nil { + return + } + + // Dependencies for issues in this repository + if _, err = sess.In("issue_id", issueIDBuilder). + Delete(&IssueDependency{}); err != nil { + return + } + + // Delete dependencies for issues in other repositories + if _, err = sess.In("dependency_id", issueIDBuilder). + Delete(&IssueDependency{}); err != nil { + return + } + + if _, err = sess.In("issue_id", issueIDBuilder). + Delete(&IssueUser{}); err != nil { + return + } + + if _, err = sess.In("issue_id", issueIDBuilder). + Delete(&Reaction{}); err != nil { + return + } + + if _, err = sess.In("issue_id", issueIDBuilder). + Delete(&IssueWatch{}); err != nil { + return + } + + if _, err = sess.In("issue_id", issueIDBuilder). + Delete(&Stopwatch{}); err != nil { + return + } + + if _, err = sess.In("issue_id", issueIDBuilder). + Delete(&TrackedTime{}); err != nil { + return + } + + var attachments []*Attachment + if err = sess.In("issue_id", issueIDBuilder). + Find(&attachments); err != nil { + return + } + for j := range attachments { + attachmentPaths = append(attachmentPaths, attachments[j].LocalPath()) + } + + if _, err = sess.In("issue_id", issueIDBuilder). + Delete(&Attachment{}); err != nil { + return + } + + if _, err = sess.In("id", issueIDBuilder).Delete(&Issue{}); err != nil { + return + } + + return +} diff --git a/models/repo.go b/models/repo.go index 438066e0da31..abb005340449 100644 --- a/models/repo.go +++ b/models/repo.go @@ -1575,69 +1575,10 @@ func DeleteRepository(doer *User, uid, repoID int64) error { return fmt.Errorf("deleteBeans: %v", err) } + // Delete Issues and related objects deleteCond := builder.Select("id").From("issue").Where(builder.Eq{"repo_id": repoID}) - // Delete comments and attachments - if _, err = sess.In("issue_id", deleteCond). - Delete(&Comment{}); err != nil { - return err - } - - // Dependencies for issues in this repository - if _, err = sess.In("issue_id", deleteCond). - Delete(&IssueDependency{}); err != nil { - return err - } - - // Delete dependencies for issues in other repositories - if _, err = sess.In("dependency_id", deleteCond). - Delete(&IssueDependency{}); err != nil { - return err - } - - if _, err = sess.In("issue_id", deleteCond). - Delete(&IssueUser{}); err != nil { - return err - } - - if _, err = sess.In("issue_id", deleteCond). - Delete(&Reaction{}); err != nil { - return err - } - - if _, err = sess.In("issue_id", deleteCond). - Delete(&IssueWatch{}); err != nil { - return err - } - - if _, err = sess.In("issue_id", deleteCond). - Delete(&Stopwatch{}); err != nil { - return err - } - - if _, err = sess.In("issue_id", deleteCond). - Delete(&TrackedTime{}); err != nil { - return err - } - - attachments = attachments[:0] - if err = sess.Join("INNER", "issue", "issue.id = attachment.issue_id"). - Where("issue.repo_id = ?", repoID). - Find(&attachments); err != nil { - return err - } - attachmentPaths := make([]string, 0, len(attachments)) - for j := range attachments { - attachmentPaths = append(attachmentPaths, attachments[j].LocalPath()) - } - - if _, err = sess.In("issue_id", deleteCond). - Delete(&Attachment{}); err != nil { - return err - } - - if _, err = sess.Delete(&Issue{RepoID: repoID}); err != nil { - return err - } + var attachmentPaths []string + attachmentPaths, err = deleteIssuesByBuilder(sess, deleteCond) if _, err = sess.Where("repo_id = ?", repoID).Delete(new(RepoUnit)); err != nil { return err From b8b5de3c9e1068cbf769dc5423bde0c26ad39630 Mon Sep 17 00:00:00 2001 From: 6543 <6543@obermui.de> Date: Sun, 19 Apr 2020 02:32:46 +0200 Subject: [PATCH 05/27] use DeleteIssuesByIDs --- cmd/doctor.go | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/cmd/doctor.go b/cmd/doctor.go index ac0c36da0c31..3d91d570353e 100644 --- a/cmd/doctor.go +++ b/cmd/doctor.go @@ -521,13 +521,17 @@ func runDoctorCheckDBConsistency(ctx *cli.Context) ([]string, error) { } if count > 0 { if ctx.Bool("fix") { - if _, err = sess.In("id", builder.Select("issue.id"). - From("issue"). + var ids []int64 + + sess.Table("issue").Select("issue.id"). Join("LEFT", "repository", "issue.repo_id=repository.id"). - Where(builder.IsNull{"repository.id"})). - Delete(models.Issue{}); err != nil { + Where("repository.id is NULL"). + Find(&ids) + + if err = models.DeleteIssuesByIDs(ids); err != nil { return nil, err } + results = append(results, fmt.Sprintf("%d issues without existing repository deleted", count)) } else { results = append(results, fmt.Sprintf("%d issues without existing repository", count)) From 17589b9310cf67658a4c087bdd5ad83b75dca2a0 Mon Sep 17 00:00:00 2001 From: 6543 <6543@obermui.de> Date: Sun, 19 Apr 2020 02:56:43 +0200 Subject: [PATCH 06/27] use err --- models/repo.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/models/repo.go b/models/repo.go index abb005340449..7d429759adbd 100644 --- a/models/repo.go +++ b/models/repo.go @@ -1578,7 +1578,9 @@ func DeleteRepository(doer *User, uid, repoID int64) error { // Delete Issues and related objects deleteCond := builder.Select("id").From("issue").Where(builder.Eq{"repo_id": repoID}) var attachmentPaths []string - attachmentPaths, err = deleteIssuesByBuilder(sess, deleteCond) + if attachmentPaths, err = deleteIssuesByBuilder(sess, deleteCond); err != nil { + return err + } if _, err = sess.Where("repo_id = ?", repoID).Delete(new(RepoUnit)); err != nil { return err From d6f1f0e4398d4a37dd30e5cbcc241438060d4906 Mon Sep 17 00:00:00 2001 From: 6543 <6543@obermui.de> Date: Sun, 19 Apr 2020 03:06:38 +0200 Subject: [PATCH 07/27] just another one --- cmd/doctor.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/cmd/doctor.go b/cmd/doctor.go index 3d91d570353e..ec483f4a547f 100644 --- a/cmd/doctor.go +++ b/cmd/doctor.go @@ -523,10 +523,12 @@ func runDoctorCheckDBConsistency(ctx *cli.Context) ([]string, error) { if ctx.Bool("fix") { var ids []int64 - sess.Table("issue").Select("issue.id"). + if err = sess.Table("issue").Select("issue.id"). Join("LEFT", "repository", "issue.repo_id=repository.id"). Where("repository.id is NULL"). - Find(&ids) + Find(&ids); err != nil { + return nil, err + } if err = models.DeleteIssuesByIDs(ids); err != nil { return nil, err From 8a6fbfb7fae296477ce816aa0bc8893cbf7bee12 Mon Sep 17 00:00:00 2001 From: 6543 <6543@obermui.de> Date: Tue, 5 May 2020 10:54:42 +0200 Subject: [PATCH 08/27] move things into models --- cmd/doctor.go | 62 ++++++++-------------- models/consistency.go | 117 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 139 insertions(+), 40 deletions(-) diff --git a/cmd/doctor.go b/cmd/doctor.go index ec483f4a547f..c2e9f0011276 100644 --- a/cmd/doctor.go +++ b/cmd/doctor.go @@ -503,37 +503,34 @@ func runDoctorScriptType(ctx *cli.Context) ([]string, error) { } func runDoctorCheckDBConsistency(ctx *cli.Context) ([]string, error) { - _, committer, err := models.TxDBContext() - if err != nil { - return nil, err - } - sess := committer.(models.Engine) - defer committer.Close() var results []string - //find issues without existing repository - count, err := sess.Table("issue"). - Join("LEFT", "repository", "issue.repo_id=repository.id"). - Where("repository.id is NULL"). - Count("id") + //find labels without existing repo or org + count, err := models.CountCorruptLabels() if err != nil { return nil, err } if count > 0 { if ctx.Bool("fix") { - var ids []int64 - - if err = sess.Table("issue").Select("issue.id"). - Join("LEFT", "repository", "issue.repo_id=repository.id"). - Where("repository.id is NULL"). - Find(&ids); err != nil { + if err = models.DeleteCorruptLabels(); err != nil { return nil, err } + results = append(results, fmt.Sprintf("%d issues without existing repository deleted", count)) + } else { + results = append(results, fmt.Sprintf("%d issues without existing repository", count)) + } + } - if err = models.DeleteIssuesByIDs(ids); err != nil { + //find issues without existing repository + count, err = models.CountCorruptIssues() + if err != nil { + return nil, err + } + if count > 0 { + if ctx.Bool("fix") { + if err = models.DeleteCorruptIssues(); err != nil { return nil, err } - results = append(results, fmt.Sprintf("%d issues without existing repository deleted", count)) } else { results = append(results, fmt.Sprintf("%d issues without existing repository", count)) @@ -541,20 +538,13 @@ func runDoctorCheckDBConsistency(ctx *cli.Context) ([]string, error) { } //find pulls without existing issues - count, err = sess.Table("pull_request"). - Join("LEFT", "issue", "pull_request.issue_id=issue.id"). - Where("issue.id is NULL"). - Count("id") + count, err = models.CountCorruptObject("pull_request", "issue", "pull_request.issue_id=issue.id") if err != nil { return nil, err } if count > 0 { if ctx.Bool("fix") { - if _, err = sess.In("id", builder.Select("pull_request.id"). - From("pull_request"). - Join("LEFT", "issue", "pull_request.issue_id=issue.id"). - Where(builder.IsNull{"issue.id"})). - Delete(models.PullRequest{}); err != nil { + if err = models.DeleteCorruptObject("pull_request", "issue", "pull_request.issue_id=issue.id"); err != nil { return nil, err } results = append(results, fmt.Sprintf("%d pull requests without existing issue deleted", count)) @@ -564,20 +554,13 @@ func runDoctorCheckDBConsistency(ctx *cli.Context) ([]string, error) { } //find tracked times without existing issues/pulls - count, err = sess.Table("tracked_time"). - Join("LEFT", "issue", "tracked_time.issue_id=issue.id"). - Where("issue.id is NULL"). - Count("id") + count, err = models.CountCorruptObject("tracked_time", "issue", "tracked_time.issue_id=issue.id") if err != nil { return nil, err } if count > 0 { if ctx.Bool("fix") { - if _, err = sess.In("id", builder.Select("tracked_time.id"). - From("tracked_time"). - Join("LEFT", "issue", "tracked_time.issue_id=issue.id"). - Where(builder.IsNull{"issue.id"})). - Delete(models.TrackedTime{}); err != nil { + if err = models.DeleteCorruptObject("tracked_time", "issue", "tracked_time.issue_id=issue.id"); err != nil { return nil, err } results = append(results, fmt.Sprintf("%d tracked times without existing issue deleted", count)) @@ -586,8 +569,7 @@ func runDoctorCheckDBConsistency(ctx *cli.Context) ([]string, error) { } } - if ctx.Bool("fix") { - return results, committer.Commit() - } + //ToDo: function to recalc all counters + return results, nil } diff --git a/models/consistency.go b/models/consistency.go index 62d1d2e874a6..034748359fb5 100644 --- a/models/consistency.go +++ b/models/consistency.go @@ -10,6 +10,7 @@ import ( "testing" "github.com/stretchr/testify/assert" + "xorm.io/builder" ) // consistencyCheckable a type that can be tested for database consistency @@ -167,3 +168,119 @@ func (action *Action) checkForConsistency(t *testing.T) { repo := AssertExistsAndLoadBean(t, &Repository{ID: action.RepoID}).(*Repository) assert.Equal(t, repo.IsPrivate, action.IsPrivate, "action: %+v", action) } + +// CountCorruptLabels return count of labels witch are broken and not accessible via ui anymore +func CountCorruptLabels() (int64, error) { + noref, err := x.Table("label").Where("repo_id=? AND org_id=?", 0, 0).Count("label.id") + if err != nil { + return 0, err + } + norepo, err := x.Table("label"). + Join("LEFT", "repository", "label.repo_id=repository.id"). + Where("repository.id is NULL AND label.repo_id > 0"). + Count("id") + if err != nil { + return 0, err + } + noorg, err := x.Table("label"). + Join("LEFT", "repository", "label.repo_id=user.id"). + Where("user.id is NULL AND label.org_id > 0"). + Count("id") + if err != nil { + return 0, err + } + if err != nil { + return 0, err + } + + return noref + norepo + noorg, nil +} + +// DeleteCorruptLabels delete labels witch are broken and not accessible via ui anymore +func DeleteCorruptLabels() error { + // delete labels with no reference + if _, err := x.Table("label").Where("repo_id=? AND org_id=?", 0, 0).Delete(new(Label)); err != nil { + return err + } + + // delete labels with none existing repos + if _, err := x.In("id", builder.Select("label.id"). + From("label"). + Join("LEFT", "repository", "label.repo_id=repository.id"). + Where(builder.IsNull{"repository.id"}).And(builder.Gt{"label.repo_id": 0})). + Delete(Label{}); err != nil { + return err + } + + // delete labels with none existing orgs + if _, err := x.In("id", builder.Select("label.id"). + From("label"). + Join("LEFT", "user", "label.org_id=user.id"). + Where(builder.IsNull{"user.id"}).And(builder.Gt{"label.org_id": 0})). + Delete(Label{}); err != nil { + return err + } + + return nil +} + +// CountCorruptIssues count issues without a repo +func CountCorruptIssues() (int64, error) { + return x.Table("issue"). + Join("LEFT", "repository", "issue.repo_id=repository.id"). + Where("repository.id is NULL"). + Count("id") +} + +// DeleteCorruptIssues delete issues without a repo +func DeleteCorruptIssues() error { + sess := x.NewSession() + defer sess.Close() + if err := sess.Begin(); err != nil { + return err + } + + var ids []int64 + var err error + + if err = sess.Table("issue").Select("issue.id"). + Join("LEFT", "repository", "issue.repo_id=repository.id"). + Where("repository.id is NULL"). + Find(&ids); err != nil { + return err + } + + deleteCond := builder.Select("id").From("issue").Where(builder.In("id", ids)) + var attachmentPaths []string + if attachmentPaths, err = deleteIssuesByBuilder(sess, deleteCond); err != nil { + return err + } + + if err = sess.Commit(); err != nil { + return err + } + + // Remove issue attachment files. + for i := range attachmentPaths { + removeAllWithNotice(x, "Delete issue attachment", attachmentPaths[i]) + } + return nil +} + +// CountCorruptObject count subjects with have no existing refobject anymore +func CountCorruptObject(subject, refobject, joinCond string) (int64, error) { + return x.Table(subject). + Join("LEFT", refobject, joinCond). + Where(refobject + ".id is NULL"). + Count("id") +} + +// DeleteCorruptObject delete subjects with have no existing refobject anymore +func DeleteCorruptObject(subject, refobject, joinCond string) error { + _, err := x.In("id", builder.Select(subject+".id"). + From(subject). + Join("LEFT", refobject, joinCond). + Where(builder.IsNull{refobject + ".id"})). + Delete(subject) + return err +} From f738ee91548b10ea795e78ea15609566c1568b49 Mon Sep 17 00:00:00 2001 From: 6543 <6543@obermui.de> Date: Wed, 6 May 2020 10:45:58 +0200 Subject: [PATCH 09/27] test ... --- models/consistency.go | 4 ++++ models/issue.go | 31 ------------------------------- models/repo.go | 4 ++++ 3 files changed, 8 insertions(+), 31 deletions(-) diff --git a/models/consistency.go b/models/consistency.go index 034748359fb5..8e44cec6ef3f 100644 --- a/models/consistency.go +++ b/models/consistency.go @@ -256,6 +256,10 @@ func DeleteCorruptIssues() error { return err } + if _, err = sess.In("id", ids).Delete(&Issue{}); err != nil { + return err + } + if err = sess.Commit(); err != nil { return err } diff --git a/models/issue.go b/models/issue.go index 9071fa6d29fb..8874875bae7c 100644 --- a/models/issue.go +++ b/models/issue.go @@ -1882,33 +1882,6 @@ func UpdateReactionsMigrationsByType(gitServiceType structs.GitServiceType, orig return err } -// DeleteIssuesByIDs and Objects who depend on them -func DeleteIssuesByIDs(ids []int64) (err error) { - - sess := x.NewSession() - defer sess.Close() - if err = sess.Begin(); err != nil { - return err - } - - deleteCond := builder.Select("id").From("issue").Where(builder.In("id", ids)) - var attachmentPaths []string - if attachmentPaths, err = deleteIssuesByBuilder(sess, deleteCond); err != nil { - return err - } - - if err = sess.Commit(); err != nil { - return err - } - - // Remove issue attachment files. - for i := range attachmentPaths { - removeAllWithNotice(x, "Delete issue attachment", attachmentPaths[i]) - } - - return nil -} - func deleteIssuesByBuilder(sess Engine, issueIDBuilder *builder.Builder) (attachmentPaths []string, err error) { // Delete comments and attachments @@ -1968,9 +1941,5 @@ func deleteIssuesByBuilder(sess Engine, issueIDBuilder *builder.Builder) (attach return } - if _, err = sess.In("id", issueIDBuilder).Delete(&Issue{}); err != nil { - return - } - return } diff --git a/models/repo.go b/models/repo.go index 7d429759adbd..2d3820f2eb63 100644 --- a/models/repo.go +++ b/models/repo.go @@ -1582,6 +1582,10 @@ func DeleteRepository(doer *User, uid, repoID int64) error { return err } + if _, err = sess.Delete(&Issue{RepoID: repoID}); err != nil { + return err + } + if _, err = sess.Where("repo_id = ?", repoID).Delete(new(RepoUnit)); err != nil { return err } From f6f44e82c1571651f5810667d0c8d2a43de36acf Mon Sep 17 00:00:00 2001 From: 6543 <6543@obermui.de> Date: Wed, 6 May 2020 15:29:12 +0200 Subject: [PATCH 10/27] restructure things ... --- cmd/doctor.go | 4 ++-- models/consistency.go | 38 +++++++++++++++++--------------------- models/issue.go | 27 ++++++++++++++++----------- models/repo.go | 8 +------- 4 files changed, 36 insertions(+), 41 deletions(-) diff --git a/cmd/doctor.go b/cmd/doctor.go index c2e9f0011276..83bf473d83c6 100644 --- a/cmd/doctor.go +++ b/cmd/doctor.go @@ -515,9 +515,9 @@ func runDoctorCheckDBConsistency(ctx *cli.Context) ([]string, error) { if err = models.DeleteCorruptLabels(); err != nil { return nil, err } - results = append(results, fmt.Sprintf("%d issues without existing repository deleted", count)) + results = append(results, fmt.Sprintf("%d labels without existing repository/organisation deleted", count)) } else { - results = append(results, fmt.Sprintf("%d issues without existing repository", count)) + results = append(results, fmt.Sprintf("%d labels without existing repository/organisation", count)) } } diff --git a/models/consistency.go b/models/consistency.go index 8e44cec6ef3f..c424ff4d080d 100644 --- a/models/consistency.go +++ b/models/consistency.go @@ -177,14 +177,14 @@ func CountCorruptLabels() (int64, error) { } norepo, err := x.Table("label"). Join("LEFT", "repository", "label.repo_id=repository.id"). - Where("repository.id is NULL AND label.repo_id > 0"). + Where(builder.IsNull{"repository.id"}).And(builder.Gt{"label.repo_id": 0}). Count("id") if err != nil { return 0, err } noorg, err := x.Table("label"). - Join("LEFT", "repository", "label.repo_id=user.id"). - Where("user.id is NULL AND label.org_id > 0"). + Join("LEFT", "`user`", "label.org_id=`user`.id"). + Where(builder.IsNull{"`user`.id"}).And(builder.Gt{"label.org_id": 0}). Count("id") if err != nil { return 0, err @@ -204,8 +204,7 @@ func DeleteCorruptLabels() error { } // delete labels with none existing repos - if _, err := x.In("id", builder.Select("label.id"). - From("label"). + if _, err := x.In("id", builder.Select("label.id").From("label"). Join("LEFT", "repository", "label.repo_id=repository.id"). Where(builder.IsNull{"repository.id"}).And(builder.Gt{"label.repo_id": 0})). Delete(Label{}); err != nil { @@ -213,10 +212,9 @@ func DeleteCorruptLabels() error { } // delete labels with none existing orgs - if _, err := x.In("id", builder.Select("label.id"). - From("label"). - Join("LEFT", "user", "label.org_id=user.id"). - Where(builder.IsNull{"user.id"}).And(builder.Gt{"label.org_id": 0})). + if _, err := x.In("id", builder.Select("label.id").From("label"). + Join("LEFT", "`user`", "label.org_id=`user`.id"). + Where(builder.IsNull{"`user`.id"}).And(builder.Gt{"label.org_id": 0})). Delete(Label{}); err != nil { return err } @@ -228,7 +226,7 @@ func DeleteCorruptLabels() error { func CountCorruptIssues() (int64, error) { return x.Table("issue"). Join("LEFT", "repository", "issue.repo_id=repository.id"). - Where("repository.id is NULL"). + Where(builder.IsNull{"repository.id"}). Count("id") } @@ -241,26 +239,24 @@ func DeleteCorruptIssues() error { } var ids []int64 - var err error - if err = sess.Table("issue").Select("issue.id"). + if err := sess.Table("issue").Select("issue.repo_id"). Join("LEFT", "repository", "issue.repo_id=repository.id"). - Where("repository.id is NULL"). + Where("repository.id is NULL").GroupBy("issue.repo_id"). Find(&ids); err != nil { return err } - deleteCond := builder.Select("id").From("issue").Where(builder.In("id", ids)) var attachmentPaths []string - if attachmentPaths, err = deleteIssuesByBuilder(sess, deleteCond); err != nil { - return err - } - - if _, err = sess.In("id", ids).Delete(&Issue{}); err != nil { - return err + for i := range ids { + paths, err := deleteIssuesByRepoID(sess, ids[i]) + if err != nil { + return err + } + attachmentPaths = append(attachmentPaths, paths...) } - if err = sess.Commit(); err != nil { + if err := sess.Commit(); err != nil { return err } diff --git a/models/issue.go b/models/issue.go index 8874875bae7c..e3e635b4e90b 100644 --- a/models/issue.go +++ b/models/issue.go @@ -1882,53 +1882,54 @@ func UpdateReactionsMigrationsByType(gitServiceType structs.GitServiceType, orig return err } -func deleteIssuesByBuilder(sess Engine, issueIDBuilder *builder.Builder) (attachmentPaths []string, err error) { +func deleteIssuesByRepoID(sess Engine, repoID int64) (attachmentPaths []string, err error) { + deleteCond := builder.Select("id").From("issue").Where(builder.Eq{"issue.repo_id": repoID}) // Delete comments and attachments - if _, err = sess.In("issue_id", issueIDBuilder). + if _, err = sess.In("issue_id", deleteCond). Delete(&Comment{}); err != nil { return } // Dependencies for issues in this repository - if _, err = sess.In("issue_id", issueIDBuilder). + if _, err = sess.In("issue_id", deleteCond). Delete(&IssueDependency{}); err != nil { return } // Delete dependencies for issues in other repositories - if _, err = sess.In("dependency_id", issueIDBuilder). + if _, err = sess.In("dependency_id", deleteCond). Delete(&IssueDependency{}); err != nil { return } - if _, err = sess.In("issue_id", issueIDBuilder). + if _, err = sess.In("issue_id", deleteCond). Delete(&IssueUser{}); err != nil { return } - if _, err = sess.In("issue_id", issueIDBuilder). + if _, err = sess.In("issue_id", deleteCond). Delete(&Reaction{}); err != nil { return } - if _, err = sess.In("issue_id", issueIDBuilder). + if _, err = sess.In("issue_id", deleteCond). Delete(&IssueWatch{}); err != nil { return } - if _, err = sess.In("issue_id", issueIDBuilder). + if _, err = sess.In("issue_id", deleteCond). Delete(&Stopwatch{}); err != nil { return } - if _, err = sess.In("issue_id", issueIDBuilder). + if _, err = sess.In("issue_id", deleteCond). Delete(&TrackedTime{}); err != nil { return } var attachments []*Attachment - if err = sess.In("issue_id", issueIDBuilder). + if err = sess.In("issue_id", deleteCond). Find(&attachments); err != nil { return } @@ -1936,10 +1937,14 @@ func deleteIssuesByBuilder(sess Engine, issueIDBuilder *builder.Builder) (attach attachmentPaths = append(attachmentPaths, attachments[j].LocalPath()) } - if _, err = sess.In("issue_id", issueIDBuilder). + if _, err = sess.In("issue_id", deleteCond). Delete(&Attachment{}); err != nil { return } + if _, err = sess.Delete(&Issue{RepoID: repoID}); err != nil { + return + } + return } diff --git a/models/repo.go b/models/repo.go index 2d3820f2eb63..ee01ef38e08f 100644 --- a/models/repo.go +++ b/models/repo.go @@ -35,7 +35,6 @@ import ( "code.gitea.io/gitea/modules/util" "github.com/unknwon/com" - "xorm.io/builder" ) var ( @@ -1576,13 +1575,8 @@ func DeleteRepository(doer *User, uid, repoID int64) error { } // Delete Issues and related objects - deleteCond := builder.Select("id").From("issue").Where(builder.Eq{"repo_id": repoID}) var attachmentPaths []string - if attachmentPaths, err = deleteIssuesByBuilder(sess, deleteCond); err != nil { - return err - } - - if _, err = sess.Delete(&Issue{RepoID: repoID}); err != nil { + if attachmentPaths, err = deleteIssuesByRepoID(sess, repoID); err != nil { return err } From 44ec589b2315f79c3aee29c5a4f07ce06f22703f Mon Sep 17 00:00:00 2001 From: 6543 <6543@obermui.de> Date: Wed, 6 May 2020 15:40:37 +0200 Subject: [PATCH 11/27] delete cracy thing --- models/consistency.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/models/consistency.go b/models/consistency.go index c424ff4d080d..8a1d01d395bd 100644 --- a/models/consistency.go +++ b/models/consistency.go @@ -175,6 +175,7 @@ func CountCorruptLabels() (int64, error) { if err != nil { return 0, err } + norepo, err := x.Table("label"). Join("LEFT", "repository", "label.repo_id=repository.id"). Where(builder.IsNull{"repository.id"}).And(builder.Gt{"label.repo_id": 0}). @@ -182,6 +183,7 @@ func CountCorruptLabels() (int64, error) { if err != nil { return 0, err } + noorg, err := x.Table("label"). Join("LEFT", "`user`", "label.org_id=`user`.id"). Where(builder.IsNull{"`user`.id"}).And(builder.Gt{"label.org_id": 0}). @@ -189,9 +191,6 @@ func CountCorruptLabels() (int64, error) { if err != nil { return 0, err } - if err != nil { - return 0, err - } return noref + norepo + noorg, nil } From 51503eb104f6eca1030f77c99a4d45cd5d882f32 Mon Sep 17 00:00:00 2001 From: 6543 <6543@obermui.de> Date: Sat, 9 May 2020 09:44:29 +0200 Subject: [PATCH 12/27] as per @guillep2k --- models/consistency.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/models/consistency.go b/models/consistency.go index 8a1d01d395bd..dd6457145eea 100644 --- a/models/consistency.go +++ b/models/consistency.go @@ -268,18 +268,18 @@ func DeleteCorruptIssues() error { // CountCorruptObject count subjects with have no existing refobject anymore func CountCorruptObject(subject, refobject, joinCond string) (int64, error) { - return x.Table(subject). + return x.Table("`"+subject+"`"). Join("LEFT", refobject, joinCond). - Where(refobject + ".id is NULL"). + Where("`" + refobject + "`.id is NULL"). Count("id") } // DeleteCorruptObject delete subjects with have no existing refobject anymore func DeleteCorruptObject(subject, refobject, joinCond string) error { - _, err := x.In("id", builder.Select(subject+".id"). - From(subject). - Join("LEFT", refobject, joinCond). - Where(builder.IsNull{refobject + ".id"})). - Delete(subject) + _, err := x.In("id", builder.Select("`"+subject+"`.id"). + From("`"+subject+"`"). + Join("LEFT", "`"+refobject+"`", joinCond). + Where(builder.IsNull{"`" + refobject + "`.id"})). + Delete("`" + subject + "`") return err } From a0f626e960ff8fb1cd3534110cf38663ab4ce958 Mon Sep 17 00:00:00 2001 From: 6543 <6543@obermui.de> Date: Sat, 16 May 2020 18:59:29 +0200 Subject: [PATCH 13/27] make sure DB version is uptodate --- cmd/doctor.go | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/cmd/doctor.go b/cmd/doctor.go index 83bf473d83c6..bc7a59b2f1fe 100644 --- a/cmd/doctor.go +++ b/cmd/doctor.go @@ -504,6 +504,13 @@ func runDoctorScriptType(ctx *cli.Context) ([]string, error) { func runDoctorCheckDBConsistency(ctx *cli.Context) ([]string, error) { var results []string + var upToDate = true + + // make sure DB version is uptodate + if err := models.NewEngine(context.Background(), migrations.EnsureUpToDate); err != nil { + upToDate = false + results = append(results, err.Error()) + } //find labels without existing repo or org count, err := models.CountCorruptLabels() @@ -511,7 +518,7 @@ func runDoctorCheckDBConsistency(ctx *cli.Context) ([]string, error) { return nil, err } if count > 0 { - if ctx.Bool("fix") { + if upToDate && ctx.Bool("fix") { if err = models.DeleteCorruptLabels(); err != nil { return nil, err } @@ -527,7 +534,7 @@ func runDoctorCheckDBConsistency(ctx *cli.Context) ([]string, error) { return nil, err } if count > 0 { - if ctx.Bool("fix") { + if upToDate && ctx.Bool("fix") { if err = models.DeleteCorruptIssues(); err != nil { return nil, err } @@ -543,7 +550,7 @@ func runDoctorCheckDBConsistency(ctx *cli.Context) ([]string, error) { return nil, err } if count > 0 { - if ctx.Bool("fix") { + if upToDate && ctx.Bool("fix") { if err = models.DeleteCorruptObject("pull_request", "issue", "pull_request.issue_id=issue.id"); err != nil { return nil, err } @@ -559,7 +566,7 @@ func runDoctorCheckDBConsistency(ctx *cli.Context) ([]string, error) { return nil, err } if count > 0 { - if ctx.Bool("fix") { + if upToDate && ctx.Bool("fix") { if err = models.DeleteCorruptObject("tracked_time", "issue", "tracked_time.issue_id=issue.id"); err != nil { return nil, err } From 0fffab563082f1bae77aafcf113d3ff5472a329e Mon Sep 17 00:00:00 2001 From: 6543 <6543@obermui.de> Date: Thu, 21 May 2020 14:54:43 +0200 Subject: [PATCH 14/27] move DB tasks next to each other --- cmd/doctor.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/cmd/doctor.go b/cmd/doctor.go index bc7a59b2f1fe..9ed350835f8a 100644 --- a/cmd/doctor.go +++ b/cmd/doctor.go @@ -90,6 +90,12 @@ var checklist = []check{ f: runDoctorCheckDBVersion, abortIfFailed: true, }, + { + title: "Check consistency of database", + name: "check-db-consistency", + isDefault: false, + f: runDoctorCheckDBConsistency, + }, { title: "Check if OpenSSH authorized_keys file is up-to-date", name: "authorized_keys", @@ -114,12 +120,6 @@ var checklist = []check{ isDefault: false, f: runDoctorPRMergeBase, }, - { - title: "Check consistency of database", - name: "check-db-consistency", - isDefault: false, - f: runDoctorCheckDBConsistency, - }, // more checks please append here } From 90731303d49474507c1ac32441d7199186891e73 Mon Sep 17 00:00:00 2001 From: 6543 <6543@obermui.de> Date: Thu, 21 May 2020 14:59:28 +0200 Subject: [PATCH 15/27] return error if try to fix an outdated DB and do nothing --- cmd/doctor.go | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/cmd/doctor.go b/cmd/doctor.go index 9ed350835f8a..7f26a989c65b 100644 --- a/cmd/doctor.go +++ b/cmd/doctor.go @@ -510,6 +510,10 @@ func runDoctorCheckDBConsistency(ctx *cli.Context) ([]string, error) { if err := models.NewEngine(context.Background(), migrations.EnsureUpToDate); err != nil { upToDate = false results = append(results, err.Error()) + if ctx.Bool("fix") { + results = append(results, "DB Version does not match the current one, cant fix Consistency") + return results, nil + } } //find labels without existing repo or org @@ -534,7 +538,7 @@ func runDoctorCheckDBConsistency(ctx *cli.Context) ([]string, error) { return nil, err } if count > 0 { - if upToDate && ctx.Bool("fix") { + if ctx.Bool("fix") { if err = models.DeleteCorruptIssues(); err != nil { return nil, err } @@ -550,7 +554,7 @@ func runDoctorCheckDBConsistency(ctx *cli.Context) ([]string, error) { return nil, err } if count > 0 { - if upToDate && ctx.Bool("fix") { + if ctx.Bool("fix") { if err = models.DeleteCorruptObject("pull_request", "issue", "pull_request.issue_id=issue.id"); err != nil { return nil, err } @@ -566,7 +570,7 @@ func runDoctorCheckDBConsistency(ctx *cli.Context) ([]string, error) { return nil, err } if count > 0 { - if upToDate && ctx.Bool("fix") { + if ctx.Bool("fix") { if err = models.DeleteCorruptObject("tracked_time", "issue", "tracked_time.issue_id=issue.id"); err != nil { return nil, err } From 2b4bd51c68e8083898c7b99a80ed3aa882bda9ef Mon Sep 17 00:00:00 2001 From: 6543 <6543@obermui.de> Date: Thu, 21 May 2020 15:02:58 +0200 Subject: [PATCH 16/27] renamed as per @guillep2k --- cmd/doctor.go | 16 ++++++++-------- models/consistency.go | 24 ++++++++++++------------ 2 files changed, 20 insertions(+), 20 deletions(-) diff --git a/cmd/doctor.go b/cmd/doctor.go index 7f26a989c65b..81d1a2f2429c 100644 --- a/cmd/doctor.go +++ b/cmd/doctor.go @@ -517,13 +517,13 @@ func runDoctorCheckDBConsistency(ctx *cli.Context) ([]string, error) { } //find labels without existing repo or org - count, err := models.CountCorruptLabels() + count, err := models.CountOrphanedLabels() if err != nil { return nil, err } if count > 0 { if upToDate && ctx.Bool("fix") { - if err = models.DeleteCorruptLabels(); err != nil { + if err = models.DeleteOrphanedLabels(); err != nil { return nil, err } results = append(results, fmt.Sprintf("%d labels without existing repository/organisation deleted", count)) @@ -533,13 +533,13 @@ func runDoctorCheckDBConsistency(ctx *cli.Context) ([]string, error) { } //find issues without existing repository - count, err = models.CountCorruptIssues() + count, err = models.CountOrphanedIssues() if err != nil { return nil, err } if count > 0 { if ctx.Bool("fix") { - if err = models.DeleteCorruptIssues(); err != nil { + if err = models.DeleteOrphanedIssues(); err != nil { return nil, err } results = append(results, fmt.Sprintf("%d issues without existing repository deleted", count)) @@ -549,13 +549,13 @@ func runDoctorCheckDBConsistency(ctx *cli.Context) ([]string, error) { } //find pulls without existing issues - count, err = models.CountCorruptObject("pull_request", "issue", "pull_request.issue_id=issue.id") + count, err = models.CountOrphanedObjects("pull_request", "issue", "pull_request.issue_id=issue.id") if err != nil { return nil, err } if count > 0 { if ctx.Bool("fix") { - if err = models.DeleteCorruptObject("pull_request", "issue", "pull_request.issue_id=issue.id"); err != nil { + if err = models.DeleteOrphanedObjects("pull_request", "issue", "pull_request.issue_id=issue.id"); err != nil { return nil, err } results = append(results, fmt.Sprintf("%d pull requests without existing issue deleted", count)) @@ -565,13 +565,13 @@ func runDoctorCheckDBConsistency(ctx *cli.Context) ([]string, error) { } //find tracked times without existing issues/pulls - count, err = models.CountCorruptObject("tracked_time", "issue", "tracked_time.issue_id=issue.id") + count, err = models.CountOrphanedObjects("tracked_time", "issue", "tracked_time.issue_id=issue.id") if err != nil { return nil, err } if count > 0 { if ctx.Bool("fix") { - if err = models.DeleteCorruptObject("tracked_time", "issue", "tracked_time.issue_id=issue.id"); err != nil { + if err = models.DeleteOrphanedObjects("tracked_time", "issue", "tracked_time.issue_id=issue.id"); err != nil { return nil, err } results = append(results, fmt.Sprintf("%d tracked times without existing issue deleted", count)) diff --git a/models/consistency.go b/models/consistency.go index dd6457145eea..8a2f7313b65f 100644 --- a/models/consistency.go +++ b/models/consistency.go @@ -169,8 +169,8 @@ func (action *Action) checkForConsistency(t *testing.T) { assert.Equal(t, repo.IsPrivate, action.IsPrivate, "action: %+v", action) } -// CountCorruptLabels return count of labels witch are broken and not accessible via ui anymore -func CountCorruptLabels() (int64, error) { +// CountOrphanedLabels return count of labels witch are broken and not accessible via ui anymore +func CountOrphanedLabels() (int64, error) { noref, err := x.Table("label").Where("repo_id=? AND org_id=?", 0, 0).Count("label.id") if err != nil { return 0, err @@ -195,8 +195,8 @@ func CountCorruptLabels() (int64, error) { return noref + norepo + noorg, nil } -// DeleteCorruptLabels delete labels witch are broken and not accessible via ui anymore -func DeleteCorruptLabels() error { +// DeleteOrphanedLabels delete labels witch are broken and not accessible via ui anymore +func DeleteOrphanedLabels() error { // delete labels with no reference if _, err := x.Table("label").Where("repo_id=? AND org_id=?", 0, 0).Delete(new(Label)); err != nil { return err @@ -221,16 +221,16 @@ func DeleteCorruptLabels() error { return nil } -// CountCorruptIssues count issues without a repo -func CountCorruptIssues() (int64, error) { +// CountOrphanedIssues count issues without a repo +func CountOrphanedIssues() (int64, error) { return x.Table("issue"). Join("LEFT", "repository", "issue.repo_id=repository.id"). Where(builder.IsNull{"repository.id"}). Count("id") } -// DeleteCorruptIssues delete issues without a repo -func DeleteCorruptIssues() error { +// DeleteOrphanedIssues delete issues without a repo +func DeleteOrphanedIssues() error { sess := x.NewSession() defer sess.Close() if err := sess.Begin(); err != nil { @@ -266,16 +266,16 @@ func DeleteCorruptIssues() error { return nil } -// CountCorruptObject count subjects with have no existing refobject anymore -func CountCorruptObject(subject, refobject, joinCond string) (int64, error) { +// CountOrphanedObjects count subjects with have no existing refobject anymore +func CountOrphanedObjects(subject, refobject, joinCond string) (int64, error) { return x.Table("`"+subject+"`"). Join("LEFT", refobject, joinCond). Where("`" + refobject + "`.id is NULL"). Count("id") } -// DeleteCorruptObject delete subjects with have no existing refobject anymore -func DeleteCorruptObject(subject, refobject, joinCond string) error { +// DeleteOrphanedObjects delete subjects with have no existing refobject anymore +func DeleteOrphanedObjects(subject, refobject, joinCond string) error { _, err := x.In("id", builder.Select("`"+subject+"`.id"). From("`"+subject+"`"). Join("LEFT", "`"+refobject+"`", joinCond). From 3fe8d16b02d6e12fccf9c9e0d0b9ce9647d87862 Mon Sep 17 00:00:00 2001 From: 6543 <6543@obermui.de> Date: Thu, 21 May 2020 15:06:03 +0200 Subject: [PATCH 17/27] use Distinct --- models/consistency.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/consistency.go b/models/consistency.go index 8a2f7313b65f..223da2adc618 100644 --- a/models/consistency.go +++ b/models/consistency.go @@ -239,7 +239,7 @@ func DeleteOrphanedIssues() error { var ids []int64 - if err := sess.Table("issue").Select("issue.repo_id"). + if err := sess.Table("issue").Distinct("issue.repo_id"). Join("LEFT", "repository", "issue.repo_id=repository.id"). Where("repository.id is NULL").GroupBy("issue.repo_id"). Find(&ids); err != nil { From 50808fa2035bd5e7dc12a07bb8438f985ac5aac0 Mon Sep 17 00:00:00 2001 From: 6543 <6543@obermui.de> Date: Thu, 21 May 2020 15:26:54 +0200 Subject: [PATCH 18/27] Fix: converting NULL to int64 is unsupported --- models/consistency.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/models/consistency.go b/models/consistency.go index 223da2adc618..d6a158840b24 100644 --- a/models/consistency.go +++ b/models/consistency.go @@ -241,7 +241,7 @@ func DeleteOrphanedIssues() error { if err := sess.Table("issue").Distinct("issue.repo_id"). Join("LEFT", "repository", "issue.repo_id=repository.id"). - Where("repository.id is NULL").GroupBy("issue.repo_id"). + Where(builder.IsNull{"repository.id"}).GroupBy("issue.repo_id"). Find(&ids); err != nil { return err } @@ -270,7 +270,7 @@ func DeleteOrphanedIssues() error { func CountOrphanedObjects(subject, refobject, joinCond string) (int64, error) { return x.Table("`"+subject+"`"). Join("LEFT", refobject, joinCond). - Where("`" + refobject + "`.id is NULL"). + Where(builder.IsNull{"`" + refobject + "`.id"}). Count("id") } From 2e8d900926778980ec1edee17e1ebc5df05745c7 Mon Sep 17 00:00:00 2001 From: 6543 <6543@obermui.de> Date: Thu, 21 May 2020 15:35:30 +0200 Subject: [PATCH 19/27] refactor --- cmd/doctor.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/cmd/doctor.go b/cmd/doctor.go index 81d1a2f2429c..f5610a4f2e74 100644 --- a/cmd/doctor.go +++ b/cmd/doctor.go @@ -504,11 +504,9 @@ func runDoctorScriptType(ctx *cli.Context) ([]string, error) { func runDoctorCheckDBConsistency(ctx *cli.Context) ([]string, error) { var results []string - var upToDate = true // make sure DB version is uptodate if err := models.NewEngine(context.Background(), migrations.EnsureUpToDate); err != nil { - upToDate = false results = append(results, err.Error()) if ctx.Bool("fix") { results = append(results, "DB Version does not match the current one, cant fix Consistency") @@ -522,7 +520,7 @@ func runDoctorCheckDBConsistency(ctx *cli.Context) ([]string, error) { return nil, err } if count > 0 { - if upToDate && ctx.Bool("fix") { + if ctx.Bool("fix") { if err = models.DeleteOrphanedLabels(); err != nil { return nil, err } From 9a324b5f023e050294a914b8ddfecc82e8b77988 Mon Sep 17 00:00:00 2001 From: 6543 <6543@obermui.de> Date: Fri, 22 May 2020 00:24:08 +0200 Subject: [PATCH 20/27] outdatedDB message Update Co-authored-by: guillep2k <18600385+guillep2k@users.noreply.github.com> --- cmd/doctor.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/doctor.go b/cmd/doctor.go index f5610a4f2e74..dab4ca6c45d3 100644 --- a/cmd/doctor.go +++ b/cmd/doctor.go @@ -509,7 +509,7 @@ func runDoctorCheckDBConsistency(ctx *cli.Context) ([]string, error) { if err := models.NewEngine(context.Background(), migrations.EnsureUpToDate); err != nil { results = append(results, err.Error()) if ctx.Bool("fix") { - results = append(results, "DB Version does not match the current one, cant fix Consistency") + results = append(results, "Warning: model version on the database does not match the current Gitea version. Model consistency can be checked but not fixed until the database is upgraded.") return results, nil } } From b862ee8c6b01495f69cdd993cef0e818ddc56005 Mon Sep 17 00:00:00 2001 From: 6543 <6543@obermui.de> Date: Fri, 22 May 2020 00:26:35 +0200 Subject: [PATCH 21/27] remove redundant error & always warn on old version --- cmd/doctor.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/cmd/doctor.go b/cmd/doctor.go index dab4ca6c45d3..a281ae6043bf 100644 --- a/cmd/doctor.go +++ b/cmd/doctor.go @@ -507,9 +507,8 @@ func runDoctorCheckDBConsistency(ctx *cli.Context) ([]string, error) { // make sure DB version is uptodate if err := models.NewEngine(context.Background(), migrations.EnsureUpToDate); err != nil { - results = append(results, err.Error()) + results = append(results, "Warning: model version on the database does not match the current Gitea version. Model consistency can be checked but not fixed until the database is upgraded.") if ctx.Bool("fix") { - results = append(results, "Warning: model version on the database does not match the current Gitea version. Model consistency can be checked but not fixed until the database is upgraded.") return results, nil } } From cba33901db881db1bada8649cd6329b29ab029b6 Mon Sep 17 00:00:00 2001 From: 6543 <6543@obermui.de> Date: Sun, 24 May 2020 01:34:39 +0200 Subject: [PATCH 22/27] whatever it takes --- cmd/doctor.go | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/cmd/doctor.go b/cmd/doctor.go index a281ae6043bf..389e753f8d48 100644 --- a/cmd/doctor.go +++ b/cmd/doctor.go @@ -504,13 +504,12 @@ func runDoctorScriptType(ctx *cli.Context) ([]string, error) { func runDoctorCheckDBConsistency(ctx *cli.Context) ([]string, error) { var results []string + var outdatedDB bool // make sure DB version is uptodate if err := models.NewEngine(context.Background(), migrations.EnsureUpToDate); err != nil { results = append(results, "Warning: model version on the database does not match the current Gitea version. Model consistency can be checked but not fixed until the database is upgraded.") - if ctx.Bool("fix") { - return results, nil - } + outdatedDB = true } //find labels without existing repo or org @@ -518,7 +517,7 @@ func runDoctorCheckDBConsistency(ctx *cli.Context) ([]string, error) { if err != nil { return nil, err } - if count > 0 { + if !outdatedDB && count > 0 { if ctx.Bool("fix") { if err = models.DeleteOrphanedLabels(); err != nil { return nil, err @@ -534,7 +533,7 @@ func runDoctorCheckDBConsistency(ctx *cli.Context) ([]string, error) { if err != nil { return nil, err } - if count > 0 { + if !outdatedDB && count > 0 { if ctx.Bool("fix") { if err = models.DeleteOrphanedIssues(); err != nil { return nil, err @@ -550,7 +549,7 @@ func runDoctorCheckDBConsistency(ctx *cli.Context) ([]string, error) { if err != nil { return nil, err } - if count > 0 { + if !outdatedDB && count > 0 { if ctx.Bool("fix") { if err = models.DeleteOrphanedObjects("pull_request", "issue", "pull_request.issue_id=issue.id"); err != nil { return nil, err @@ -566,7 +565,7 @@ func runDoctorCheckDBConsistency(ctx *cli.Context) ([]string, error) { if err != nil { return nil, err } - if count > 0 { + if !outdatedDB && count > 0 { if ctx.Bool("fix") { if err = models.DeleteOrphanedObjects("tracked_time", "issue", "tracked_time.issue_id=issue.id"); err != nil { return nil, err From a9dcb3e84907e9515f3680d7262459b67f3c5170 Mon Sep 17 00:00:00 2001 From: 6543 <6543@obermui.de> Date: Wed, 27 May 2020 11:44:23 +0200 Subject: [PATCH 23/27] only sync if needed --- cmd/doctor.go | 6 +++--- cmd/migrate.go | 2 +- contrib/pr/checkout.go | 2 +- integrations/migration-test/migration_test.go | 2 +- models/models.go | 8 +++++--- routers/init.go | 2 +- 6 files changed, 12 insertions(+), 10 deletions(-) diff --git a/cmd/doctor.go b/cmd/doctor.go index 389e753f8d48..dcf7b3776fa1 100644 --- a/cmd/doctor.go +++ b/cmd/doctor.go @@ -365,9 +365,9 @@ func runDoctorAuthorizedKeys(ctx *cli.Context) ([]string, error) { } func runDoctorCheckDBVersion(ctx *cli.Context) ([]string, error) { - if err := models.NewEngine(context.Background(), migrations.EnsureUpToDate); err != nil { + if err := models.NewEngine(context.Background(), migrations.EnsureUpToDate, false); err != nil { if ctx.Bool("fix") { - return []string{fmt.Sprintf("WARN: Got Error %v during ensure up to date", err), "Attempting to migrate to the latest DB version to fix this."}, models.NewEngine(context.Background(), migrations.Migrate) + return []string{fmt.Sprintf("WARN: Got Error %v during ensure up to date", err), "Attempting to migrate to the latest DB version to fix this."}, models.NewEngine(context.Background(), migrations.Migrate, true) } return nil, err } @@ -507,7 +507,7 @@ func runDoctorCheckDBConsistency(ctx *cli.Context) ([]string, error) { var outdatedDB bool // make sure DB version is uptodate - if err := models.NewEngine(context.Background(), migrations.EnsureUpToDate); err != nil { + if err := models.NewEngine(context.Background(), migrations.EnsureUpToDate, false); err != nil { results = append(results, "Warning: model version on the database does not match the current Gitea version. Model consistency can be checked but not fixed until the database is upgraded.") outdatedDB = true } diff --git a/cmd/migrate.go b/cmd/migrate.go index 2428925887c9..c2ed96ecc3e9 100644 --- a/cmd/migrate.go +++ b/cmd/migrate.go @@ -34,7 +34,7 @@ func runMigrate(ctx *cli.Context) error { log.Trace("Log path: %s", setting.LogRootPath) setting.InitDBConfig() - if err := models.NewEngine(context.Background(), migrations.Migrate); err != nil { + if err := models.NewEngine(context.Background(), migrations.Migrate, true); err != nil { log.Fatal("Failed to initialize ORM engine: %v", err) return err } diff --git a/contrib/pr/checkout.go b/contrib/pr/checkout.go index 5085f0a67a67..9bad5f9790db 100644 --- a/contrib/pr/checkout.go +++ b/contrib/pr/checkout.go @@ -99,7 +99,7 @@ func runPR() { var helper testfixtures.Helper = &testfixtures.SQLite{} models.NewEngine(context.Background(), func(_ *xorm.Engine) error { return nil - }) + }, true) models.HasEngine = true //x.ShowSQL(true) err = models.InitFixtures( diff --git a/integrations/migration-test/migration_test.go b/integrations/migration-test/migration_test.go index 4ee045db3834..9258a2cf7900 100644 --- a/integrations/migration-test/migration_test.go +++ b/integrations/migration-test/migration_test.go @@ -254,7 +254,7 @@ func doMigrationTest(t *testing.T, version string) { err := models.SetEngine() assert.NoError(t, err) - err = models.NewEngine(context.Background(), wrappedMigrate) + err = models.NewEngine(context.Background(), wrappedMigrate, true) assert.NoError(t, err) currentEngine.Close() } diff --git a/models/models.go b/models/models.go index c818c651007b..bc3b3f941f5c 100644 --- a/models/models.go +++ b/models/models.go @@ -182,7 +182,7 @@ func SetEngine() (err error) { } // NewEngine initializes a new xorm.Engine -func NewEngine(ctx context.Context, migrateFunc func(*xorm.Engine) error) (err error) { +func NewEngine(ctx context.Context, migrateFunc func(*xorm.Engine) error, sync bool) (err error) { if err = SetEngine(); err != nil { return err } @@ -197,8 +197,10 @@ func NewEngine(ctx context.Context, migrateFunc func(*xorm.Engine) error) (err e return fmt.Errorf("migrate: %v", err) } - if err = x.StoreEngine("InnoDB").Sync2(tables...); err != nil { - return fmt.Errorf("sync database struct error: %v", err) + if sync { + if err = x.StoreEngine("InnoDB").Sync2(tables...); err != nil { + return fmt.Errorf("sync database struct error: %v", err) + } } return nil diff --git a/routers/init.go b/routers/init.go index c891ec314962..0354e325bdaf 100644 --- a/routers/init.go +++ b/routers/init.go @@ -69,7 +69,7 @@ func initDBEngine(ctx context.Context) (err error) { default: } log.Info("ORM engine initialization attempt #%d/%d...", i+1, setting.Database.DBConnectRetries) - if err = models.NewEngine(ctx, migrations.Migrate); err == nil { + if err = models.NewEngine(ctx, migrations.Migrate, true); err == nil { break } else if i == setting.Database.DBConnectRetries-1 { return err From a76c2f2d27d6855c653e088afbc453dc3ea45eef Mon Sep 17 00:00:00 2001 From: 6543 <6543@obermui.de> Date: Thu, 28 May 2020 20:46:10 +0200 Subject: [PATCH 24/27] Revert "only sync if needed" This reverts commit a9dcb3e84907e9515f3680d7262459b67f3c5170. --- cmd/doctor.go | 6 +++--- cmd/migrate.go | 2 +- contrib/pr/checkout.go | 2 +- integrations/migration-test/migration_test.go | 2 +- models/models.go | 8 +++----- routers/init.go | 2 +- 6 files changed, 10 insertions(+), 12 deletions(-) diff --git a/cmd/doctor.go b/cmd/doctor.go index dcf7b3776fa1..389e753f8d48 100644 --- a/cmd/doctor.go +++ b/cmd/doctor.go @@ -365,9 +365,9 @@ func runDoctorAuthorizedKeys(ctx *cli.Context) ([]string, error) { } func runDoctorCheckDBVersion(ctx *cli.Context) ([]string, error) { - if err := models.NewEngine(context.Background(), migrations.EnsureUpToDate, false); err != nil { + if err := models.NewEngine(context.Background(), migrations.EnsureUpToDate); err != nil { if ctx.Bool("fix") { - return []string{fmt.Sprintf("WARN: Got Error %v during ensure up to date", err), "Attempting to migrate to the latest DB version to fix this."}, models.NewEngine(context.Background(), migrations.Migrate, true) + return []string{fmt.Sprintf("WARN: Got Error %v during ensure up to date", err), "Attempting to migrate to the latest DB version to fix this."}, models.NewEngine(context.Background(), migrations.Migrate) } return nil, err } @@ -507,7 +507,7 @@ func runDoctorCheckDBConsistency(ctx *cli.Context) ([]string, error) { var outdatedDB bool // make sure DB version is uptodate - if err := models.NewEngine(context.Background(), migrations.EnsureUpToDate, false); err != nil { + if err := models.NewEngine(context.Background(), migrations.EnsureUpToDate); err != nil { results = append(results, "Warning: model version on the database does not match the current Gitea version. Model consistency can be checked but not fixed until the database is upgraded.") outdatedDB = true } diff --git a/cmd/migrate.go b/cmd/migrate.go index c2ed96ecc3e9..2428925887c9 100644 --- a/cmd/migrate.go +++ b/cmd/migrate.go @@ -34,7 +34,7 @@ func runMigrate(ctx *cli.Context) error { log.Trace("Log path: %s", setting.LogRootPath) setting.InitDBConfig() - if err := models.NewEngine(context.Background(), migrations.Migrate, true); err != nil { + if err := models.NewEngine(context.Background(), migrations.Migrate); err != nil { log.Fatal("Failed to initialize ORM engine: %v", err) return err } diff --git a/contrib/pr/checkout.go b/contrib/pr/checkout.go index 9bad5f9790db..5085f0a67a67 100644 --- a/contrib/pr/checkout.go +++ b/contrib/pr/checkout.go @@ -99,7 +99,7 @@ func runPR() { var helper testfixtures.Helper = &testfixtures.SQLite{} models.NewEngine(context.Background(), func(_ *xorm.Engine) error { return nil - }, true) + }) models.HasEngine = true //x.ShowSQL(true) err = models.InitFixtures( diff --git a/integrations/migration-test/migration_test.go b/integrations/migration-test/migration_test.go index 9258a2cf7900..4ee045db3834 100644 --- a/integrations/migration-test/migration_test.go +++ b/integrations/migration-test/migration_test.go @@ -254,7 +254,7 @@ func doMigrationTest(t *testing.T, version string) { err := models.SetEngine() assert.NoError(t, err) - err = models.NewEngine(context.Background(), wrappedMigrate, true) + err = models.NewEngine(context.Background(), wrappedMigrate) assert.NoError(t, err) currentEngine.Close() } diff --git a/models/models.go b/models/models.go index bc3b3f941f5c..c818c651007b 100644 --- a/models/models.go +++ b/models/models.go @@ -182,7 +182,7 @@ func SetEngine() (err error) { } // NewEngine initializes a new xorm.Engine -func NewEngine(ctx context.Context, migrateFunc func(*xorm.Engine) error, sync bool) (err error) { +func NewEngine(ctx context.Context, migrateFunc func(*xorm.Engine) error) (err error) { if err = SetEngine(); err != nil { return err } @@ -197,10 +197,8 @@ func NewEngine(ctx context.Context, migrateFunc func(*xorm.Engine) error, sync b return fmt.Errorf("migrate: %v", err) } - if sync { - if err = x.StoreEngine("InnoDB").Sync2(tables...); err != nil { - return fmt.Errorf("sync database struct error: %v", err) - } + if err = x.StoreEngine("InnoDB").Sync2(tables...); err != nil { + return fmt.Errorf("sync database struct error: %v", err) } return nil diff --git a/routers/init.go b/routers/init.go index 0354e325bdaf..c891ec314962 100644 --- a/routers/init.go +++ b/routers/init.go @@ -69,7 +69,7 @@ func initDBEngine(ctx context.Context) (err error) { default: } log.Info("ORM engine initialization attempt #%d/%d...", i+1, setting.Database.DBConnectRetries) - if err = models.NewEngine(ctx, migrations.Migrate, true); err == nil { + if err = models.NewEngine(ctx, migrations.Migrate); err == nil { break } else if i == setting.Database.DBConnectRetries-1 { return err From 704e359489b1dc5356106c0a62af23ccd1e0e256 Mon Sep 17 00:00:00 2001 From: 6543 <6543@obermui.de> Date: Thu, 28 May 2020 20:49:33 +0200 Subject: [PATCH 25/27] add documentation --- models/models.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/models/models.go b/models/models.go index c818c651007b..7f12d6260ad0 100644 --- a/models/models.go +++ b/models/models.go @@ -182,6 +182,10 @@ func SetEngine() (err error) { } // NewEngine initializes a new xorm.Engine +// This function must never call .Sync2() if the provided migration function fails. +// When called from the "doctor" command, the migration function is a version check +// that prevents the doctor from fixing anything in the database if the migration level +// is different from the expected value. func NewEngine(ctx context.Context, migrateFunc func(*xorm.Engine) error) (err error) { if err = SetEngine(); err != nil { return err From 0ae896d68a5fd7bb7d595b587373476c038a9414 Mon Sep 17 00:00:00 2001 From: 6543 <6543@obermui.de> Date: Thu, 28 May 2020 23:24:41 +0200 Subject: [PATCH 26/27] its Nr 11111 --- cmd/doctor.go | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/cmd/doctor.go b/cmd/doctor.go index 389e753f8d48..d8647d8ec900 100644 --- a/cmd/doctor.go +++ b/cmd/doctor.go @@ -504,12 +504,10 @@ func runDoctorScriptType(ctx *cli.Context) ([]string, error) { func runDoctorCheckDBConsistency(ctx *cli.Context) ([]string, error) { var results []string - var outdatedDB bool // make sure DB version is uptodate if err := models.NewEngine(context.Background(), migrations.EnsureUpToDate); err != nil { - results = append(results, "Warning: model version on the database does not match the current Gitea version. Model consistency can be checked but not fixed until the database is upgraded.") - outdatedDB = true + return []string{"Warning: model version on the database does not match the current Gitea version. Model consistency can be checked but not fixed until the database is upgraded."}, err } //find labels without existing repo or org @@ -517,7 +515,7 @@ func runDoctorCheckDBConsistency(ctx *cli.Context) ([]string, error) { if err != nil { return nil, err } - if !outdatedDB && count > 0 { + if count > 0 { if ctx.Bool("fix") { if err = models.DeleteOrphanedLabels(); err != nil { return nil, err @@ -533,7 +531,7 @@ func runDoctorCheckDBConsistency(ctx *cli.Context) ([]string, error) { if err != nil { return nil, err } - if !outdatedDB && count > 0 { + if count > 0 { if ctx.Bool("fix") { if err = models.DeleteOrphanedIssues(); err != nil { return nil, err @@ -549,7 +547,7 @@ func runDoctorCheckDBConsistency(ctx *cli.Context) ([]string, error) { if err != nil { return nil, err } - if !outdatedDB && count > 0 { + if count > 0 { if ctx.Bool("fix") { if err = models.DeleteOrphanedObjects("pull_request", "issue", "pull_request.issue_id=issue.id"); err != nil { return nil, err @@ -565,7 +563,7 @@ func runDoctorCheckDBConsistency(ctx *cli.Context) ([]string, error) { if err != nil { return nil, err } - if !outdatedDB && count > 0 { + if count > 0 { if ctx.Bool("fix") { if err = models.DeleteOrphanedObjects("tracked_time", "issue", "tracked_time.issue_id=issue.id"); err != nil { return nil, err From d2e438d232db8c5b81cfd32d11133ba7ce7094ad Mon Sep 17 00:00:00 2001 From: 6543 <6543@obermui.de> Date: Thu, 28 May 2020 23:34:14 +0200 Subject: [PATCH 27/27] its an error now --- cmd/doctor.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/doctor.go b/cmd/doctor.go index d8647d8ec900..3456f26f7020 100644 --- a/cmd/doctor.go +++ b/cmd/doctor.go @@ -88,7 +88,7 @@ var checklist = []check{ name: "check-db-version", isDefault: true, f: runDoctorCheckDBVersion, - abortIfFailed: true, + abortIfFailed: false, }, { title: "Check consistency of database", @@ -507,7 +507,7 @@ func runDoctorCheckDBConsistency(ctx *cli.Context) ([]string, error) { // make sure DB version is uptodate if err := models.NewEngine(context.Background(), migrations.EnsureUpToDate); err != nil { - return []string{"Warning: model version on the database does not match the current Gitea version. Model consistency can be checked but not fixed until the database is upgraded."}, err + return nil, fmt.Errorf("model version on the database does not match the current Gitea version. Model consistency will not be checked until the database is upgraded") } //find labels without existing repo or org