From f52310a8411fcd4a8068408bd452799cccaf5248 Mon Sep 17 00:00:00 2001 From: Thomas Miceli <27960254+thomiceli@users.noreply.github.com> Date: Tue, 2 Jan 2024 04:01:45 +0100 Subject: [PATCH] Add 2 new admin actions (#191) * Synchronize all gists previews * Reset Git server hooks for all repositories --- internal/actions/actions.go | 156 +++++++++++++++++++++++++++++++ internal/db/gist.go | 11 ++- internal/git/commands.go | 10 +- internal/i18n/locales/en-US.yml | 4 +- internal/ssh/git_ssh.go | 2 +- internal/web/admin.go | 96 +++++-------------- internal/web/gist.go | 4 +- internal/web/git_http.go | 2 +- internal/web/server.go | 2 + templates/pages/admin_index.html | 12 +++ 10 files changed, 213 insertions(+), 86 deletions(-) create mode 100644 internal/actions/actions.go diff --git a/internal/actions/actions.go b/internal/actions/actions.go new file mode 100644 index 00000000..93dc28c0 --- /dev/null +++ b/internal/actions/actions.go @@ -0,0 +1,156 @@ +package actions + +import ( + "github.com/rs/zerolog/log" + "github.com/thomiceli/opengist/internal/config" + "github.com/thomiceli/opengist/internal/db" + "github.com/thomiceli/opengist/internal/git" + "os" + "path/filepath" + "strings" + "sync" +) + +type ActionStatus struct { + Running bool +} + +const ( + SyncReposFromFS = iota + SyncReposFromDB = iota + GitGcRepos = iota + SyncGistPreviews = iota + ResetHooks = iota +) + +var ( + mutex sync.Mutex + actions = make(map[int]ActionStatus) +) + +func updateActionStatus(actionType int, running bool) { + actions[actionType] = ActionStatus{ + Running: running, + } +} + +func IsRunning(actionType int) bool { + mutex.Lock() + defer mutex.Unlock() + return actions[actionType].Running +} + +func Run(actionType int) { + mutex.Lock() + + if actions[actionType].Running { + mutex.Unlock() + return + } + + updateActionStatus(actionType, true) + mutex.Unlock() + + defer func() { + mutex.Lock() + updateActionStatus(actionType, false) + mutex.Unlock() + }() + + var functionToRun func() + switch actionType { + case SyncReposFromFS: + functionToRun = syncReposFromFS + case SyncReposFromDB: + functionToRun = syncReposFromDB + case GitGcRepos: + functionToRun = gitGcRepos + case SyncGistPreviews: + functionToRun = syncGistPreviews + case ResetHooks: + functionToRun = resetHooks + default: + panic("unhandled default case") + } + + functionToRun() +} + +func syncReposFromFS() { + log.Info().Msg("Syncing repositories from filesystem...") + gists, err := db.GetAllGistsRows() + if err != nil { + log.Error().Err(err).Msg("Cannot get gists") + return + } + for _, gist := range gists { + // if repository does not exist, delete gist from database + if _, err := os.Stat(git.RepositoryPath(gist.User.Username, gist.Uuid)); err != nil && !os.IsExist(err) { + if err2 := gist.Delete(); err2 != nil { + log.Error().Err(err2).Msgf("Cannot delete gist %d", gist.ID) + return + } + } + } +} + +func syncReposFromDB() { + log.Info().Msg("Syncing repositories from database...") + entries, err := filepath.Glob(filepath.Join(config.GetHomeDir(), "repos", "*", "*")) + if err != nil { + log.Error().Err(err).Msg("Cannot read repos directories") + return + } + + for _, e := range entries { + path := strings.Split(e, string(os.PathSeparator)) + gist, _ := db.GetGist(path[len(path)-2], path[len(path)-1]) + + if gist.ID == 0 { + if err := git.DeleteRepository(path[len(path)-2], path[len(path)-1]); err != nil { + log.Error().Err(err).Msgf("Cannot delete repository %s/%s", path[len(path)-2], path[len(path)-1]) + return + } + } + } +} + +func gitGcRepos() { + log.Info().Msg("Garbage collecting all repositories...") + if err := git.GcRepos(); err != nil { + log.Error().Err(err).Msg("Error garbage collecting repositories") + } +} + +func syncGistPreviews() { + log.Info().Msg("Syncing all Gist previews...") + + gists, err := db.GetAllGistsRows() + if err != nil { + log.Error().Err(err).Msg("Cannot get gists") + return + } + for _, gist := range gists { + if err = gist.UpdatePreviewAndCount(false); err != nil { + log.Error().Err(err).Msgf("Cannot update preview and count for gist %d", gist.ID) + return + } + } +} + +func resetHooks() { + log.Info().Msg("Resetting Git server hooks for all repositories...") + entries, err := filepath.Glob(filepath.Join(config.GetHomeDir(), "repos", "*", "*")) + if err != nil { + log.Error().Err(err).Msg("Cannot read repos directories") + return + } + + for _, e := range entries { + path := strings.Split(e, string(os.PathSeparator)) + if err := git.CreateDotGitFiles(path[len(path)-2], path[len(path)-1]); err != nil { + log.Error().Err(err).Msgf("Cannot reset hooks for repository %s/%s", path[len(path)-2], path[len(path)-1]) + return + } + } +} diff --git a/internal/db/gist.go b/internal/db/gist.go index 136beee2..73a89ab7 100644 --- a/internal/db/gist.go +++ b/internal/db/gist.go @@ -226,6 +226,10 @@ func (gist *Gist) Update() error { return db.Omit("forked_id").Save(&gist).Error } +func (gist *Gist) UpdateNoTimestamps() error { + return db.Omit("forked_id", "updated_at").Save(&gist).Error +} + func (gist *Gist) Delete() error { err := gist.DeleteRepository() if err != nil { @@ -419,7 +423,7 @@ func (gist *Gist) RPC(service string) ([]byte, error) { return git.RPC(gist.User.Username, gist.Uuid, service) } -func (gist *Gist) UpdatePreviewAndCount() error { +func (gist *Gist) UpdatePreviewAndCount(withTimestampUpdate bool) error { filesStr, err := git.GetFilesOfRepository(gist.User.Username, gist.Uuid, "HEAD") if err != nil { return err @@ -445,7 +449,10 @@ func (gist *Gist) UpdatePreviewAndCount() error { gist.PreviewFilename = file.Filename } - return gist.Update() + if withTimestampUpdate { + return gist.Update() + } + return gist.UpdateNoTimestamps() } func (gist *Gist) VisibilityStr() string { diff --git a/internal/git/commands.go b/internal/git/commands.go index b176eb24..e4530c2b 100644 --- a/internal/git/commands.go +++ b/internal/git/commands.go @@ -77,7 +77,7 @@ func InitRepository(user string, gist string) error { return err } - return createDotGitFiles(repositoryPath) + return CreateDotGitFiles(user, gist) } func InitRepositoryViaInit(user string, gist string, ctx echo.Context) error { @@ -371,7 +371,7 @@ func ForkClone(userSrc string, gistSrc string, userDst string, gistDst string) e return err } - return createDotGitFiles(repositoryPathDst) + return CreateDotGitFiles(userDst, gistDst) } func SetFileContent(gistTmpId string, filename string, content string) error { @@ -525,7 +525,9 @@ func GetGitVersion() (string, error) { return versionFields[2], nil } -func createDotGitFiles(repositoryPath string) error { +func CreateDotGitFiles(user string, gist string) error { + repositoryPath := RepositoryPath(user, gist) + f1, err := os.OpenFile(filepath.Join(repositoryPath, "git-daemon-export-ok"), os.O_RDONLY|os.O_CREATE, 0644) if err != nil { return err @@ -540,7 +542,7 @@ func createDotGitFiles(repositoryPath string) error { } func createDotGitHookFile(repositoryPath string, hook string, content string) error { - preReceiveDst, err := os.OpenFile(filepath.Join(repositoryPath, "hooks", hook), os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0744) + preReceiveDst, err := os.OpenFile(filepath.Join(repositoryPath, "hooks", hook), os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0744) if err != nil { return err } diff --git a/internal/i18n/locales/en-US.yml b/internal/i18n/locales/en-US.yml index 628fd646..0c136bb8 100644 --- a/internal/i18n/locales/en-US.yml +++ b/internal/i18n/locales/en-US.yml @@ -161,7 +161,9 @@ admin.stats: Stats admin.actions: Actions admin.actions.sync-fs: Synchronize gists from filesystem admin.actions.sync-db: Synchronize gists from database -admin.actions.git-gc: Garbage collect git repositories +admin.actions.git-gc: Garbage collect all git repositories +admin.actions.sync-previews: Synchronize all gists previews +admin.actions.reset-hooks: Reset Git server hooks for all repositories admin.id: ID admin.user: User admin.delete: Delete diff --git a/internal/ssh/git_ssh.go b/internal/ssh/git_ssh.go index 7847b170..9c3f544c 100644 --- a/internal/ssh/git_ssh.go +++ b/internal/ssh/git_ssh.go @@ -94,7 +94,7 @@ func runGitCommand(ch ssh.Channel, gitCmd string, key string, ip string) error { // updatedAt is updated only if serviceType is receive-pack if verb == "receive-pack" { _ = gist.SetLastActiveNow() - _ = gist.UpdatePreviewAndCount() + _ = gist.UpdatePreviewAndCount(false) } return nil diff --git a/internal/web/admin.go b/internal/web/admin.go index 444b4a06..cfeb45dc 100644 --- a/internal/web/admin.go +++ b/internal/web/admin.go @@ -2,21 +2,12 @@ package web import ( "github.com/labstack/echo/v4" - "github.com/rs/zerolog/log" + "github.com/thomiceli/opengist/internal/actions" "github.com/thomiceli/opengist/internal/config" "github.com/thomiceli/opengist/internal/db" "github.com/thomiceli/opengist/internal/git" - "os" - "path/filepath" "runtime" "strconv" - "strings" -) - -var ( - syncReposFromFS = false - syncReposFromDB = false - gitGcRepos = false ) func adminIndex(ctx echo.Context) error { @@ -50,9 +41,11 @@ func adminIndex(ctx echo.Context) error { } setData(ctx, "countKeys", countKeys) - setData(ctx, "syncReposFromFS", syncReposFromFS) - setData(ctx, "syncReposFromDB", syncReposFromDB) - setData(ctx, "gitGcRepos", gitGcRepos) + setData(ctx, "syncReposFromFS", actions.IsRunning(actions.SyncReposFromFS)) + setData(ctx, "syncReposFromDB", actions.IsRunning(actions.SyncReposFromDB)) + setData(ctx, "gitGcRepos", actions.IsRunning(actions.GitGcRepos)) + setData(ctx, "syncGistPreviews", actions.IsRunning(actions.SyncGistPreviews)) + setData(ctx, "resetHooks", actions.IsRunning(actions.ResetHooks)) return html(ctx, "admin_index.html") } @@ -129,78 +122,31 @@ func adminGistDelete(ctx echo.Context) error { func adminSyncReposFromFS(ctx echo.Context) error { addFlash(ctx, "Syncing repositories from filesystem...", "success") - go func() { - if syncReposFromFS { - return - } - syncReposFromFS = true - - gists, err := db.GetAllGistsRows() - if err != nil { - log.Error().Err(err).Msg("Cannot get gists") - syncReposFromFS = false - return - } - for _, gist := range gists { - // if repository does not exist, delete gist from database - if _, err := os.Stat(git.RepositoryPath(gist.User.Username, gist.Uuid)); err != nil && !os.IsExist(err) { - if err2 := gist.Delete(); err2 != nil { - log.Error().Err(err2).Msg("Cannot delete gist") - syncReposFromFS = false - return - } - } - } - syncReposFromFS = false - }() + go actions.Run(actions.SyncReposFromFS) return redirect(ctx, "/admin-panel") } func adminSyncReposFromDB(ctx echo.Context) error { addFlash(ctx, "Syncing repositories from database...", "success") - go func() { - if syncReposFromDB { - return - } - syncReposFromDB = true - entries, err := filepath.Glob(filepath.Join(config.GetHomeDir(), "repos", "*", "*")) - if err != nil { - log.Error().Err(err).Msg("Cannot read repos directories") - syncReposFromDB = false - return - } - - for _, e := range entries { - path := strings.Split(e, string(os.PathSeparator)) - gist, _ := db.GetGist(path[len(path)-2], path[len(path)-1]) - - if gist.ID == 0 { - if err := git.DeleteRepository(path[len(path)-2], path[len(path)-1]); err != nil { - log.Error().Err(err).Msg("Cannot delete repository") - syncReposFromDB = false - return - } - } - } - syncReposFromDB = false - }() + go actions.Run(actions.SyncReposFromDB) return redirect(ctx, "/admin-panel") } func adminGcRepos(ctx echo.Context) error { addFlash(ctx, "Garbage collecting repositories...", "success") - go func() { - if gitGcRepos { - return - } - gitGcRepos = true - if err := git.GcRepos(); err != nil { - log.Error().Err(err).Msg("Error garbage collecting repositories") - gitGcRepos = false - return - } - gitGcRepos = false - }() + go actions.Run(actions.GitGcRepos) + return redirect(ctx, "/admin-panel") +} + +func adminSyncGistPreviews(ctx echo.Context) error { + addFlash(ctx, "Syncing Gist previews...", "success") + go actions.Run(actions.SyncGistPreviews) + return redirect(ctx, "/admin-panel") +} + +func adminResetHooks(ctx echo.Context) error { + addFlash(ctx, "Resetting Git server hooks for all repositories...", "success") + go actions.Run(actions.ResetHooks) return redirect(ctx, "/admin-panel") } diff --git a/internal/web/gist.go b/internal/web/gist.go index f53ce9eb..ec8da501 100644 --- a/internal/web/gist.go +++ b/internal/web/gist.go @@ -542,7 +542,7 @@ func toggleVisibility(ctx echo.Context) error { gist := getData(ctx, "gist").(*db.Gist) gist.Private = (gist.Private + 1) % 3 - if err := gist.Update(); err != nil { + if err := gist.UpdateNoTimestamps(); err != nil { return errorRes(500, "Error updating this gist", err) } @@ -806,7 +806,7 @@ func checkbox(ctx echo.Context) error { return errorRes(500, "Error adding and committing files", err) } - if err = gist.UpdatePreviewAndCount(); err != nil { + if err = gist.UpdatePreviewAndCount(true); err != nil { return errorRes(500, "Error updating the gist", err) } diff --git a/internal/web/git_http.go b/internal/web/git_http.go index 4d69dcae..7363dc67 100644 --- a/internal/web/git_http.go +++ b/internal/web/git_http.go @@ -217,7 +217,7 @@ func pack(ctx echo.Context, serviceType string) error { } _ = gist.SetLastActiveNow() - _ = gist.UpdatePreviewAndCount() + _ = gist.UpdatePreviewAndCount(false) } return nil } diff --git a/internal/web/server.go b/internal/web/server.go index 0fdd0e39..71af827a 100644 --- a/internal/web/server.go +++ b/internal/web/server.go @@ -235,6 +235,8 @@ func NewServer(isDev bool) *Server { g2.POST("/sync-fs", adminSyncReposFromFS) g2.POST("/sync-db", adminSyncReposFromDB) g2.POST("/gc-repos", adminGcRepos) + g2.POST("/sync-previews", adminSyncGistPreviews) + g2.POST("/reset-hooks", adminResetHooks) g2.GET("/configuration", adminConfig) g2.PUT("/set-config", adminSetConfig) } diff --git a/templates/pages/admin_index.html b/templates/pages/admin_index.html index 7c878174..806b3039 100644 --- a/templates/pages/admin_index.html +++ b/templates/pages/admin_index.html @@ -74,6 +74,18 @@ {{ .locale.Tr "admin.actions.git-gc" }} +
+