Optimize multiple file rendering (thomiceli#189)

internal/render/highlight.go

@@ -11,6 +11,7 @@ import (
  "github.com/rs/zerolog/log"
  "github.com/thomiceli/opengist/internal/db"
  "github.com/thomiceli/opengist/internal/git"
+ "sync"
 )
 
 type RenderedFile struct {
@@ -67,17 +68,36 @@ func HighlightFile(file *git.File) (RenderedFile, error) {
  return rendered, err
 }
 
-func HighlightFiles(files []*git.File) ([]RenderedFile, error) {
- renderedFiles := make([]RenderedFile, 0, len(files))
- for _, file := range files {
- rendered, err := HighlightFile(file)
- if err != nil {
- log.Warn().Err(err).Msg("Error rendering gist preview for " + file.Filename)
+func HighlightFiles(files []*git.File) []RenderedFile {
+ const numWorkers = 10
+ jobs := make(chan int, numWorkers)
+ renderedFiles := make([]RenderedFile, len(files))
+ var wg sync.WaitGroup
+
+ worker := func() {
+ for idx := range jobs {
+ rendered, err := HighlightFile(files[idx])
+ if err != nil {
+ log.Error().Err(err).Msg("Error rendering gist preview for " + files[idx].Filename)
+ }
+ renderedFiles[idx] = rendered
  }
- renderedFiles = append(renderedFiles, rendered)
+ wg.Done()
+ }
+
+ for i := 0; i < numWorkers; i++ {
+ wg.Add(1)
+ go worker()
  }
 
- return renderedFiles, nil
+ for i := range files {
+ jobs <- i
+ }
+ close(jobs)
+
+ wg.Wait()
+
+ return renderedFiles
 }
 
 func HighlightGistPreview(gist *db.Gist) (RenderedGist, error) {

internal/web/gist.go

@@ -294,10 +294,7 @@ func gistIndex(ctx echo.Context) error {
  return errorRes(500, "Error fetching files", err)
  }
 
- renderedFiles, err := render.HighlightFiles(files)
- if err != nil {
- return errorRes(500, "Error rendering files", err)
- }
+ renderedFiles := render.HighlightFiles(files)
 
  setData(ctx, "page", "code")
  setData(ctx, "commit", revision)
@@ -314,11 +311,7 @@ func gistJson(ctx echo.Context) error {
  return errorRes(500, "Error fetching files", err)
  }
 
- renderedFiles, err := render.HighlightFiles(files)
- if err != nil {
- return errorRes(500, "Error rendering files", err)
- }
-
+ renderedFiles := render.HighlightFiles(files)
  setData(ctx, "files", renderedFiles)
 
  htmlbuf := bytes.Buffer{}
@@ -362,11 +355,7 @@ func gistJs(ctx echo.Context) error {
  return errorRes(500, "Error fetching files", err)
  }
 
- renderedFiles, err := render.HighlightFiles(files)
- if err != nil {
- return errorRes(500, "Error rendering files", err)
- }
-
+ renderedFiles := render.HighlightFiles(files)
  setData(ctx, "files", renderedFiles)
 
  htmlbuf := bytes.Buffer{}