Merge remote-tracking branch 'giteaofficial/main'

* giteaofficial/main:
  [skip ci] Updated translations via Crowdin
  Properly flush unique queues on startup (go-gitea#23154)
  Use context parameter in services/repository (go-gitea#23186)
  Pass `--global` when calling `git config --get`, for consistency with `git config --set` (go-gitea#23157)
  Make `gitea serv` respect git binary home (go-gitea#23138)
  Write Gitpod `app.ini` only once (go-gitea#23192)
  Avoid too long names for actions (go-gitea#23162)

.gitpod.yml

@@ -10,9 +10,12 @@ tasks:
  - name: Run backend
  command: |
  gp sync-await setup
- mkdir -p custom/conf/
- echo -e "[server]\nROOT_URL=$(gp url 3000)/" > custom/conf/app.ini
- echo -e "\n[database]\nDB_TYPE = sqlite3\nPATH = $GITPOD_REPO_ROOT/data/gitea.db" >> custom/conf/app.ini
+ if [ ! -f custom/conf/app.ini ]
+ then
+ mkdir -p custom/conf/
+ echo -e "[server]\nROOT_URL=$(gp url 3000)/" > custom/conf/app.ini
+ echo -e "\n[database]\nDB_TYPE = sqlite3\nPATH = $GITPOD_REPO_ROOT/data/gitea.db" >> custom/conf/app.ini
+ fi
  export TAGS="sqlite sqlite_unlock_notify"
  make watch-backend
  - name: Run frontend

cmd/serv.go

@@ -11,6 +11,7 @@ import (
  "net/url"
  "os"
  "os/exec"
+ "path/filepath"
  "regexp"
  "strconv"
  "strings"
@@ -290,17 +291,21 @@ func runServ(c *cli.Context) error {
  return nil
  }
 
- // Special handle for Windows.
- if setting.IsWindows {
- verb = strings.Replace(verb, "-", " ", 1)
- }
-
  var gitcmd *exec.Cmd
- verbs := strings.Split(verb, " ")
- if len(verbs) == 2 {
- gitcmd = exec.CommandContext(ctx, verbs[0], verbs[1], repoPath)
- } else {
- gitcmd = exec.CommandContext(ctx, verb, repoPath)
+ gitBinPath := filepath.Dir(git.GitExecutable) // e.g. /usr/bin
+ gitBinVerb := filepath.Join(gitBinPath, verb) // e.g. /usr/bin/git-upload-pack
+ if _, err := os.Stat(gitBinVerb); err != nil {
+ // if the command "git-upload-pack" doesn't exist, try to split "git-upload-pack" to use the sub-command with git
+ // ps: Windows only has "git.exe" in the bin path, so Windows always uses this way
+ verbFields := strings.SplitN(verb, "-", 2)
+ if len(verbFields) == 2 {
+ // use git binary with the sub-command part: "C:\...\bin\git.exe", "upload-pack", ...
+ gitcmd = exec.CommandContext(ctx, git.GitExecutable, verbFields[1], repoPath)
+ }
+ }
+ if gitcmd == nil {
+ // by default, use the verb (it has been checked above by allowedCommands)
+ gitcmd = exec.CommandContext(ctx, gitBinVerb, repoPath)
  }
 
  process.SetSysProcAttribute(gitcmd)

models/actions/run.go

@@ -192,6 +192,7 @@ func InsertRun(ctx context.Context, run *ActionRun, jobs []*jobparser.SingleWork
  if len(needs) > 0 || run.NeedApproval {
  status = StatusBlocked
  }
+ job.Name, _ = util.SplitStringAtByteN(job.Name, 255)
  runJobs = append(runJobs, &ActionRunJob{
  RunID: run.ID,
  RepoID: run.RepoID,

models/actions/task.go

@@ -298,8 +298,9 @@ func CreateTaskForRunner(ctx context.Context, runner *ActionRunner) (*ActionTask
  if len(workflowJob.Steps) > 0 {
  steps := make([]*ActionTaskStep, len(workflowJob.Steps))
  for i, v := range workflowJob.Steps {
+ name, _ := util.SplitStringAtByteN(v.String(), 255)
  steps[i] = &ActionTaskStep{
- Name: v.String(),
+ Name: name,
  TaskID: task.ID,
  Index: int64(i),
  RepoID: task.RepoID,

modules/git/git.go

@@ -312,7 +312,7 @@ func CheckGitVersionAtLeast(atLeast string) error {
 }
 
 func configSet(key, value string) error {
- stdout, _, err := NewCommand(DefaultContext, "config", "--get").AddDynamicArguments(key).RunStdString(nil)
+ stdout, _, err := NewCommand(DefaultContext, "config", "--global", "--get").AddDynamicArguments(key).RunStdString(nil)
  if err != nil && !err.IsExitCode(1) {
  return fmt.Errorf("failed to get git config %s, err: %w", key, err)
  }
@@ -331,7 +331,7 @@ func configSet(key, value string) error {
 }
 
 func configSetNonExist(key, value string) error {
- _, _, err := NewCommand(DefaultContext, "config", "--get").AddDynamicArguments(key).RunStdString(nil)
+ _, _, err := NewCommand(DefaultContext, "config", "--global", "--get").AddDynamicArguments(key).RunStdString(nil)
  if err == nil {
  // already exist
  return nil
@@ -349,7 +349,7 @@ func configSetNonExist(key, value string) error {
 }
 
 func configAddNonExist(key, value string) error {
- _, _, err := NewCommand(DefaultContext, "config", "--get").AddDynamicArguments(key, regexp.QuoteMeta(value)).RunStdString(nil)
+ _, _, err := NewCommand(DefaultContext, "config", "--global", "--get").AddDynamicArguments(key, regexp.QuoteMeta(value)).RunStdString(nil)
  if err == nil {
  // already exist
  return nil
@@ -366,7 +366,7 @@ func configAddNonExist(key, value string) error {
 }
 
 func configUnsetAll(key, value string) error {
- _, _, err := NewCommand(DefaultContext, "config", "--get").AddDynamicArguments(key).RunStdString(nil)
+ _, _, err := NewCommand(DefaultContext, "config", "--global", "--get").AddDynamicArguments(key).RunStdString(nil)
  if err == nil {
  // exist, need to remove
  _, _, err = NewCommand(DefaultContext, "config", "--global", "--unset-all").AddDynamicArguments(key, regexp.QuoteMeta(value)).RunStdString(nil)

modules/queue/queue_channel.go

@@ -124,7 +124,10 @@ func (q *ChannelQueue) Shutdown() {
  log.Trace("ChannelQueue: %s Flushing", q.name)
  // We can't use Cleanup here because that will close the channel
  if err := q.FlushWithContext(q.terminateCtx); err != nil {
- log.Warn("ChannelQueue: %s Terminated before completed flushing", q.name)
+ count := atomic.LoadInt64(&q.numInQueue)
+ if count > 0 {
+ log.Warn("ChannelQueue: %s Terminated before completed flushing", q.name)
+ }
  return
  }
  log.Debug("ChannelQueue: %s Flushed", q.name)

modules/queue/queue_disk_channel.go

@@ -94,7 +94,8 @@ func NewPersistableChannelQueue(handle HandlerFunc, cfg, exemplar interface{}) (
  },
  Workers: 0,
  },
- DataDir: config.DataDir,
+ DataDir: config.DataDir,
+ QueueName: config.Name + "-level",
  }
 
  levelQueue, err := NewLevelQueue(wrappedHandle, levelCfg, exemplar)
@@ -172,16 +173,18 @@ func (q *PersistableChannelQueue) Run(atShutdown, atTerminate func(func())) {
  atShutdown(q.Shutdown)
  atTerminate(q.Terminate)
 
- if lq, ok := q.internal.(*LevelQueue); ok && lq.byteFIFO.Len(lq.shutdownCtx) != 0 {
+ if lq, ok := q.internal.(*LevelQueue); ok && lq.byteFIFO.Len(lq.terminateCtx) != 0 {
  // Just run the level queue - we shut it down once it's flushed
  go q.internal.Run(func(_ func()) {}, func(_ func()) {})
  go func() {
- for !q.IsEmpty() {
- _ = q.internal.Flush(0)
+ for !lq.IsEmpty() {
+ _ = lq.Flush(0)
  select {
  case <-time.After(100 * time.Millisecond):
- case <-q.internal.(*LevelQueue).shutdownCtx.Done():
- log.Warn("LevelQueue: %s shut down before completely flushed", q.internal.(*LevelQueue).Name())
+ case <-lq.shutdownCtx.Done():
+ if lq.byteFIFO.Len(lq.terminateCtx) > 0 {
+ log.Warn("LevelQueue: %s shut down before completely flushed", q.internal.(*LevelQueue).Name())
+ }
  return
  }
  }
@@ -316,10 +319,22 @@ func (q *PersistableChannelQueue) Shutdown() {
  // Redirect all remaining data in the chan to the internal channel
  log.Trace("PersistableChannelQueue: %s Redirecting remaining data", q.delayedStarter.name)
  close(q.channelQueue.dataChan)
+ countOK, countLost := 0, 0
  for data := range q.channelQueue.dataChan {
- _ = q.internal.Push(data)
+ err := q.internal.Push(data)
+ if err != nil {
+ log.Error("PersistableChannelQueue: %s Unable redirect %v due to: %v", q.delayedStarter.name, data, err)
+ countLost++
+ } else {
+ countOK++
+ }
  atomic.AddInt64(&q.channelQueue.numInQueue, -1)
  }
+ if countLost > 0 {
+ log.Warn("PersistableChannelQueue: %s %d will be restored on restart, %d lost", q.delayedStarter.name, countOK, countLost)
+ } else if countOK > 0 {
+ log.Warn("PersistableChannelQueue: %s %d will be restored on restart", q.delayedStarter.name, countOK)
+ }
  log.Trace("PersistableChannelQueue: %s Done Redirecting remaining data", q.delayedStarter.name)
 
  log.Debug("PersistableChannelQueue: %s Shutdown", q.delayedStarter.name)

modules/queue/queue_disk_channel_test.go

@@ -39,7 +39,7 @@ func TestPersistableChannelQueue(t *testing.T) {
  Workers: 1,
  BoostWorkers: 0,
  MaxWorkers: 10,
- Name: "first",
+ Name: "test-queue",
  }, &testData{})
  assert.NoError(t, err)
 
@@ -135,7 +135,7 @@ func TestPersistableChannelQueue(t *testing.T) {
  Workers: 1,
  BoostWorkers: 0,
  MaxWorkers: 10,
- Name: "second",
+ Name: "test-queue",
  }, &testData{})
  assert.NoError(t, err)
 
@@ -227,7 +227,7 @@ func TestPersistableChannelQueue_Pause(t *testing.T) {
  Workers: 1,
  BoostWorkers: 0,
  MaxWorkers: 10,
- Name: "first",
+ Name: "test-queue",
  }, &testData{})
  assert.NoError(t, err)
 
@@ -433,7 +433,7 @@ func TestPersistableChannelQueue_Pause(t *testing.T) {
  Workers: 1,
  BoostWorkers: 0,
  MaxWorkers: 10,
- Name: "second",
+ Name: "test-queue",
  }, &testData{})
  assert.NoError(t, err)
  pausable, ok = queue.(Pausable)

modules/queue/unique_queue_channel.go

@@ -177,7 +177,9 @@ func (q *ChannelUniqueQueue) Shutdown() {
  go func() {
  log.Trace("ChannelUniqueQueue: %s Flushing", q.name)
  if err := q.FlushWithContext(q.terminateCtx); err != nil {
- log.Warn("ChannelUniqueQueue: %s Terminated before completed flushing", q.name)
+ if !q.IsEmpty() {
+ log.Warn("ChannelUniqueQueue: %s Terminated before completed flushing", q.name)
+ }
  return
  }
  log.Debug("ChannelUniqueQueue: %s Flushed", q.name)

modules/queue/unique_queue_channel_test.go

@@ -8,10 +8,13 @@ import (
  "testing"
  "time"
 
+ "code.gitea.io/gitea/modules/log"
+
  "github.com/stretchr/testify/assert"
 )
 
 func TestChannelUniqueQueue(t *testing.T) {
+ _ = log.NewLogger(1000, "console", "console", `{"level":"warn","stacktracelevel":"NONE","stderr":true}`)
  handleChan := make(chan *testData)
  handle := func(data ...Data) []Data {
  for _, datum := range data {
@@ -52,6 +55,8 @@ func TestChannelUniqueQueue(t *testing.T) {
 }
 
 func TestChannelUniqueQueue_Batch(t *testing.T) {
+ _ = log.NewLogger(1000, "console", "console", `{"level":"warn","stacktracelevel":"NONE","stderr":true}`)
+
  handleChan := make(chan *testData)
  handle := func(data ...Data) []Data {
  for _, datum := range data {
@@ -98,6 +103,8 @@ func TestChannelUniqueQueue_Batch(t *testing.T) {
 }
 
 func TestChannelUniqueQueue_Pause(t *testing.T) {
+ _ = log.NewLogger(1000, "console", "console", `{"level":"warn","stacktracelevel":"NONE","stderr":true}`)
+
  lock := sync.Mutex{}
  var queue Queue
  var err error

modules/queue/unique_queue_disk_channel.go

@@ -94,7 +94,8 @@ func NewPersistableChannelUniqueQueue(handle HandlerFunc, cfg, exemplar interfac
  },
  Workers: 0,
  },
- DataDir: config.DataDir,
+ DataDir: config.DataDir,
+ QueueName: config.Name + "-level",
  }
 
  queue.channelQueue = channelUniqueQueue.(*ChannelUniqueQueue)
@@ -209,17 +210,29 @@ func (q *PersistableChannelUniqueQueue) Run(atShutdown, atTerminate func(func())
  atTerminate(q.Terminate)
  _ = q.channelQueue.AddWorkers(q.channelQueue.workers, 0)
 
- if luq, ok := q.internal.(*LevelUniqueQueue); ok && luq.ByteFIFOUniqueQueue.byteFIFO.Len(luq.shutdownCtx) != 0 {
+ if luq, ok := q.internal.(*LevelUniqueQueue); ok && !luq.IsEmpty() {
  // Just run the level queue - we shut it down once it's flushed
- go q.internal.Run(func(_ func()) {}, func(_ func()) {})
+ go luq.Run(func(_ func()) {}, func(_ func()) {})
  go func() {
- _ = q.internal.Flush(0)
- log.Debug("LevelUniqueQueue: %s flushed so shutting down", q.internal.(*LevelUniqueQueue).Name())
- q.internal.(*LevelUniqueQueue).Shutdown()
- GetManager().Remove(q.internal.(*LevelUniqueQueue).qid)
+ _ = luq.Flush(0)
+ for !luq.IsEmpty() {
+ _ = luq.Flush(0)
+ select {
+ case <-time.After(100 * time.Millisecond):
+ case <-luq.shutdownCtx.Done():
+ if luq.byteFIFO.Len(luq.terminateCtx) > 0 {
+ log.Warn("LevelUniqueQueue: %s shut down before completely flushed", luq.Name())
+ }
+ return
+ }
+ }
+ log.Debug("LevelUniqueQueue: %s flushed so shutting down", luq.Name())
+ luq.Shutdown()
+ GetManager().Remove(luq.qid)
  }()
  } else {
  log.Debug("PersistableChannelUniqueQueue: %s Skipping running the empty level queue", q.delayedStarter.name)
+ _ = q.internal.Flush(0)
  q.internal.(*LevelUniqueQueue).Shutdown()
  GetManager().Remove(q.internal.(*LevelUniqueQueue).qid)
  }
@@ -285,8 +298,20 @@ func (q *PersistableChannelUniqueQueue) Shutdown() {
  // Redirect all remaining data in the chan to the internal channel
  close(q.channelQueue.dataChan)
  log.Trace("PersistableChannelUniqueQueue: %s Redirecting remaining data", q.delayedStarter.name)
+ countOK, countLost := 0, 0
  for data := range q.channelQueue.dataChan {
- _ = q.internal.Push(data)
+ err := q.internal.(*LevelUniqueQueue).Push(data)
+ if err != nil {
+ log.Error("PersistableChannelUniqueQueue: %s Unable redirect %v due to: %v", q.delayedStarter.name, data, err)
+ countLost++
+ } else {
+ countOK++
+ }
+ }
+ if countLost > 0 {
+ log.Warn("PersistableChannelUniqueQueue: %s %d will be restored on restart, %d lost", q.delayedStarter.name, countOK, countLost)
+ } else if countOK > 0 {
+ log.Warn("PersistableChannelUniqueQueue: %s %d will be restored on restart", q.delayedStarter.name, countOK)
  }
  log.Trace("PersistableChannelUniqueQueue: %s Done Redirecting remaining data", q.delayedStarter.name)