From d519a39302f38bb4dcd4174639d5699dd23f5baa Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Wed, 1 Nov 2023 19:17:18 +0800 Subject: [PATCH 1/4] Support storage base path as prefix (#27827) This PR adds a prefix path for all minio storage and override base path will override the path. The previous behavior is undefined officially, so it will be marked as breaking. --- modules/setting/storage.go | 17 +++++++++-- modules/setting/storage_test.go | 53 +++++++++++++++++++++++++++++++++ 2 files changed, 67 insertions(+), 3 deletions(-) diff --git a/modules/setting/storage.go b/modules/setting/storage.go index cc3a2899d76e..f937c7cff399 100644 --- a/modules/setting/storage.go +++ b/modules/setting/storage.go @@ -7,6 +7,7 @@ import ( "errors" "fmt" "path/filepath" + "strings" ) // StorageType is a type of Storage @@ -249,14 +250,24 @@ func getStorageForMinio(targetSec, overrideSec ConfigSection, tp targetSecType, return nil, fmt.Errorf("map minio config failed: %v", err) } - if storage.MinioConfig.BasePath == "" { - storage.MinioConfig.BasePath = name + "/" + var defaultPath string + if storage.MinioConfig.BasePath != "" { + if tp == targetSecIsStorage || tp == targetSecIsDefault { + defaultPath = strings.TrimSuffix(storage.MinioConfig.BasePath, "/") + "/" + name + "/" + } else { + defaultPath = storage.MinioConfig.BasePath + } + } + if defaultPath == "" { + defaultPath = name + "/" } if overrideSec != nil { storage.MinioConfig.ServeDirect = ConfigSectionKeyBool(overrideSec, "SERVE_DIRECT", storage.MinioConfig.ServeDirect) - storage.MinioConfig.BasePath = ConfigSectionKeyString(overrideSec, "MINIO_BASE_PATH", storage.MinioConfig.BasePath) + storage.MinioConfig.BasePath = ConfigSectionKeyString(overrideSec, "MINIO_BASE_PATH", defaultPath) storage.MinioConfig.Bucket = ConfigSectionKeyString(overrideSec, "MINIO_BUCKET", storage.MinioConfig.Bucket) + } else { + storage.MinioConfig.BasePath = defaultPath } return &storage, nil } diff --git a/modules/setting/storage_test.go b/modules/setting/storage_test.go index 20886d4c4e95..6f38bf1d5575 100644 --- a/modules/setting/storage_test.go +++ b/modules/setting/storage_test.go @@ -412,3 +412,56 @@ MINIO_USE_SSL = true assert.EqualValues(t, true, RepoArchive.Storage.MinioConfig.UseSSL) assert.EqualValues(t, "repo-archive/", RepoArchive.Storage.MinioConfig.BasePath) } + +func Test_getStorageConfiguration28(t *testing.T) { + cfg, err := NewConfigProviderFromData(` +[storage] +STORAGE_TYPE = minio +MINIO_ACCESS_KEY_ID = my_access_key +MINIO_SECRET_ACCESS_KEY = my_secret_key +MINIO_USE_SSL = true +MINIO_BASE_PATH = /prefix +`) + assert.NoError(t, err) + assert.NoError(t, loadRepoArchiveFrom(cfg)) + assert.EqualValues(t, "my_access_key", RepoArchive.Storage.MinioConfig.AccessKeyID) + assert.EqualValues(t, "my_secret_key", RepoArchive.Storage.MinioConfig.SecretAccessKey) + assert.EqualValues(t, true, RepoArchive.Storage.MinioConfig.UseSSL) + assert.EqualValues(t, "/prefix/repo-archive/", RepoArchive.Storage.MinioConfig.BasePath) + + cfg, err = NewConfigProviderFromData(` +[storage] +STORAGE_TYPE = minio +MINIO_ACCESS_KEY_ID = my_access_key +MINIO_SECRET_ACCESS_KEY = my_secret_key +MINIO_USE_SSL = true +MINIO_BASE_PATH = /prefix + +[lfs] +MINIO_BASE_PATH = /lfs +`) + assert.NoError(t, err) + assert.NoError(t, loadLFSFrom(cfg)) + assert.EqualValues(t, "my_access_key", LFS.Storage.MinioConfig.AccessKeyID) + assert.EqualValues(t, "my_secret_key", LFS.Storage.MinioConfig.SecretAccessKey) + assert.EqualValues(t, true, LFS.Storage.MinioConfig.UseSSL) + assert.EqualValues(t, "/lfs", LFS.Storage.MinioConfig.BasePath) + + cfg, err = NewConfigProviderFromData(` +[storage] +STORAGE_TYPE = minio +MINIO_ACCESS_KEY_ID = my_access_key +MINIO_SECRET_ACCESS_KEY = my_secret_key +MINIO_USE_SSL = true +MINIO_BASE_PATH = /prefix + +[storage.lfs] +MINIO_BASE_PATH = /lfs +`) + assert.NoError(t, err) + assert.NoError(t, loadLFSFrom(cfg)) + assert.EqualValues(t, "my_access_key", LFS.Storage.MinioConfig.AccessKeyID) + assert.EqualValues(t, "my_secret_key", LFS.Storage.MinioConfig.SecretAccessKey) + assert.EqualValues(t, true, LFS.Storage.MinioConfig.UseSSL) + assert.EqualValues(t, "/lfs", LFS.Storage.MinioConfig.BasePath) +} From 665d12cf8485d8be2189208b60f371365d56bb0e Mon Sep 17 00:00:00 2001 From: nodiscc Date: Wed, 1 Nov 2023 15:14:40 +0000 Subject: [PATCH 2/4] doc: actions/act-runner: document running as a systemd service (#27844) This documents running `act-runner` as a systemd service under a dedicated user account. --- .../content/usage/actions/act-runner.en-us.md | 34 +++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/docs/content/usage/actions/act-runner.en-us.md b/docs/content/usage/actions/act-runner.en-us.md index 33813f5910d6..6e99beb870ac 100644 --- a/docs/content/usage/actions/act-runner.en-us.md +++ b/docs/content/usage/actions/act-runner.en-us.md @@ -268,6 +268,40 @@ The runner will fetch jobs from the Gitea instance and run them automatically. Since act runner is still in development, it is recommended to check the latest version and upgrade it regularly. +## Systemd service + +It is also possible to run act-runner as a [systemd](https://en.wikipedia.org/wiki/Systemd) service. Create an unprivileged `act_runner` user on your system, and the following file in `/etc/systemd/system/act_runner.service`. The paths in `ExecStart` and `WorkingDirectory` may need to be adjusted depending on where you installed the `act_runner` binary, its configuration file, and the home directory of the `act_runner` user. + +```ini +[Unit] +Description=Gitea Actions runner +Documentation=https://gitea.com/gitea/act_runner +After=docker.service + +[Service] +ExecStart=/usr/local/bin/act_runner daemon --config /etc/act_runner/config.yaml +ExecReload=/bin/kill -s HUP $MAINPID +WorkingDirectory=/var/lib/act_runner +TimeoutSec=0 +RestartSec=10 +Restart=always +User=act_runner + +[Install] +WantedBy=multi-user.target +``` + +Then: + +```bash +# load the new systemd unit file +sudo systemctl daemon-reload +# start the service and enable it at boot +sudo systemctl enable act_runner --now +``` + +If using Docker, the `act_runner` user should also be added to the `docker` group before starting the service. Keep in mind that this effectively gives `act_runner` root access to the system [[1]](https://docs.docker.com/engine/security/#docker-daemon-attack-surface). + ## Configuration variable You can create configuration variables on the user, organization and repository level. From 9b6e77c489edf3af3a9782c977cc04fde7aa2fd9 Mon Sep 17 00:00:00 2001 From: Moritz Poldrack <33086936+mpldr@users.noreply.github.com> Date: Wed, 1 Nov 2023 19:00:20 +0100 Subject: [PATCH 3/4] refactor postgres connection string building (#27723) This patchset changes the connection string builder to use net.URL and the host/port parser to use the stdlib function for splitting host from port. It also adds a footnote about a potentially required portnumber for postgres UNIX sockets. Fixes: #24552 --- .../config-cheat-sheet.en-us.md | 4 +- modules/setting/database.go | 40 ++++++++++++------- modules/setting/database_test.go | 33 ++++++++------- 3 files changed, 47 insertions(+), 30 deletions(-) diff --git a/docs/content/administration/config-cheat-sheet.en-us.md b/docs/content/administration/config-cheat-sheet.en-us.md index 617715fbaa12..715bd6d325f5 100644 --- a/docs/content/administration/config-cheat-sheet.en-us.md +++ b/docs/content/administration/config-cheat-sheet.en-us.md @@ -424,7 +424,7 @@ The following configuration set `Content-Type: application/vnd.android.package-a ## Database (`database`) - `DB_TYPE`: **mysql**: The database type in use \[mysql, postgres, mssql, sqlite3\]. -- `HOST`: **127.0.0.1:3306**: Database host address and port or absolute path for unix socket \[mysql, postgres\] (ex: /var/run/mysqld/mysqld.sock). +- `HOST`: **127.0.0.1:3306**: Database host address and port or absolute path for unix socket \[mysql, postgres[^1]\] (ex: /var/run/mysqld/mysqld.sock). - `NAME`: **gitea**: Database name. - `USER`: **root**: Database username. - `PASSWD`: **_empty_**: Database user password. Use \`your password\` or """your password""" for quoting if you use special characters in the password. @@ -455,6 +455,8 @@ The following configuration set `Content-Type: application/vnd.android.package-a - `CONN_MAX_LIFETIME` **0 or 3s**: Sets the maximum amount of time a DB connection may be reused - default is 0, meaning there is no limit (except on MySQL where it is 3s - see #6804 & #7071). - `AUTO_MIGRATION` **true**: Whether execute database models migrations automatically. +[^1]: It may be necessary to specify a hostport even when listening on a unix socket, as the port is part of the socket name. see [#24552](https://github.com/go-gitea/gitea/issues/24552#issuecomment-1681649367) for additional details. + Please see #8540 & #8273 for further discussion of the appropriate values for `MAX_OPEN_CONNS`, `MAX_IDLE_CONNS` & `CONN_MAX_LIFETIME` and their relation to port exhaustion. diff --git a/modules/setting/database.go b/modules/setting/database.go index 709655368c67..aa42f506bc51 100644 --- a/modules/setting/database.go +++ b/modules/setting/database.go @@ -6,6 +6,7 @@ package setting import ( "errors" "fmt" + "net" "net/url" "os" "path" @@ -135,15 +136,18 @@ func DBConnStr() (string, error) { // parsePostgreSQLHostPort parses given input in various forms defined in // https://www.postgresql.org/docs/current/static/libpq-connect.html#LIBPQ-CONNSTRING // and returns proper host and port number. -func parsePostgreSQLHostPort(info string) (string, string) { - host, port := "127.0.0.1", "5432" - if strings.Contains(info, ":") && !strings.HasSuffix(info, "]") { - idx := strings.LastIndex(info, ":") - host = info[:idx] - port = info[idx+1:] - } else if len(info) > 0 { +func parsePostgreSQLHostPort(info string) (host, port string) { + if h, p, err := net.SplitHostPort(info); err == nil { + host, port = h, p + } else { + // treat the "info" as "host", if it's an IPv6 address, remove the wrapper host = info + if strings.HasPrefix(host, "[") && strings.HasSuffix(host, "]") { + host = host[1 : len(host)-1] + } } + + // set fallback values if host == "" { host = "127.0.0.1" } @@ -155,14 +159,22 @@ func parsePostgreSQLHostPort(info string) (string, string) { func getPostgreSQLConnectionString(dbHost, dbUser, dbPasswd, dbName, dbParam, dbsslMode string) (connStr string) { host, port := parsePostgreSQLHostPort(dbHost) - if host[0] == '/' { // looks like a unix socket - connStr = fmt.Sprintf("postgres://%s:%s@:%s/%s%ssslmode=%s&host=%s", - url.PathEscape(dbUser), url.PathEscape(dbPasswd), port, dbName, dbParam, dbsslMode, host) - } else { - connStr = fmt.Sprintf("postgres://%s:%s@%s:%s/%s%ssslmode=%s", - url.PathEscape(dbUser), url.PathEscape(dbPasswd), host, port, dbName, dbParam, dbsslMode) + connURL := url.URL{ + Scheme: "postgres", + User: url.UserPassword(dbUser, dbPasswd), + Host: net.JoinHostPort(host, port), + Path: dbName, + OmitHost: false, + RawQuery: dbParam, + } + query := connURL.Query() + if dbHost[0] == '/' { // looks like a unix socket + query.Add("host", dbHost) + connURL.Host = ":" + port } - return connStr + query.Set("sslmode", dbsslMode) + connURL.RawQuery = query.Encode() + return connURL.String() } // ParseMSSQLHostPort splits the host into host and port diff --git a/modules/setting/database_test.go b/modules/setting/database_test.go index 481ca969b1fb..85271c36cb38 100644 --- a/modules/setting/database_test.go +++ b/modules/setting/database_test.go @@ -10,46 +10,49 @@ import ( ) func Test_parsePostgreSQLHostPort(t *testing.T) { - tests := []struct { + tests := map[string]struct { HostPort string Host string Port string }{ - { + "host-port": { HostPort: "127.0.0.1:1234", Host: "127.0.0.1", Port: "1234", }, - { + "no-port": { HostPort: "127.0.0.1", Host: "127.0.0.1", Port: "5432", }, - { + "ipv6-port": { HostPort: "[::1]:1234", - Host: "[::1]", + Host: "::1", Port: "1234", }, - { + "ipv6-no-port": { HostPort: "[::1]", - Host: "[::1]", + Host: "::1", Port: "5432", }, - { + "unix-socket": { HostPort: "/tmp/pg.sock:1234", Host: "/tmp/pg.sock", Port: "1234", }, - { + "unix-socket-no-port": { HostPort: "/tmp/pg.sock", Host: "/tmp/pg.sock", Port: "5432", }, } - for _, test := range tests { - host, port := parsePostgreSQLHostPort(test.HostPort) - assert.Equal(t, test.Host, host) - assert.Equal(t, test.Port, port) + for k, test := range tests { + t.Run(k, func(t *testing.T) { + t.Log(test.HostPort) + host, port := parsePostgreSQLHostPort(test.HostPort) + assert.Equal(t, test.Host, host) + assert.Equal(t, test.Port, port) + }) } } @@ -72,7 +75,7 @@ func Test_getPostgreSQLConnectionString(t *testing.T) { Name: "gitea", Param: "", SSLMode: "false", - Output: "postgres://testuser:space%20space%20%21%23$%25%5E%5E%25%5E%60%60%60-=%3F=@:5432/giteasslmode=false&host=/tmp/pg.sock", + Output: "postgres://testuser:space%20space%20%21%23$%25%5E%5E%25%5E%60%60%60-=%3F=@:5432/gitea?host=%2Ftmp%2Fpg.sock&sslmode=false", }, { Host: "localhost", @@ -82,7 +85,7 @@ func Test_getPostgreSQLConnectionString(t *testing.T) { Name: "gitea", Param: "", SSLMode: "true", - Output: "postgres://pgsqlusername:I%20love%20Gitea%21@localhost:5432/giteasslmode=true", + Output: "postgres://pgsqlusername:I%20love%20Gitea%21@localhost:5432/gitea?sslmode=true", }, } From e378545f3083990eb36ff5d72477662d9787280d Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Thu, 2 Nov 2023 00:46:26 +0100 Subject: [PATCH 4/4] Filter inactive auth sources (#27870) Fix nil access for inactive auth sources. > Render failed, failed to render template: user/settings/security/security, error: template error: builtin(static):user/settings/security/accountlinks:32:20 : executing "user/settings/security/accountlinks" at <$providerData.IconHTML>: nil pointer evaluating oauth2.Provider.IconHTML Code tries to access the auth source of an `ExternalLoginUser` but the list contains only the active auth sources. --- routers/web/user/setting/security/security.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/routers/web/user/setting/security/security.go b/routers/web/user/setting/security/security.go index c687f7314d40..58c637e2b336 100644 --- a/routers/web/user/setting/security/security.go +++ b/routers/web/user/setting/security/security.go @@ -82,7 +82,7 @@ func loadSecurityData(ctx *context.Context) { // map the provider display name with the AuthSource sources := make(map[*auth_model.Source]string) for _, externalAccount := range accountLinks { - if authSource, err := auth_model.GetSourceByID(ctx, externalAccount.LoginSourceID); err == nil { + if authSource, err := auth_model.GetSourceByID(ctx, externalAccount.LoginSourceID); err == nil && authSource.IsActive { var providerDisplayName string type DisplayNamed interface {