Add support for database schema in PostgreSQL (#8819)

* Add support for database schema * Require setting search_path for the db user * Add schema setting to admin/config.tmpl * Use a schema different from default for psql tests * Update postgres scripts to use custom schema * Update to xorm/core 0.7.3 and xorm/xorm c37aff9b3a * Fix migration test Co-authored-by: Antoine GIRARD <[email protected]> Co-authored-by: Lunny Xiao <[email protected]>
go-gitea · Jan 20, 2020 · ad1b6d4 · ad1b6d4
1 parent 6d6f1d5
commit ad1b6d4
Show file tree

Hide file tree

Showing 28 changed files with 177 additions and 407 deletions.
diff --git a/Makefile b/Makefile
@@ -79,6 +79,7 @@ TEST_PGSQL_HOST ?= pgsql:5432
 TEST_PGSQL_DBNAME ?= testgitea
 TEST_PGSQL_USERNAME ?= postgres
 TEST_PGSQL_PASSWORD ?= postgres
+TEST_PGSQL_SCHEMA ?= gtestschema
 TEST_MSSQL_HOST ?= mssql:1433
 TEST_MSSQL_DBNAME ?= gitea
 TEST_MSSQL_USERNAME ?= sa
@@ -306,6 +307,7 @@ generate-ini-pgsql:
  -e 's|{{TEST_PGSQL_DBNAME}}|${TEST_PGSQL_DBNAME}|g' \
  -e 's|{{TEST_PGSQL_USERNAME}}|${TEST_PGSQL_USERNAME}|g' \
  -e 's|{{TEST_PGSQL_PASSWORD}}|${TEST_PGSQL_PASSWORD}|g' \
+ -e 's|{{TEST_PGSQL_SCHEMA}}|${TEST_PGSQL_SCHEMA}|g' \
  integrations/pgsql.ini.tmpl > integrations/pgsql.ini
 
 .PHONY: test-pgsql

diff --git a/custom/conf/app.ini.sample b/custom/conf/app.ini.sample
@@ -336,6 +336,10 @@ NAME = gitea
 USER = root
 ; Use PASSWD = `your password` for quoting if you use special characters in the password.
 PASSWD =
+; For Postgres, schema to use if different from "public". The schema must exist beforehand,
+; the user must have creation privileges on it, and the user search path must be set
+; to the look into the schema first. e.g.:ALTER USER user SET SEARCH_PATH = schema_name,"$user",public;
+SCHEMA =
 ; For Postgres, either "disable" (default), "require", or "verify-full"
 ; For MySQL, either "false" (default), "true", or "skip-verify"
 SSL_MODE = disable

diff --git a/docs/content/doc/advanced/config-cheat-sheet.en-us.md b/docs/content/doc/advanced/config-cheat-sheet.en-us.md
@@ -209,6 +209,9 @@ Values containing `#` or `;` must be quoted using `` ` `` or `"""`.
 - `NAME`: **gitea**: Database name.
 - `USER`: **root**: Database username.
 - `PASSWD`: **\<empty\>**: Database user password. Use \`your password\` for quoting if you use special characters in the password.
+- `SCHEMA`: **\<empty\>**: For PostgreSQL only, schema to use if different from "public". The schema must exist beforehand,
+ the user must have creation privileges on it, and the user search path must be set to the look into the schema first 
+ (e.g. `ALTER USER user SET SEARCH_PATH = schema_name,"$user",public;`).
 - `SSL_MODE`: **disable**: For PostgreSQL and MySQL only.
 - `CHARSET`: **utf8**: For MySQL only, either "utf8" or "utf8mb4", default is "utf8". NOTICE: for "utf8mb4" you must use MySQL InnoDB > 5.6. Gitea is unable to check this.
 - `PATH`: **data/gitea.db**: For SQLite3 only, the database file path.

diff --git a/go.mod b/go.mod
@@ -112,6 +112,6 @@ require (
  mvdan.cc/xurls/v2 v2.1.0
  strk.kbt.io/projects/go/libravatar v0.0.0-20191008002943-06d1c002b251
  xorm.io/builder v0.3.6
- xorm.io/core v0.7.2
- xorm.io/xorm v0.8.1
+ xorm.io/core v0.7.3
+ xorm.io/xorm v0.8.2-0.20200120024500-c37aff9b3a4a
 )
diff --git a/go.sum b/go.sum
@@ -760,7 +760,9 @@ xorm.io/builder v0.3.6 h1:ha28mQ2M+TFx96Hxo+iq6tQgnkC9IZkM6D8w9sKHHF8=
 xorm.io/builder v0.3.6/go.mod h1:LEFAPISnRzG+zxaxj2vPicRwz67BdhFreKg8yv8/TgU=
 xorm.io/core v0.7.2 h1:mEO22A2Z7a3fPaZMk6gKL/jMD80iiyNwRrX5HOv3XLw=
 xorm.io/core v0.7.2/go.mod h1:jJfd0UAEzZ4t87nbQYtVjmqpIODugN6PD2D9E+dJvdM=
+xorm.io/core v0.7.3 h1:W8ws1PlrnkS1CZU1YWaYLMQcQilwAmQXU0BJDJon+H0=
+xorm.io/core v0.7.3/go.mod h1:jJfd0UAEzZ4t87nbQYtVjmqpIODugN6PD2D9E+dJvdM=
 xorm.io/xorm v0.8.0 h1:iALxgJrX8O00f8Jk22GbZwPmxJNgssV5Mv4uc2HL9PM=
 xorm.io/xorm v0.8.0/go.mod h1:ZkJLEYLoVyg7amJK/5r779bHyzs2AU8f8VMiP6BM7uY=
-xorm.io/xorm v0.8.1 h1:4f2KXuQxVdaX3RdI3Fw81NzMiSpZeyCZt8m3sEVeIkQ=
-xorm.io/xorm v0.8.1/go.mod h1:ZkJLEYLoVyg7amJK/5r779bHyzs2AU8f8VMiP6BM7uY=
+xorm.io/xorm v0.8.2-0.20200120024500-c37aff9b3a4a h1:hzGd080rlkZ5a7v6Tr3x8PJJnWPfKxGMMl92c8DNcww=
+xorm.io/xorm v0.8.2-0.20200120024500-c37aff9b3a4a/go.mod h1:ZkJLEYLoVyg7amJK/5r779bHyzs2AU8f8VMiP6BM7uY=
diff --git a/integrations/integration_test.go b/integrations/integration_test.go
@@ -153,18 +153,53 @@ func initIntegrationTest() {
  if err != nil {
  log.Fatalf("sql.Open: %v", err)
  }
- rows, err := db.Query(fmt.Sprintf("SELECT 1 FROM pg_database WHERE datname = '%s'", setting.Database.Name))
+ dbrows, err := db.Query(fmt.Sprintf("SELECT 1 FROM pg_database WHERE datname = '%s'", setting.Database.Name))
  if err != nil {
  log.Fatalf("db.Query: %v", err)
  }
- defer rows.Close()
+ defer dbrows.Close()
 
- if rows.Next() {
+ if !dbrows.Next() {
+ if _, err = db.Exec(fmt.Sprintf("CREATE DATABASE %s", setting.Database.Name)); err != nil {
+ log.Fatalf("db.Exec: CREATE DATABASE: %v", err)
+ }
+ }
+ // Check if we need to setup a specific schema
+ if len(setting.Database.Schema) == 0 {
  break
  }
- if _, err = db.Exec(fmt.Sprintf("CREATE DATABASE %s", setting.Database.Name)); err != nil {
- log.Fatalf("db.Exec: %v", err)
+ db.Close()
+
+ db, err = sql.Open("postgres", fmt.Sprintf("postgres:https://%s:%s@%s/%s?sslmode=%s",
+ setting.Database.User, setting.Database.Passwd, setting.Database.Host, setting.Database.Name, setting.Database.SSLMode))
+ // This is a different db object; requires a different Close()
+ defer db.Close()
+ if err != nil {
+ log.Fatalf("sql.Open: %v", err)
+ }
+ schrows, err := db.Query(fmt.Sprintf("SELECT 1 FROM information_schema.schemata WHERE schema_name = '%s'", setting.Database.Schema))
+ if err != nil {
+ log.Fatalf("db.Query: %v", err)
+ }
+ defer schrows.Close()
+
+ if !schrows.Next() {
+ // Create and setup a DB schema
+ if _, err = db.Exec(fmt.Sprintf("CREATE SCHEMA %s", setting.Database.Schema)); err != nil {
+ log.Fatalf("db.Exec: CREATE SCHEMA: %v", err)
+ }
+ }
+
+ // Make the user's default search path the created schema; this will affect new connections
+ if _, err = db.Exec(fmt.Sprintf(`ALTER USER "%s" SET search_path = %s`, setting.Database.User, setting.Database.Schema)); err != nil {
+ log.Fatalf("db.Exec: ALTER USER SET search_path: %v", err)
+ }
+
+ // Make the current connection's search the created schema
+ if _, err = db.Exec(fmt.Sprintf(`SET search_path = %s`, setting.Database.Schema)); err != nil {
+ log.Fatalf("db.Exec: ALTER USER SET search_path: %v", err)
  }
+
  case setting.Database.UseMSSQL:
  host, port := setting.ParseMSSQLHostPort(setting.Database.Host)
  db, err := sql.Open("mssql", fmt.Sprintf("server=%s; port=%s; database=%s; user id=%s; password=%s;",

diff --git a/integrations/migration-test/gitea-v1.5.3.postgres.sql.gz b/integrations/migration-test/gitea-v1.5.3.postgres.sql.gz
diff --git a/integrations/migration-test/gitea-v1.6.4.postgres.sql.gz b/integrations/migration-test/gitea-v1.6.4.postgres.sql.gz
diff --git a/integrations/migration-test/gitea-v1.7.0.postgres.sql.gz b/integrations/migration-test/gitea-v1.7.0.postgres.sql.gz
diff --git a/integrations/migration-test/migration_test.go b/integrations/migration-test/migration_test.go
@@ -168,6 +168,32 @@ func restoreOldDB(t *testing.T, version string) bool {
  assert.NoError(t, err)
  db.Close()
 
+ // Check if we need to setup a specific schema
+ if len(setting.Database.Schema) != 0 {
+ db, err = sql.Open("postgres", fmt.Sprintf("postgres:https://%s:%s@%s/%s?sslmode=%s",
+ setting.Database.User, setting.Database.Passwd, setting.Database.Host, setting.Database.Name, setting.Database.SSLMode))
+ if !assert.NoError(t, err) {
+ return false
+ }
+ schrows, err := db.Query(fmt.Sprintf("SELECT 1 FROM information_schema.schemata WHERE schema_name = '%s'", setting.Database.Schema))
+ if !assert.NoError(t, err) || !assert.NotEmpty(t, schrows) {
+ return false
+ }
+
+ if !schrows.Next() {
+ // Create and setup a DB schema
+ _, err = db.Exec(fmt.Sprintf("CREATE SCHEMA %s", setting.Database.Schema))
+ assert.NoError(t, err)
+ }
+ schrows.Close()
+
+ // Make the user's default search path the created schema; this will affect new connections
+ _, err = db.Exec(fmt.Sprintf(`ALTER USER "%s" SET search_path = %s`, setting.Database.User, setting.Database.Schema))
+ assert.NoError(t, err)
+
+ db.Close()
+ }
+
  db, err = sql.Open("postgres", fmt.Sprintf("postgres:https://%s:%s@%s/%s?sslmode=%s",
  setting.Database.User, setting.Database.Passwd, setting.Database.Host, setting.Database.Name, setting.Database.SSLMode))
  assert.NoError(t, err)

diff --git a/integrations/pgsql.ini.tmpl b/integrations/pgsql.ini.tmpl
@@ -7,6 +7,7 @@ HOST = {{TEST_PGSQL_HOST}}
 NAME = {{TEST_PGSQL_DBNAME}}
 USER = {{TEST_PGSQL_USERNAME}}
 PASSWD = {{TEST_PGSQL_PASSWORD}}
+SCHEMA = {{TEST_PGSQL_SCHEMA}}
 SSL_MODE = disable
 
 [indexer]

diff --git a/models/models.go b/models/models.go
@@ -128,7 +128,12 @@ func getEngine() (*xorm.Engine, error) {
  return nil, err
  }
 
- return xorm.NewEngine(setting.Database.Type, connStr)
+ engine, err := xorm.NewEngine(setting.Database.Type, connStr)
+ if err != nil {
+ return nil, err
+ }
+ engine.SetSchema(setting.Database.Schema)
+ return engine, nil
 }
 
 // NewTestEngine sets a new test xorm.Engine

diff --git a/modules/auth/user_form.go b/modules/auth/user_form.go
@@ -25,6 +25,7 @@ type InstallForm struct {
  SSLMode string
  Charset string `binding:"Required;In(utf8,utf8mb4)"`
  DbPath string
+ DbSchema string
 
  AppName string `binding:"Required" locale:"install.app_name"`
  RepoRootPath string `binding:"Required"`

diff --git a/modules/setting/database.go b/modules/setting/database.go
@@ -30,6 +30,7 @@ var (
  Name string
  User string
  Passwd string
+ Schema string
  SSLMode string
  Path string
  LogSQL bool
@@ -75,6 +76,7 @@ func InitDBConfig() {
  if len(Database.Passwd) == 0 {
  Database.Passwd = sec.Key("PASSWD").String()
  }
+ Database.Schema = sec.Key("SCHEMA").String()
  Database.SSLMode = sec.Key("SSL_MODE").MustString("disable")
  Database.Charset = sec.Key("CHARSET").In("utf8", []string{"utf8", "utf8mb4"})
  Database.Path = sec.Key("PATH").MustString(filepath.Join(AppDataPath, "gitea.db"))

diff --git a/options/locale/locale_en-US.ini b/options/locale/locale_en-US.ini
@@ -102,6 +102,8 @@ user = Username
 password = Password
 db_name = Database Name
 db_helper = Note to MySQL users: please use the InnoDB storage engine and if you use "utf8mb4", your InnoDB version must be greater than 5.6 .
+db_schema = Schema
+db_schema_helper = Leave blank for database default ("public").
 ssl_mode = SSL
 charset = Charset
 path = Path
@@ -1953,6 +1955,7 @@ config.db_type = Type
 config.db_host = Host
 config.db_name = Name
 config.db_user = Username
+config.db_schema = Schema
 config.db_ssl_mode = SSL
 config.db_path = Path
 

diff --git a/routers/install.go b/routers/install.go
@@ -54,6 +54,7 @@ func Install(ctx *context.Context) {
  form.DbPasswd = setting.Database.Passwd
  form.DbName = setting.Database.Name
  form.DbPath = setting.Database.Path
+ form.DbSchema = setting.Database.Schema
  form.Charset = setting.Database.Charset
 
  ctx.Data["CurDbOption"] = "MySQL"
@@ -147,6 +148,7 @@ func InstallPost(ctx *context.Context, form auth.InstallForm) {
  setting.Database.User = form.DbUser
  setting.Database.Passwd = form.DbPasswd
  setting.Database.Name = form.DbName
+ setting.Database.Schema = form.DbSchema
  setting.Database.SSLMode = form.SSLMode
  setting.Database.Charset = form.Charset
  setting.Database.Path = form.DbPath
@@ -267,6 +269,7 @@ func InstallPost(ctx *context.Context, form auth.InstallForm) {
  cfg.Section("database").Key("NAME").SetValue(setting.Database.Name)
  cfg.Section("database").Key("USER").SetValue(setting.Database.User)
  cfg.Section("database").Key("PASSWD").SetValue(setting.Database.Passwd)
+ cfg.Section("database").Key("SCHEMA").SetValue(setting.Database.Schema)
  cfg.Section("database").Key("SSL_MODE").SetValue(setting.Database.SSLMode)
  cfg.Section("database").Key("CHARSET").SetValue(setting.Database.Charset)
  cfg.Section("database").Key("PATH").SetValue(setting.Database.Path)

diff --git a/templates/admin/config.tmpl b/templates/admin/config.tmpl
@@ -128,6 +128,8 @@
  <dd>{{if .DbCfg.User}}{{.DbCfg.User}}{{else}}-{{end}}</dd>
  {{end}}
  {{if eq .DbCfg.Type "postgres"}}
+ <dt>{{.i18n.Tr "admin.config.db_schema"}}</dt>
+ <dd>{{if .DbCfg.Schema}}{{.DbCfg.Schema}}{{else}}-{{end}}</dd>
  <dt>{{.i18n.Tr "admin.config.db_ssl_mode"}}</dt>
  <dd>{{if .DbCfg.SSLMode}}{{.DbCfg.SSLMode}}{{else}}-{{end}}</dd>
  {{end}}

diff --git a/templates/install.tmpl b/templates/install.tmpl
@@ -62,6 +62,11 @@
  </div>
  </div>
  </div>
+ <div class="inline field {{if .Err_DbSetting}}error{{end}}">
+ <label for="db_schema">{{.i18n.Tr "install.db_schema"}}</label>
+ <input id="db_schema" name="db_schema" value="{{.db_schema}}">
+ <span class="help">{{.i18n.Tr "install.db_schema_helper"}}</span>
+ </div>
  </div>
 
  <div id="mysql_settings" class="{{if not (eq .CurDbOption "MySQL")}}hide{{end}}">

diff --git a/vendor/modules.txt b/vendor/modules.txt
@@ -613,7 +613,7 @@ mvdan.cc/xurls/v2
 strk.kbt.io/projects/go/libravatar
 # xorm.io/builder v0.3.6
 xorm.io/builder
-# xorm.io/core v0.7.2
+# xorm.io/core v0.7.3
 xorm.io/core
-# xorm.io/xorm v0.8.1
+# xorm.io/xorm v0.8.2-0.20200120024500-c37aff9b3a4a
 xorm.io/xorm
diff --git a/vendor/xorm.io/core/.drone.yml b/vendor/xorm.io/core/.drone.yml