Browse Source

Use buffersize to reduce database connection when iterate (#2724)

* use buffersize to reduce database connection when iterate

* fix typo

* add default value on app.ini comment
Lunny Xiao 2 years ago
parent
commit
985a39590b
7 changed files with 19 additions and 15 deletions
  1. 2 0
      conf/app.ini
  2. 1 1
      models/migrations/v19.go
  3. 1 1
      models/migrations/v22.go
  4. 1 1
      models/migrations/v26.go
  5. 2 2
      models/repo.go
  6. 1 1
      models/user.go
  7. 11 9
      modules/setting/setting.go

+ 2 - 0
conf/app.ini

@@ -184,6 +184,8 @@ SSL_MODE = disable
184 184
 PATH = data/gitea.db
185 185
 ; For "sqlite3" only. Query timeout
186 186
 SQLITE_TIMEOUT = 500
187
+; For iterate buffer, default is 50
188
+ITERATE_BUFFER_SIZE = 50
187 189
 
188 190
 [indexer]
189 191
 ISSUE_INDEXER_PATH = indexers/issues.bleve

+ 1 - 1
models/migrations/v19.go

@@ -42,7 +42,7 @@ func generateAndMigrateGitHooks(x *xorm.Engine) (err error) {
42 42
 		}
43 43
 	)
44 44
 
45
-	return x.Where("id > 0").Iterate(new(Repository),
45
+	return x.Where("id > 0").BufferSize(setting.IterateBufferSize).Iterate(new(Repository),
46 46
 		func(idx int, bean interface{}) error {
47 47
 			repo := bean.(*Repository)
48 48
 			user := new(User)

+ 1 - 1
models/migrations/v22.go

@@ -42,7 +42,7 @@ func generateAndMigrateWikiGitHooks(x *xorm.Engine) (err error) {
42 42
 		}
43 43
 	)
44 44
 
45
-	return x.Where("id > 0").Iterate(new(Repository),
45
+	return x.Where("id > 0").BufferSize(setting.IterateBufferSize).Iterate(new(Repository),
46 46
 		func(idx int, bean interface{}) error {
47 47
 			repo := bean.(*Repository)
48 48
 			user := new(User)

+ 1 - 1
models/migrations/v26.go

@@ -36,7 +36,7 @@ func generateAndMigrateGitHookChains(x *xorm.Engine) (err error) {
36 36
 		hookTpl   = fmt.Sprintf("#!/usr/bin/env %s\ndata=$(cat)\nexitcodes=\"\"\nhookname=$(basename $0)\nGIT_DIR=${GIT_DIR:-$(dirname $0)}\n\nfor hook in ${GIT_DIR}/hooks/${hookname}.d/*; do\ntest -x \"${hook}\" || continue\necho \"${data}\" | \"${hook}\"\nexitcodes=\"${exitcodes} $?\"\ndone\n\nfor i in ${exitcodes}; do\n[ ${i} -eq 0 ] || exit ${i}\ndone\n", setting.ScriptType)
37 37
 	)
38 38
 
39
-	return x.Where("id > 0").Iterate(new(Repository),
39
+	return x.Where("id > 0").BufferSize(setting.IterateBufferSize).Iterate(new(Repository),
40 40
 		func(idx int, bean interface{}) error {
41 41
 			repo := bean.(*Repository)
42 42
 			user := new(User)

+ 2 - 2
models/repo.go

@@ -2145,7 +2145,7 @@ func GitFsck() {
2145 2145
 	log.Trace("Doing: GitFsck")
2146 2146
 
2147 2147
 	if err := x.
2148
-		Where("id>0").
2148
+		Where("id>0").BufferSize(setting.IterateBufferSize).
2149 2149
 		Iterate(new(Repository),
2150 2150
 			func(idx int, bean interface{}) error {
2151 2151
 				repo := bean.(*Repository)
@@ -2167,7 +2167,7 @@ func GitFsck() {
2167 2167
 func GitGcRepos() error {
2168 2168
 	args := append([]string{"gc"}, setting.Git.GCArgs...)
2169 2169
 	return x.
2170
-		Where("id > 0").
2170
+		Where("id > 0").BufferSize(setting.IterateBufferSize).
2171 2171
 		Iterate(new(Repository),
2172 2172
 			func(idx int, bean interface{}) error {
2173 2173
 				repo := bean.(*Repository)

+ 1 - 1
models/user.go

@@ -816,7 +816,7 @@ func ChangeUserName(u *User, newUserName string) (err error) {
816 816
 	}
817 817
 
818 818
 	// Delete all local copies of repository wiki that user owns.
819
-	if err = x.
819
+	if err = x.BufferSize(setting.IterateBufferSize).
820 820
 		Where("owner_id=?", u.ID).
821 821
 		Iterate(new(Repository), func(idx int, bean interface{}) error {
822 822
 			repo := bean.(*Repository)

+ 11 - 9
modules/setting/setting.go

@@ -480,15 +480,16 @@ var (
480 480
 	ShowFooterTemplateLoadTime bool
481 481
 
482 482
 	// Global setting objects
483
-	Cfg           *ini.File
484
-	CustomPath    string // Custom directory path
485
-	CustomConf    string
486
-	CustomPID     string
487
-	ProdMode      bool
488
-	RunUser       string
489
-	IsWindows     bool
490
-	HasRobotsTxt  bool
491
-	InternalToken string // internal access token
483
+	Cfg               *ini.File
484
+	CustomPath        string // Custom directory path
485
+	CustomConf        string
486
+	CustomPID         string
487
+	ProdMode          bool
488
+	RunUser           string
489
+	IsWindows         bool
490
+	HasRobotsTxt      bool
491
+	InternalToken     string // internal access token
492
+	IterateBufferSize int
492 493
 )
493 494
 
494 495
 // DateLang transforms standard language locale name to corresponding value in datetime plugin.
@@ -873,6 +874,7 @@ func NewContext() {
873 874
 			log.Fatal(4, "Error saving generated JWT Secret to custom config: %v", err)
874 875
 		}
875 876
 	}
877
+	IterateBufferSize = Cfg.Section("database").Key("ITERATE_BUFFER_SIZE").MustInt(50)
876 878
 
877 879
 	sec = Cfg.Section("attachment")
878 880
 	AttachmentPath = sec.Key("PATH").MustString(path.Join(AppDataPath, "attachments"))