Fork to maintain patches against the official gitea for https://code.ceondo.com https://github.com/go-gitea/gitea

repo_mirror.go 6.6KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254
  1. // Copyright 2016 The Gogs Authors. All rights reserved.
  2. // Use of this source code is governed by a MIT-style
  3. // license that can be found in the LICENSE file.
  4. package models
  5. import (
  6. "fmt"
  7. "strings"
  8. "time"
  9. "github.com/Unknwon/com"
  10. "github.com/go-xorm/xorm"
  11. "gopkg.in/ini.v1"
  12. "code.gitea.io/gitea/modules/log"
  13. "code.gitea.io/gitea/modules/process"
  14. "code.gitea.io/gitea/modules/setting"
  15. "code.gitea.io/gitea/modules/sync"
  16. )
  17. // MirrorQueue holds an UniqueQueue object of the mirror
  18. var MirrorQueue = sync.NewUniqueQueue(setting.Repository.MirrorQueueLength)
  19. // Mirror represents mirror information of a repository.
  20. type Mirror struct {
  21. ID int64 `xorm:"pk autoincr"`
  22. RepoID int64 `xorm:"INDEX"`
  23. Repo *Repository `xorm:"-"`
  24. Interval int // Hour.
  25. EnablePrune bool `xorm:"NOT NULL DEFAULT true"`
  26. Updated time.Time `xorm:"-"`
  27. UpdatedUnix int64 `xorm:"INDEX"`
  28. NextUpdate time.Time `xorm:"-"`
  29. NextUpdateUnix int64 `xorm:"INDEX"`
  30. address string `xorm:"-"`
  31. }
  32. // BeforeInsert will be invoked by XORM before inserting a record
  33. func (m *Mirror) BeforeInsert() {
  34. m.UpdatedUnix = time.Now().Unix()
  35. m.NextUpdateUnix = m.NextUpdate.Unix()
  36. }
  37. // BeforeUpdate is invoked from XORM before updating this object.
  38. func (m *Mirror) BeforeUpdate() {
  39. m.UpdatedUnix = time.Now().Unix()
  40. m.NextUpdateUnix = m.NextUpdate.Unix()
  41. }
  42. // AfterSet is invoked from XORM after setting the value of a field of this object.
  43. func (m *Mirror) AfterSet(colName string, _ xorm.Cell) {
  44. var err error
  45. switch colName {
  46. case "repo_id":
  47. m.Repo, err = GetRepositoryByID(m.RepoID)
  48. if err != nil {
  49. log.Error(3, "GetRepositoryByID[%d]: %v", m.ID, err)
  50. }
  51. case "updated_unix":
  52. m.Updated = time.Unix(m.UpdatedUnix, 0).Local()
  53. case "next_updated_unix":
  54. m.NextUpdate = time.Unix(m.NextUpdateUnix, 0).Local()
  55. }
  56. }
  57. // ScheduleNextUpdate calculates and sets next update time.
  58. func (m *Mirror) ScheduleNextUpdate() {
  59. m.NextUpdate = time.Now().Add(time.Duration(m.Interval) * time.Hour)
  60. }
  61. func (m *Mirror) readAddress() {
  62. if len(m.address) > 0 {
  63. return
  64. }
  65. cfg, err := ini.Load(m.Repo.GitConfigPath())
  66. if err != nil {
  67. log.Error(4, "Load: %v", err)
  68. return
  69. }
  70. m.address = cfg.Section("remote \"origin\"").Key("url").Value()
  71. }
  72. // HandleCloneUserCredentials replaces user credentials from HTTP/HTTPS URL
  73. // with placeholder <credentials>.
  74. // It will fail for any other forms of clone addresses.
  75. func HandleCloneUserCredentials(url string, mosaics bool) string {
  76. i := strings.Index(url, "@")
  77. if i == -1 {
  78. return url
  79. }
  80. start := strings.Index(url, "://")
  81. if start == -1 {
  82. return url
  83. }
  84. if mosaics {
  85. return url[:start+3] + "<credentials>" + url[i:]
  86. }
  87. return url[:start+3] + url[i+1:]
  88. }
  89. // Address returns mirror address from Git repository config without credentials.
  90. func (m *Mirror) Address() string {
  91. m.readAddress()
  92. return HandleCloneUserCredentials(m.address, false)
  93. }
  94. // FullAddress returns mirror address from Git repository config.
  95. func (m *Mirror) FullAddress() string {
  96. m.readAddress()
  97. return m.address
  98. }
  99. // SaveAddress writes new address to Git repository config.
  100. func (m *Mirror) SaveAddress(addr string) error {
  101. configPath := m.Repo.GitConfigPath()
  102. cfg, err := ini.Load(configPath)
  103. if err != nil {
  104. return fmt.Errorf("Load: %v", err)
  105. }
  106. cfg.Section("remote \"origin\"").Key("url").SetValue(addr)
  107. return cfg.SaveToIndent(configPath, "\t")
  108. }
  109. // runSync returns true if sync finished without error.
  110. func (m *Mirror) runSync() bool {
  111. repoPath := m.Repo.RepoPath()
  112. wikiPath := m.Repo.WikiPath()
  113. timeout := time.Duration(setting.Git.Timeout.Mirror) * time.Second
  114. gitArgs := []string{"remote", "update"}
  115. if m.EnablePrune {
  116. gitArgs = append(gitArgs, "--prune")
  117. }
  118. if _, stderr, err := process.GetManager().ExecDir(
  119. timeout, repoPath, fmt.Sprintf("Mirror.runSync: %s", repoPath),
  120. "git", gitArgs...); err != nil {
  121. desc := fmt.Sprintf("Failed to update mirror repository '%s': %s", repoPath, stderr)
  122. log.Error(4, desc)
  123. if err = CreateRepositoryNotice(desc); err != nil {
  124. log.Error(4, "CreateRepositoryNotice: %v", err)
  125. }
  126. return false
  127. }
  128. if m.Repo.HasWiki() {
  129. if _, stderr, err := process.GetManager().ExecDir(
  130. timeout, wikiPath, fmt.Sprintf("Mirror.runSync: %s", wikiPath),
  131. "git", "remote", "update", "--prune"); err != nil {
  132. desc := fmt.Sprintf("Failed to update mirror wiki repository '%s': %s", wikiPath, stderr)
  133. log.Error(4, desc)
  134. if err = CreateRepositoryNotice(desc); err != nil {
  135. log.Error(4, "CreateRepositoryNotice: %v", err)
  136. }
  137. return false
  138. }
  139. }
  140. return true
  141. }
  142. func getMirrorByRepoID(e Engine, repoID int64) (*Mirror, error) {
  143. m := &Mirror{RepoID: repoID}
  144. has, err := e.Get(m)
  145. if err != nil {
  146. return nil, err
  147. } else if !has {
  148. return nil, ErrMirrorNotExist
  149. }
  150. return m, nil
  151. }
  152. // GetMirrorByRepoID returns mirror information of a repository.
  153. func GetMirrorByRepoID(repoID int64) (*Mirror, error) {
  154. return getMirrorByRepoID(x, repoID)
  155. }
  156. func updateMirror(e Engine, m *Mirror) error {
  157. _, err := e.Id(m.ID).AllCols().Update(m)
  158. return err
  159. }
  160. // UpdateMirror updates the mirror
  161. func UpdateMirror(m *Mirror) error {
  162. return updateMirror(x, m)
  163. }
  164. // DeleteMirrorByRepoID deletes a mirror by repoID
  165. func DeleteMirrorByRepoID(repoID int64) error {
  166. _, err := x.Delete(&Mirror{RepoID: repoID})
  167. return err
  168. }
  169. // MirrorUpdate checks and updates mirror repositories.
  170. func MirrorUpdate() {
  171. if taskStatusTable.IsRunning(mirrorUpdate) {
  172. return
  173. }
  174. taskStatusTable.Start(mirrorUpdate)
  175. defer taskStatusTable.Stop(mirrorUpdate)
  176. log.Trace("Doing: MirrorUpdate")
  177. if err := x.
  178. Where("next_update_unix<=?", time.Now().Unix()).
  179. Iterate(new(Mirror), func(idx int, bean interface{}) error {
  180. m := bean.(*Mirror)
  181. if m.Repo == nil {
  182. log.Error(4, "Disconnected mirror repository found: %d", m.ID)
  183. return nil
  184. }
  185. MirrorQueue.Add(m.RepoID)
  186. return nil
  187. }); err != nil {
  188. log.Error(4, "MirrorUpdate: %v", err)
  189. }
  190. }
  191. // SyncMirrors checks and syncs mirrors.
  192. // TODO: sync more mirrors at same time.
  193. func SyncMirrors() {
  194. // Start listening on new sync requests.
  195. for repoID := range MirrorQueue.Queue() {
  196. log.Trace("SyncMirrors [repo_id: %v]", repoID)
  197. MirrorQueue.Remove(repoID)
  198. m, err := GetMirrorByRepoID(com.StrTo(repoID).MustInt64())
  199. if err != nil {
  200. log.Error(4, "GetMirrorByRepoID [%s]: %v", repoID, err)
  201. continue
  202. }
  203. if !m.runSync() {
  204. continue
  205. }
  206. m.ScheduleNextUpdate()
  207. if err = UpdateMirror(m); err != nil {
  208. log.Error(4, "UpdateMirror [%s]: %v", repoID, err)
  209. continue
  210. }
  211. }
  212. }
  213. // InitSyncMirrors initializes a go routine to sync the mirrors
  214. func InitSyncMirrors() {
  215. go SyncMirrors()
  216. }