Fork to maintain patches against the official gitea for https://code.ceondo.com https://github.com/go-gitea/gitea

repo_mirror.go 7.5KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287
  1. // Copyright 2016 The Gogs Authors. All rights reserved.
  2. // Use of this source code is governed by a MIT-style
  3. // license that can be found in the LICENSE file.
  4. package models
  5. import (
  6. "fmt"
  7. "time"
  8. "code.gitea.io/git"
  9. "code.gitea.io/gitea/modules/log"
  10. "code.gitea.io/gitea/modules/process"
  11. "code.gitea.io/gitea/modules/setting"
  12. "code.gitea.io/gitea/modules/sync"
  13. "code.gitea.io/gitea/modules/util"
  14. "github.com/Unknwon/com"
  15. "github.com/go-xorm/xorm"
  16. "gopkg.in/ini.v1"
  17. )
  18. // MirrorQueue holds an UniqueQueue object of the mirror
  19. var MirrorQueue = sync.NewUniqueQueue(setting.Repository.MirrorQueueLength)
  20. // Mirror represents mirror information of a repository.
  21. type Mirror struct {
  22. ID int64 `xorm:"pk autoincr"`
  23. RepoID int64 `xorm:"INDEX"`
  24. Repo *Repository `xorm:"-"`
  25. Interval time.Duration
  26. EnablePrune bool `xorm:"NOT NULL DEFAULT true"`
  27. Updated time.Time `xorm:"-"`
  28. UpdatedUnix int64 `xorm:"INDEX"`
  29. NextUpdate time.Time `xorm:"-"`
  30. NextUpdateUnix int64 `xorm:"INDEX"`
  31. address string `xorm:"-"`
  32. }
  33. // BeforeInsert will be invoked by XORM before inserting a record
  34. func (m *Mirror) BeforeInsert() {
  35. if m != nil {
  36. m.UpdatedUnix = time.Now().Unix()
  37. m.NextUpdateUnix = m.NextUpdate.Unix()
  38. }
  39. }
  40. // BeforeUpdate is invoked from XORM before updating this object.
  41. func (m *Mirror) BeforeUpdate() {
  42. if m != nil {
  43. m.UpdatedUnix = m.Updated.Unix()
  44. m.NextUpdateUnix = m.NextUpdate.Unix()
  45. }
  46. }
  47. // AfterLoad is invoked from XORM after setting the values of all fields of this object.
  48. func (m *Mirror) AfterLoad(session *xorm.Session) {
  49. if m == nil {
  50. return
  51. }
  52. var err error
  53. m.Repo, err = getRepositoryByID(session, m.RepoID)
  54. if err != nil {
  55. log.Error(3, "getRepositoryByID[%d]: %v", m.ID, err)
  56. }
  57. m.Updated = time.Unix(m.UpdatedUnix, 0).Local()
  58. m.NextUpdate = time.Unix(m.NextUpdateUnix, 0).Local()
  59. }
  60. // ScheduleNextUpdate calculates and sets next update time.
  61. func (m *Mirror) ScheduleNextUpdate() {
  62. m.NextUpdate = time.Now().Add(m.Interval)
  63. }
  64. func remoteAddress(repoPath string) (string, error) {
  65. cfg, err := ini.Load(GitConfigPath(repoPath))
  66. if err != nil {
  67. return "", err
  68. }
  69. return cfg.Section("remote \"origin\"").Key("url").Value(), nil
  70. }
  71. func (m *Mirror) readAddress() {
  72. if len(m.address) > 0 {
  73. return
  74. }
  75. var err error
  76. m.address, err = remoteAddress(m.Repo.RepoPath())
  77. if err != nil {
  78. log.Error(4, "remoteAddress: %v", err)
  79. }
  80. }
  81. // sanitizeOutput sanitizes output of a command, replacing occurrences of the
  82. // repository's remote address with a sanitized version.
  83. func sanitizeOutput(output, repoPath string) (string, error) {
  84. remoteAddr, err := remoteAddress(repoPath)
  85. if err != nil {
  86. // if we're unable to load the remote address, then we're unable to
  87. // sanitize.
  88. return "", err
  89. }
  90. return util.SanitizeMessage(output, remoteAddr), nil
  91. }
  92. // Address returns mirror address from Git repository config without credentials.
  93. func (m *Mirror) Address() string {
  94. m.readAddress()
  95. return util.SanitizeURLCredentials(m.address, false)
  96. }
  97. // FullAddress returns mirror address from Git repository config.
  98. func (m *Mirror) FullAddress() string {
  99. m.readAddress()
  100. return m.address
  101. }
  102. // SaveAddress writes new address to Git repository config.
  103. func (m *Mirror) SaveAddress(addr string) error {
  104. configPath := m.Repo.GitConfigPath()
  105. cfg, err := ini.Load(configPath)
  106. if err != nil {
  107. return fmt.Errorf("Load: %v", err)
  108. }
  109. cfg.Section("remote \"origin\"").Key("url").SetValue(addr)
  110. return cfg.SaveToIndent(configPath, "\t")
  111. }
  112. // runSync returns true if sync finished without error.
  113. func (m *Mirror) runSync() bool {
  114. repoPath := m.Repo.RepoPath()
  115. wikiPath := m.Repo.WikiPath()
  116. timeout := time.Duration(setting.Git.Timeout.Mirror) * time.Second
  117. gitArgs := []string{"remote", "update"}
  118. if m.EnablePrune {
  119. gitArgs = append(gitArgs, "--prune")
  120. }
  121. if _, stderr, err := process.GetManager().ExecDir(
  122. timeout, repoPath, fmt.Sprintf("Mirror.runSync: %s", repoPath),
  123. "git", gitArgs...); err != nil {
  124. // sanitize the output, since it may contain the remote address, which may
  125. // contain a password
  126. message, err := sanitizeOutput(stderr, repoPath)
  127. if err != nil {
  128. log.Error(4, "sanitizeOutput: %v", err)
  129. return false
  130. }
  131. desc := fmt.Sprintf("Failed to update mirror repository '%s': %s", repoPath, message)
  132. log.Error(4, desc)
  133. if err = CreateRepositoryNotice(desc); err != nil {
  134. log.Error(4, "CreateRepositoryNotice: %v", err)
  135. }
  136. return false
  137. }
  138. gitRepo, err := git.OpenRepository(repoPath)
  139. if err != nil {
  140. log.Error(4, "OpenRepository: %v", err)
  141. return false
  142. }
  143. if err = SyncReleasesWithTags(m.Repo, gitRepo); err != nil {
  144. log.Error(4, "Failed to synchronize tags to releases for repository: %v", err)
  145. }
  146. if err := m.Repo.UpdateSize(); err != nil {
  147. log.Error(4, "Failed to update size for mirror repository: %v", err)
  148. }
  149. if m.Repo.HasWiki() {
  150. if _, stderr, err := process.GetManager().ExecDir(
  151. timeout, wikiPath, fmt.Sprintf("Mirror.runSync: %s", wikiPath),
  152. "git", "remote", "update", "--prune"); err != nil {
  153. // sanitize the output, since it may contain the remote address, which may
  154. // contain a password
  155. message, err := sanitizeOutput(stderr, wikiPath)
  156. if err != nil {
  157. log.Error(4, "sanitizeOutput: %v", err)
  158. return false
  159. }
  160. desc := fmt.Sprintf("Failed to update mirror wiki repository '%s': %s", wikiPath, message)
  161. log.Error(4, desc)
  162. if err = CreateRepositoryNotice(desc); err != nil {
  163. log.Error(4, "CreateRepositoryNotice: %v", err)
  164. }
  165. return false
  166. }
  167. }
  168. m.Updated = time.Now()
  169. return true
  170. }
  171. func getMirrorByRepoID(e Engine, repoID int64) (*Mirror, error) {
  172. m := &Mirror{RepoID: repoID}
  173. has, err := e.Get(m)
  174. if err != nil {
  175. return nil, err
  176. } else if !has {
  177. return nil, ErrMirrorNotExist
  178. }
  179. return m, nil
  180. }
  181. // GetMirrorByRepoID returns mirror information of a repository.
  182. func GetMirrorByRepoID(repoID int64) (*Mirror, error) {
  183. return getMirrorByRepoID(x, repoID)
  184. }
  185. func updateMirror(e Engine, m *Mirror) error {
  186. _, err := e.ID(m.ID).AllCols().Update(m)
  187. return err
  188. }
  189. // UpdateMirror updates the mirror
  190. func UpdateMirror(m *Mirror) error {
  191. return updateMirror(x, m)
  192. }
  193. // DeleteMirrorByRepoID deletes a mirror by repoID
  194. func DeleteMirrorByRepoID(repoID int64) error {
  195. _, err := x.Delete(&Mirror{RepoID: repoID})
  196. return err
  197. }
  198. // MirrorUpdate checks and updates mirror repositories.
  199. func MirrorUpdate() {
  200. if !taskStatusTable.StartIfNotRunning(mirrorUpdate) {
  201. return
  202. }
  203. defer taskStatusTable.Stop(mirrorUpdate)
  204. log.Trace("Doing: MirrorUpdate")
  205. if err := x.
  206. Where("next_update_unix<=?", time.Now().Unix()).
  207. Iterate(new(Mirror), func(idx int, bean interface{}) error {
  208. m := bean.(*Mirror)
  209. if m.Repo == nil {
  210. log.Error(4, "Disconnected mirror repository found: %d", m.ID)
  211. return nil
  212. }
  213. MirrorQueue.Add(m.RepoID)
  214. return nil
  215. }); err != nil {
  216. log.Error(4, "MirrorUpdate: %v", err)
  217. }
  218. }
  219. // SyncMirrors checks and syncs mirrors.
  220. // TODO: sync more mirrors at same time.
  221. func SyncMirrors() {
  222. // Start listening on new sync requests.
  223. for repoID := range MirrorQueue.Queue() {
  224. log.Trace("SyncMirrors [repo_id: %v]", repoID)
  225. MirrorQueue.Remove(repoID)
  226. m, err := GetMirrorByRepoID(com.StrTo(repoID).MustInt64())
  227. if err != nil {
  228. log.Error(4, "GetMirrorByRepoID [%s]: %v", repoID, err)
  229. continue
  230. }
  231. if !m.runSync() {
  232. continue
  233. }
  234. m.ScheduleNextUpdate()
  235. if err = UpdateMirror(m); err != nil {
  236. log.Error(4, "UpdateMirror [%s]: %v", repoID, err)
  237. continue
  238. }
  239. }
  240. }
  241. // InitSyncMirrors initializes a go routine to sync the mirrors
  242. func InitSyncMirrors() {
  243. go SyncMirrors()
  244. }