Fork to maintain patches against the official gitea for https://code.ceondo.com https://github.com/go-gitea/gitea

repo_mirror.go 8.0KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306
  1. // Copyright 2016 The Gogs Authors. All rights reserved.
  2. // Use of this source code is governed by a MIT-style
  3. // license that can be found in the LICENSE file.
  4. package models
  5. import (
  6. "fmt"
  7. "strings"
  8. "time"
  9. "github.com/Unknwon/com"
  10. "github.com/go-xorm/xorm"
  11. "gopkg.in/ini.v1"
  12. "code.gitea.io/git"
  13. "code.gitea.io/gitea/modules/log"
  14. "code.gitea.io/gitea/modules/process"
  15. "code.gitea.io/gitea/modules/setting"
  16. "code.gitea.io/gitea/modules/sync"
  17. )
  18. // MirrorQueue holds an UniqueQueue object of the mirror
  19. var MirrorQueue = sync.NewUniqueQueue(setting.Repository.MirrorQueueLength)
  20. // Mirror represents mirror information of a repository.
  21. type Mirror struct {
  22. ID int64 `xorm:"pk autoincr"`
  23. RepoID int64 `xorm:"INDEX"`
  24. Repo *Repository `xorm:"-"`
  25. Interval time.Duration
  26. EnablePrune bool `xorm:"NOT NULL DEFAULT true"`
  27. Updated time.Time `xorm:"-"`
  28. UpdatedUnix int64 `xorm:"INDEX"`
  29. NextUpdate time.Time `xorm:"-"`
  30. NextUpdateUnix int64 `xorm:"INDEX"`
  31. address string `xorm:"-"`
  32. }
  33. // BeforeInsert will be invoked by XORM before inserting a record
  34. func (m *Mirror) BeforeInsert() {
  35. if m != nil {
  36. m.UpdatedUnix = time.Now().Unix()
  37. m.NextUpdateUnix = m.NextUpdate.Unix()
  38. }
  39. }
  40. // BeforeUpdate is invoked from XORM before updating this object.
  41. func (m *Mirror) BeforeUpdate() {
  42. if m != nil {
  43. m.UpdatedUnix = m.Updated.Unix()
  44. m.NextUpdateUnix = m.NextUpdate.Unix()
  45. }
  46. }
  47. // AfterLoad is invoked from XORM after setting the values of all fields of this object.
  48. func (m *Mirror) AfterLoad(session *xorm.Session) {
  49. if m == nil {
  50. return
  51. }
  52. var err error
  53. m.Repo, err = getRepositoryByID(session, m.RepoID)
  54. if err != nil {
  55. log.Error(3, "getRepositoryByID[%d]: %v", m.ID, err)
  56. }
  57. m.Updated = time.Unix(m.UpdatedUnix, 0).Local()
  58. m.NextUpdate = time.Unix(m.NextUpdateUnix, 0).Local()
  59. }
  60. // ScheduleNextUpdate calculates and sets next update time.
  61. func (m *Mirror) ScheduleNextUpdate() {
  62. m.NextUpdate = time.Now().Add(m.Interval)
  63. }
  64. func remoteAddress(repoPath string) (string, error) {
  65. cfg, err := ini.Load(GitConfigPath(repoPath))
  66. if err != nil {
  67. return "", err
  68. }
  69. return cfg.Section("remote \"origin\"").Key("url").Value(), nil
  70. }
  71. func (m *Mirror) readAddress() {
  72. if len(m.address) > 0 {
  73. return
  74. }
  75. var err error
  76. m.address, err = remoteAddress(m.Repo.RepoPath())
  77. if err != nil {
  78. log.Error(4, "remoteAddress: %v", err)
  79. }
  80. }
  81. // HandleCloneUserCredentials replaces user credentials from HTTP/HTTPS URL
  82. // with placeholder <credentials>.
  83. // It will fail for any other forms of clone addresses.
  84. func HandleCloneUserCredentials(url string, mosaics bool) string {
  85. i := strings.Index(url, "@")
  86. if i == -1 {
  87. return url
  88. }
  89. start := strings.Index(url, "://")
  90. if start == -1 {
  91. return url
  92. }
  93. if mosaics {
  94. return url[:start+3] + "<credentials>" + url[i:]
  95. }
  96. return url[:start+3] + url[i+1:]
  97. }
  98. // sanitizeOutput sanitizes output of a command, replacing occurrences of the
  99. // repository's remote address with a sanitized version.
  100. func sanitizeOutput(output, repoPath string) (string, error) {
  101. remoteAddr, err := remoteAddress(repoPath)
  102. if err != nil {
  103. // if we're unable to load the remote address, then we're unable to
  104. // sanitize.
  105. return "", err
  106. }
  107. sanitized := HandleCloneUserCredentials(remoteAddr, true)
  108. return strings.Replace(output, remoteAddr, sanitized, -1), nil
  109. }
  110. // Address returns mirror address from Git repository config without credentials.
  111. func (m *Mirror) Address() string {
  112. m.readAddress()
  113. return HandleCloneUserCredentials(m.address, false)
  114. }
  115. // FullAddress returns mirror address from Git repository config.
  116. func (m *Mirror) FullAddress() string {
  117. m.readAddress()
  118. return m.address
  119. }
  120. // SaveAddress writes new address to Git repository config.
  121. func (m *Mirror) SaveAddress(addr string) error {
  122. configPath := m.Repo.GitConfigPath()
  123. cfg, err := ini.Load(configPath)
  124. if err != nil {
  125. return fmt.Errorf("Load: %v", err)
  126. }
  127. cfg.Section("remote \"origin\"").Key("url").SetValue(addr)
  128. return cfg.SaveToIndent(configPath, "\t")
  129. }
  130. // runSync returns true if sync finished without error.
  131. func (m *Mirror) runSync() bool {
  132. repoPath := m.Repo.RepoPath()
  133. wikiPath := m.Repo.WikiPath()
  134. timeout := time.Duration(setting.Git.Timeout.Mirror) * time.Second
  135. gitArgs := []string{"remote", "update"}
  136. if m.EnablePrune {
  137. gitArgs = append(gitArgs, "--prune")
  138. }
  139. if _, stderr, err := process.GetManager().ExecDir(
  140. timeout, repoPath, fmt.Sprintf("Mirror.runSync: %s", repoPath),
  141. "git", gitArgs...); err != nil {
  142. // sanitize the output, since it may contain the remote address, which may
  143. // contain a password
  144. message, err := sanitizeOutput(stderr, repoPath)
  145. if err != nil {
  146. log.Error(4, "sanitizeOutput: %v", err)
  147. return false
  148. }
  149. desc := fmt.Sprintf("Failed to update mirror repository '%s': %s", repoPath, message)
  150. log.Error(4, desc)
  151. if err = CreateRepositoryNotice(desc); err != nil {
  152. log.Error(4, "CreateRepositoryNotice: %v", err)
  153. }
  154. return false
  155. }
  156. gitRepo, err := git.OpenRepository(repoPath)
  157. if err != nil {
  158. log.Error(4, "OpenRepository: %v", err)
  159. return false
  160. }
  161. if err = SyncReleasesWithTags(m.Repo, gitRepo); err != nil {
  162. log.Error(4, "Failed to synchronize tags to releases for repository: %v", err)
  163. }
  164. if err := m.Repo.UpdateSize(); err != nil {
  165. log.Error(4, "Failed to update size for mirror repository: %v", err)
  166. }
  167. if m.Repo.HasWiki() {
  168. if _, stderr, err := process.GetManager().ExecDir(
  169. timeout, wikiPath, fmt.Sprintf("Mirror.runSync: %s", wikiPath),
  170. "git", "remote", "update", "--prune"); err != nil {
  171. // sanitize the output, since it may contain the remote address, which may
  172. // contain a password
  173. message, err := sanitizeOutput(stderr, wikiPath)
  174. if err != nil {
  175. log.Error(4, "sanitizeOutput: %v", err)
  176. return false
  177. }
  178. desc := fmt.Sprintf("Failed to update mirror wiki repository '%s': %s", wikiPath, message)
  179. log.Error(4, desc)
  180. if err = CreateRepositoryNotice(desc); err != nil {
  181. log.Error(4, "CreateRepositoryNotice: %v", err)
  182. }
  183. return false
  184. }
  185. }
  186. m.Updated = time.Now()
  187. return true
  188. }
  189. func getMirrorByRepoID(e Engine, repoID int64) (*Mirror, error) {
  190. m := &Mirror{RepoID: repoID}
  191. has, err := e.Get(m)
  192. if err != nil {
  193. return nil, err
  194. } else if !has {
  195. return nil, ErrMirrorNotExist
  196. }
  197. return m, nil
  198. }
  199. // GetMirrorByRepoID returns mirror information of a repository.
  200. func GetMirrorByRepoID(repoID int64) (*Mirror, error) {
  201. return getMirrorByRepoID(x, repoID)
  202. }
  203. func updateMirror(e Engine, m *Mirror) error {
  204. _, err := e.ID(m.ID).AllCols().Update(m)
  205. return err
  206. }
  207. // UpdateMirror updates the mirror
  208. func UpdateMirror(m *Mirror) error {
  209. return updateMirror(x, m)
  210. }
  211. // DeleteMirrorByRepoID deletes a mirror by repoID
  212. func DeleteMirrorByRepoID(repoID int64) error {
  213. _, err := x.Delete(&Mirror{RepoID: repoID})
  214. return err
  215. }
  216. // MirrorUpdate checks and updates mirror repositories.
  217. func MirrorUpdate() {
  218. if !taskStatusTable.StartIfNotRunning(mirrorUpdate) {
  219. return
  220. }
  221. defer taskStatusTable.Stop(mirrorUpdate)
  222. log.Trace("Doing: MirrorUpdate")
  223. if err := x.
  224. Where("next_update_unix<=?", time.Now().Unix()).
  225. Iterate(new(Mirror), func(idx int, bean interface{}) error {
  226. m := bean.(*Mirror)
  227. if m.Repo == nil {
  228. log.Error(4, "Disconnected mirror repository found: %d", m.ID)
  229. return nil
  230. }
  231. MirrorQueue.Add(m.RepoID)
  232. return nil
  233. }); err != nil {
  234. log.Error(4, "MirrorUpdate: %v", err)
  235. }
  236. }
  237. // SyncMirrors checks and syncs mirrors.
  238. // TODO: sync more mirrors at same time.
  239. func SyncMirrors() {
  240. // Start listening on new sync requests.
  241. for repoID := range MirrorQueue.Queue() {
  242. log.Trace("SyncMirrors [repo_id: %v]", repoID)
  243. MirrorQueue.Remove(repoID)
  244. m, err := GetMirrorByRepoID(com.StrTo(repoID).MustInt64())
  245. if err != nil {
  246. log.Error(4, "GetMirrorByRepoID [%s]: %v", repoID, err)
  247. continue
  248. }
  249. if !m.runSync() {
  250. continue
  251. }
  252. m.ScheduleNextUpdate()
  253. if err = UpdateMirror(m); err != nil {
  254. log.Error(4, "UpdateMirror [%s]: %v", repoID, err)
  255. continue
  256. }
  257. }
  258. }
  259. // InitSyncMirrors initializes a go routine to sync the mirrors
  260. func InitSyncMirrors() {
  261. go SyncMirrors()
  262. }