Fork to maintain patches against the official gitea for https://code.ceondo.com https://github.com/go-gitea/gitea

repo_indexer.go 7.7KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303
  1. // Copyright 2017 The Gitea Authors. All rights reserved.
  2. // Use of this source code is governed by a MIT-style
  3. // license that can be found in the LICENSE file.
  4. package models
  5. import (
  6. "io/ioutil"
  7. "os"
  8. "path"
  9. "strconv"
  10. "strings"
  11. "code.gitea.io/git"
  12. "code.gitea.io/gitea/modules/base"
  13. "code.gitea.io/gitea/modules/indexer"
  14. "code.gitea.io/gitea/modules/log"
  15. "code.gitea.io/gitea/modules/setting"
  16. "github.com/Unknwon/com"
  17. )
  18. // RepoIndexerStatus status of a repo's entry in the repo indexer
  19. // For now, implicitly refers to default branch
  20. type RepoIndexerStatus struct {
  21. ID int64 `xorm:"pk autoincr"`
  22. RepoID int64 `xorm:"INDEX"`
  23. CommitSha string `xorm:"VARCHAR(40)"`
  24. }
  25. func (repo *Repository) getIndexerStatus() error {
  26. if repo.IndexerStatus != nil {
  27. return nil
  28. }
  29. status := &RepoIndexerStatus{RepoID: repo.ID}
  30. has, err := x.Get(status)
  31. if err != nil {
  32. return err
  33. } else if !has {
  34. status.CommitSha = ""
  35. }
  36. repo.IndexerStatus = status
  37. return nil
  38. }
  39. func (repo *Repository) updateIndexerStatus(sha string) error {
  40. if err := repo.getIndexerStatus(); err != nil {
  41. return err
  42. }
  43. if len(repo.IndexerStatus.CommitSha) == 0 {
  44. repo.IndexerStatus.CommitSha = sha
  45. _, err := x.Insert(repo.IndexerStatus)
  46. return err
  47. }
  48. repo.IndexerStatus.CommitSha = sha
  49. _, err := x.ID(repo.IndexerStatus.ID).Cols("commit_sha").
  50. Update(repo.IndexerStatus)
  51. return err
  52. }
  53. type repoIndexerOperation struct {
  54. repo *Repository
  55. deleted bool
  56. }
  57. var repoIndexerOperationQueue chan repoIndexerOperation
  58. // InitRepoIndexer initialize the repo indexer
  59. func InitRepoIndexer() {
  60. if !setting.Indexer.RepoIndexerEnabled {
  61. return
  62. }
  63. indexer.InitRepoIndexer(populateRepoIndexer)
  64. repoIndexerOperationQueue = make(chan repoIndexerOperation, setting.Indexer.UpdateQueueLength)
  65. go processRepoIndexerOperationQueue()
  66. }
  67. // populateRepoIndexer populate the repo indexer with data
  68. func populateRepoIndexer() error {
  69. log.Info("Populating repository indexer (this may take a while)")
  70. for page := 1; ; page++ {
  71. repos, _, err := SearchRepositoryByName(&SearchRepoOptions{
  72. Page: page,
  73. PageSize: 10,
  74. OrderBy: SearchOrderByID,
  75. Private: true,
  76. })
  77. if err != nil {
  78. return err
  79. } else if len(repos) == 0 {
  80. return nil
  81. }
  82. for _, repo := range repos {
  83. if err = updateRepoIndexer(repo); err != nil {
  84. // only log error, since this should not prevent
  85. // gitea from starting up
  86. log.Error(4, "updateRepoIndexer: repoID=%d, %v", repo.ID, err)
  87. }
  88. }
  89. }
  90. }
  91. type updateBatch struct {
  92. updates []indexer.RepoIndexerUpdate
  93. }
  94. func updateRepoIndexer(repo *Repository) error {
  95. changes, err := getRepoChanges(repo)
  96. if err != nil {
  97. return err
  98. } else if changes == nil {
  99. return nil
  100. }
  101. batch := indexer.RepoIndexerBatch()
  102. for _, filename := range changes.UpdatedFiles {
  103. if err := addUpdate(filename, repo, batch); err != nil {
  104. return err
  105. }
  106. }
  107. for _, filename := range changes.RemovedFiles {
  108. if err := addDelete(filename, repo, batch); err != nil {
  109. return err
  110. }
  111. }
  112. if err = batch.Flush(); err != nil {
  113. return err
  114. }
  115. return updateLastIndexSync(repo)
  116. }
  117. // repoChanges changes (file additions/updates/removals) to a repo
  118. type repoChanges struct {
  119. UpdatedFiles []string
  120. RemovedFiles []string
  121. }
  122. // getRepoChanges returns changes to repo since last indexer update
  123. func getRepoChanges(repo *Repository) (*repoChanges, error) {
  124. repoWorkingPool.CheckIn(com.ToStr(repo.ID))
  125. defer repoWorkingPool.CheckOut(com.ToStr(repo.ID))
  126. if err := repo.UpdateLocalCopyBranch(""); err != nil {
  127. return nil, err
  128. } else if !git.IsBranchExist(repo.LocalCopyPath(), repo.DefaultBranch) {
  129. // repo does not have any commits yet, so nothing to update
  130. return nil, nil
  131. } else if err = repo.UpdateLocalCopyBranch(repo.DefaultBranch); err != nil {
  132. return nil, err
  133. } else if err = repo.getIndexerStatus(); err != nil {
  134. return nil, err
  135. }
  136. if len(repo.IndexerStatus.CommitSha) == 0 {
  137. return genesisChanges(repo)
  138. }
  139. return nonGenesisChanges(repo)
  140. }
  141. func addUpdate(filename string, repo *Repository, batch *indexer.Batch) error {
  142. filepath := path.Join(repo.LocalCopyPath(), filename)
  143. if stat, err := os.Stat(filepath); err != nil {
  144. return err
  145. } else if stat.Size() > setting.Indexer.MaxIndexerFileSize {
  146. return nil
  147. }
  148. fileContents, err := ioutil.ReadFile(filepath)
  149. if err != nil {
  150. return err
  151. } else if !base.IsTextFile(fileContents) {
  152. return nil
  153. }
  154. return batch.Add(indexer.RepoIndexerUpdate{
  155. Filepath: filename,
  156. Op: indexer.RepoIndexerOpUpdate,
  157. Data: &indexer.RepoIndexerData{
  158. RepoID: repo.ID,
  159. Content: string(fileContents),
  160. },
  161. })
  162. }
  163. func addDelete(filename string, repo *Repository, batch *indexer.Batch) error {
  164. return batch.Add(indexer.RepoIndexerUpdate{
  165. Filepath: filename,
  166. Op: indexer.RepoIndexerOpDelete,
  167. Data: &indexer.RepoIndexerData{
  168. RepoID: repo.ID,
  169. },
  170. })
  171. }
  172. // genesisChanges get changes to add repo to the indexer for the first time
  173. func genesisChanges(repo *Repository) (*repoChanges, error) {
  174. var changes repoChanges
  175. stdout, err := git.NewCommand("ls-files").RunInDir(repo.LocalCopyPath())
  176. if err != nil {
  177. return nil, err
  178. }
  179. for _, line := range strings.Split(stdout, "\n") {
  180. filename := strings.TrimSpace(line)
  181. if len(filename) == 0 {
  182. continue
  183. } else if filename[0] == '"' {
  184. filename, err = strconv.Unquote(filename)
  185. if err != nil {
  186. return nil, err
  187. }
  188. }
  189. changes.UpdatedFiles = append(changes.UpdatedFiles, filename)
  190. }
  191. return &changes, nil
  192. }
  193. // nonGenesisChanges get changes since the previous indexer update
  194. func nonGenesisChanges(repo *Repository) (*repoChanges, error) {
  195. diffCmd := git.NewCommand("diff", "--name-status",
  196. repo.IndexerStatus.CommitSha, "HEAD")
  197. stdout, err := diffCmd.RunInDir(repo.LocalCopyPath())
  198. if err != nil {
  199. // previous commit sha may have been removed by a force push, so
  200. // try rebuilding from scratch
  201. if err = indexer.DeleteRepoFromIndexer(repo.ID); err != nil {
  202. return nil, err
  203. }
  204. return genesisChanges(repo)
  205. }
  206. var changes repoChanges
  207. for _, line := range strings.Split(stdout, "\n") {
  208. line = strings.TrimSpace(line)
  209. if len(line) == 0 {
  210. continue
  211. }
  212. filename := strings.TrimSpace(line[1:])
  213. if len(filename) == 0 {
  214. continue
  215. } else if filename[0] == '"' {
  216. filename, err = strconv.Unquote(filename)
  217. if err != nil {
  218. return nil, err
  219. }
  220. }
  221. switch status := line[0]; status {
  222. case 'M', 'A':
  223. changes.UpdatedFiles = append(changes.UpdatedFiles, filename)
  224. case 'D':
  225. changes.RemovedFiles = append(changes.RemovedFiles, filename)
  226. default:
  227. log.Warn("Unrecognized status: %c (line=%s)", status, line)
  228. }
  229. }
  230. return &changes, nil
  231. }
  232. func updateLastIndexSync(repo *Repository) error {
  233. stdout, err := git.NewCommand("rev-parse", "HEAD").RunInDir(repo.LocalCopyPath())
  234. if err != nil {
  235. return err
  236. }
  237. sha := strings.TrimSpace(stdout)
  238. return repo.updateIndexerStatus(sha)
  239. }
  240. func processRepoIndexerOperationQueue() {
  241. for {
  242. op := <-repoIndexerOperationQueue
  243. if op.deleted {
  244. if err := indexer.DeleteRepoFromIndexer(op.repo.ID); err != nil {
  245. log.Error(4, "DeleteRepoFromIndexer: %v", err)
  246. }
  247. } else {
  248. if err := updateRepoIndexer(op.repo); err != nil {
  249. log.Error(4, "updateRepoIndexer: %v", err)
  250. }
  251. }
  252. }
  253. }
  254. // DeleteRepoFromIndexer remove all of a repository's entries from the indexer
  255. func DeleteRepoFromIndexer(repo *Repository) {
  256. addOperationToQueue(repoIndexerOperation{repo: repo, deleted: true})
  257. }
  258. // UpdateRepoIndexer update a repository's entries in the indexer
  259. func UpdateRepoIndexer(repo *Repository) {
  260. addOperationToQueue(repoIndexerOperation{repo: repo, deleted: false})
  261. }
  262. func addOperationToQueue(op repoIndexerOperation) {
  263. if !setting.Indexer.RepoIndexerEnabled {
  264. return
  265. }
  266. select {
  267. case repoIndexerOperationQueue <- op:
  268. break
  269. default:
  270. go func() {
  271. repoIndexerOperationQueue <- op
  272. }()
  273. }
  274. }