Fork to maintain patches against the official gitea for https://code.ceondo.com https://github.com/go-gitea/gitea

issue_indexer.go 4.9KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185
  1. // Copyright 2017 The Gitea Authors. All rights reserved.
  2. // Use of this source code is governed by a MIT-style
  3. // license that can be found in the LICENSE file.
  4. package models
  5. import (
  6. "fmt"
  7. "os"
  8. "strconv"
  9. "strings"
  10. "code.gitea.io/gitea/modules/log"
  11. "code.gitea.io/gitea/modules/setting"
  12. "code.gitea.io/gitea/modules/util"
  13. "github.com/blevesearch/bleve"
  14. "github.com/blevesearch/bleve/analysis/analyzer/simple"
  15. "github.com/blevesearch/bleve/search/query"
  16. )
  17. // issueIndexerUpdateQueue queue of issues that need to be updated in the issues
  18. // indexer
  19. var issueIndexerUpdateQueue chan *Issue
  20. // issueIndexer (thread-safe) index for searching issues
  21. var issueIndexer bleve.Index
  22. // issueIndexerData data stored in the issue indexer
  23. type issueIndexerData struct {
  24. ID int64
  25. RepoID int64
  26. Title string
  27. Content string
  28. }
  29. // numericQuery an numeric-equality query for the given value and field
  30. func numericQuery(value int64, field string) *query.NumericRangeQuery {
  31. f := float64(value)
  32. tru := true
  33. q := bleve.NewNumericRangeInclusiveQuery(&f, &f, &tru, &tru)
  34. q.SetField(field)
  35. return q
  36. }
  37. // SearchIssuesByKeyword searches for issues by given conditions.
  38. // Returns the matching issue IDs
  39. func SearchIssuesByKeyword(repoID int64, keyword string) ([]int64, error) {
  40. terms := strings.Fields(strings.ToLower(keyword))
  41. indexerQuery := bleve.NewConjunctionQuery(
  42. numericQuery(repoID, "RepoID"),
  43. bleve.NewDisjunctionQuery(
  44. bleve.NewPhraseQuery(terms, "Title"),
  45. bleve.NewPhraseQuery(terms, "Content"),
  46. ))
  47. search := bleve.NewSearchRequestOptions(indexerQuery, 2147483647, 0, false)
  48. search.Fields = []string{"ID"}
  49. result, err := issueIndexer.Search(search)
  50. if err != nil {
  51. return nil, err
  52. }
  53. issueIDs := make([]int64, len(result.Hits))
  54. for i, hit := range result.Hits {
  55. issueIDs[i] = int64(hit.Fields["ID"].(float64))
  56. }
  57. return issueIDs, nil
  58. }
  59. // InitIssueIndexer initialize issue indexer
  60. func InitIssueIndexer() {
  61. _, err := os.Stat(setting.Indexer.IssuePath)
  62. if err != nil {
  63. if os.IsNotExist(err) {
  64. if err = createIssueIndexer(); err != nil {
  65. log.Fatal(4, "CreateIssuesIndexer: %v", err)
  66. }
  67. if err = populateIssueIndexer(); err != nil {
  68. log.Fatal(4, "PopulateIssuesIndex: %v", err)
  69. }
  70. } else {
  71. log.Fatal(4, "InitIssuesIndexer: %v", err)
  72. }
  73. } else {
  74. issueIndexer, err = bleve.Open(setting.Indexer.IssuePath)
  75. if err != nil {
  76. log.Fatal(4, "InitIssuesIndexer, open index: %v", err)
  77. }
  78. }
  79. issueIndexerUpdateQueue = make(chan *Issue, setting.Indexer.UpdateQueueLength)
  80. go processIssueIndexerUpdateQueue()
  81. // TODO close issueIndexer when Gitea closes
  82. }
  83. // createIssueIndexer create an issue indexer if one does not already exist
  84. func createIssueIndexer() error {
  85. mapping := bleve.NewIndexMapping()
  86. docMapping := bleve.NewDocumentMapping()
  87. docMapping.AddFieldMappingsAt("ID", bleve.NewNumericFieldMapping())
  88. docMapping.AddFieldMappingsAt("RepoID", bleve.NewNumericFieldMapping())
  89. textFieldMapping := bleve.NewTextFieldMapping()
  90. textFieldMapping.Analyzer = simple.Name
  91. docMapping.AddFieldMappingsAt("Title", textFieldMapping)
  92. docMapping.AddFieldMappingsAt("Content", textFieldMapping)
  93. mapping.AddDocumentMapping("issues", docMapping)
  94. var err error
  95. issueIndexer, err = bleve.New(setting.Indexer.IssuePath, mapping)
  96. return err
  97. }
  98. // populateIssueIndexer populate the issue indexer with issue data
  99. func populateIssueIndexer() error {
  100. for page := 1; ; page++ {
  101. repos, _, err := Repositories(&SearchRepoOptions{
  102. Page: page,
  103. PageSize: 10,
  104. })
  105. if err != nil {
  106. return fmt.Errorf("Repositories: %v", err)
  107. }
  108. if len(repos) == 0 {
  109. return nil
  110. }
  111. batch := issueIndexer.NewBatch()
  112. for _, repo := range repos {
  113. issues, err := Issues(&IssuesOptions{
  114. RepoID: repo.ID,
  115. IsClosed: util.OptionalBoolNone,
  116. IsPull: util.OptionalBoolNone,
  117. Page: -1, // do not page
  118. })
  119. if err != nil {
  120. return fmt.Errorf("Issues: %v", err)
  121. }
  122. for _, issue := range issues {
  123. err = batch.Index(issue.indexUID(), issue.issueData())
  124. if err != nil {
  125. return fmt.Errorf("batch.Index: %v", err)
  126. }
  127. }
  128. }
  129. if err = issueIndexer.Batch(batch); err != nil {
  130. return fmt.Errorf("index.Batch: %v", err)
  131. }
  132. }
  133. }
  134. func processIssueIndexerUpdateQueue() {
  135. for {
  136. select {
  137. case issue := <-issueIndexerUpdateQueue:
  138. if err := issueIndexer.Index(issue.indexUID(), issue.issueData()); err != nil {
  139. log.Error(4, "issuesIndexer.Index: %v", err)
  140. }
  141. }
  142. }
  143. }
  144. // indexUID a unique identifier for an issue used in full-text indices
  145. func (issue *Issue) indexUID() string {
  146. return strconv.FormatInt(issue.ID, 36)
  147. }
  148. func (issue *Issue) issueData() *issueIndexerData {
  149. return &issueIndexerData{
  150. ID: issue.ID,
  151. RepoID: issue.RepoID,
  152. Title: issue.Title,
  153. Content: issue.Content,
  154. }
  155. }
  156. // UpdateIssueIndexer add/update an issue to the issue indexer
  157. func UpdateIssueIndexer(issue *Issue) {
  158. go func() {
  159. issueIndexerUpdateQueue <- issue
  160. }()
  161. }