Browse Source

Improve issue search (#2387)

* Improve issue indexer

* Fix new issue sqlite bug

* Different test indexer paths for each db

* Add integration indexer paths to make clean
Ethan Koenig 2 years ago
parent
commit
b0f7457d9e
100 changed files with 3290 additions and 1123 deletions
  1. 3 0
      .gitignore
  2. 5 1
      Makefile
  3. 8 1
      integrations/integration_test.go
  4. 38 5
      integrations/issue_test.go
  5. 3 0
      integrations/mysql.ini.tmpl
  6. 3 0
      integrations/pgsql.ini.tmpl
  7. 3 0
      integrations/sqlite.ini
  8. 5 5
      models/fixtures/issue.yml
  9. 17 12
      models/issue.go
  10. 20 4
      models/issue_comment.go
  11. 40 124
      models/issue_indexer.go
  12. 2 0
      models/pull.go
  13. 33 4
      modules/indexer/indexer.go
  14. 143 0
      modules/indexer/issue.go
  15. 1 2
      routers/init.go
  16. 2 1
      routers/repo/issue.go
  17. 25 20
      vendor/github.com/blevesearch/bleve/README.md
  18. 145 0
      vendor/github.com/blevesearch/bleve/analysis/analyzer/custom/custom.go
  19. 0 46
      vendor/github.com/blevesearch/bleve/analysis/analyzer/simple/simple.go
  20. 79 0
      vendor/github.com/blevesearch/bleve/analysis/token/unicodenorm/unicodenorm.go
  21. 0 76
      vendor/github.com/blevesearch/bleve/analysis/tokenizer/character/character.go
  22. 0 33
      vendor/github.com/blevesearch/bleve/analysis/tokenizer/letter/letter.go
  23. 0 23
      vendor/github.com/blevesearch/bleve/config_app.go
  24. 137 0
      vendor/github.com/blevesearch/bleve/document/field_geopoint.go
  25. 9 0
      vendor/github.com/blevesearch/bleve/geo/README.md
  26. 170 0
      vendor/github.com/blevesearch/bleve/geo/geo.go
  27. 98 0
      vendor/github.com/blevesearch/bleve/geo/geo_dist.go
  28. 140 0
      vendor/github.com/blevesearch/bleve/geo/parse.go
  29. 212 0
      vendor/github.com/blevesearch/bleve/geo/sloppy.go
  30. 18 4
      vendor/github.com/blevesearch/bleve/index.go
  31. 3 1
      vendor/github.com/blevesearch/bleve/index/index.go
  32. 3 3
      vendor/github.com/blevesearch/bleve/index/upsidedown/analysis.go
  33. 6 4
      vendor/github.com/blevesearch/bleve/index/upsidedown/dump.go
  34. 28 14
      vendor/github.com/blevesearch/bleve/index/upsidedown/index_reader.go
  35. 37 26
      vendor/github.com/blevesearch/bleve/index/upsidedown/reader.go
  36. 284 25
      vendor/github.com/blevesearch/bleve/index/upsidedown/row.go
  37. 78 43
      vendor/github.com/blevesearch/bleve/index/upsidedown/upsidedown.go
  38. 73 69
      vendor/github.com/blevesearch/bleve/index/upsidedown/upsidedown.pb.go
  39. 4 4
      vendor/github.com/blevesearch/bleve/index/upsidedown/upsidedown.proto
  40. 9 8
      vendor/github.com/blevesearch/bleve/index_alias_impl.go
  41. 30 1
      vendor/github.com/blevesearch/bleve/index_impl.go
  42. 4 0
      vendor/github.com/blevesearch/bleve/mapping.go
  43. 50 1
      vendor/github.com/blevesearch/bleve/mapping/document.go
  44. 25 0
      vendor/github.com/blevesearch/bleve/mapping/field.go
  45. 6 1
      vendor/github.com/blevesearch/bleve/mapping/index.go
  46. 11 2
      vendor/github.com/blevesearch/bleve/mapping/mapping.go
  47. 43 0
      vendor/github.com/blevesearch/bleve/numeric/bin.go
  48. 32 0
      vendor/github.com/blevesearch/bleve/query.go
  49. 63 29
      vendor/github.com/blevesearch/bleve/search.go
  50. 17 13
      vendor/github.com/blevesearch/bleve/search/collector/heap.go
  51. 12 3
      vendor/github.com/blevesearch/bleve/search/collector/list.go
  52. 12 3
      vendor/github.com/blevesearch/bleve/search/collector/slice.go
  53. 53 30
      vendor/github.com/blevesearch/bleve/search/collector/topn.go
  54. 26 27
      vendor/github.com/blevesearch/bleve/search/facet/facet_builder_datetime.go
  55. 26 27
      vendor/github.com/blevesearch/bleve/search/facet/facet_builder_numeric.go
  56. 15 14
      vendor/github.com/blevesearch/bleve/search/facet/facet_builder_terms.go
  57. 20 21
      vendor/github.com/blevesearch/bleve/search/facets_builder.go
  58. 1 1
      vendor/github.com/blevesearch/bleve/search/highlight/format/html/html.go
  59. 1 1
      vendor/github.com/blevesearch/bleve/search/highlight/highlighter/simple/fragment_scorer_simple.go
  60. 1 1
      vendor/github.com/blevesearch/bleve/search/highlight/highlighter/simple/highlighter_simple.go
  61. 1 13
      vendor/github.com/blevesearch/bleve/search/highlight/term_locations.go
  62. 8 4
      vendor/github.com/blevesearch/bleve/search/pool.go
  63. 6 7
      vendor/github.com/blevesearch/bleve/search/query/bool_field.go
  64. 54 15
      vendor/github.com/blevesearch/bleve/search/query/boolean.go
  65. 17 9
      vendor/github.com/blevesearch/bleve/search/query/conjunction.go
  66. 4 5
      vendor/github.com/blevesearch/bleve/search/query/date_range.go
  67. 18 11
      vendor/github.com/blevesearch/bleve/search/query/disjunction.go
  68. 3 3
      vendor/github.com/blevesearch/bleve/search/query/docid.go
  69. 4 4
      vendor/github.com/blevesearch/bleve/search/query/fuzzy.go
  70. 113 0
      vendor/github.com/blevesearch/bleve/search/query/geo_boundingbox.go
  71. 100 0
      vendor/github.com/blevesearch/bleve/search/query/geo_distance.go
  72. 6 6
      vendor/github.com/blevesearch/bleve/search/query/match.go
  73. 3 5
      vendor/github.com/blevesearch/bleve/search/query/match_all.go
  74. 2 2
      vendor/github.com/blevesearch/bleve/search/query/match_none.go
  75. 9 12
      vendor/github.com/blevesearch/bleve/search/query/match_phrase.go
  76. 80 0
      vendor/github.com/blevesearch/bleve/search/query/multi_phrase.go
  77. 4 4
      vendor/github.com/blevesearch/bleve/search/query/numeric_range.go
  78. 9 29
      vendor/github.com/blevesearch/bleve/search/query/phrase.go
  79. 4 4
      vendor/github.com/blevesearch/bleve/search/query/prefix.go
  80. 40 12
      vendor/github.com/blevesearch/bleve/search/query/query.go
  81. 7 3
      vendor/github.com/blevesearch/bleve/search/query/query_string.go
  82. 51 12
      vendor/github.com/blevesearch/bleve/search/query/query_string.y
  83. 95 53
      vendor/github.com/blevesearch/bleve/search/query/query_string.y.go
  84. 8 2
      vendor/github.com/blevesearch/bleve/search/query/query_string_parser.go
  85. 13 11
      vendor/github.com/blevesearch/bleve/search/query/regexp.go
  86. 4 4
      vendor/github.com/blevesearch/bleve/search/query/term.go
  87. 95 0
      vendor/github.com/blevesearch/bleve/search/query/term_range.go
  88. 5 5
      vendor/github.com/blevesearch/bleve/search/query/wildcard.go
  89. 6 6
      vendor/github.com/blevesearch/bleve/search/scorer/scorer_conjunction.go
  90. 7 7
      vendor/github.com/blevesearch/bleve/search/scorer/scorer_constant.go
  91. 7 7
      vendor/github.com/blevesearch/bleve/search/scorer/scorer_disjunction.go
  92. 29 27
      vendor/github.com/blevesearch/bleve/search/scorer/scorer_term.go
  93. 22 16
      vendor/github.com/blevesearch/bleve/search/search.go
  94. 2 2
      vendor/github.com/blevesearch/bleve/search/searcher/search_boolean.go
  95. 4 4
      vendor/github.com/blevesearch/bleve/search/searcher/search_conjunction.go
  96. 19 6
      vendor/github.com/blevesearch/bleve/search/searcher/search_disjunction.go
  97. 2 2
      vendor/github.com/blevesearch/bleve/search/searcher/search_docid.go
  98. 88 0
      vendor/github.com/blevesearch/bleve/search/searcher/search_filter.go
  99. 9 78
      vendor/github.com/blevesearch/bleve/search/searcher/search_fuzzy.go
  100. 0 0
      vendor/github.com/blevesearch/bleve/search/searcher/search_geoboundingbox.go

+ 3 - 0
.gitignore

@@ -53,5 +53,8 @@ coverage.all
53 53
 /integrations/gitea-integration-mysql
54 54
 /integrations/gitea-integration-pgsql
55 55
 /integrations/gitea-integration-sqlite
56
+/integrations/indexers-mysql
57
+/integrations/indexers-pgsql
58
+/integrations/indexers-sqlite
56 59
 /integrations/mysql.ini
57 60
 /integrations/pgsql.ini

+ 5 - 1
Makefile

@@ -63,7 +63,11 @@ all: build
63 63
 .PHONY: clean
64 64
 clean:
65 65
 	$(GO) clean -i ./...
66
-	rm -rf $(EXECUTABLE) $(DIST) $(BINDATA) integrations*.test integrations/gitea-integration-pgsql/ integrations/gitea-integration-mysql/ integrations/gitea-integration-sqlite/ integrations/mysql.ini integrations/pgsql.ini
66
+	rm -rf $(EXECUTABLE) $(DIST) $(BINDATA) \
67
+		integrations*.test \
68
+		integrations/gitea-integration-pgsql/ integrations/gitea-integration-mysql/ integrations/gitea-integration-sqlite/ \
69
+		integrations/indexers-mysql/ integrations/indexers-pgsql integrations/indexers-sqlite \
70
+		integrations/mysql.ini integrations/pgsql.ini
67 71
 
68 72
 required-gofmt-version:
69 73
 	@$(GO) version  | grep -q '\(1.7\|1.8\)' || { echo "We require go version 1.7 or 1.8 to format code" >&2 && exit 1; }

+ 8 - 1
integrations/integration_test.go

@@ -57,7 +57,14 @@ func TestMain(m *testing.M) {
57 57
 		fmt.Printf("Error initializing test database: %v\n", err)
58 58
 		os.Exit(1)
59 59
 	}
60
-	os.Exit(m.Run())
60
+	exitCode := m.Run()
61
+
62
+	if err = os.RemoveAll(setting.Indexer.IssuePath); err != nil {
63
+		fmt.Printf("os.RemoveAll: %v\n", err)
64
+		os.Exit(1)
65
+	}
66
+
67
+	os.Exit(exitCode)
61 68
 }
62 69
 
63 70
 func initIntegrationTest() {

+ 38 - 5
integrations/issue_test.go

@@ -18,8 +18,10 @@ import (
18 18
 	"github.com/stretchr/testify/assert"
19 19
 )
20 20
 
21
-func getIssuesSelection(htmlDoc *HTMLDoc) *goquery.Selection {
22
-	return htmlDoc.doc.Find(".issue.list").Find("li").Find(".title")
21
+func getIssuesSelection(t testing.TB, htmlDoc *HTMLDoc) *goquery.Selection {
22
+	issueList := htmlDoc.doc.Find(".issue.list")
23
+	assert.EqualValues(t, 1, issueList.Length())
24
+	return issueList.Find("li").Find(".title")
23 25
 }
24 26
 
25 27
 func getIssue(t *testing.T, repoID int64, issueSelection *goquery.Selection) *models.Issue {
@@ -31,6 +33,18 @@ func getIssue(t *testing.T, repoID int64, issueSelection *goquery.Selection) *mo
31 33
 	return models.AssertExistsAndLoadBean(t, &models.Issue{RepoID: repoID, Index: int64(index)}).(*models.Issue)
32 34
 }
33 35
 
36
+func assertMatch(t testing.TB, issue *models.Issue, keyword string) {
37
+	matches := strings.Contains(strings.ToLower(issue.Title), keyword) ||
38
+		strings.Contains(strings.ToLower(issue.Content), keyword)
39
+	for _, comment := range issue.Comments {
40
+		matches = matches || strings.Contains(
41
+			strings.ToLower(comment.Content),
42
+			keyword,
43
+		)
44
+	}
45
+	assert.True(t, matches)
46
+}
47
+
34 48
 func TestNoLoginViewIssues(t *testing.T) {
35 49
 	prepareTestEnv(t)
36 50
 
@@ -38,19 +52,18 @@ func TestNoLoginViewIssues(t *testing.T) {
38 52
 	MakeRequest(t, req, http.StatusOK)
39 53
 }
40 54
 
41
-func TestNoLoginViewIssuesSortByType(t *testing.T) {
55
+func TestViewIssuesSortByType(t *testing.T) {
42 56
 	prepareTestEnv(t)
43 57
 
44 58
 	user := models.AssertExistsAndLoadBean(t, &models.User{ID: 1}).(*models.User)
45 59
 	repo := models.AssertExistsAndLoadBean(t, &models.Repository{ID: 1}).(*models.Repository)
46
-	repo.Owner = models.AssertExistsAndLoadBean(t, &models.User{ID: repo.OwnerID}).(*models.User)
47 60
 
48 61
 	session := loginUser(t, user.Name)
49 62
 	req := NewRequest(t, "GET", repo.RelLink()+"/issues?type=created_by")
50 63
 	resp := session.MakeRequest(t, req, http.StatusOK)
51 64
 
52 65
 	htmlDoc := NewHTMLParser(t, resp.Body)
53
-	issuesSelection := getIssuesSelection(htmlDoc)
66
+	issuesSelection := getIssuesSelection(t, htmlDoc)
54 67
 	expectedNumIssues := models.GetCount(t,
55 68
 		&models.Issue{RepoID: repo.ID, PosterID: user.ID},
56 69
 		models.Cond("is_closed=?", false),
@@ -67,6 +80,26 @@ func TestNoLoginViewIssuesSortByType(t *testing.T) {
67 80
 	})
68 81
 }
69 82
 
83
+func TestViewIssuesKeyword(t *testing.T) {
84
+	prepareTestEnv(t)
85
+
86
+	repo := models.AssertExistsAndLoadBean(t, &models.Repository{ID: 1}).(*models.Repository)
87
+
88
+	const keyword = "first"
89
+	req := NewRequestf(t, "GET", "%s/issues?q=%s", repo.RelLink(), keyword)
90
+	resp := MakeRequest(t, req, http.StatusOK)
91
+
92
+	htmlDoc := NewHTMLParser(t, resp.Body)
93
+	issuesSelection := getIssuesSelection(t, htmlDoc)
94
+	assert.EqualValues(t, 1, issuesSelection.Length())
95
+	issuesSelection.Each(func(_ int, selection *goquery.Selection) {
96
+		issue := getIssue(t, repo.ID, selection)
97
+		assert.False(t, issue.IsClosed)
98
+		assert.False(t, issue.IsPull)
99
+		assertMatch(t, issue, keyword)
100
+	})
101
+}
102
+
70 103
 func TestNoLoginViewIssue(t *testing.T) {
71 104
 	prepareTestEnv(t)
72 105
 

+ 3 - 0
integrations/mysql.ini.tmpl

@@ -10,6 +10,9 @@ PASSWD   = {{TEST_MYSQL_PASSWORD}}
10 10
 SSL_MODE = disable
11 11
 PATH     = data/gitea.db
12 12
 
13
+[indexer]
14
+ISSUE_INDEXER_PATH = integrations/indexers-mysql/issues.bleve
15
+
13 16
 [repository]
14 17
 ROOT = integrations/gitea-integration-mysql/gitea-repositories
15 18
 

+ 3 - 0
integrations/pgsql.ini.tmpl

@@ -10,6 +10,9 @@ PASSWD   = {{TEST_PGSQL_PASSWORD}}
10 10
 SSL_MODE = disable
11 11
 PATH     = data/gitea.db
12 12
 
13
+[indexer]
14
+ISSUE_INDEXER_PATH = integrations/indexers-pgsql/issues.bleve
15
+
13 16
 [repository]
14 17
 ROOT = integrations/gitea-integration-pgsql/gitea-repositories
15 18
 

+ 3 - 0
integrations/sqlite.ini

@@ -5,6 +5,9 @@ RUN_MODE = prod
5 5
 DB_TYPE  = sqlite3
6 6
 PATH     = :memory:
7 7
 
8
+[indexer]
9
+ISSUE_INDEXER_PATH = integrations/indexers-sqlite/issues.bleve
10
+
8 11
 [repository]
9 12
 ROOT = integrations/gitea-integration-sqlite/gitea-repositories
10 13
 

+ 5 - 5
models/fixtures/issue.yml

@@ -5,7 +5,7 @@
5 5
   poster_id: 1
6 6
   assignee_id: 1
7 7
   name: issue1
8
-  content: content1
8
+  content: content for the first issue
9 9
   is_closed: false
10 10
   is_pull: false
11 11
   num_comments: 2
@@ -18,7 +18,7 @@
18 18
   index: 2
19 19
   poster_id: 1
20 20
   name: issue2
21
-  content: content2
21
+  content: content for the second issue
22 22
   milestone_id: 1
23 23
   is_closed: false
24 24
   is_pull: true
@@ -32,7 +32,7 @@
32 32
   index: 3
33 33
   poster_id: 1
34 34
   name: issue3
35
-  content: content4
35
+  content: content for the third issue
36 36
   is_closed: false
37 37
   is_pull: true
38 38
   created_unix: 946684820
@@ -44,7 +44,7 @@
44 44
   index: 1
45 45
   poster_id: 2
46 46
   name: issue4
47
-  content: content4
47
+  content: content for the fourth issue
48 48
   is_closed: true
49 49
   is_pull: false
50 50
 
@@ -54,7 +54,7 @@
54 54
   index: 4
55 55
   poster_id: 2
56 56
   name: issue5
57
-  content: content5
57
+  content: content for the fifth issue
58 58
   is_closed: true
59 59
   is_pull: false
60 60
 -

+ 17 - 12
models/issue.go

@@ -155,6 +155,17 @@ func (issue *Issue) loadPullRequest(e Engine) (err error) {
155 155
 	return nil
156 156
 }
157 157
 
158
+func (issue *Issue) loadComments(e Engine) (err error) {
159
+	if issue.Comments != nil {
160
+		return nil
161
+	}
162
+	issue.Comments, err = findComments(e, FindCommentsOptions{
163
+		IssueID: issue.ID,
164
+		Type:    CommentTypeUnknown,
165
+	})
166
+	return err
167
+}
168
+
158 169
 func (issue *Issue) loadAttributes(e Engine) (err error) {
159 170
 	if err = issue.loadRepo(e); err != nil {
160 171
 		return
@@ -191,14 +202,8 @@ func (issue *Issue) loadAttributes(e Engine) (err error) {
191 202
 		}
192 203
 	}
193 204
 
194
-	if issue.Comments == nil {
195
-		issue.Comments, err = findComments(e, FindCommentsOptions{
196
-			IssueID: issue.ID,
197
-			Type:    CommentTypeUnknown,
198
-		})
199
-		if err != nil {
200
-			return fmt.Errorf("getCommentsByIssueID [%d]: %v", issue.ID, err)
201
-		}
205
+	if err = issue.loadComments(e); err != nil {
206
+		return
202 207
 	}
203 208
 
204 209
 	return nil
@@ -577,7 +582,7 @@ func updateIssueCols(e Engine, issue *Issue, cols ...string) error {
577 582
 	if _, err := e.Id(issue.ID).Cols(cols...).Update(issue); err != nil {
578 583
 		return err
579 584
 	}
580
-	UpdateIssueIndexer(issue)
585
+	UpdateIssueIndexer(issue.ID)
581 586
 	return nil
582 587
 }
583 588
 
@@ -907,8 +912,6 @@ func newIssue(e *xorm.Session, doer *User, opts NewIssueOptions) (err error) {
907 912
 		return err
908 913
 	}
909 914
 
910
-	UpdateIssueIndexer(opts.Issue)
911
-
912 915
 	if len(opts.Attachments) > 0 {
913 916
 		attachments, err := getAttachmentsByUUIDs(e, opts.Attachments)
914 917
 		if err != nil {
@@ -947,6 +950,8 @@ func NewIssue(repo *Repository, issue *Issue, labelIDs []int64, uuids []string)
947 950
 		return fmt.Errorf("Commit: %v", err)
948 951
 	}
949 952
 
953
+	UpdateIssueIndexer(issue.ID)
954
+
950 955
 	if err = NotifyWatchers(&Action{
951 956
 		ActUserID: issue.Poster.ID,
952 957
 		ActUser:   issue.Poster,
@@ -1448,7 +1453,7 @@ func updateIssue(e Engine, issue *Issue) error {
1448 1453
 	if err != nil {
1449 1454
 		return err
1450 1455
 	}
1451
-	UpdateIssueIndexer(issue)
1456
+	UpdateIssueIndexer(issue.ID)
1452 1457
 	return nil
1453 1458
 }
1454 1459
 

+ 20 - 4
models/issue_comment.go

@@ -520,7 +520,14 @@ func CreateComment(opts *CreateCommentOptions) (comment *Comment, err error) {
520 520
 		return nil, err
521 521
 	}
522 522
 
523
-	return comment, sess.Commit()
523
+	if err = sess.Commit(); err != nil {
524
+		return nil, err
525
+	}
526
+
527
+	if opts.Type == CommentTypeComment {
528
+		UpdateIssueIndexer(opts.Issue.ID)
529
+	}
530
+	return comment, nil
524 531
 }
525 532
 
526 533
 // CreateIssueComment creates a plain issue comment.
@@ -645,8 +652,12 @@ func GetCommentsByRepoIDSince(repoID, since int64) ([]*Comment, error) {
645 652
 
646 653
 // UpdateComment updates information of comment.
647 654
 func UpdateComment(c *Comment) error {
648
-	_, err := x.Id(c.ID).AllCols().Update(c)
649
-	return err
655
+	if _, err := x.Id(c.ID).AllCols().Update(c); err != nil {
656
+		return err
657
+	} else if c.Type == CommentTypeComment {
658
+		UpdateIssueIndexer(c.IssueID)
659
+	}
660
+	return nil
650 661
 }
651 662
 
652 663
 // DeleteComment deletes the comment
@@ -672,5 +683,10 @@ func DeleteComment(comment *Comment) error {
672 683
 		return err
673 684
 	}
674 685
 
675
-	return sess.Commit()
686
+	if err := sess.Commit(); err != nil {
687
+		return err
688
+	} else if comment.Type == CommentTypeComment {
689
+		UpdateIssueIndexer(comment.IssueID)
690
+	}
691
+	return nil
676 692
 }

+ 40 - 124
models/issue_indexer.go

@@ -6,112 +6,21 @@ package models
6 6
 
7 7
 import (
8 8
 	"fmt"
9
-	"os"
10
-	"strconv"
11
-	"strings"
12 9
 
10
+	"code.gitea.io/gitea/modules/indexer"
13 11
 	"code.gitea.io/gitea/modules/log"
14 12
 	"code.gitea.io/gitea/modules/setting"
15 13
 	"code.gitea.io/gitea/modules/util"
16
-
17
-	"github.com/blevesearch/bleve"
18
-	"github.com/blevesearch/bleve/analysis/analyzer/simple"
19
-	"github.com/blevesearch/bleve/search/query"
20 14
 )
21 15
 
22
-// issueIndexerUpdateQueue queue of issues that need to be updated in the issues
23
-// indexer
24
-var issueIndexerUpdateQueue chan *Issue
25
-
26
-// issueIndexer (thread-safe) index for searching issues
27
-var issueIndexer bleve.Index
28
-
29
-// issueIndexerData data stored in the issue indexer
30
-type issueIndexerData struct {
31
-	ID     int64
32
-	RepoID int64
33
-
34
-	Title   string
35
-	Content string
36
-}
37
-
38
-// numericQuery an numeric-equality query for the given value and field
39
-func numericQuery(value int64, field string) *query.NumericRangeQuery {
40
-	f := float64(value)
41
-	tru := true
42
-	q := bleve.NewNumericRangeInclusiveQuery(&f, &f, &tru, &tru)
43
-	q.SetField(field)
44
-	return q
45
-}
46
-
47
-// SearchIssuesByKeyword searches for issues by given conditions.
48
-// Returns the matching issue IDs
49
-func SearchIssuesByKeyword(repoID int64, keyword string) ([]int64, error) {
50
-	terms := strings.Fields(strings.ToLower(keyword))
51
-	indexerQuery := bleve.NewConjunctionQuery(
52
-		numericQuery(repoID, "RepoID"),
53
-		bleve.NewDisjunctionQuery(
54
-			bleve.NewPhraseQuery(terms, "Title"),
55
-			bleve.NewPhraseQuery(terms, "Content"),
56
-		))
57
-	search := bleve.NewSearchRequestOptions(indexerQuery, 2147483647, 0, false)
58
-	search.Fields = []string{"ID"}
59
-
60
-	result, err := issueIndexer.Search(search)
61
-	if err != nil {
62
-		return nil, err
63
-	}
64
-
65
-	issueIDs := make([]int64, len(result.Hits))
66
-	for i, hit := range result.Hits {
67
-		issueIDs[i] = int64(hit.Fields["ID"].(float64))
68
-	}
69
-	return issueIDs, nil
70
-}
16
+// issueIndexerUpdateQueue queue of issue ids to be updated
17
+var issueIndexerUpdateQueue chan int64
71 18
 
72 19
 // InitIssueIndexer initialize issue indexer
73 20
 func InitIssueIndexer() {
74
-	_, err := os.Stat(setting.Indexer.IssuePath)
75
-	if err != nil {
76
-		if os.IsNotExist(err) {
77
-			if err = createIssueIndexer(); err != nil {
78
-				log.Fatal(4, "CreateIssuesIndexer: %v", err)
79
-			}
80
-			if err = populateIssueIndexer(); err != nil {
81
-				log.Fatal(4, "PopulateIssuesIndex: %v", err)
82
-			}
83
-		} else {
84
-			log.Fatal(4, "InitIssuesIndexer: %v", err)
85
-		}
86
-	} else {
87
-		issueIndexer, err = bleve.Open(setting.Indexer.IssuePath)
88
-		if err != nil {
89
-			log.Fatal(4, "InitIssuesIndexer, open index: %v", err)
90
-		}
91
-	}
92
-	issueIndexerUpdateQueue = make(chan *Issue, setting.Indexer.UpdateQueueLength)
21
+	indexer.InitIssueIndexer(populateIssueIndexer)
22
+	issueIndexerUpdateQueue = make(chan int64, setting.Indexer.UpdateQueueLength)
93 23
 	go processIssueIndexerUpdateQueue()
94
-	// TODO close issueIndexer when Gitea closes
95
-}
96
-
97
-// createIssueIndexer create an issue indexer if one does not already exist
98
-func createIssueIndexer() error {
99
-	mapping := bleve.NewIndexMapping()
100
-	docMapping := bleve.NewDocumentMapping()
101
-
102
-	docMapping.AddFieldMappingsAt("ID", bleve.NewNumericFieldMapping())
103
-	docMapping.AddFieldMappingsAt("RepoID", bleve.NewNumericFieldMapping())
104
-
105
-	textFieldMapping := bleve.NewTextFieldMapping()
106
-	textFieldMapping.Analyzer = simple.Name
107
-	docMapping.AddFieldMappingsAt("Title", textFieldMapping)
108
-	docMapping.AddFieldMappingsAt("Content", textFieldMapping)
109
-
110
-	mapping.AddDocumentMapping("issues", docMapping)
111
-
112
-	var err error
113
-	issueIndexer, err = bleve.New(setting.Indexer.IssuePath, mapping)
114
-	return err
115 24
 }
116 25
 
117 26
 // populateIssueIndexer populate the issue indexer with issue data
@@ -127,57 +36,64 @@ func populateIssueIndexer() error {
127 36
 		if len(repos) == 0 {
128 37
 			return nil
129 38
 		}
130
-		batch := issueIndexer.NewBatch()
131 39
 		for _, repo := range repos {
132 40
 			issues, err := Issues(&IssuesOptions{
133 41
 				RepoID:   repo.ID,
134 42
 				IsClosed: util.OptionalBoolNone,
135 43
 				IsPull:   util.OptionalBoolNone,
136 44
 			})
137
-			if err != nil {
138
-				return fmt.Errorf("Issues: %v", err)
45
+			updates := make([]indexer.IssueIndexerUpdate, len(issues))
46
+			for i, issue := range issues {
47
+				updates[i] = issue.update()
139 48
 			}
140
-			for _, issue := range issues {
141
-				err = batch.Index(issue.indexUID(), issue.issueData())
142
-				if err != nil {
143
-					return fmt.Errorf("batch.Index: %v", err)
144
-				}
49
+			if err = indexer.BatchUpdateIssues(updates...); err != nil {
50
+				return fmt.Errorf("BatchUpdate: %v", err)
145 51
 			}
146 52
 		}
147
-		if err = issueIndexer.Batch(batch); err != nil {
148
-			return fmt.Errorf("index.Batch: %v", err)
149
-		}
150 53
 	}
151 54
 }
152 55
 
153 56
 func processIssueIndexerUpdateQueue() {
154 57
 	for {
155 58
 		select {
156
-		case issue := <-issueIndexerUpdateQueue:
157
-			if err := issueIndexer.Index(issue.indexUID(), issue.issueData()); err != nil {
59
+		case issueID := <-issueIndexerUpdateQueue:
60
+			issue, err := GetIssueByID(issueID)
61
+			if err != nil {
62
+				log.Error(4, "issuesIndexer.Index: %v", err)
63
+				continue
64
+			}
65
+			if err = indexer.UpdateIssue(issue.update()); err != nil {
158 66
 				log.Error(4, "issuesIndexer.Index: %v", err)
159 67
 			}
160 68
 		}
161 69
 	}
162 70
 }
163 71
 
164
-// indexUID a unique identifier for an issue used in full-text indices
165
-func (issue *Issue) indexUID() string {
166
-	return strconv.FormatInt(issue.ID, 36)
167
-}
168
-
169
-func (issue *Issue) issueData() *issueIndexerData {
170
-	return &issueIndexerData{
171
-		ID:      issue.ID,
172
-		RepoID:  issue.RepoID,
173
-		Title:   issue.Title,
174
-		Content: issue.Content,
72
+func (issue *Issue) update() indexer.IssueIndexerUpdate {
73
+	comments := make([]string, 0, 5)
74
+	for _, comment := range issue.Comments {
75
+		if comment.Type == CommentTypeComment {
76
+			comments = append(comments, comment.Content)
77
+		}
78
+	}
79
+	return indexer.IssueIndexerUpdate{
80
+		IssueID: issue.ID,
81
+		Data: &indexer.IssueIndexerData{
82
+			RepoID:   issue.RepoID,
83
+			Title:    issue.Title,
84
+			Content:  issue.Content,
85
+			Comments: comments,
86
+		},
175 87
 	}
176 88
 }
177 89
 
178 90
 // UpdateIssueIndexer add/update an issue to the issue indexer
179
-func UpdateIssueIndexer(issue *Issue) {
180
-	go func() {
181
-		issueIndexerUpdateQueue <- issue
182
-	}()
91
+func UpdateIssueIndexer(issueID int64) {
92
+	select {
93
+	case issueIndexerUpdateQueue <- issueID:
94
+	default:
95
+		go func() {
96
+			issueIndexerUpdateQueue <- issueID
97
+		}()
98
+	}
183 99
 }

+ 2 - 0
models/pull.go

@@ -640,6 +640,8 @@ func NewPullRequest(repo *Repository, pull *Issue, labelIDs []int64, uuids []str
640 640
 		return fmt.Errorf("Commit: %v", err)
641 641
 	}
642 642
 
643
+	UpdateIssueIndexer(pull.ID)
644
+
643 645
 	if err = NotifyWatchers(&Action{
644 646
 		ActUserID: pull.Poster.ID,
645 647
 		ActUser:   pull.Poster,

+ 33 - 4
modules/indexer/indexer.go

@@ -5,10 +5,39 @@
5 5
 package indexer
6 6
 
7 7
 import (
8
-	"code.gitea.io/gitea/models"
8
+	"fmt"
9
+	"strconv"
10
+
11
+	"github.com/blevesearch/bleve"
12
+	"github.com/blevesearch/bleve/search/query"
9 13
 )
10 14
 
11
-// NewContext start indexer service
12
-func NewContext() {
13
-	models.InitIssueIndexer()
15
+// indexerID a bleve-compatible unique identifier for an integer id
16
+func indexerID(id int64) string {
17
+	return strconv.FormatInt(id, 36)
18
+}
19
+
20
+// idOfIndexerID the integer id associated with an indexer id
21
+func idOfIndexerID(indexerID string) (int64, error) {
22
+	id, err := strconv.ParseInt(indexerID, 36, 64)
23
+	if err != nil {
24
+		return 0, fmt.Errorf("Unexpected indexer ID %s: %v", indexerID, err)
25
+	}
26
+	return id, nil
27
+}
28
+
29
+// numericEqualityQuery a numeric equality query for the given value and field
30
+func numericEqualityQuery(value int64, field string) *query.NumericRangeQuery {
31
+	f := float64(value)
32
+	tru := true
33
+	q := bleve.NewNumericRangeInclusiveQuery(&f, &f, &tru, &tru)
34
+	q.SetField(field)
35
+	return q
36
+}
37
+
38
+func newMatchPhraseQuery(matchPhrase, field, analyzer string) *query.MatchPhraseQuery {
39
+	q := bleve.NewMatchPhraseQuery(matchPhrase)
40
+	q.FieldVal = field
41
+	q.Analyzer = analyzer
42
+	return q
14 43
 }

+ 143 - 0
modules/indexer/issue.go

@@ -0,0 +1,143 @@
1
+// Copyright 2017 The Gitea Authors. All rights reserved.
2
+// Use of this source code is governed by a MIT-style
3
+// license that can be found in the LICENSE file.
4
+
5
+package indexer
6
+
7
+import (
8
+	"os"
9
+
10
+	"code.gitea.io/gitea/modules/log"
11
+	"code.gitea.io/gitea/modules/setting"
12
+
13
+	"github.com/blevesearch/bleve"
14
+	"github.com/blevesearch/bleve/analysis/analyzer/custom"
15
+	"github.com/blevesearch/bleve/analysis/token/lowercase"
16
+	"github.com/blevesearch/bleve/analysis/token/unicodenorm"
17
+	"github.com/blevesearch/bleve/analysis/tokenizer/unicode"
18
+)
19
+
20
+// issueIndexer (thread-safe) index for searching issues
21
+var issueIndexer bleve.Index
22
+
23
+// IssueIndexerData data stored in the issue indexer
24
+type IssueIndexerData struct {
25
+	RepoID   int64
26
+	Title    string
27
+	Content  string
28
+	Comments []string
29
+}
30
+
31
+// IssueIndexerUpdate an update to the issue indexer
32
+type IssueIndexerUpdate struct {
33
+	IssueID int64
34
+	Data    *IssueIndexerData
35
+}
36
+
37
+const issueIndexerAnalyzer = "issueIndexer"
38
+
39
+// InitIssueIndexer initialize issue indexer
40
+func InitIssueIndexer(populateIndexer func() error) {
41
+	_, err := os.Stat(setting.Indexer.IssuePath)
42
+	if err != nil {
43
+		if os.IsNotExist(err) {
44
+			if err = createIssueIndexer(); err != nil {
45
+				log.Fatal(4, "CreateIssuesIndexer: %v", err)
46
+			}
47
+			if err = populateIndexer(); err != nil {
48
+				log.Fatal(4, "PopulateIssuesIndex: %v", err)
49
+			}
50
+		} else {
51
+			log.Fatal(4, "InitIssuesIndexer: %v", err)
52
+		}
53
+	} else {
54
+		issueIndexer, err = bleve.Open(setting.Indexer.IssuePath)
55
+		if err != nil {
56
+			log.Error(4, "Unable to open issues indexer (%s)."+
57
+				" If the error is due to incompatible versions, try deleting the indexer files;"+
58
+				" gitea will recreate them with the appropriate version the next time it runs."+
59
+				" Deleting the indexer files will not result in loss of data.",
60
+				setting.Indexer.IssuePath)
61
+			log.Fatal(4, "InitIssuesIndexer, open index: %v", err)
62
+		}
63
+	}
64
+}
65
+
66
+// createIssueIndexer create an issue indexer if one does not already exist
67
+func createIssueIndexer() error {
68
+	mapping := bleve.NewIndexMapping()
69
+	docMapping := bleve.NewDocumentMapping()
70
+
71
+	docMapping.AddFieldMappingsAt("RepoID", bleve.NewNumericFieldMapping())
72
+
73
+	textFieldMapping := bleve.NewTextFieldMapping()
74
+	docMapping.AddFieldMappingsAt("Title", textFieldMapping)
75
+	docMapping.AddFieldMappingsAt("Content", textFieldMapping)
76
+	docMapping.AddFieldMappingsAt("Comments", textFieldMapping)
77
+
78
+	const unicodeNormNFC = "unicodeNormNFC"
79
+	if err := mapping.AddCustomTokenFilter(unicodeNormNFC, map[string]interface{}{
80
+		"type": unicodenorm.Name,
81
+		"form": unicodenorm.NFC,
82
+	}); err != nil {
83
+		return err
84
+	} else if err = mapping.AddCustomAnalyzer(issueIndexerAnalyzer, map[string]interface{}{
85
+		"type":          custom.Name,
86
+		"char_filters":  []string{},
87
+		"tokenizer":     unicode.Name,
88
+		"token_filters": []string{unicodeNormNFC, lowercase.Name},
89
+	}); err != nil {
90
+		return err
91
+	}
92
+
93
+	mapping.DefaultAnalyzer = issueIndexerAnalyzer
94
+	mapping.AddDocumentMapping("issues", docMapping)
95
+
96
+	var err error
97
+	issueIndexer, err = bleve.New(setting.Indexer.IssuePath, mapping)
98
+	return err
99
+}
100
+
101
+// UpdateIssue update the issue indexer
102
+func UpdateIssue(update IssueIndexerUpdate) error {
103
+	return issueIndexer.Index(indexerID(update.IssueID), update.Data)
104
+}
105
+
106
+// BatchUpdateIssues perform a batch update of the issue indexer
107
+func BatchUpdateIssues(updates ...IssueIndexerUpdate) error {
108
+	batch := issueIndexer.NewBatch()
109
+	for _, update := range updates {
110
+		err := batch.Index(indexerID(update.IssueID), update.Data)
111
+		if err != nil {
112
+			return err
113
+		}
114
+	}
115
+	return issueIndexer.Batch(batch)
116
+}
117
+
118
+// SearchIssuesByKeyword searches for issues by given conditions.
119
+// Returns the matching issue IDs
120
+func SearchIssuesByKeyword(repoID int64, keyword string) ([]int64, error) {
121
+	indexerQuery := bleve.NewConjunctionQuery(
122
+		numericEqualityQuery(repoID, "RepoID"),
123
+		bleve.NewDisjunctionQuery(
124
+			newMatchPhraseQuery(keyword, "Title", issueIndexerAnalyzer),
125
+			newMatchPhraseQuery(keyword, "Content", issueIndexerAnalyzer),
126
+			newMatchPhraseQuery(keyword, "Comments", issueIndexerAnalyzer),
127
+		))
128
+	search := bleve.NewSearchRequestOptions(indexerQuery, 2147483647, 0, false)
129
+
130
+	result, err := issueIndexer.Search(search)
131
+	if err != nil {
132
+		return nil, err
133
+	}
134
+
135
+	issueIDs := make([]int64, len(result.Hits))
136
+	for i, hit := range result.Hits {
137
+		issueIDs[i], err = idOfIndexerID(hit.ID)
138
+		if err != nil {
139
+			return nil, err
140
+		}
141
+	}
142
+	return issueIDs, nil
143
+}

+ 1 - 2
routers/init.go

@@ -13,7 +13,6 @@ import (
13 13
 	"code.gitea.io/gitea/models/migrations"
14 14
 	"code.gitea.io/gitea/modules/cron"
15 15
 	"code.gitea.io/gitea/modules/highlight"
16
-	"code.gitea.io/gitea/modules/indexer"
17 16
 	"code.gitea.io/gitea/modules/log"
18 17
 	"code.gitea.io/gitea/modules/mailer"
19 18
 	"code.gitea.io/gitea/modules/markup"
@@ -63,7 +62,7 @@ func GlobalInit() {
63 62
 
64 63
 		// Booting long running goroutines.
65 64
 		cron.NewContext()
66
-		indexer.NewContext()
65
+		models.InitIssueIndexer()
67 66
 		models.InitSyncMirrors()
68 67
 		models.InitDeliverHooks()
69 68
 		models.InitTestPullRequests()

+ 2 - 1
routers/repo/issue.go

@@ -22,6 +22,7 @@ import (
22 22
 	"code.gitea.io/gitea/modules/auth"
23 23
 	"code.gitea.io/gitea/modules/base"
24 24
 	"code.gitea.io/gitea/modules/context"
25
+	"code.gitea.io/gitea/modules/indexer"
25 26
 	"code.gitea.io/gitea/modules/log"
26 27
 	"code.gitea.io/gitea/modules/markdown"
27 28
 	"code.gitea.io/gitea/modules/notification"
@@ -142,7 +143,7 @@ func Issues(ctx *context.Context) {
142 143
 	var issueIDs []int64
143 144
 	var err error
144 145
 	if len(keyword) > 0 {
145
-		issueIDs, err = models.SearchIssuesByKeyword(repo.ID, keyword)
146
+		issueIDs, err = indexer.SearchIssuesByKeyword(repo.ID, keyword)
146 147
 		if len(issueIDs) == 0 {
147 148
 			forceEmpty = true
148 149
 		}

+ 25 - 20
vendor/github.com/blevesearch/bleve/README.md

@@ -4,6 +4,7 @@
4 4
 [![Join the chat at https://gitter.im/blevesearch/bleve](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/blevesearch/bleve?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge)
5 5
 [![codebeat](https://codebeat.co/badges/38a7cbc9-9cf5-41c0-a315-0746178230f4)](https://codebeat.co/projects/github-com-blevesearch-bleve)
6 6
 [![Go Report Card](https://goreportcard.com/badge/blevesearch/bleve)](https://goreportcard.com/report/blevesearch/bleve)
7
+[![Sourcegraph](https://sourcegraph.com/github.com/blevesearch/bleve/-/badge.svg)](https://sourcegraph.com/github.com/blevesearch/bleve?badge)  [![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0)
7 8
 
8 9
 modern text indexing in go - [blevesearch.com](http://www.blevesearch.com/)
9 10
 
@@ -33,29 +34,33 @@ Discuss usage and development of bleve in the [google group](https://groups.goog
33 34
 
34 35
 ## Indexing
35 36
 
36
-		message := struct{
37
-			Id   string
38
-			From string
39
-			Body string
40
-		}{
41
-			Id:   "example",
42
-			From: "marty.schoch@gmail.com",
43
-			Body: "bleve indexing is easy",
44
-		}
45
-
46
-		mapping := bleve.NewIndexMapping()
47
-		index, err := bleve.New("example.bleve", mapping)
48
-		if err != nil {
49
-			panic(err)
50
-		}
51
-		index.Index(message.Id, message)
37
+```go
38
+message := struct{
39
+	Id   string
40
+	From string
41
+	Body string
42
+}{
43
+	Id:   "example",
44
+	From: "marty.schoch@gmail.com",
45
+	Body: "bleve indexing is easy",
46
+}
47
+
48
+mapping := bleve.NewIndexMapping()
49
+index, err := bleve.New("example.bleve", mapping)
50
+if err != nil {
51
+	panic(err)
52
+}
53
+index.Index(message.Id, message)
54
+```
52 55
 
53 56
 ## Querying
54 57
 
55
-		index, _ := bleve.Open("example.bleve")
56
-		query := bleve.NewQueryStringQuery("bleve")
57
-		searchRequest := bleve.NewSearchRequest(query)
58
-		searchResult, _ := index.Search(searchRequest)
58
+```go
59
+index, _ := bleve.Open("example.bleve")
60
+query := bleve.NewQueryStringQuery("bleve")
61
+searchRequest := bleve.NewSearchRequest(query)
62
+searchResult, _ := index.Search(searchRequest)
63
+```
59 64
 
60 65
 ## License
61 66
 

+ 145 - 0
vendor/github.com/blevesearch/bleve/analysis/analyzer/custom/custom.go

@@ -0,0 +1,145 @@
1
+//  Copyright (c) 2014 Couchbase, Inc.
2
+//
3
+// Licensed under the Apache License, Version 2.0 (the "License");
4
+// you may not use this file except in compliance with the License.
5
+// You may obtain a copy of the License at
6
+//
7
+// 		http://www.apache.org/licenses/LICENSE-2.0
8
+//
9
+// Unless required by applicable law or agreed to in writing, software
10
+// distributed under the License is distributed on an "AS IS" BASIS,
11
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+// See the License for the specific language governing permissions and
13
+// limitations under the License.
14
+
15
+package custom
16
+
17
+import (
18
+	"fmt"
19
+
20
+	"github.com/blevesearch/bleve/analysis"
21
+	"github.com/blevesearch/bleve/registry"
22
+)
23
+
24
+const Name = "custom"
25
+
26
+func AnalyzerConstructor(config map[string]interface{}, cache *registry.Cache) (*analysis.Analyzer, error) {
27
+
28
+	var err error
29
+	var charFilters []analysis.CharFilter
30
+	charFiltersValue, ok := config["char_filters"]
31
+	if ok {
32
+		switch charFiltersValue := charFiltersValue.(type) {
33
+		case []string:
34
+			charFilters, err = getCharFilters(charFiltersValue, cache)
35
+			if err != nil {
36
+				return nil, err
37
+			}
38
+		case []interface{}:
39
+			charFiltersNames, err := convertInterfaceSliceToStringSlice(charFiltersValue, "char filter")
40
+			if err != nil {
41
+				return nil, err
42
+			}
43
+			charFilters, err = getCharFilters(charFiltersNames, cache)
44
+			if err != nil {
45
+				return nil, err
46
+			}
47
+		default:
48
+			return nil, fmt.Errorf("unsupported type for char_filters, must be slice")
49
+		}
50
+	}
51
+
52
+	var tokenizerName string
53
+	tokenizerValue, ok := config["tokenizer"]
54
+	if ok {
55
+		tokenizerName, ok = tokenizerValue.(string)
56
+		if !ok {
57
+			return nil, fmt.Errorf("must specify tokenizer as string")
58
+		}
59
+	} else {
60
+		return nil, fmt.Errorf("must specify tokenizer")
61
+	}
62
+
63
+	tokenizer, err := cache.TokenizerNamed(tokenizerName)
64
+	if err != nil {
65
+		return nil, err
66
+	}
67
+
68
+	var tokenFilters []analysis.TokenFilter
69
+	tokenFiltersValue, ok := config["token_filters"]
70
+	if ok {
71
+		switch tokenFiltersValue := tokenFiltersValue.(type) {
72
+		case []string:
73
+			tokenFilters, err = getTokenFilters(tokenFiltersValue, cache)
74
+			if err != nil {
75
+				return nil, err
76
+			}
77
+		case []interface{}:
78
+			tokenFiltersNames, err := convertInterfaceSliceToStringSlice(tokenFiltersValue, "token filter")
79
+			if err != nil {
80
+				return nil, err
81
+			}
82
+			tokenFilters, err = getTokenFilters(tokenFiltersNames, cache)
83
+			if err != nil {
84
+				return nil, err
85
+			}
86
+		default:
87
+			return nil, fmt.Errorf("unsupported type for token_filters, must be slice")
88
+		}
89
+	}
90
+
91
+	rv := analysis.Analyzer{
92
+		Tokenizer: tokenizer,
93
+	}
94
+	if charFilters != nil {
95
+		rv.CharFilters = charFilters
96
+	}
97
+	if tokenFilters != nil {
98
+		rv.TokenFilters = tokenFilters
99
+	}
100
+	return &rv, nil
101
+}
102
+
103
+func init() {
104
+	registry.RegisterAnalyzer(Name, AnalyzerConstructor)
105
+}
106
+
107
+func getCharFilters(charFilterNames []string, cache *registry.Cache) ([]analysis.CharFilter, error) {
108
+	charFilters := make([]analysis.CharFilter, len(charFilterNames))
109
+	for i, charFilterName := range charFilterNames {
110
+		charFilter, err := cache.CharFilterNamed(charFilterName)
111
+		if err != nil {
112
+			return nil, err
113
+		}
114
+		charFilters[i] = charFilter
115
+	}
116
+
117
+	return charFilters, nil
118
+}
119
+
120
+func getTokenFilters(tokenFilterNames []string, cache *registry.Cache) ([]analysis.TokenFilter, error) {
121
+	tokenFilters := make([]analysis.TokenFilter, len(tokenFilterNames))
122
+	for i, tokenFilterName := range tokenFilterNames {
123
+		tokenFilter, err := cache.TokenFilterNamed(tokenFilterName)
124
+		if err != nil {
125
+			return nil, err
126
+		}
127
+		tokenFilters[i] = tokenFilter
128
+	}
129
+
130
+	return tokenFilters, nil
131
+}
132
+
133
+func convertInterfaceSliceToStringSlice(interfaceSlice []interface{}, objType string) ([]string, error) {
134
+	stringSlice := make([]string, len(interfaceSlice))
135
+	for i, interfaceObj := range interfaceSlice {
136
+		stringObj, ok := interfaceObj.(string)
137
+		if ok {
138
+			stringSlice[i] = stringObj
139
+		} else {
140
+			return nil, fmt.Errorf(objType + " name must be a string")
141
+		}
142
+	}
143
+
144
+	return stringSlice, nil
145
+}

+ 0 - 46
vendor/github.com/blevesearch/bleve/analysis/analyzer/simple/simple.go

@@ -1,46 +0,0 @@
1
-//  Copyright (c) 2014 Couchbase, Inc.
2
-//
3
-// Licensed under the Apache License, Version 2.0 (the "License");
4
-// you may not use this file except in compliance with the License.
5
-// You may obtain a copy of the License at
6
-//
7
-// 		http://www.apache.org/licenses/LICENSE-2.0
8
-//
9
-// Unless required by applicable law or agreed to in writing, software
10
-// distributed under the License is distributed on an "AS IS" BASIS,
11
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
-// See the License for the specific language governing permissions and
13
-// limitations under the License.
14
-
15
-package simple
16
-
17
-import (
18
-	"github.com/blevesearch/bleve/analysis"
19
-	"github.com/blevesearch/bleve/analysis/token/lowercase"
20
-	"github.com/blevesearch/bleve/analysis/tokenizer/letter"
21
-	"github.com/blevesearch/bleve/registry"
22
-)
23
-
24
-const Name = "simple"
25
-
26
-func AnalyzerConstructor(config map[string]interface{}, cache *registry.Cache) (*analysis.Analyzer, error) {
27
-	tokenizer, err := cache.TokenizerNamed(letter.Name)
28
-	if err != nil {
29
-		return nil, err
30
-	}
31
-	toLowerFilter, err := cache.TokenFilterNamed(lowercase.Name)
32
-	if err != nil {
33
-		return nil, err
34
-	}
35
-	rv := analysis.Analyzer{
36
-		Tokenizer: tokenizer,
37
-		TokenFilters: []analysis.TokenFilter{
38
-			toLowerFilter,
39
-		},
40
-	}
41
-	return &rv, nil
42
-}
43
-
44
-func init() {
45
-	registry.RegisterAnalyzer(Name, AnalyzerConstructor)
46
-}

+ 79 - 0
vendor/github.com/blevesearch/bleve/analysis/token/unicodenorm/unicodenorm.go

@@ -0,0 +1,79 @@
1
+//  Copyright (c) 2014 Couchbase, Inc.
2
+//
3
+// Licensed under the Apache License, Version 2.0 (the "License");
4
+// you may not use this file except in compliance with the License.
5
+// You may obtain a copy of the License at
6
+//
7
+// 		http://www.apache.org/licenses/LICENSE-2.0
8
+//
9
+// Unless required by applicable law or agreed to in writing, software
10
+// distributed under the License is distributed on an "AS IS" BASIS,
11
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+// See the License for the specific language governing permissions and
13
+// limitations under the License.
14
+
15
+package unicodenorm
16
+
17
+import (
18
+	"fmt"
19
+
20
+	"github.com/blevesearch/bleve/analysis"
21
+	"github.com/blevesearch/bleve/registry"
22
+	"golang.org/x/text/unicode/norm"
23
+)
24
+
25
+const Name = "normalize_unicode"
26
+
27
+const NFC = "nfc"
28
+const NFD = "nfd"
29
+const NFKC = "nfkc"
30
+const NFKD = "nfkd"
31
+
32
+var forms = map[string]norm.Form{
33
+	NFC:  norm.NFC,
34
+	NFD:  norm.NFD,
35
+	NFKC: norm.NFKC,
36
+	NFKD: norm.NFKD,
37
+}
38
+
39
+type UnicodeNormalizeFilter struct {
40
+	form norm.Form
41
+}
42
+
43
+func NewUnicodeNormalizeFilter(formName string) (*UnicodeNormalizeFilter, error) {
44
+	form, ok := forms[formName]
45
+	if !ok {
46
+		return nil, fmt.Errorf("no form named %s", formName)
47
+	}
48
+	return &UnicodeNormalizeFilter{
49
+		form: form,
50
+	}, nil
51
+}
52
+
53
+func MustNewUnicodeNormalizeFilter(formName string) *UnicodeNormalizeFilter {
54
+	filter, err := NewUnicodeNormalizeFilter(formName)
55
+	if err != nil {
56
+		panic(err)
57
+	}
58
+	return filter
59
+}
60
+
61
+func (s *UnicodeNormalizeFilter) Filter(input analysis.TokenStream) analysis.TokenStream {
62
+	for _, token := range input {
63
+		token.Term = s.form.Bytes(token.Term)
64
+	}
65
+	return input
66
+}
67
+
68
+func UnicodeNormalizeFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) {
69
+	formVal, ok := config["form"].(string)
70
+	if !ok {
71
+		return nil, fmt.Errorf("must specify form")
72
+	}
73
+	form := formVal
74
+	return NewUnicodeNormalizeFilter(form)
75
+}
76
+
77
+func init() {
78
+	registry.RegisterTokenFilter(Name, UnicodeNormalizeFilterConstructor)
79
+}

+ 0 - 76
vendor/github.com/blevesearch/bleve/analysis/tokenizer/character/character.go

@@ -1,76 +0,0 @@
1
-//  Copyright (c) 2016 Couchbase, Inc.
2
-//
3
-// Licensed under the Apache License, Version 2.0 (the "License");
4
-// you may not use this file except in compliance with the License.
5
-// You may obtain a copy of the License at
6
-//
7
-// 		http://www.apache.org/licenses/LICENSE-2.0
8
-//
9
-// Unless required by applicable law or agreed to in writing, software
10
-// distributed under the License is distributed on an "AS IS" BASIS,
11
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
-// See the License for the specific language governing permissions and
13
-// limitations under the License.
14
-
15
-package character
16
-
17
-import (
18
-	"unicode/utf8"
19
-
20
-	"github.com/blevesearch/bleve/analysis"
21
-)
22
-
23
-type IsTokenRune func(r rune) bool
24
-
25
-type CharacterTokenizer struct {
26
-	isTokenRun IsTokenRune
27
-}
28
-
29
-func NewCharacterTokenizer(f IsTokenRune) *CharacterTokenizer {
30
-	return &CharacterTokenizer{
31
-		isTokenRun: f,
32
-	}
33
-}
34
-
35
-func (c *CharacterTokenizer) Tokenize(input []byte) analysis.TokenStream {
36
-
37
-	rv := make(analysis.TokenStream, 0, 1024)
38
-
39
-	offset := 0
40
-	start := 0
41
-	end := 0
42
-	count := 0
43
-	for currRune, size := utf8.DecodeRune(input[offset:]); currRune != utf8.RuneError; currRune, size = utf8.DecodeRune(input[offset:]) {
44
-		isToken := c.isTokenRun(currRune)
45
-		if isToken {
46
-			end = offset + size
47
-		} else {
48
-			if end-start > 0 {
49
-				// build token
50
-				rv = append(rv, &analysis.Token{
51
-					Term:     input[start:end],
52
-					Start:    start,
53
-					End:      end,
54
-					Position: count + 1,
55
-					Type:     analysis.AlphaNumeric,
56
-				})
57
-				count++
58
-			}
59
-			start = offset + size
60
-			end = start
61
-		}
62
-		offset += size
63
-	}
64
-	// if we ended in the middle of a token, finish it
65
-	if end-start > 0 {
66
-		// build token
67
-		rv = append(rv, &analysis.Token{
68
-			Term:     input[start:end],
69
-			Start:    start,
70
-			End:      end,
71
-			Position: count + 1,
72
-			Type:     analysis.AlphaNumeric,
73
-		})
74
-	}
75
-	return rv
76
-}

+ 0 - 33
vendor/github.com/blevesearch/bleve/analysis/tokenizer/letter/letter.go

@@ -1,33 +0,0 @@
1
-//  Copyright (c) 2016 Couchbase, Inc.
2
-//
3
-// Licensed under the Apache License, Version 2.0 (the "License");
4
-// you may not use this file except in compliance with the License.
5
-// You may obtain a copy of the License at
6
-//
7
-// 		http://www.apache.org/licenses/LICENSE-2.0
8
-//
9
-// Unless required by applicable law or agreed to in writing, software
10
-// distributed under the License is distributed on an "AS IS" BASIS,
11
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
-// See the License for the specific language governing permissions and
13
-// limitations under the License.
14
-
15
-package letter
16
-
17
-import (
18
-	"unicode"
19
-
20
-	"github.com/blevesearch/bleve/analysis"
21
-	"github.com/blevesearch/bleve/analysis/tokenizer/character"
22
-	"github.com/blevesearch/bleve/registry"
23
-)
24
-
25
-const Name = "letter"
26
-
27
-func TokenizerConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.Tokenizer, error) {
28
-	return character.NewCharacterTokenizer(unicode.IsLetter), nil
29
-}
30
-
31
-func init() {
32
-	registry.RegisterTokenizer(Name, TokenizerConstructor)
33
-}

+ 0 - 23
vendor/github.com/blevesearch/bleve/config_app.go

@@ -1,23 +0,0 @@
1
-//  Copyright (c) 2014 Couchbase, Inc.
2
-//
3
-// Licensed under the Apache License, Version 2.0 (the "License");
4
-// you may not use this file except in compliance with the License.
5
-// You may obtain a copy of the License at
6
-//
7
-// 		http://www.apache.org/licenses/LICENSE-2.0
8
-//
9
-// Unless required by applicable law or agreed to in writing, software
10
-// distributed under the License is distributed on an "AS IS" BASIS,
11
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
-// See the License for the specific language governing permissions and
13
-// limitations under the License.
14
-
15
-// +build appengine appenginevm
16
-
17
-package bleve
18
-
19
-// in the appengine environment we cannot support disk based indexes
20
-// so we do no extra configuration in this method
21
-func initDisk() {
22
-
23
-}

+ 137 - 0
vendor/github.com/blevesearch/bleve/document/field_geopoint.go

@@ -0,0 +1,137 @@
1
+//  Copyright (c) 2017 Couchbase, Inc.
2
+//
3
+// Licensed under the Apache License, Version 2.0 (the "License");
4
+// you may not use this file except in compliance with the License.
5
+// You may obtain a copy of the License at
6
+//
7
+// 		http://www.apache.org/licenses/LICENSE-2.0
8
+//
9
+// Unless required by applicable law or agreed to in writing, software
10
+// distributed under the License is distributed on an "AS IS" BASIS,
11
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+// See the License for the specific language governing permissions and
13
+// limitations under the License.
14
+
15
+package document
16
+
17
+import (
18
+	"fmt"
19
+
20
+	"github.com/blevesearch/bleve/analysis"
21
+	"github.com/blevesearch/bleve/geo"
22
+	"github.com/blevesearch/bleve/numeric"
23
+)
24
+
25
+var GeoPrecisionStep uint = 9
26
+
27
+type GeoPointField struct {
28
+	name              string
29
+	arrayPositions    []uint64
30
+	options           IndexingOptions
31
+	value             numeric.PrefixCoded
32
+	numPlainTextBytes uint64
33
+}
34
+
35
+func (n *GeoPointField) Name() string {
36
+	return n.name
37
+}
38
+
39
+func (n *GeoPointField) ArrayPositions() []uint64 {
40
+	return n.arrayPositions
41
+}
42
+
43
+func (n *GeoPointField) Options() IndexingOptions {
44
+	return n.options
45
+}
46
+
47
+func (n *GeoPointField) Analyze() (int, analysis.TokenFrequencies) {
48
+	tokens := make(analysis.TokenStream, 0)
49
+	tokens = append(tokens, &analysis.Token{
50
+		Start:    0,
51
+		End:      len(n.value),
52
+		Term:     n.value,
53
+		Position: 1,
54
+		Type:     analysis.Numeric,
55
+	})
56
+
57
+	original, err := n.value.Int64()
58
+	if err == nil {
59
+
60
+		shift := GeoPrecisionStep
61
+		for shift < 64 {
62
+			shiftEncoded, err := numeric.NewPrefixCodedInt64(original, shift)
63
+			if err != nil {
64
+				break
65
+			}
66
+			token := analysis.Token{
67
+				Start:    0,
68
+				End:      len(shiftEncoded),
69
+				Term:     shiftEncoded,
70
+				Position: 1,
71
+				Type:     analysis.Numeric,
72
+			}
73
+			tokens = append(tokens, &token)
74
+			shift += GeoPrecisionStep
75
+		}
76
+	}
77
+
78
+	fieldLength := len(tokens)
79
+	tokenFreqs := analysis.TokenFrequency(tokens, n.arrayPositions, n.options.IncludeTermVectors())
80
+	return fieldLength, tokenFreqs
81
+}
82
+
83
+func (n *GeoPointField) Value() []byte {
84
+	return n.value
85
+}
86
+
87
+func (n *GeoPointField) Lon() (float64, error) {
88
+	i64, err := n.value.Int64()
89
+	if err != nil {
90
+		return 0.0, err
91
+	}
92
+	return geo.MortonUnhashLon(uint64(i64)), nil
93
+}
94
+
95
+func (n *GeoPointField) Lat() (float64, error) {
96
+	i64, err := n.value.Int64()
97
+	if err != nil {
98
+		return 0.0, err
99
+	}
100
+	return geo.MortonUnhashLat(uint64(i64)), nil
101
+}
102
+
103
+func (n *GeoPointField) GoString() string {
104
+	return fmt.Sprintf("&document.GeoPointField{Name:%s, Options: %s, Value: %s}", n.name, n.options, n.value)
105
+}
106
+
107
+func (n *GeoPointField) NumPlainTextBytes() uint64 {
108
+	return n.numPlainTextBytes
109
+}
110
+
111
+func NewGeoPointFieldFromBytes(name string, arrayPositions []uint64, value []byte) *GeoPointField {
112
+	return &GeoPointField{
113
+		name:              name,
114
+		arrayPositions:    arrayPositions,
115
+		value:             value,
116
+		options:           DefaultNumericIndexingOptions,
117
+		numPlainTextBytes: uint64(len(value)),
118
+	}
119
+}
120
+
121
+func NewGeoPointField(name string, arrayPositions []uint64, lon, lat float64) *GeoPointField {
122
+	return NewGeoPointFieldWithIndexingOptions(name, arrayPositions, lon, lat, DefaultNumericIndexingOptions)
123
+}
124
+
125
+func NewGeoPointFieldWithIndexingOptions(name string, arrayPositions []uint64, lon, lat float64, options IndexingOptions) *GeoPointField {
126
+	mhash := geo.MortonHash(lon, lat)
127
+	prefixCoded := numeric.MustNewPrefixCodedInt64(int64(mhash), 0)
128
+	return &GeoPointField{
129
+		name:           name,
130
+		arrayPositions: arrayPositions,
131
+		value:          prefixCoded,
132
+		options:        options,
133
+		// not correct, just a place holder until we revisit how fields are
134
+		// represented and can fix this better
135
+		numPlainTextBytes: uint64(8),
136
+	}
137
+}

+ 9 - 0
vendor/github.com/blevesearch/bleve/geo/README.md

@@ -0,0 +1,9 @@
1
+# geo support in bleve
2
+
3
+First, all of this geo code is a Go adaptation of the [Lucene 5.3.2 sandbox geo support](https://lucene.apache.org/core/5_3_2/sandbox/org/apache/lucene/util/package-summary.html).
4
+
5
+## Notes
6
+
7
+- All of the APIs will use float64 for lon/lat values.
8
+- When describing a point in function arguments or return values, we always use the order lon, lat.
9
+- High level APIs will use TopLeft and BottomRight to describe bounding boxes.  This may not map cleanly to min/max lon/lat when crossing the dateline.  The lower level APIs will use min/max lon/lat and require the higher-level code to split boxes accordingly.

+ 170 - 0
vendor/github.com/blevesearch/bleve/geo/geo.go

@@ -0,0 +1,170 @@
1
+//  Copyright (c) 2017 Couchbase, Inc.
2
+//
3
+// Licensed under the Apache License, Version 2.0 (the "License");
4
+// you may not use this file except in compliance with the License.
5
+// You may obtain a copy of the License at
6
+//
7
+// 		http://www.apache.org/licenses/LICENSE-2.0
8
+//
9
+// Unless required by applicable law or agreed to in writing, software
10
+// distributed under the License is distributed on an "AS IS" BASIS,
11
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+// See the License for the specific language governing permissions and
13
+// limitations under the License.
14
+
15
+package geo
16
+
17
+import (
18
+	"fmt"
19
+	"math"
20
+
21
+	"github.com/blevesearch/bleve/numeric"
22
+)
23
+
24
+// GeoBits is the number of bits used for a single geo point
25
+// Currently this is 32bits for lon and 32bits for lat
26
+var GeoBits uint = 32
27
+
28
+var minLon = -180.0
29
+var minLat = -90.0
30
+var maxLon = 180.0
31
+var maxLat = 90.0
32
+var minLonRad = minLon * degreesToRadian
33
+var minLatRad = minLat * degreesToRadian
34
+var maxLonRad = maxLon * degreesToRadian
35
+var maxLatRad = maxLat * degreesToRadian
36
+var geoTolerance = 1E-6
37
+var lonScale = float64((uint64(0x1)<<GeoBits)-1) / 360.0
38
+var latScale = float64((uint64(0x1)<<GeoBits)-1) / 180.0
39
+
40
+// MortonHash computes the morton hash value for the provided geo point
41
+// This point is ordered as lon, lat.
42
+func MortonHash(lon, lat float64) uint64 {
43
+	return numeric.Interleave(scaleLon(lon), scaleLat(lat))
44
+}
45
+
46
+func scaleLon(lon float64) uint64 {
47
+	rv := uint64((lon - minLon) * lonScale)
48
+	return rv
49
+}
50
+
51
+func scaleLat(lat float64) uint64 {
52
+	rv := uint64((lat - minLat) * latScale)
53
+	return rv
54
+}
55
+
56
+// MortonUnhashLon extracts the longitude value from the provided morton hash.
57
+func MortonUnhashLon(hash uint64) float64 {
58
+	return unscaleLon(numeric.Deinterleave(hash))
59
+}
60
+
61
+// MortonUnhashLat extracts the latitude value from the provided morton hash.
62
+func MortonUnhashLat(hash uint64) float64 {
63
+	return unscaleLat(numeric.Deinterleave(hash >> 1))
64
+}
65
+
66
+func unscaleLon(lon uint64) float64 {
67
+	return (float64(lon) / lonScale) + minLon
68
+}
69
+
70
+func unscaleLat(lat uint64) float64 {
71
+	return (float64(lat) / latScale) + minLat
72
+}
73
+
74
+// compareGeo will compare two float values and see if they are the same
75
+// taking into consideration a known geo tolerance.
76
+func compareGeo(a, b float64) float64 {
77
+	compare := a - b
78
+	if math.Abs(compare) <= geoTolerance {
79
+		return 0
80
+	}
81
+	return compare
82
+}
83
+
84
+// RectIntersects checks whether rectangles a and b intersect
85
+func RectIntersects(aMinX, aMinY, aMaxX, aMaxY, bMinX, bMinY, bMaxX, bMaxY float64) bool {
86
+	return !(aMaxX < bMinX || aMinX > bMaxX || aMaxY < bMinY || aMinY > bMaxY)
87
+}
88
+
89
+// RectWithin checks whether box a is within box b
90
+func RectWithin(aMinX, aMinY, aMaxX, aMaxY, bMinX, bMinY, bMaxX, bMaxY float64) bool {
91
+	rv := !(aMinX < bMinX || aMinY < bMinY || aMaxX > bMaxX || aMaxY > bMaxY)
92
+	return rv
93
+}
94
+
95
+// BoundingBoxContains checks whether the lon/lat point is within the box
96
+func BoundingBoxContains(lon, lat, minLon, minLat, maxLon, maxLat float64) bool {
97
+	return compareGeo(lon, minLon) >= 0 && compareGeo(lon, maxLon) <= 0 &&
98
+		compareGeo(lat, minLat) >= 0 && compareGeo(lat, maxLat) <= 0
99
+}
100
+
101
+const degreesToRadian = math.Pi / 180
102
+const radiansToDegrees = 180 / math.Pi
103
+
104
+// DegreesToRadians converts an angle in degrees to radians
105
+func DegreesToRadians(d float64) float64 {
106
+	return d * degreesToRadian
107
+}
108
+
109
+// RadiansToDegrees converts an angle in radians to degress
110
+func RadiansToDegrees(r float64) float64 {
111
+	return r * radiansToDegrees
112
+}
113
+
114
+var earthMeanRadiusMeters = 6371008.7714
115
+
116
+func RectFromPointDistance(lon, lat, dist float64) (float64, float64, float64, float64, error) {
117
+	err := checkLongitude(lon)
118
+	if err != nil {
119
+		return 0, 0, 0, 0, err
120
+	}
121
+	err = checkLatitude(lat)
122
+	if err != nil {
123
+		return 0, 0, 0, 0, err
124
+	}
125
+	radLon := DegreesToRadians(lon)
126
+	radLat := DegreesToRadians(lat)
127
+	radDistance := (dist + 7e-2) / earthMeanRadiusMeters
128
+
129
+	minLatL := radLat - radDistance
130
+	maxLatL := radLat + radDistance
131
+
132
+	var minLonL, maxLonL float64
133
+	if minLatL > minLatRad && maxLatL < maxLatRad {
134
+		deltaLon := asin(sin(radDistance) / cos(radLat))
135
+		minLonL = radLon - deltaLon
136
+		if minLonL < minLonRad {
137
+			minLonL += 2 * math.Pi
138
+		}
139
+		maxLonL = radLon + deltaLon
140
+		if maxLonL > maxLonRad {
141
+			maxLonL -= 2 * math.Pi
142
+		}
143
+	} else {
144
+		// pole is inside distance
145
+		minLatL = math.Max(minLatL, minLatRad)
146
+		maxLatL = math.Min(maxLatL, maxLatRad)
147
+		minLonL = minLonRad
148
+		maxLonL = maxLonRad
149
+	}
150
+
151
+	return RadiansToDegrees(minLonL),
152
+		RadiansToDegrees(maxLatL),
153
+		RadiansToDegrees(maxLonL),
154
+		RadiansToDegrees(minLatL),
155
+		nil
156
+}
157
+
158
+func checkLatitude(latitude float64) error {
159
+	if math.IsNaN(latitude) || latitude < minLat || latitude > maxLat {
160
+		return fmt.Errorf("invalid latitude %f; must be between %f and %f", latitude, minLat, maxLat)
161
+	}
162
+	return nil
163
+}
164
+
165
+func checkLongitude(longitude float64) error {
166
+	if math.IsNaN(longitude) || longitude < minLon || longitude > maxLon {
167
+		return fmt.Errorf("invalid longitude %f; must be between %f and %f", longitude, minLon, maxLon)
168
+	}
169
+	return nil
170
+}

+ 98 - 0
vendor/github.com/blevesearch/bleve/geo/geo_dist.go

@@ -0,0 +1,98 @@
1
+//  Copyright (c) 2017 Couchbase, Inc.
2
+//
3
+// Licensed under the Apache License, Version 2.0 (the "License");
4
+// you may not use this file except in compliance with the License.
5
+// You may obtain a copy of the License at
6
+//
7
+// 		http://www.apache.org/licenses/LICENSE-2.0
8
+//
9
+// Unless required by applicable law or agreed to in writing, software
10
+// distributed under the License is distributed on an "AS IS" BASIS,
11
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+// See the License for the specific language governing permissions and
13
+// limitations under the License.
14
+
15
+package geo
16
+
17
+import (
18
+	"fmt"
19
+	"math"
20
+	"strconv"
21
+	"strings"
22
+)
23
+
24
+type distanceUnit struct {
25
+	conv     float64
26
+	suffixes []string
27
+}
28
+
29
+var inch = distanceUnit{0.0254, []string{"in", "inch"}}
30
+var yard = distanceUnit{0.9144, []string{"yd", "yards"}}
31
+var feet = distanceUnit{0.3048, []string{"ft", "feet"}}
32
+var kilom = distanceUnit{1000, []string{"km", "kilometers"}}
33
+var nauticalm = distanceUnit{1852.0, []string{"nm", "nauticalmiles"}}
34
+var millim = distanceUnit{0.001, []string{"mm", "millimeters"}}
35
+var centim = distanceUnit{0.01, []string{"cm", "centimeters"}}
36
+var miles = distanceUnit{1609.344, []string{"mi", "miles"}}
37
+var meters = distanceUnit{1, []string{"m", "meters"}}
38
+
39
+var distanceUnits = []*distanceUnit{
40
+	&inch, &yard, &feet, &kilom, &nauticalm, &millim, &centim, &miles, &meters,
41
+}
42
+
43
+// ParseDistance attempts to parse a distance string and return distance in
44
+// meters.  Example formats supported:
45
+// "5in" "5inch" "7yd" "7yards" "9ft" "9feet" "11km" "11kilometers"
46
+// "3nm" "3nauticalmiles" "13mm" "13millimeters" "15cm" "15centimeters"
47
+// "17mi" "17miles" "19m" "19meters"
48
+// If the unit cannot be determined, the entire string is parsed and the
49
+// unit of meters is assumed.
50
+// If the number portion cannot be parsed, 0 and the parse error are returned.
51
+func ParseDistance(d string) (float64, error) {
52
+	for _, unit := range distanceUnits {
53
+		for _, unitSuffix := range unit.suffixes {
54
+			if strings.HasSuffix(d, unitSuffix) {
55
+				parsedNum, err := strconv.ParseFloat(d[0:len(d)-len(unitSuffix)], 64)
56
+				if err != nil {
57
+					return 0, err
58
+				}
59
+				return parsedNum * unit.conv, nil
60
+			}
61
+		}
62
+	}
63
+	// no unit matched, try assuming meters?
64
+	parsedNum, err := strconv.ParseFloat(d, 64)
65
+	if err != nil {
66
+		return 0, err
67
+	}
68
+	return parsedNum, nil
69
+}
70
+
71
+// ParseDistanceUnit attempts to parse a distance unit and return the
72
+// multiplier for converting this to meters.  If the unit cannot be parsed
73
+// then 0 and the error message is returned.
74
+func ParseDistanceUnit(u string) (float64, error) {
75
+	for _, unit := range distanceUnits {
76
+		for _, unitSuffix := range unit.suffixes {
77
+			if u == unitSuffix {
78
+				return unit.conv, nil
79
+			}
80
+		}
81
+	}
82
+	return 0, fmt.Errorf("unknown distance unit: %s", u)
83
+}
84
+
85
+// Haversin computes the distance between two points.
86
+// This implemenation uses the sloppy math implemenations which trade off
87
+// accuracy for performance.  The distance returned is in kilometers.
88
+func Haversin(lon1, lat1, lon2, lat2 float64) float64 {
89
+	x1 := lat1 * degreesToRadian
90
+	x2 := lat2 * degreesToRadian
91
+	h1 := 1 - cos(x1-x2)
92
+	h2 := 1 - cos((lon1-lon2)*degreesToRadian)
93
+	h := (h1 + cos(x1)*cos(x2)*h2) / 2
94
+	avgLat := (x1 + x2) / 2
95
+	diameter := earthDiameter(avgLat)
96
+
97
+	return diameter * asin(math.Min(1, math.Sqrt(h)))
98
+}

+ 140 - 0
vendor/github.com/blevesearch/bleve/geo/parse.go

@@ -0,0 +1,140 @@
1
+//  Copyright (c) 2017 Couchbase, Inc.
2
+//
3
+// Licensed under the Apache License, Version 2.0 (the "License");
4
+// you may not use this file except in compliance with the License.
5
+// You may obtain a copy of the License at
6
+//
7
+// 		http://www.apache.org/licenses/LICENSE-2.0
8
+//
9
+// Unless required by applicable law or agreed to in writing, software
10
+// distributed under the License is distributed on an "AS IS" BASIS,
11
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+// See the License for the specific language governing permissions and
13
+// limitations under the License.
14
+
15
+package geo
16
+
17
+import (
18
+	"reflect"
19
+	"strings"
20
+)
21
+
22
+// ExtractGeoPoint takes an arbitrary interface{} and tries it's best to
23
+// interpret it is as geo point.  Supported formats:
24
+// Container:
25
+// slice length 2 (GeoJSON)
26
+//  first element lon, second element lat
27
+// map[string]interface{}
28
+//  exact keys lat and lon or lng
29
+// struct
30
+//  w/exported fields case-insensitive match on lat and lon or lng
31
+// struct
32
+//  satisfying Later and Loner or Lnger interfaces
33
+//
34
+// in all cases values must be some sort of numeric-like thing: int/uint/float
35
+func ExtractGeoPoint(thing interface{}) (lon, lat float64, success bool) {
36
+	var foundLon, foundLat bool
37
+
38
+	thingVal := reflect.ValueOf(thing)
39
+	thingTyp := thingVal.Type()
40
+
41
+	// is it a slice
42
+	if thingVal.IsValid() && thingVal.Kind() == reflect.Slice {
43
+		// must be length 2
44
+		if thingVal.Len() == 2 {
45
+			first := thingVal.Index(0)
46
+			if first.CanInterface() {
47
+				firstVal := first.Interface()
48
+				lon, foundLon = extractNumericVal(firstVal)
49
+			}
50
+			second := thingVal.Index(1)
51
+			if second.CanInterface() {
52
+				secondVal := second.Interface()
53
+				lat, foundLat = extractNumericVal(secondVal)
54
+			}
55
+		}
56
+	}
57
+
58
+	// is it a map
59
+	if l, ok := thing.(map[string]interface{}); ok {
60
+		if lval, ok := l["lon"]; ok {
61
+			lon, foundLon = extractNumericVal(lval)
62
+		} else if lval, ok := l["lng"]; ok {
63
+			lon, foundLon = extractNumericVal(lval)
64
+		}
65
+		if lval, ok := l["lat"]; ok {
66
+			lat, foundLat = extractNumericVal(lval)
67
+		}
68
+	}
69
+
70
+	// now try reflection on struct fields
71
+	if thingVal.IsValid() && thingVal.Kind() == reflect.Struct {
72
+		for i := 0; i < thingVal.NumField(); i++ {
73
+			fieldName := thingTyp.Field(i).Name
74
+			if strings.HasPrefix(strings.ToLower(fieldName), "lon") {
75
+				if thingVal.Field(i).CanInterface() {
76
+					fieldVal := thingVal.Field(i).Interface()
77
+					lon, foundLon = extractNumericVal(fieldVal)
78
+				}
79
+			}
80
+			if strings.HasPrefix(strings.ToLower(fieldName), "lng") {
81
+				if thingVal.Field(i).CanInterface() {
82
+					fieldVal := thingVal.Field(i).Interface()
83
+					lon, foundLon = extractNumericVal(fieldVal)
84
+				}
85
+			}
86
+			if strings.HasPrefix(strings.ToLower(fieldName), "lat") {
87
+				if thingVal.Field(i).CanInterface() {
88
+					fieldVal := thingVal.Field(i).Interface()
89
+					lat, foundLat = extractNumericVal(fieldVal)
90
+				}
91
+			}
92
+		}
93
+	}
94
+
95
+	// last hope, some interfaces
96
+	// lon
97
+	if l, ok := thing.(loner); ok {
98
+		lon = l.Lon()
99
+		foundLon = true
100
+	} else if l, ok := thing.(lnger); ok {
101
+		lon = l.Lng()
102
+		foundLon = true
103
+	}
104
+	// lat
105
+	if l, ok := thing.(later); ok {
106
+		lat = l.Lat()
107
+		foundLat = true
108
+	}
109
+
110
+	return lon, lat, foundLon && foundLat
111
+}
112
+
113
+// extract numeric value (if possible) and returns a float64
114
+func extractNumericVal(v interface{}) (float64, bool) {
115
+	val := reflect.ValueOf(v)
116
+	typ := val.Type()
117
+	switch typ.Kind() {
118
+	case reflect.Float32, reflect.Float64:
119
+		return val.Float(), true
120
+	case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
121
+		return float64(val.Int()), true
122
+	case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
123
+		return float64(val.Uint()), true
124
+	}
125
+
126
+	return 0, false
127
+}
128
+
129
+// various support interfaces which can be used to find lat/lon
130
+type loner interface {
131
+	Lon() float64
132
+}
133
+
134
+type later interface {
135
+	Lat() float64
136
+}
137
+
138
+type lnger interface {
139
+	Lng() float64
140
+}

+ 212 - 0
vendor/github.com/blevesearch/bleve/geo/sloppy.go

@@ -0,0 +1,212 @@
1
+//  Copyright (c) 2017 Couchbase, Inc.
2
+//
3
+// Licensed under the Apache License, Version 2.0 (the "License");
4
+// you may not use this file except in compliance with the License.
5
+// You may obtain a copy of the License at
6
+//
7
+// 		http://www.apache.org/licenses/LICENSE-2.0
8
+//
9
+// Unless required by applicable law or agreed to in writing, software
10
+// distributed under the License is distributed on an "AS IS" BASIS,
11
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+// See the License for the specific language governing permissions and
13
+// limitations under the License.
14
+
15
+package geo
16
+
17
+import (
18
+	"math"
19
+)
20
+
21
+var earthDiameterPerLatitude []float64
22
+var sinTab []float64
23
+var cosTab []float64
24
+var asinTab []float64
25
+var asinDer1DivF1Tab []float64
26
+var asinDer2DivF2Tab []float64
27
+var asinDer3DivF3Tab []float64
28
+var asinDer4DivF4Tab []float64
29
+
30
+const radiusTabsSize = (1 << 10) + 1
31
+const radiusDelta = (math.Pi / 2) / (radiusTabsSize - 1)
32
+const radiusIndexer = 1 / radiusDelta
33
+const sinCosTabsSize = (1 << 11) + 1
34
+const asinTabsSize = (1 << 13) + 1
35
+const oneDivF2 = 1 / 2.0
36
+const oneDivF3 = 1 / 6.0
37
+const oneDivF4 = 1 / 24.0
38
+
39
+// 1.57079632673412561417e+00 first 33 bits of pi/2
40
+var pio2Hi = math.Float64frombits(0x3FF921FB54400000)
41
+
42
+// 6.07710050650619224932e-11 pi/2 - PIO2_HI
43
+var pio2Lo = math.Float64frombits(0x3DD0B4611A626331)
44
+
45
+var asinPio2Hi = math.Float64frombits(0x3FF921FB54442D18) // 1.57079632679489655800e+00
46
+var asinPio2Lo = math.Float64frombits(0x3C91A62633145C07) // 6.12323399573676603587e-17
47
+var asinPs0 = math.Float64frombits(0x3fc5555555555555)    //  1.66666666666666657415e-01
48
+var asinPs1 = math.Float64frombits(0xbfd4d61203eb6f7d)    // -3.25565818622400915405e-01
49
+var asinPs2 = math.Float64frombits(0x3fc9c1550e884455)    //  2.01212532134862925881e-01
50
+var asinPs3 = math.Float64frombits(0xbfa48228b5688f3b)    // -4.00555345006794114027e-02
51
+var asinPs4 = math.Float64frombits(0x3f49efe07501b288)    //  7.91534994289814532176e-04
52
+var asinPs5 = math.Float64frombits(0x3f023de10dfdf709)    //  3.47933107596021167570e-05
53
+var asinQs1 = math.Float64frombits(0xc0033a271c8a2d4b)    // -2.40339491173441421878e+00
54
+var asinQs2 = math.Float64frombits(0x40002ae59c598ac8)    //  2.02094576023350569471e+00
55
+var asinQs3 = math.Float64frombits(0xbfe6066c1b8d0159)    // -6.88283971605453293030e-01
56
+var asinQs4 = math.Float64frombits(0x3fb3b8c5b12e9282)    //  7.70381505559019352791e-02
57
+
58
+var twoPiHi = 4 * pio2Hi
59
+var twoPiLo = 4 * pio2Lo
60
+var sinCosDeltaHi = twoPiHi/sinCosTabsSize - 1
61
+var sinCosDeltaLo = twoPiLo/sinCosTabsSize - 1
62
+var sinCosIndexer = 1 / (sinCosDeltaHi + sinCosDeltaLo)
63
+var sinCosMaxValueForIntModulo = ((math.MaxInt64 >> 9) / sinCosIndexer) * 0.99
64
+var asinMaxValueForTabs = math.Sin(73.0 * degreesToRadian)
65
+
66
+var asinDelta = asinMaxValueForTabs / (asinTabsSize - 1)
67
+var asinIndexer = 1 / asinDelta
68
+
69
+func init() {
70
+	// initializes the tables used for the sloppy math functions
71
+
72
+	// sin and cos
73
+	sinTab = make([]float64, sinCosTabsSize)
74
+	cosTab = make([]float64, sinCosTabsSize)
75
+	sinCosPiIndex := (sinCosTabsSize - 1) / 2
76
+	sinCosPiMul2Index := 2 * sinCosPiIndex
77
+	sinCosPiMul05Index := sinCosPiIndex / 2
78
+	sinCosPiMul15Index := 3 * sinCosPiIndex / 2
79
+	for i := 0; i < sinCosTabsSize; i++ {
80
+		// angle: in [0,2*PI].
81
+		angle := float64(i)*sinCosDeltaHi + float64(i)*sinCosDeltaLo
82
+		sinAngle := math.Sin(angle)
83
+		cosAngle := math.Cos(angle)
84
+		// For indexes corresponding to null cosine or sine, we make sure the value is zero
85
+		// and not an epsilon. This allows for a much better accuracy for results close to zero.
86
+		if i == sinCosPiIndex {
87
+			sinAngle = 0.0
88
+		} else if i == sinCosPiMul2Index {
89
+			sinAngle = 0.0
90
+		} else if i == sinCosPiMul05Index {
91
+			sinAngle = 0.0
92
+		} else if i == sinCosPiMul15Index {
93
+			sinAngle = 0.0
94
+		}
95
+		sinTab[i] = sinAngle
96
+		cosTab[i] = cosAngle
97
+	}
98
+
99
+	// asin
100
+	asinTab = make([]float64, asinTabsSize)
101
+	asinDer1DivF1Tab = make([]float64, asinTabsSize)
102
+	asinDer2DivF2Tab = make([]float64, asinTabsSize)
103
+	asinDer3DivF3Tab = make([]float64, asinTabsSize)
104
+	asinDer4DivF4Tab = make([]float64, asinTabsSize)
105
+	for i := 0; i < asinTabsSize; i++ {
106
+		// x: in [0,ASIN_MAX_VALUE_FOR_TABS].
107
+		x := float64(i) * asinDelta
108
+		asinTab[i] = math.Asin(x)
109
+		oneMinusXSqInv := 1.0 / (1 - x*x)
110
+		oneMinusXSqInv05 := math.Sqrt(oneMinusXSqInv)
111
+		oneMinusXSqInv15 := oneMinusXSqInv05 * oneMinusXSqInv
112
+		oneMinusXSqInv25 := oneMinusXSqInv15 * oneMinusXSqInv
113
+		oneMinusXSqInv35 := oneMinusXSqInv25 * oneMinusXSqInv
114
+		asinDer1DivF1Tab[i] = oneMinusXSqInv05
115
+		asinDer2DivF2Tab[i] = (x * oneMinusXSqInv15) * oneDivF2
116
+		asinDer3DivF3Tab[i] = ((1 + 2*x*x) * oneMinusXSqInv25) * oneDivF3
117
+		asinDer4DivF4Tab[i] = ((5 + 2*x*(2+x*(5-2*x))) * oneMinusXSqInv35) * oneDivF4
118
+	}
119
+
120
+	// earth radius
121
+	a := 6378137.0
122
+	b := 6356752.31420
123
+	a2 := a * a
124
+	b2 := b * b
125
+	earthDiameterPerLatitude = make([]float64, radiusTabsSize)
126
+	earthDiameterPerLatitude[0] = 2.0 * a / 1000
127
+	earthDiameterPerLatitude[radiusTabsSize-1] = 2.0 * b / 1000
128
+	for i := 1; i < radiusTabsSize-1; i++ {
129
+		lat := math.Pi * float64(i) / (2*radiusTabsSize - 1)
130
+		one := math.Pow(a2*math.Cos(lat), 2)
131
+		two := math.Pow(b2*math.Sin(lat), 2)
132
+		three := math.Pow(float64(a)*math.Cos(lat), 2)
133
+		four := math.Pow(b*math.Sin(lat), 2)
134
+		radius := math.Sqrt((one + two) / (three + four))
135
+		earthDiameterPerLatitude[i] = 2 * radius / 1000
136
+	}
137
+}
138
+
139
+// earthDiameter returns an estimation of the earth's diameter at the specified
140
+// latitude in kilometers
141
+func earthDiameter(lat float64) float64 {
142
+	index := math.Mod(math.Abs(lat)*radiusIndexer+0.5, float64(len(earthDiameterPerLatitude)))
143
+	if math.IsNaN(index) {
144
+		return 0
145
+	}
146
+	return earthDiameterPerLatitude[int(index)]
147
+}
148
+
149
+var pio2 = math.Pi / 2
150
+
151
+func sin(a float64) float64 {
152
+	return cos(a - pio2)
153
+}
154
+
155
+// cos is a sloppy math (faster) implementation of math.Cos
156
+func cos(a float64) float64 {
157
+	if a < 0.0 {
158
+		a = -a
159
+	}
160
+	if a > sinCosMaxValueForIntModulo {
161
+		return math.Cos(a)
162
+	}
163
+	// index: possibly outside tables range.
164
+	index := int(a*sinCosIndexer + 0.5)
165
+	delta := (a - float64(index)*sinCosDeltaHi) - float64(index)*sinCosDeltaLo
166
+	// Making sure index is within tables range.
167
+	// Last value of each table is the same than first, so we ignore it (tabs size minus one) for modulo.
168
+	index &= (sinCosTabsSize - 2) // index % (SIN_COS_TABS_SIZE-1)
169
+	indexCos := cosTab[index]
170
+	indexSin := sinTab[index]
171
+	return indexCos + delta*(-indexSin+delta*(-indexCos*oneDivF2+delta*(indexSin*oneDivF3+delta*indexCos*oneDivF4)))
172
+}
173
+
174
+// asin is a sloppy math (faster) implementation of math.Asin
175
+func asin(a float64) float64 {
176
+	var negateResult bool
177
+	if a < 0 {
178
+		a = -a
179
+		negateResult = true
180
+	}
181
+	if a <= asinMaxValueForTabs {
182
+		index := int(a*asinIndexer + 0.5)
183
+		delta := a - float64(index)*asinDelta
184
+		result := asinTab[index] + delta*(asinDer1DivF1Tab[index]+delta*(asinDer2DivF2Tab[index]+delta*(asinDer3DivF3Tab[index]+delta*asinDer4DivF4Tab[index])))
185
+		if negateResult {
186
+			return -result
187
+		}
188
+		return result
189
+	}
190
+	// value > ASIN_MAX_VALUE_FOR_TABS, or value is NaN
191
+	// This part is derived from fdlibm.
192
+	if a < 1 {
193
+		t := (1.0 - a) * 0.5
194
+		p := t * (asinPs0 + t*(asinPs1+t*(asinPs2+t*(asinPs3+t*(asinPs4+t+asinPs5)))))
195
+		q := 1.0 + t*(asinQs1+t*(asinQs2+t*(asinQs3+t*asinQs4)))
196
+		s := math.Sqrt(t)
197
+		z := s + s*(p/q)
198
+		result := asinPio2Hi - ((z + z) - asinPio2Lo)
199
+		if negateResult {
200
+			return -result
201
+		}
202
+		return result
203
+	}
204
+	// value >= 1.0, or value is NaN
205
+	if a == 1.0 {
206
+		if negateResult {
207
+			return -math.Pi / 2
208
+		}
209
+		return math.Pi / 2
210
+	}
211
+	return math.NaN()
212
+}

+ 18 - 4
vendor/github.com/blevesearch/bleve/index.go

@@ -49,6 +49,17 @@ func (b *Batch) Index(id string, data interface{}) error {
49 49
 	return nil
50 50
 }
51 51
 
52
+// IndexAdvanced adds the specified index operation to the
53
+// batch which skips the mapping.  NOTE: the bleve Index is not updated
54
+// until the batch is executed.
55
+func (b *Batch) IndexAdvanced(doc *document.Document) (err error) {
56
+	if doc.ID == "" {
57
+		return ErrorEmptyID
58
+	}
59
+	b.internal.Update(doc)
60
+	return nil
61
+}
62
+
52 63
 // Delete adds the specified delete operation to the
53 64
 // batch.  NOTE: the bleve Index is not updated until
54 65
 // the batch is executed.
@@ -99,12 +110,15 @@ func (b *Batch) Reset() {
99 110
 // them.
100 111
 //
101 112
 // The DocumentMapping used to index a value is deduced by the following rules:
102
-// 1) If value implements Classifier interface, resolve the mapping from Type().
103
-// 2) If value has a string field or value at IndexMapping.TypeField.
113
+// 1) If value implements mapping.bleveClassifier interface, resolve the mapping
114
+//    from BleveType().
115
+// 2) If value implements mapping.Classifier interface, resolve the mapping
116
+//    from Type().
117
+// 3) If value has a string field or value at IndexMapping.TypeField.
104 118
 // (defaulting to "_type"), use it to resolve the mapping. Fields addressing
105 119
 // is described below.
106
-// 3) If IndexMapping.DefaultType is registered, return it.
107
-// 4) Return IndexMapping.DefaultMapping.
120
+// 4) If IndexMapping.DefaultType is registered, return it.
121
+// 5) Return IndexMapping.DefaultMapping.
108 122
 //
109 123
 // Each field or nested field of the value is identified by a string path, then
110 124
 // mapped to one or several FieldMappings which extract the result for analysis.

+ 3 - 1
vendor/github.com/blevesearch/bleve/index/index.go

@@ -48,6 +48,8 @@ type Index interface {
48 48
 	Advanced() (store.KVStore, error)
49 49
 }
50 50
 
51
+type DocumentFieldTermVisitor func(field string, term []byte)
52
+
51 53
 type IndexReader interface {
52 54
 	TermFieldReader(term []byte, field string, includeFreq, includeNorm, includeTermVectors bool) (TermFieldReader, error)
53 55
 
@@ -64,7 +66,7 @@ type IndexReader interface {
64 66
 	FieldDictPrefix(field string, termPrefix []byte) (FieldDict, error)
65 67
 
66 68
 	Document(id string) (*document.Document, error)
67
-	DocumentFieldTerms(id IndexInternalID, fields []string) (FieldTerms, error)
69
+	DocumentVisitFieldTerms(id IndexInternalID, fields []string, visitor DocumentFieldTermVisitor) error
68 70
 
69 71
 	Fields() ([]string, error)
70 72
 

+ 3 - 3
vendor/github.com/blevesearch/bleve/index/upsidedown/analysis.go

@@ -90,7 +90,7 @@ func (udc *UpsideDownCouch) Analyze(d *document.Document) *index.AnalysisResult
90 90
 
91 91
 	rv.Rows = append(make([]index.IndexRow, 0, rowsCapNeeded), rv.Rows...)
92 92
 
93
-	backIndexTermEntries := make([]*BackIndexTermEntry, 0, rowsCapNeeded)
93
+	backIndexTermsEntries := make([]*BackIndexTermsEntry, 0, len(fieldTermFreqs))
94 94
 
95 95
 	// walk through the collated information and process
96 96
 	// once for each indexed field (unique name)
@@ -99,11 +99,11 @@ func (udc *UpsideDownCouch) Analyze(d *document.Document) *index.AnalysisResult
99 99
 		includeTermVectors := fieldIncludeTermVectors[fieldIndex]
100 100
 
101 101
 		// encode this field
102
-		rv.Rows, backIndexTermEntries = udc.indexField(docIDBytes, includeTermVectors, fieldIndex, fieldLength, tokenFreqs, rv.Rows, backIndexTermEntries)
102
+		rv.Rows, backIndexTermsEntries = udc.indexField(docIDBytes, includeTermVectors, fieldIndex, fieldLength, tokenFreqs, rv.Rows, backIndexTermsEntries)
103 103
 	}
104 104
 
105 105
 	// build the back index row
106
-	backIndexRow := NewBackIndexRow(docIDBytes, backIndexTermEntries, backIndexStoredEntries)
106
+	backIndexRow := NewBackIndexRow(docIDBytes, backIndexTermsEntries, backIndexStoredEntries)
107 107
 	rv.Rows = append(rv.Rows, backIndexRow)
108 108
 
109 109
 	return rv

+ 6 - 4
vendor/github.com/blevesearch/bleve/index/upsidedown/dump.go

@@ -127,10 +127,12 @@ func (i *IndexReader) DumpDoc(id string) chan interface{} {
127 127
 		}
128 128
 		// build sorted list of term keys
129 129
 		keys := make(keyset, 0)
130
-		for _, entry := range back.termEntries {
131
-			tfr := NewTermFrequencyRow([]byte(*entry.Term), uint16(*entry.Field), idBytes, 0, 0)
132
-			key := tfr.Key()
133
-			keys = append(keys, key)
130
+		for _, entry := range back.termsEntries {
131
+			for i := range entry.Terms {
132
+				tfr := NewTermFrequencyRow([]byte(entry.Terms[i]), uint16(*entry.Field), idBytes, 0, 0)
133
+				key := tfr.Key()
134
+				keys = append(keys, key)
135
+			}
134 136
 		}
135 137
 		sort.Sort(keys)
136 138
 

+ 28 - 14
vendor/github.com/blevesearch/bleve/index/upsidedown/index_reader.go

@@ -101,15 +101,7 @@ func (i *IndexReader) Document(id string) (doc *document.Document, err error) {
101 101
 	return
102 102
 }
103 103
 
104
-func (i *IndexReader) DocumentFieldTerms(id index.IndexInternalID, fields []string) (index.FieldTerms, error) {
105
-	back, err := backIndexRowForDoc(i.kvreader, id)
106
-	if err != nil {
107
-		return nil, err
108
-	}
109
-	if back == nil {
110
-		return nil, nil
111
-	}
112
-	rv := make(index.FieldTerms, len(fields))
104
+func (i *IndexReader) DocumentVisitFieldTerms(id index.IndexInternalID, fields []string, visitor index.DocumentFieldTermVisitor) error {
113 105
 	fieldsMap := make(map[uint16]string, len(fields))
114 106
 	for _, f := range fields {
115 107
 		id, ok := i.index.fieldCache.FieldNamed(f, false)
@@ -117,12 +109,34 @@ func (i *IndexReader) DocumentFieldTerms(id index.IndexInternalID, fields []stri
117 109
 			fieldsMap[id] = f
118 110
 		}
119 111
 	}
120
-	for _, entry := range back.termEntries {
121
-		if field, ok := fieldsMap[uint16(*entry.Field)]; ok {
122
-			rv[field] = append(rv[field], *entry.Term)
123
-		}
112
+
113
+	tempRow := BackIndexRow{
114
+		doc: id,
115
+	}
116
+
117
+	keyBuf := GetRowBuffer()
118
+	if tempRow.KeySize() > len(keyBuf) {
119
+		keyBuf = make([]byte, 2*tempRow.KeySize())
124 120
 	}
125
-	return rv, nil
121
+	defer PutRowBuffer(keyBuf)
122
+	keySize, err := tempRow.KeyTo(keyBuf)
123
+	if err != nil {
124
+		return err
125
+	}
126
+
127
+	value, err := i.kvreader.Get(keyBuf[:keySize])
128
+	if err != nil {
129
+		return err
130
+	}
131
+	if value == nil {
132
+		return nil
133
+	}
134
+
135
+	return visitBackIndexRow(value, func(field uint32, term []byte) {
136
+		if field, ok := fieldsMap[uint16(field)]; ok {
137
+			visitor(field, term)
138
+		}
139
+	})
126 140
 }
127 141
 
128 142
 func (i *IndexReader) Fields() (fields []string, err error) {

+ 37 - 26
vendor/github.com/blevesearch/bleve/index/upsidedown/reader.go

@@ -24,46 +24,57 @@ import (
24 24
 )
25 25
 
26 26
 type UpsideDownCouchTermFieldReader struct {
27
-	count       uint64
28
-	indexReader *IndexReader
29
-	iterator    store.KVIterator
30
-	term        []byte
31
-	tfrNext     *TermFrequencyRow
32
-	keyBuf      []byte
33
-	field       uint16
27
+	count              uint64
28
+	indexReader        *IndexReader
29
+	iterator           store.KVIterator
30
+	term               []byte
31
+	tfrNext            *TermFrequencyRow
32
+	tfrPrealloc        TermFrequencyRow
33
+	keyBuf             []byte
34
+	field              uint16
35
+	includeTermVectors bool
34 36
 }
35 37
 
36 38
 func newUpsideDownCouchTermFieldReader(indexReader *IndexReader, term []byte, field uint16, includeFreq, includeNorm, includeTermVectors bool) (*UpsideDownCouchTermFieldReader, error) {
37
-	dictionaryRow := NewDictionaryRow(term, field, 0)
38
-	val, err := indexReader.kvreader.Get(dictionaryRow.Key())
39
+	bufNeeded := termFrequencyRowKeySize(term, nil)
40
+	if bufNeeded < dictionaryRowKeySize(term) {
41
+		bufNeeded = dictionaryRowKeySize(term)
42
+	}
43
+	buf := make([]byte, bufNeeded)
44
+
45
+	bufUsed := dictionaryRowKeyTo(buf, field, term)
46
+	val, err := indexReader.kvreader.Get(buf[:bufUsed])
39 47
 	if err != nil {
40 48
 		return nil, err
41 49
 	}
42 50
 	if val == nil {
43 51
 		atomic.AddUint64(&indexReader.index.stats.termSearchersStarted, uint64(1))
44
-		return &UpsideDownCouchTermFieldReader{
45
-			count:   0,
46
-			term:    term,
47
-			tfrNext: &TermFrequencyRow{},
48
-			field:   field,
49
-		}, nil
52
+		rv := &UpsideDownCouchTermFieldReader{
53
+			count:              0,
54
+			term:               term,
55
+			field:              field,
56
+			includeTermVectors: includeTermVectors,
57
+		}
58
+		rv.tfrNext = &rv.tfrPrealloc
59
+		return rv, nil
50 60
 	}
51 61
 
52
-	err = dictionaryRow.parseDictionaryV(val)
62
+	count, err := dictionaryRowParseV(val)
53 63
 	if err != nil {
54 64
 		return nil, err
55 65
 	}
56 66
 
57
-	tfr := NewTermFrequencyRow(term, field, []byte{}, 0, 0)
58
-	it := indexReader.kvreader.PrefixIterator(tfr.Key())
67
+	bufUsed = termFrequencyRowKeyTo(buf, field, term, nil)
68
+	it := indexReader.kvreader.PrefixIterator(buf[:bufUsed])
59 69
 
60 70
 	atomic.AddUint64(&indexReader.index.stats.termSearchersStarted, uint64(1))
61 71
 	return &UpsideDownCouchTermFieldReader{
62
-		indexReader: indexReader,
63
-		iterator:    it,
64
-		count:       dictionaryRow.count,
65
-		term:        term,
66
-		field:       field,
72
+		indexReader:        indexReader,
73
+		iterator:           it,
74
+		count:              count,
75
+		term:               term,
76
+		field:              field,
77
+		includeTermVectors: includeTermVectors,
67 78
 	}, nil
68 79
 }
69 80
 
@@ -79,7 +90,7 @@ func (r *UpsideDownCouchTermFieldReader) Next(preAlloced *index.TermFieldDoc) (*
79 90
 		if r.tfrNext != nil {
80 91
 			r.iterator.Next()
81 92
 		} else {
82
-			r.tfrNext = &TermFrequencyRow{}
93
+			r.tfrNext = &r.tfrPrealloc
83 94
 		}
84 95
 		key, val, valid := r.iterator.Current()
85 96
 		if valid {
@@ -88,7 +99,7 @@ func (r *UpsideDownCouchTermFieldReader) Next(preAlloced *index.TermFieldDoc) (*
88 99
 			if err != nil {
89 100
 				return nil, err
90 101
 			}
91
-			err = tfr.parseV(val)
102
+			err = tfr.parseV(val, r.includeTermVectors)
92 103
 			if err != nil {
93 104
 				return nil, err
94 105
 			}
@@ -125,7 +136,7 @@ func (r *UpsideDownCouchTermFieldReader) Advance(docID index.IndexInternalID, pr
125 136
 			if err != nil {
126 137
 				return nil, err
127 138
 			}
128
-			err = tfr.parseV(val)
139
+			err = tfr.parseV(val, r.includeTermVectors)
129 140
 			if err != nil {
130 141
 				return nil, err
131 142
 			}

+ 284 - 25
vendor/github.com/blevesearch/bleve/index/upsidedown/row.go

@@ -254,14 +254,22 @@ func (dr *DictionaryRow) Key() []byte {
254 254
 }
255 255
 
256 256
 func (dr *DictionaryRow) KeySize() int {
257
-	return len(dr.term) + 3
257
+	return dictionaryRowKeySize(dr.term)
258
+}
259
+
260
+func dictionaryRowKeySize(term []byte) int {
261
+	return len(term) + 3
258 262
 }
259 263
 
260 264
 func (dr *DictionaryRow) KeyTo(buf []byte) (int, error) {
265
+	return dictionaryRowKeyTo(buf, dr.field, dr.term), nil
266
+}
267
+
268
+func dictionaryRowKeyTo(buf []byte, field uint16, term []byte) int {
261 269
 	buf[0] = 'd'
262
-	binary.LittleEndian.PutUint16(buf[1:3], dr.field)
263
-	size := copy(buf[3:], dr.term)
264
-	return size + 3, nil
270
+	binary.LittleEndian.PutUint16(buf[1:3], field)
271
+	size := copy(buf[3:], term)
272
+	return size + 3
265 273
 }
266 274
 
267 275
 func (dr *DictionaryRow) Value() []byte {
@@ -324,14 +332,22 @@ func (dr *DictionaryRow) parseDictionaryK(key []byte) error {
324 332
 }
325 333
 
326 334
 func (dr *DictionaryRow) parseDictionaryV(value []byte) error {
327
-	count, nread := binary.Uvarint(value)
328
-	if nread <= 0 {
329
-		return fmt.Errorf("DictionaryRow parse Uvarint error, nread: %d", nread)
335
+	count, err := dictionaryRowParseV(value)
336
+	if err != nil {
337
+		return err
330 338
 	}
331 339
 	dr.count = count
332 340
 	return nil
333 341
 }
334 342
 
343
+func dictionaryRowParseV(value []byte) (uint64, error) {
344
+	count, nread := binary.Uvarint(value)
345
+	if nread <= 0 {
346
+		return 0, fmt.Errorf("DictionaryRow parse Uvarint error, nread: %d", nread)
347
+	}
348
+	return count, nil
349
+}
350
+
335 351
 // TERM FIELD FREQUENCY
336 352
 
337 353
 type TermVector struct {
@@ -394,16 +410,24 @@ func (tfr *TermFrequencyRow) Key() []byte {
394 410
 }
395 411
 
396 412
 func (tfr *TermFrequencyRow) KeySize() int {
397
-	return 3 + len(tfr.term) + 1 + len(tfr.doc)
413
+	return termFrequencyRowKeySize(tfr.term, tfr.doc)
414
+}
415
+
416
+func termFrequencyRowKeySize(term, doc []byte) int {
417
+	return 3 + len(term) + 1 + len(doc)
398 418
 }
399 419
 
400 420
 func (tfr *TermFrequencyRow) KeyTo(buf []byte) (int, error) {
421
+	return termFrequencyRowKeyTo(buf, tfr.field, tfr.term, tfr.doc), nil
422
+}
423
+
424
+func termFrequencyRowKeyTo(buf []byte, field uint16, term, doc []byte) int {
401 425
 	buf[0] = 't'
402
-	binary.LittleEndian.PutUint16(buf[1:3], tfr.field)
403
-	termLen := copy(buf[3:], tfr.term)
426
+	binary.LittleEndian.PutUint16(buf[1:3], field)
427
+	termLen := copy(buf[3:], term)
404 428
 	buf[3+termLen] = ByteSeparator
405
-	docLen := copy(buf[3+termLen+1:], tfr.doc)
406
-	return 3 + termLen + 1 + docLen, nil
429
+	docLen := copy(buf[3+termLen+1:], doc)
430
+	return 3 + termLen + 1 + docLen
407 431
 }
408 432
 
409 433
 func (tfr *TermFrequencyRow) KeyAppendTo(buf []byte) ([]byte, error) {
@@ -538,7 +562,7 @@ func (tfr *TermFrequencyRow) parseKDoc(key []byte, term []byte) error {
538 562
 	return nil
539 563
 }
540 564
 
541
-func (tfr *TermFrequencyRow) parseV(value []byte) error {
565
+func (tfr *TermFrequencyRow) parseV(value []byte, includeTermVectors bool) error {
542 566
 	var bytesRead int
543 567
 	tfr.freq, bytesRead = binary.Uvarint(value)
544 568
 	if bytesRead <= 0 {
@@ -556,6 +580,10 @@ func (tfr *TermFrequencyRow) parseV(value []byte) error {
556 580
 	tfr.norm = math.Float32frombits(uint32(norm))
557 581
 
558 582
 	tfr.vectors = nil
583
+	if !includeTermVectors {
584
+		return nil
585
+	}
586
+
559 587
 	var field uint64
560 588
 	field, bytesRead = binary.Uvarint(value[currOffset:])
561 589
 	for bytesRead > 0 {
@@ -620,7 +648,7 @@ func NewTermFrequencyRowKV(key, value []byte) (*TermFrequencyRow, error) {
620 648
 		return nil, err
621 649
 	}
622 650
 
623
-	err = rv.parseV(value)
651
+	err = rv.parseV(value, true)
624 652
 	if err != nil {
625 653
 		return nil, err
626 654
 	}
@@ -630,7 +658,7 @@ func NewTermFrequencyRowKV(key, value []byte) (*TermFrequencyRow, error) {
630 658
 
631 659
 type BackIndexRow struct {
632 660
 	doc           []byte
633
-	termEntries   []*BackIndexTermEntry
661
+	termsEntries  []*BackIndexTermsEntry
634 662
 	storedEntries []*BackIndexStoreEntry
635 663
 }
636 664
 
@@ -638,10 +666,12 @@ func (br *BackIndexRow) AllTermKeys() [][]byte {
638 666
 	if br == nil {
639 667
 		return nil
640 668
 	}
641
-	rv := make([][]byte, len(br.termEntries))
642
-	for i, termEntry := range br.termEntries {
643
-		termRow := NewTermFrequencyRow([]byte(termEntry.GetTerm()), uint16(termEntry.GetField()), br.doc, 0, 0)
644
-		rv[i] = termRow.Key()
669
+	rv := make([][]byte, 0, len(br.termsEntries)) // FIXME this underestimates severely
670
+	for _, termsEntry := range br.termsEntries {
671
+		for i := range termsEntry.Terms {
672
+			termRow := NewTermFrequencyRow([]byte(termsEntry.Terms[i]), uint16(termsEntry.GetField()), br.doc, 0, 0)
673
+			rv = append(rv, termRow.Key())
674
+		}
645 675
 	}
646 676
 	return rv
647 677
 }
@@ -682,7 +712,7 @@ func (br *BackIndexRow) Value() []byte {
682 712
 
683 713
 func (br *BackIndexRow) ValueSize() int {
684 714
 	birv := &BackIndexRowValue{
685
-		TermEntries:   br.termEntries,
715
+		TermsEntries:  br.termsEntries,
686 716
 		StoredEntries: br.storedEntries,
687 717
 	}
688 718
 	return birv.Size()
@@ -690,20 +720,20 @@ func (br *BackIndexRow) ValueSize() int {
690 720
 
691 721
 func (br *BackIndexRow) ValueTo(buf []byte) (int, error) {
692 722
 	birv := &BackIndexRowValue{
693
-		TermEntries:   br.termEntries,
723
+		TermsEntries:  br.termsEntries,
694 724
 		StoredEntries: br.storedEntries,
695 725
 	}
696 726
 	return birv.MarshalTo(buf)
697 727
 }
698 728
 
699 729
 func (br *BackIndexRow) String() string {
700
-	return fmt.Sprintf("Backindex DocId: `%s` Term Entries: %v, Stored Entries: %v", string(br.doc), br.termEntries, br.storedEntries)
730
+	return fmt.Sprintf("Backindex DocId: `%s` Terms Entries: %v, Stored Entries: %v", string(br.doc), br.termsEntries, br.storedEntries)
701 731
 }
702 732
 
703
-func NewBackIndexRow(docID []byte, entries []*BackIndexTermEntry, storedFields []*BackIndexStoreEntry) *BackIndexRow {
733
+func NewBackIndexRow(docID []byte, entries []*BackIndexTermsEntry, storedFields []*BackIndexStoreEntry) *BackIndexRow {
704 734
 	return &BackIndexRow{
705 735
 		doc:           docID,
706
-		termEntries:   entries,
736
+		termsEntries:  entries,
707 737
 		storedEntries: storedFields,
708 738
 	}
709 739
 }
@@ -732,7 +762,7 @@ func NewBackIndexRowKV(key, value []byte) (*BackIndexRow, error) {
732 762
 	if err != nil {
733 763
 		return nil, err
734 764
 	}
735
-	rv.termEntries = birv.TermEntries
765
+	rv.termsEntries = birv.TermsEntries
736 766
 	rv.storedEntries = birv.StoredEntries
737 767
 
738 768
 	return &rv, nil
@@ -851,3 +881,232 @@ func NewStoredRowKV(key, value []byte) (*StoredRow, error) {
851 881
 	rv.value = value[1:]
852 882
 	return rv, nil
853 883
 }
884
+
885
+type backIndexFieldTermVisitor func(field uint32, term []byte)
886
+
887
+// visitBackIndexRow is designed to process a protobuf encoded
888
+// value, without creating unnecessary garbage.  Instead values are passed
889
+// to a callback, inspected first, and only copied if necessary.
890
+// Due to the fact that this borrows from generated code, it must be marnually
891
+// updated if the protobuf definition changes.
892
+//
893
+// This code originates from:
894
+// func (m *BackIndexRowValue) Unmarshal(data []byte) error
895
+// the sections which create garbage or parse unintersting sections
896
+// have been commented out.  This was done by design to allow for easier
897
+// merging in the future if that original function is regenerated
898
+func visitBackIndexRow(data []byte, callback backIndexFieldTermVisitor) error {
899
+	l := len(data)
900
+	iNdEx := 0
901
+	for iNdEx < l {
902
+		var wire uint64
903
+		for shift := uint(0); ; shift += 7 {
904
+			if iNdEx >= l {
905
+				return io.ErrUnexpectedEOF
906
+			}
907
+			b := data[iNdEx]
908
+			iNdEx++
909
+			wire |= (uint64(b) & 0x7F) << shift
910
+			if b < 0x80 {
911
+				break
912
+			}
913
+		}
914
+		fieldNum := int32(wire >> 3)
915
+		wireType := int(wire & 0x7)
916
+		switch fieldNum {
917
+		case 1:
918
+			if wireType != 2 {
919
+				return fmt.Errorf("proto: wrong wireType = %d for field TermsEntries", wireType)
920
+			}
921
+			var msglen int
922
+			for shift := uint(0); ; shift += 7 {
923
+				if iNdEx >= l {
924
+					return io.ErrUnexpectedEOF
925
+				}
926
+				b := data[iNdEx]
927
+				iNdEx++
928
+				msglen |= (int(b) & 0x7F) << shift
929
+				if b < 0x80 {
930
+					break
931
+				}
932
+			}
933
+			postIndex := iNdEx + msglen
934
+			if msglen < 0 {
935
+				return ErrInvalidLengthUpsidedown
936
+			}
937
+			if postIndex > l {
938
+				return io.ErrUnexpectedEOF
939
+			}
940
+			// dont parse term entries
941
+			// m.TermsEntries = append(m.TermsEntries, &BackIndexTermsEntry{})
942
+			// if err := m.TermsEntries[len(m.TermsEntries)-1].Unmarshal(data[iNdEx:postIndex]); err != nil {
943
+			// 	return err
944
+			// }
945
+			// instead, inspect them
946
+			if err := visitBackIndexRowFieldTerms(data[iNdEx:postIndex], callback); err != nil {
947
+				return err
948
+			}
949
+			iNdEx = postIndex
950
+		case 2:
951
+			if wireType != 2 {
952
+				return fmt.Errorf("proto: wrong wireType = %d for field StoredEntries", wireType)
953
+			}
954
+			var msglen int
955
+			for shift := uint(0); ; shift += 7 {
956
+				if iNdEx >= l {
957
+					return io.ErrUnexpectedEOF
958
+				}
959
+				b := data[iNdEx]
960
+				iNdEx++
961
+				msglen |= (int(b) & 0x7F) << shift
962
+				if b < 0x80 {
963
+					break
964
+				}
965
+			}
966
+			postIndex := iNdEx + msglen
967
+			if msglen < 0 {
968
+				return ErrInvalidLengthUpsidedown
969
+			}
970
+			if postIndex > l {
971
+				return io.ErrUnexpectedEOF
972
+			}
973
+			// don't parse stored entries
974
+			// m.StoredEntries = append(m.StoredEntries, &BackIndexStoreEntry{})
975
+			// if err := m.StoredEntries[len(m.StoredEntries)-1].Unmarshal(data[iNdEx:postIndex]); err != nil {
976
+			// 	return err