Browse Source

Code/repo search (#2582)

Indexed search of repository contents (for default branch only)
Ethan Koenig 1 year ago
parent
commit
5866eb2321

+ 4 - 0
conf/app.ini

@@ -192,7 +192,11 @@ ITERATE_BUFFER_SIZE = 50
192 192
 
193 193
 [indexer]
194 194
 ISSUE_INDEXER_PATH = indexers/issues.bleve
195
+; repo indexer by default disabled, since it uses a lot of disk space
196
+REPO_INDEXER_ENABLED = false
197
+REPO_INDEXER_PATH = indexers/repos.bleve
195 198
 UPDATE_BUFFER_LEN = 20
199
+MAX_FILE_SIZE = 1048576
196 200
 
197 201
 [admin]
198 202
 ; Disable regular (non-admin) users to create organizations

+ 4 - 0
integrations/integration_test.go

@@ -63,6 +63,10 @@ func TestMain(m *testing.M) {
63 63
 		fmt.Printf("os.RemoveAll: %v\n", err)
64 64
 		os.Exit(1)
65 65
 	}
66
+	if err = os.RemoveAll(setting.Indexer.RepoPath); err != nil {
67
+		fmt.Printf("Unable to remove repo indexer: %v\n", err)
68
+		os.Exit(1)
69
+	}
66 70
 
67 71
 	os.Exit(exitCode)
68 72
 }

+ 2 - 0
integrations/mysql.ini.tmpl

@@ -12,6 +12,8 @@ PATH     = data/gitea.db
12 12
 
13 13
 [indexer]
14 14
 ISSUE_INDEXER_PATH = integrations/indexers-mysql/issues.bleve
15
+REPO_INDEXER_ENABLED = true
16
+REPO_INDEXER_PATH = integrations/indexers-mysql/repos.bleve
15 17
 
16 18
 [repository]
17 19
 ROOT = integrations/gitea-integration-mysql/gitea-repositories

+ 2 - 0
integrations/pgsql.ini.tmpl

@@ -12,6 +12,8 @@ PATH     = data/gitea.db
12 12
 
13 13
 [indexer]
14 14
 ISSUE_INDEXER_PATH = integrations/indexers-pgsql/issues.bleve
15
+REPO_INDEXER_ENABLED = true
16
+REPO_INDEXER_PATH = integrations/indexers-pgsql/repos.bleve
15 17
 
16 18
 [repository]
17 19
 ROOT = integrations/gitea-integration-pgsql/gitea-repositories

+ 35 - 0
integrations/repo_search_test.go

@@ -0,0 +1,35 @@
1
+// Copyright 2017 The Gitea Authors. All rights reserved.
2
+// Use of this source code is governed by a MIT-style
3
+// license that can be found in the LICENSE file.
4
+
5
+package integrations
6
+
7
+import (
8
+	"net/http"
9
+	"testing"
10
+
11
+	"github.com/PuerkitoBio/goquery"
12
+	"github.com/stretchr/testify/assert"
13
+)
14
+
15
+func resultFilenames(t testing.TB, doc *HTMLDoc) []string {
16
+	resultsSelection := doc.doc.Find(".repository.search")
17
+	assert.EqualValues(t, 1, resultsSelection.Length(),
18
+		"Invalid template (repo search template has changed?)")
19
+	filenameSelections := resultsSelection.Find(".repo-search-result").Find(".header").Find("span.file")
20
+	result := make([]string, filenameSelections.Length())
21
+	filenameSelections.Each(func(i int, selection *goquery.Selection) {
22
+		result[i] = selection.Text()
23
+	})
24
+	return result
25
+}
26
+
27
+func TestSearchRepo(t *testing.T) {
28
+	prepareTestEnv(t)
29
+
30
+	req := NewRequestf(t, "GET", "/user2/repo1/search?q=Description&page=1")
31
+	resp := MakeRequest(t, req, http.StatusOK)
32
+
33
+	filenames := resultFilenames(t, NewHTMLParser(t, resp.Body))
34
+	assert.EqualValues(t, []string{"README.md"}, filenames)
35
+}

+ 2 - 0
integrations/sqlite.ini

@@ -7,6 +7,8 @@ PATH     = :memory:
7 7
 
8 8
 [indexer]
9 9
 ISSUE_INDEXER_PATH = integrations/indexers-sqlite/issues.bleve
10
+REPO_INDEXER_ENABLED = true
11
+REPO_INDEXER_PATH = integrations/indexers-sqlite/repos.bleve
10 12
 
11 13
 [repository]
12 14
 ROOT = integrations/gitea-integration-sqlite/gitea-repositories

+ 1 - 0
models/fixtures/repo_indexer_status.yml

@@ -0,0 +1 @@
1
+[] # empty

+ 2 - 0
models/migrations/migrations.go

@@ -144,6 +144,8 @@ var migrations = []Migration{
144 144
 	NewMigration("remove organization watch repositories", removeOrganizationWatchRepo),
145 145
 	// v47 -> v48
146 146
 	NewMigration("add deleted branches", addDeletedBranch),
147
+	// v48 -> v49
148
+	NewMigration("add repo indexer status", addRepoIndexerStatus),
147 149
 }
148 150
 
149 151
 // Migrate database to current version

+ 25 - 0
models/migrations/v48.go

@@ -0,0 +1,25 @@
1
+// Copyright 2017 The Gitea Authors. All rights reserved.
2
+// Use of this source code is governed by a MIT-style
3
+// license that can be found in the LICENSE file.
4
+
5
+package migrations
6
+
7
+import (
8
+	"fmt"
9
+
10
+	"github.com/go-xorm/xorm"
11
+)
12
+
13
+func addRepoIndexerStatus(x *xorm.Engine) error {
14
+	// RepoIndexerStatus see models/repo_indexer.go
15
+	type RepoIndexerStatus struct {
16
+		ID        int64  `xorm:"pk autoincr"`
17
+		RepoID    int64  `xorm:"INDEX NOT NULL"`
18
+		CommitSha string `xorm:"VARCHAR(40)"`
19
+	}
20
+
21
+	if err := x.Sync2(new(RepoIndexerStatus)); err != nil {
22
+		return fmt.Errorf("Sync2: %v", err)
23
+	}
24
+	return nil
25
+}

+ 20 - 4
models/models.go

@@ -13,6 +13,10 @@ import (
13 13
 	"path"
14 14
 	"strings"
15 15
 
16
+	"code.gitea.io/gitea/modules/log"
17
+	"code.gitea.io/gitea/modules/setting"
18
+	"code.gitea.io/gitea/modules/util"
19
+
16 20
 	// Needed for the MySQL driver
17 21
 	_ "github.com/go-sql-driver/mysql"
18 22
 	"github.com/go-xorm/core"
@@ -23,9 +27,6 @@ import (
23 27
 
24 28
 	// Needed for the MSSSQL driver
25 29
 	_ "github.com/denisenkom/go-mssqldb"
26
-
27
-	"code.gitea.io/gitea/modules/log"
28
-	"code.gitea.io/gitea/modules/setting"
29 30
 )
30 31
 
31 32
 // Engine represents a xorm engine or session.
@@ -115,6 +116,7 @@ func init() {
115 116
 		new(Stopwatch),
116 117
 		new(TrackedTime),
117 118
 		new(DeletedBranch),
119
+		new(RepoIndexerStatus),
118 120
 	)
119 121
 
120 122
 	gonicNames := []string{"SSL", "UID"}
@@ -150,8 +152,13 @@ func LoadConfigs() {
150 152
 	DbCfg.Timeout = sec.Key("SQLITE_TIMEOUT").MustInt(500)
151 153
 
152 154
 	sec = setting.Cfg.Section("indexer")
153
-	setting.Indexer.IssuePath = sec.Key("ISSUE_INDEXER_PATH").MustString("indexers/issues.bleve")
155
+	setting.Indexer.IssuePath = absolutePath(
156
+		sec.Key("ISSUE_INDEXER_PATH").MustString("indexers/issues.bleve"))
157
+	setting.Indexer.RepoIndexerEnabled = sec.Key("REPO_INDEXER_ENABLED").MustBool(false)
158
+	setting.Indexer.RepoPath = absolutePath(
159
+		sec.Key("REPO_INDEXER_PATH").MustString("indexers/repos.bleve"))
154 160
 	setting.Indexer.UpdateQueueLength = sec.Key("UPDATE_BUFFER_LEN").MustInt(20)
161
+	setting.Indexer.MaxIndexerFileSize = sec.Key("MAX_FILE_SIZE").MustInt64(512 * 1024 * 1024)
155 162
 }
156 163
 
157 164
 // parsePostgreSQLHostPort parses given input in various forms defined in
@@ -336,3 +343,12 @@ func DumpDatabase(filePath string, dbType string) error {
336 343
 	}
337 344
 	return x.DumpTablesToFile(tbs, filePath)
338 345
 }
346
+
347
+// absolutePath make path absolute if it is relative
348
+func absolutePath(path string) string {
349
+	workDir, err := setting.WorkDir()
350
+	if err != nil {
351
+		log.Fatal(4, "Failed to get work directory: %v", err)
352
+	}
353
+	return util.EnsureAbsolutePath(path, workDir)
354
+}

+ 11 - 6
models/repo.go

@@ -205,10 +205,11 @@ type Repository struct {
205 205
 	ExternalMetas map[string]string `xorm:"-"`
206 206
 	Units         []*RepoUnit       `xorm:"-"`
207 207
 
208
-	IsFork   bool        `xorm:"INDEX NOT NULL DEFAULT false"`
209
-	ForkID   int64       `xorm:"INDEX"`
210
-	BaseRepo *Repository `xorm:"-"`
211
-	Size     int64       `xorm:"NOT NULL DEFAULT 0"`
208
+	IsFork        bool               `xorm:"INDEX NOT NULL DEFAULT false"`
209
+	ForkID        int64              `xorm:"INDEX"`
210
+	BaseRepo      *Repository        `xorm:"-"`
211
+	Size          int64              `xorm:"NOT NULL DEFAULT 0"`
212
+	IndexerStatus *RepoIndexerStatus `xorm:"-"`
212 213
 
213 214
 	Created     time.Time `xorm:"-"`
214 215
 	CreatedUnix int64     `xorm:"INDEX created"`
@@ -782,8 +783,10 @@ func UpdateLocalCopyBranch(repoPath, localPath, branch string) error {
782 783
 		if err != nil {
783 784
 			return fmt.Errorf("git fetch origin: %v", err)
784 785
 		}
785
-		if err := git.ResetHEAD(localPath, true, "origin/"+branch); err != nil {
786
-			return fmt.Errorf("git reset --hard origin/%s: %v", branch, err)
786
+		if len(branch) > 0 {
787
+			if err := git.ResetHEAD(localPath, true, "origin/"+branch); err != nil {
788
+				return fmt.Errorf("git reset --hard origin/%s: %v", branch, err)
789
+			}
787 790
 		}
788 791
 	}
789 792
 	return nil
@@ -989,6 +992,7 @@ func MigrateRepository(doer, u *User, opts MigrateRepoOptions) (*Repository, err
989 992
 		if err = SyncReleasesWithTags(repo, gitRepo); err != nil {
990 993
 			log.Error(4, "Failed to synchronize tags to releases for repository: %v", err)
991 994
 		}
995
+		UpdateRepoIndexer(repo)
992 996
 	}
993 997
 
994 998
 	if err = repo.UpdateSize(); err != nil {
@@ -1883,6 +1887,7 @@ func DeleteRepository(doer *User, uid, repoID int64) error {
1883 1887
 		go HookQueue.Add(repo.ID)
1884 1888
 	}
1885 1889
 
1890
+	DeleteRepoFromIndexer(repo)
1886 1891
 	return nil
1887 1892
 }
1888 1893
 

+ 2 - 0
models/repo_editor.go

@@ -178,6 +178,8 @@ func (repo *Repository) UpdateRepoFile(doer *User, opts UpdateRepoFileOptions) (
178 178
 	if err != nil {
179 179
 		return fmt.Errorf("PushUpdate: %v", err)
180 180
 	}
181
+	UpdateRepoIndexer(repo)
182
+
181 183
 	return nil
182 184
 }
183 185
 

+ 302 - 0
models/repo_indexer.go

@@ -0,0 +1,302 @@
1
+// Copyright 2017 The Gitea Authors. All rights reserved.
2
+// Use of this source code is governed by a MIT-style
3
+// license that can be found in the LICENSE file.
4
+
5
+package models
6
+
7
+import (
8
+	"io/ioutil"
9
+	"os"
10
+	"path"
11
+	"strconv"
12
+	"strings"
13
+
14
+	"code.gitea.io/git"
15
+	"code.gitea.io/gitea/modules/base"
16
+	"code.gitea.io/gitea/modules/indexer"
17
+	"code.gitea.io/gitea/modules/log"
18
+	"code.gitea.io/gitea/modules/setting"
19
+
20
+	"github.com/Unknwon/com"
21
+)
22
+
23
+// RepoIndexerStatus status of a repo's entry in the repo indexer
24
+// For now, implicitly refers to default branch
25
+type RepoIndexerStatus struct {
26
+	ID        int64  `xorm:"pk autoincr"`
27
+	RepoID    int64  `xorm:"INDEX"`
28
+	CommitSha string `xorm:"VARCHAR(40)"`
29
+}
30
+
31
+func (repo *Repository) getIndexerStatus() error {
32
+	if repo.IndexerStatus != nil {
33
+		return nil
34
+	}
35
+	status := &RepoIndexerStatus{RepoID: repo.ID}
36
+	has, err := x.Get(status)
37
+	if err != nil {
38
+		return err
39
+	} else if !has {
40
+		status.CommitSha = ""
41
+	}
42
+	repo.IndexerStatus = status
43
+	return nil
44
+}
45
+
46
+func (repo *Repository) updateIndexerStatus(sha string) error {
47
+	if err := repo.getIndexerStatus(); err != nil {
48
+		return err
49
+	}
50
+	if len(repo.IndexerStatus.CommitSha) == 0 {
51
+		repo.IndexerStatus.CommitSha = sha
52
+		_, err := x.Insert(repo.IndexerStatus)
53
+		return err
54
+	}
55
+	repo.IndexerStatus.CommitSha = sha
56
+	_, err := x.ID(repo.IndexerStatus.ID).Cols("commit_sha").
57
+		Update(repo.IndexerStatus)
58
+	return err
59
+}
60
+
61
+type repoIndexerOperation struct {
62
+	repo    *Repository
63
+	deleted bool
64
+}
65
+
66
+var repoIndexerOperationQueue chan repoIndexerOperation
67
+
68
+// InitRepoIndexer initialize the repo indexer
69
+func InitRepoIndexer() {
70
+	if !setting.Indexer.RepoIndexerEnabled {
71
+		return
72
+	}
73
+	indexer.InitRepoIndexer(populateRepoIndexer)
74
+	repoIndexerOperationQueue = make(chan repoIndexerOperation, setting.Indexer.UpdateQueueLength)
75
+	go processRepoIndexerOperationQueue()
76
+}
77
+
78
+// populateRepoIndexer populate the repo indexer with data
79
+func populateRepoIndexer() error {
80
+	log.Info("Populating repository indexer (this may take a while)")
81
+	for page := 1; ; page++ {
82
+		repos, _, err := SearchRepositoryByName(&SearchRepoOptions{
83
+			Page:     page,
84
+			PageSize: 10,
85
+			OrderBy:  SearchOrderByID,
86
+			Private:  true,
87
+		})
88
+		if err != nil {
89
+			return err
90
+		} else if len(repos) == 0 {
91
+			return nil
92
+		}
93
+		for _, repo := range repos {
94
+			if err = updateRepoIndexer(repo); err != nil {
95
+				// only log error, since this should not prevent
96
+				// gitea from starting up
97
+				log.Error(4, "updateRepoIndexer: repoID=%d, %v", repo.ID, err)
98
+			}
99
+		}
100
+	}
101
+}
102
+
103
+type updateBatch struct {
104
+	updates []indexer.RepoIndexerUpdate
105
+}
106
+
107
+func updateRepoIndexer(repo *Repository) error {
108
+	changes, err := getRepoChanges(repo)
109
+	if err != nil {
110
+		return err
111
+	} else if changes == nil {
112
+		return nil
113
+	}
114
+
115
+	batch := indexer.RepoIndexerBatch()
116
+	for _, filename := range changes.UpdatedFiles {
117
+		if err := addUpdate(filename, repo, batch); err != nil {
118
+			return err
119
+		}
120
+	}
121
+	for _, filename := range changes.RemovedFiles {
122
+		if err := addDelete(filename, repo, batch); err != nil {
123
+			return err
124
+		}
125
+	}
126
+	if err = batch.Flush(); err != nil {
127
+		return err
128
+	}
129
+	return updateLastIndexSync(repo)
130
+}
131
+
132
+// repoChanges changes (file additions/updates/removals) to a repo
133
+type repoChanges struct {
134
+	UpdatedFiles []string
135
+	RemovedFiles []string
136
+}
137
+
138
+// getRepoChanges returns changes to repo since last indexer update
139
+func getRepoChanges(repo *Repository) (*repoChanges, error) {
140
+	repoWorkingPool.CheckIn(com.ToStr(repo.ID))
141
+	defer repoWorkingPool.CheckOut(com.ToStr(repo.ID))
142
+
143
+	if err := repo.UpdateLocalCopyBranch(""); err != nil {
144
+		return nil, err
145
+	} else if !git.IsBranchExist(repo.LocalCopyPath(), repo.DefaultBranch) {
146
+		// repo does not have any commits yet, so nothing to update
147
+		return nil, nil
148
+	} else if err = repo.UpdateLocalCopyBranch(repo.DefaultBranch); err != nil {
149
+		return nil, err
150
+	} else if err = repo.getIndexerStatus(); err != nil {
151
+		return nil, err
152
+	}
153
+
154
+	if len(repo.IndexerStatus.CommitSha) == 0 {
155
+		return genesisChanges(repo)
156
+	}
157
+	return nonGenesisChanges(repo)
158
+}
159
+
160
+func addUpdate(filename string, repo *Repository, batch *indexer.Batch) error {
161
+	filepath := path.Join(repo.LocalCopyPath(), filename)
162
+	if stat, err := os.Stat(filepath); err != nil {
163
+		return err
164
+	} else if stat.Size() > setting.Indexer.MaxIndexerFileSize {
165
+		return nil
166
+	}
167
+	fileContents, err := ioutil.ReadFile(filepath)
168
+	if err != nil {
169
+		return err
170
+	} else if !base.IsTextFile(fileContents) {
171
+		return nil
172
+	}
173
+	return batch.Add(indexer.RepoIndexerUpdate{
174
+		Filepath: filename,
175
+		Op:       indexer.RepoIndexerOpUpdate,
176
+		Data: &indexer.RepoIndexerData{
177
+			RepoID:  repo.ID,
178
+			Content: string(fileContents),
179
+		},
180
+	})
181
+}
182
+
183
+func addDelete(filename string, repo *Repository, batch *indexer.Batch) error {
184
+	return batch.Add(indexer.RepoIndexerUpdate{
185
+		Filepath: filename,
186
+		Op:       indexer.RepoIndexerOpDelete,
187
+		Data: &indexer.RepoIndexerData{
188
+			RepoID: repo.ID,
189
+		},
190
+	})
191
+}
192
+
193
+// genesisChanges get changes to add repo to the indexer for the first time
194
+func genesisChanges(repo *Repository) (*repoChanges, error) {
195
+	var changes repoChanges
196
+	stdout, err := git.NewCommand("ls-files").RunInDir(repo.LocalCopyPath())
197
+	if err != nil {
198
+		return nil, err
199
+	}
200
+	for _, line := range strings.Split(stdout, "\n") {
201
+		filename := strings.TrimSpace(line)
202
+		if len(filename) == 0 {
203
+			continue
204
+		} else if filename[0] == '"' {
205
+			filename, err = strconv.Unquote(filename)
206
+			if err != nil {
207
+				return nil, err
208
+			}
209
+		}
210
+		changes.UpdatedFiles = append(changes.UpdatedFiles, filename)
211
+	}
212
+	return &changes, nil
213
+}
214
+
215
+// nonGenesisChanges get changes since the previous indexer update
216
+func nonGenesisChanges(repo *Repository) (*repoChanges, error) {
217
+	diffCmd := git.NewCommand("diff", "--name-status",
218
+		repo.IndexerStatus.CommitSha, "HEAD")
219
+	stdout, err := diffCmd.RunInDir(repo.LocalCopyPath())
220
+	if err != nil {
221
+		// previous commit sha may have been removed by a force push, so
222
+		// try rebuilding from scratch
223
+		if err = indexer.DeleteRepoFromIndexer(repo.ID); err != nil {
224
+			return nil, err
225
+		}
226
+		return genesisChanges(repo)
227
+	}
228
+	var changes repoChanges
229
+	for _, line := range strings.Split(stdout, "\n") {
230
+		line = strings.TrimSpace(line)
231
+		if len(line) == 0 {
232
+			continue
233
+		}
234
+		filename := strings.TrimSpace(line[1:])
235
+		if len(filename) == 0 {
236
+			continue
237
+		} else if filename[0] == '"' {
238
+			filename, err = strconv.Unquote(filename)
239
+			if err != nil {
240
+				return nil, err
241
+			}
242
+		}
243
+
244
+		switch status := line[0]; status {
245
+		case 'M', 'A':
246
+			changes.UpdatedFiles = append(changes.UpdatedFiles, filename)
247
+		case 'D':
248
+			changes.RemovedFiles = append(changes.RemovedFiles, filename)
249
+		default:
250
+			log.Warn("Unrecognized status: %c (line=%s)", status, line)
251
+		}
252
+	}
253
+	return &changes, nil
254
+}
255
+
256
+func updateLastIndexSync(repo *Repository) error {
257
+	stdout, err := git.NewCommand("rev-parse", "HEAD").RunInDir(repo.LocalCopyPath())
258
+	if err != nil {
259
+		return err
260
+	}
261
+	sha := strings.TrimSpace(stdout)
262
+	return repo.updateIndexerStatus(sha)
263
+}
264
+
265
+func processRepoIndexerOperationQueue() {
266
+	for {
267
+		op := <-repoIndexerOperationQueue
268
+		if op.deleted {
269
+			if err := indexer.DeleteRepoFromIndexer(op.repo.ID); err != nil {
270
+				log.Error(4, "DeleteRepoFromIndexer: %v", err)
271
+			}
272
+		} else {
273
+			if err := updateRepoIndexer(op.repo); err != nil {
274
+				log.Error(4, "updateRepoIndexer: %v", err)
275
+			}
276
+		}
277
+	}
278
+}
279
+
280
+// DeleteRepoFromIndexer remove all of a repository's entries from the indexer
281
+func DeleteRepoFromIndexer(repo *Repository) {
282
+	addOperationToQueue(repoIndexerOperation{repo: repo, deleted: true})
283
+}
284
+
285
+// UpdateRepoIndexer update a repository's entries in the indexer
286
+func UpdateRepoIndexer(repo *Repository) {
287
+	addOperationToQueue(repoIndexerOperation{repo: repo, deleted: false})
288
+}
289
+
290
+func addOperationToQueue(op repoIndexerOperation) {
291
+	if !setting.Indexer.RepoIndexerEnabled {
292
+		return
293
+	}
294
+	select {
295
+	case repoIndexerOperationQueue <- op:
296
+		break
297
+	default:
298
+		go func() {
299
+			repoIndexerOperationQueue <- op
300
+		}()
301
+	}
302
+}

+ 4 - 0
models/update.go

@@ -263,6 +263,10 @@ func pushUpdate(opts PushUpdateOptions) (repo *Repository, err error) {
263 263
 		commits = ListToPushCommits(l)
264 264
 	}
265 265
 
266
+	if opts.RefFullName == git.BranchPrefix+repo.DefaultBranch {
267
+		UpdateRepoIndexer(repo)
268
+	}
269
+
266 270
 	if err := CommitRepoAction(CommitRepoActionOptions{
267 271
 		PusherName:  opts.PusherName,
268 272
 		RepoOwnerID: owner.ID,

+ 1 - 0
modules/context/repo.go

@@ -367,6 +367,7 @@ func RepoAssignment() macaron.Handler {
367 367
 		ctx.Data["DisableSSH"] = setting.SSH.Disabled
368 368
 		ctx.Data["ExposeAnonSSH"] = setting.SSH.ExposeAnonymous
369 369
 		ctx.Data["DisableHTTP"] = setting.Repository.DisableHTTPGit
370
+		ctx.Data["RepoSearchEnabled"] = setting.Indexer.RepoIndexerEnabled
370 371
 		ctx.Data["CloneLink"] = repo.CloneLink()
371 372
 		ctx.Data["WikiCloneLink"] = repo.WikiCloneLink()
372 373
 

+ 199 - 0
modules/indexer/repo.go

@@ -0,0 +1,199 @@
1
+// Copyright 2017 The Gitea Authors. All rights reserved.
2
+// Use of this source code is governed by a MIT-style
3
+// license that can be found in the LICENSE file.
4
+
5
+package indexer
6
+
7
+import (
8
+	"os"
9
+	"strings"
10
+
11
+	"code.gitea.io/gitea/modules/log"
12
+	"code.gitea.io/gitea/modules/setting"
13
+
14
+	"github.com/blevesearch/bleve"
15
+	"github.com/blevesearch/bleve/analysis/analyzer/custom"
16
+	"github.com/blevesearch/bleve/analysis/token/camelcase"
17
+	"github.com/blevesearch/bleve/analysis/token/lowercase"
18
+	"github.com/blevesearch/bleve/analysis/tokenizer/unicode"
19
+)
20
+
21
+const repoIndexerAnalyzer = "repoIndexerAnalyzer"
22
+
23
+// repoIndexer (thread-safe) index for repository contents
24
+var repoIndexer bleve.Index
25
+
26
+// RepoIndexerOp type of operation to perform on repo indexer
27
+type RepoIndexerOp int
28
+
29
+const (
30
+	// RepoIndexerOpUpdate add/update a file's contents
31
+	RepoIndexerOpUpdate = iota
32
+
33
+	// RepoIndexerOpDelete delete a file
34
+	RepoIndexerOpDelete
35
+)
36
+
37
+// RepoIndexerData data stored in the repo indexer
38
+type RepoIndexerData struct {
39
+	RepoID  int64
40
+	Content string
41
+}
42
+
43
+// RepoIndexerUpdate an update to the repo indexer
44
+type RepoIndexerUpdate struct {
45
+	Filepath string
46
+	Op       RepoIndexerOp
47
+	Data     *RepoIndexerData
48
+}
49
+
50
+func (update RepoIndexerUpdate) addToBatch(batch *bleve.Batch) error {
51
+	id := filenameIndexerID(update.Data.RepoID, update.Filepath)
52
+	switch update.Op {
53
+	case RepoIndexerOpUpdate:
54
+		return batch.Index(id, update.Data)
55
+	case RepoIndexerOpDelete:
56
+		batch.Delete(id)
57
+	default:
58
+		log.Error(4, "Unrecognized repo indexer op: %d", update.Op)
59
+	}
60
+	return nil
61
+}
62
+
63
+// InitRepoIndexer initialize repo indexer
64
+func InitRepoIndexer(populateIndexer func() error) {
65
+	_, err := os.Stat(setting.Indexer.RepoPath)
66
+	if err != nil {
67
+		if os.IsNotExist(err) {
68
+			if err = createRepoIndexer(); err != nil {
69
+				log.Fatal(4, "CreateRepoIndexer: %v", err)
70
+			}
71
+			if err = populateIndexer(); err != nil {
72
+				log.Fatal(4, "PopulateRepoIndex: %v", err)
73
+			}
74
+		} else {
75
+			log.Fatal(4, "InitRepoIndexer: %v", err)
76
+		}
77
+	} else {
78
+		repoIndexer, err = bleve.Open(setting.Indexer.RepoPath)
79
+		if err != nil {
80
+			log.Fatal(4, "InitRepoIndexer, open index: %v", err)
81
+		}
82
+	}
83
+}
84
+
85
+// createRepoIndexer create a repo indexer if one does not already exist
86
+func createRepoIndexer() error {
87
+	docMapping := bleve.NewDocumentMapping()
88
+	docMapping.AddFieldMappingsAt("RepoID", bleve.NewNumericFieldMapping())
89
+
90
+	textFieldMapping := bleve.NewTextFieldMapping()
91
+	docMapping.AddFieldMappingsAt("Content", textFieldMapping)
92
+
93
+	mapping := bleve.NewIndexMapping()
94
+	if err := addUnicodeNormalizeTokenFilter(mapping); err != nil {
95
+		return err
96
+	} else if err := mapping.AddCustomAnalyzer(repoIndexerAnalyzer, map[string]interface{}{
97
+		"type":          custom.Name,
98
+		"char_filters":  []string{},
99
+		"tokenizer":     unicode.Name,
100
+		"token_filters": []string{unicodeNormalizeName, camelcase.Name, lowercase.Name},
101
+	}); err != nil {
102
+		return err
103
+	}
104
+	mapping.DefaultAnalyzer = repoIndexerAnalyzer
105
+	mapping.AddDocumentMapping("repo", docMapping)
106
+	var err error
107
+	repoIndexer, err = bleve.New(setting.Indexer.RepoPath, mapping)
108
+	return err
109
+}
110
+
111
+func filenameIndexerID(repoID int64, filename string) string {
112
+	return indexerID(repoID) + "_" + filename
113
+}
114
+
115
+func filenameOfIndexerID(indexerID string) string {
116
+	index := strings.IndexByte(indexerID, '_')
117
+	if index == -1 {
118
+		log.Error(4, "Unexpected ID in repo indexer: %s", indexerID)
119
+	}
120
+	return indexerID[index+1:]
121
+}
122
+
123
+// RepoIndexerBatch batch to add updates to
124
+func RepoIndexerBatch() *Batch {
125
+	return &Batch{
126
+		batch: repoIndexer.NewBatch(),
127
+		index: repoIndexer,
128
+	}
129
+}
130
+
131
+// DeleteRepoFromIndexer delete all of a repo's files from indexer
132
+func DeleteRepoFromIndexer(repoID int64) error {
133
+	query := numericEqualityQuery(repoID, "RepoID")
134
+	searchRequest := bleve.NewSearchRequestOptions(query, 2147483647, 0, false)
135
+	result, err := repoIndexer.Search(searchRequest)
136
+	if err != nil {
137
+		return err
138
+	}
139
+	batch := RepoIndexerBatch()
140
+	for _, hit := range result.Hits {
141
+		batch.batch.Delete(hit.ID)
142
+		if err = batch.flushIfFull(); err != nil {
143
+			return err
144
+		}
145
+	}
146
+	return batch.Flush()
147
+}
148
+
149
+// RepoSearchResult result of performing a search in a repo
150
+type RepoSearchResult struct {
151
+	StartIndex int
152
+	EndIndex   int
153
+	Filename   string
154
+	Content    string
155
+}
156
+
157
+// SearchRepoByKeyword searches for files in the specified repo.
158
+// Returns the matching file-paths
159
+func SearchRepoByKeyword(repoID int64, keyword string, page, pageSize int) (int64, []*RepoSearchResult, error) {
160
+	phraseQuery := bleve.NewMatchPhraseQuery(keyword)
161
+	phraseQuery.FieldVal = "Content"
162
+	phraseQuery.Analyzer = repoIndexerAnalyzer
163
+	indexerQuery := bleve.NewConjunctionQuery(
164
+		numericEqualityQuery(repoID, "RepoID"),
165
+		phraseQuery,
166
+	)
167
+	from := (page - 1) * pageSize
168
+	searchRequest := bleve.NewSearchRequestOptions(indexerQuery, pageSize, from, false)
169
+	searchRequest.Fields = []string{"Content"}
170
+	searchRequest.IncludeLocations = true
171
+
172
+	result, err := repoIndexer.Search(searchRequest)
173
+	if err != nil {
174
+		return 0, nil, err
175
+	}
176
+
177
+	searchResults := make([]*RepoSearchResult, len(result.Hits))
178
+	for i, hit := range result.Hits {
179
+		var startIndex, endIndex int = -1, -1
180
+		for _, locations := range hit.Locations["Content"] {
181
+			location := locations[0]
182
+			locationStart := int(location.Start)
183
+			locationEnd := int(location.End)
184
+			if startIndex < 0 || locationStart < startIndex {
185
+				startIndex = locationStart
186
+			}
187
+			if endIndex < 0 || locationEnd > endIndex {
188
+				endIndex = locationEnd
189
+			}
190
+		}
191
+		searchResults[i] = &RepoSearchResult{
192
+			StartIndex: startIndex,
193
+			EndIndex:   endIndex,
194
+			Filename:   filenameOfIndexerID(hit.ID),
195
+			Content:    hit.Fields["Content"].(string),
196
+		}
197
+	}
198
+	return int64(result.Total), searchResults, nil
199
+}

+ 128 - 0
modules/search/search.go

@@ -0,0 +1,128 @@
1
+// Copyright 2017 The Gitea Authors. All rights reserved.
2
+// Use of this source code is governed by a MIT-style
3
+// license that can be found in the LICENSE file.
4
+
5
+package search
6
+
7
+import (
8
+	"bytes"
9
+	gotemplate "html/template"
10
+	"strings"
11
+
12
+	"code.gitea.io/gitea/modules/highlight"
13
+	"code.gitea.io/gitea/modules/indexer"
14
+	"code.gitea.io/gitea/modules/util"
15
+)
16
+
17
+// Result a search result to display
18
+type Result struct {
19
+	Filename       string
20
+	HighlightClass string
21
+	LineNumbers    []int
22
+	FormattedLines gotemplate.HTML
23
+}
24
+
25
+func indices(content string, selectionStartIndex, selectionEndIndex int) (int, int) {
26
+	startIndex := selectionStartIndex
27
+	numLinesBefore := 0
28
+	for ; startIndex > 0; startIndex-- {
29
+		if content[startIndex-1] == '\n' {
30
+			if numLinesBefore == 1 {
31
+				break
32
+			}
33
+			numLinesBefore++
34
+		}
35
+	}
36
+
37
+	endIndex := selectionEndIndex
38
+	numLinesAfter := 0
39
+	for ; endIndex < len(content); endIndex++ {
40
+		if content[endIndex] == '\n' {
41
+			if numLinesAfter == 1 {
42
+				break
43
+			}
44
+			numLinesAfter++
45
+		}
46
+	}
47
+
48
+	return startIndex, endIndex
49
+}
50
+
51
+func writeStrings(buf *bytes.Buffer, strs ...string) error {
52
+	for _, s := range strs {
53
+		_, err := buf.WriteString(s)
54
+		if err != nil {
55
+			return err
56
+		}
57
+	}
58
+	return nil
59
+}
60
+
61
+func searchResult(result *indexer.RepoSearchResult, startIndex, endIndex int) (*Result, error) {
62
+	startLineNum := 1 + strings.Count(result.Content[:startIndex], "\n")
63
+
64
+	var formattedLinesBuffer bytes.Buffer
65
+
66
+	contentLines := strings.SplitAfter(result.Content[startIndex:endIndex], "\n")
67
+	lineNumbers := make([]int, len(contentLines))
68
+	index := startIndex
69
+	for i, line := range contentLines {
70
+		var err error
71
+		if index < result.EndIndex &&
72
+			result.StartIndex < index+len(line) &&
73
+			result.StartIndex < result.EndIndex {
74
+			openActiveIndex := util.Max(result.StartIndex-index, 0)
75
+			closeActiveIndex := util.Min(result.EndIndex-index, len(line))
76
+			err = writeStrings(&formattedLinesBuffer,
77
+				`<li>`,
78
+				line[:openActiveIndex],
79
+				`<span class='active'>`,
80
+				line[openActiveIndex:closeActiveIndex],
81
+				`</span>`,
82
+				line[closeActiveIndex:],
83
+				`</li>`,
84
+			)
85
+		} else {
86
+			err = writeStrings(&formattedLinesBuffer,
87
+				`<li>`,
88
+				line,
89
+				`</li>`,
90
+			)
91
+		}
92
+		if err != nil {
93
+			return nil, err
94
+		}
95
+
96
+		lineNumbers[i] = startLineNum + i
97
+		index += len(line)
98
+	}
99
+	return &Result{
100
+		Filename:       result.Filename,
101
+		HighlightClass: highlight.FileNameToHighlightClass(result.Filename),
102
+		LineNumbers:    lineNumbers,
103
+		FormattedLines: gotemplate.HTML(formattedLinesBuffer.String()),
104
+	}, nil
105
+}
106
+
107
+// PerformSearch perform a search on a repository
108
+func PerformSearch(repoID int64, keyword string, page, pageSize int) (int, []*Result, error) {
109
+	if len(keyword) == 0 {
110
+		return 0, nil, nil
111
+	}
112
+
113
+	total, results, err := indexer.SearchRepoByKeyword(repoID, keyword, page, pageSize)
114
+	if err != nil {
115
+		return 0, nil, err
116
+	}
117
+
118
+	displayResults := make([]*Result, len(results))
119
+
120
+	for i, result := range results {
121
+		startIndex, endIndex := indices(result.Content, result.StartIndex, result.EndIndex)
122
+		displayResults[i], err = searchResult(result, startIndex, endIndex)
123
+		if err != nil {
124
+			return 0, nil, err
125
+		}
126
+	}
127
+	return int(total), displayResults, nil
128
+}

+ 18 - 13
modules/setting/setting.go

@@ -140,8 +140,11 @@ var (
140 140
 
141 141
 	// Indexer settings
142 142
 	Indexer struct {
143
-		IssuePath         string
144
-		UpdateQueueLength int
143
+		IssuePath          string
144
+		RepoIndexerEnabled bool
145
+		RepoPath           string
146
+		UpdateQueueLength  int
147
+		MaxIndexerFileSize int64
145 148
 	}
146 149
 
147 150
 	// Webhook settings
@@ -234,12 +237,13 @@ var (
234 237
 
235 238
 	// UI settings
236 239
 	UI = struct {
237
-		ExplorePagingNum   int
238
-		IssuePagingNum     int
239
-		FeedMaxCommitNum   int
240
-		ThemeColorMetaTag  string
241
-		MaxDisplayFileSize int64
242
-		ShowUserEmail      bool
240
+		ExplorePagingNum    int
241
+		IssuePagingNum      int
242
+		RepoSearchPagingNum int
243
+		FeedMaxCommitNum    int
244
+		ThemeColorMetaTag   string
245
+		MaxDisplayFileSize  int64
246
+		ShowUserEmail       bool
243 247
 
244 248
 		Admin struct {
245 249
 			UserPagingNum   int
@@ -256,11 +260,12 @@ var (
256 260
 			Keywords    string
257 261
 		} `ini:"ui.meta"`
258 262
 	}{
259
-		ExplorePagingNum:   20,
260
-		IssuePagingNum:     10,
261
-		FeedMaxCommitNum:   5,
262
-		ThemeColorMetaTag:  `#6cc644`,
263
-		MaxDisplayFileSize: 8388608,
263
+		ExplorePagingNum:    20,
264
+		IssuePagingNum:      10,
265
+		RepoSearchPagingNum: 10,
266
+		FeedMaxCommitNum:    5,
267
+		ThemeColorMetaTag:   `#6cc644`,
268
+		MaxDisplayFileSize:  8388608,
264 269
 		Admin: struct {
265 270
 			UserPagingNum   int
266 271
 			RepoPagingNum   int

+ 16 - 0
modules/util/path.go

@@ -0,0 +1,16 @@
1
+// Copyright 2017 The Gitea Authors. All rights reserved.
2
+// Use of this source code is governed by a MIT-style
3
+// license that can be found in the LICENSE file.
4
+
5
+package util
6
+
7
+import "path/filepath"
8
+
9
+// EnsureAbsolutePath ensure that a path is absolute, making it
10
+// relative to absoluteBase if necessary
11
+func EnsureAbsolutePath(path string, absoluteBase string) string {
12
+	if filepath.IsAbs(path) {
13
+		return path
14
+	}
15
+	return filepath.Join(absoluteBase, path)
16
+}

+ 16 - 0
modules/util/util.go

@@ -38,3 +38,19 @@ func OptionalBoolOf(b bool) OptionalBool {
38 38
 	}
39 39
 	return OptionalBoolFalse
40 40
 }
41
+
42
+// Max max of two ints
43
+func Max(a, b int) int {
44
+	if a < b {
45
+		return b
46
+	}
47
+	return a
48
+}
49
+
50
+// Min min of two ints
51
+func Min(a, b int) int {
52
+	if a > b {
53
+		return b
54
+	}
55
+	return a
56
+}

+ 4 - 0
options/locale/locale_en-US.ini

@@ -848,6 +848,10 @@ activity.title.releases_n = %d Releases
848 848
 activity.title.releases_published_by = %s published by %s
849 849
 activity.published_release_label = Published
850 850
 
851
+search = Search
852
+search.search_repo = Search repository
853
+search.results = Search results for "%s" in <a href="%s">%s</a>
854
+
851 855
 settings = Settings
852 856
 settings.desc = Settings is where you can manage the settings for the repository
853 857
 settings.options = Options

File diff suppressed because it is too large
+ 1 - 1
public/css/index.css


+ 17 - 2
public/less/_repository.less

@@ -158,6 +158,11 @@
158 158
     }
159 159
 
160 160
     &.file.list {
161
+        .repo-description {
162
+            display: flex;
163
+            justify-content: space-between;
164
+            align-items: center;
165
+        }
161 166
         #repo-desc {
162 167
             font-size: 1.2em;
163 168
         }
@@ -226,7 +231,7 @@
226 231
                         }
227 232
         }
228 233
 
229
-        #file-content {
234
+        .non-diff-file-content {
230 235
             .header {
231 236
                 .icon {
232 237
                     font-size: 1em;
@@ -244,7 +249,7 @@
244 249
                         background: transparent;
245 250
                         border: 0;
246 251
                         outline: none;
247
-                     }
252
+                    }
248 253
                     .btn-octicon:hover {
249 254
                         color: #4078c0;
250 255
                     }
@@ -323,6 +328,9 @@
323 328
                         }
324 329
                     }
325 330
                 }
331
+                .active {
332
+                    background: #ffffdd;
333
+                }
326 334
             }
327 335
         }
328 336
 
@@ -1038,6 +1046,13 @@
1038 1046
         overflow-x: auto;
1039 1047
         overflow-y: hidden;
1040 1048
     }
1049
+    .repo-search-result {
1050
+        padding-top: 10px;
1051
+        padding-bottom: 10px;
1052
+        .lines-num a {
1053
+            color: inherit;
1054
+        }
1055
+    }
1041 1056
 
1042 1057
     &.quickstart {
1043 1058
         .guide {

+ 1 - 0
routers/init.go

@@ -66,6 +66,7 @@ func GlobalInit() {
66 66
 		// Booting long running goroutines.
67 67
 		cron.NewContext()
68 68
 		models.InitIssueIndexer()
69
+		models.InitRepoIndexer()
69 70
 		models.InitSyncMirrors()
70 71
 		models.InitDeliverHooks()
71 72
 		models.InitTestPullRequests()

+ 46 - 0
routers/repo/search.go

@@ -0,0 +1,46 @@
1
+// Copyright 2017 The Gitea Authors. All rights reserved.
2
+// Use of this source code is governed by a MIT-style
3
+// license that can be found in the LICENSE file.
4
+
5
+package repo
6
+
7
+import (
8
+	"path"
9
+	"strings"
10
+
11
+	"code.gitea.io/gitea/modules/base"
12
+	"code.gitea.io/gitea/modules/context"
13
+	"code.gitea.io/gitea/modules/search"
14
+	"code.gitea.io/gitea/modules/setting"
15
+
16
+	"github.com/Unknwon/paginater"
17
+)
18
+
19
+const tplSearch base.TplName = "repo/search"
20
+
21
+// Search render repository search page
22
+func Search(ctx *context.Context) {
23
+	if !setting.Indexer.RepoIndexerEnabled {
24
+		ctx.Redirect(ctx.Repo.RepoLink, 302)
25
+		return
26
+	}
27
+	keyword := strings.TrimSpace(ctx.Query("q"))
28
+	page := ctx.QueryInt("page")
29
+	if page <= 0 {
30
+		page = 1
31
+	}
32
+	total, searchResults, err := search.PerformSearch(ctx.Repo.Repository.ID, keyword, page, setting.UI.RepoSearchPagingNum)
33
+	if err != nil {
34
+		ctx.Handle(500, "SearchResults", err)
35
+		return
36
+	}
37
+	ctx.Data["Keyword"] = keyword
38
+	pager := paginater.New(total, setting.UI.RepoSearchPagingNum, page, 5)
39
+	ctx.Data["Page"] = pager
40
+	ctx.Data["SourcePath"] = setting.AppSubURL + "/" +
41
+		path.Join(ctx.Repo.Repository.Owner.Name, ctx.Repo.Repository.Name, "src", ctx.Repo.Repository.DefaultBranch)
42
+	ctx.Data["SearchResults"] = searchResults
43
+	ctx.Data["RequireHighlightJS"] = true
44
+	ctx.Data["PageIsViewCode"] = true
45
+	ctx.HTML(200, tplSearch)
46
+}

+ 1 - 0
routers/routes/routes.go

@@ -649,6 +649,7 @@ func RegisterRoutes(m *macaron.Macaron) {
649 649
 	m.Group("/:username/:reponame", func() {
650 650
 		m.Get("/stars", repo.Stars)
651 651
 		m.Get("/watchers", repo.Watchers)
652
+		m.Get("/search", context.CheckUnit(models.UnitTypeCode), repo.Search)
652 653
 	}, ignSignIn, context.RepoAssignment(), context.RepoRef(), context.UnitTypes(), context.LoadRepoUnits())
653 654
 
654 655
 	m.Group("/:username", func() {

+ 20 - 4
templates/repo/home.tmpl

@@ -3,10 +3,26 @@
3 3
 	{{template "repo/header" .}}
4 4
 	<div class="ui container">
5 5
 		{{template "base/alert" .}}
6
-		<p id="repo-desc">
7
-			{{if .Repository.DescriptionHTML}}<span class="description has-emoji">{{.Repository.DescriptionHTML}}</span>{{else if .IsRepositoryAdmin}}<span class="no-description text-italic">{{.i18n.Tr "repo.no_desc"}}</span>{{end}}
8
-			<a class="link" href="{{.Repository.Website}}">{{.Repository.Website}}</a>
9
-		</p>
6
+		<div class="ui repo-description">
7
+			<div id="repo-desc">
8
+				{{if .Repository.DescriptionHTML}}<span class="description has-emoji">{{.Repository.DescriptionHTML}}</span>{{else if .IsRepositoryAdmin}}<span class="no-description text-italic">{{.i18n.Tr "repo.no_desc"}}</span>{{end}}
9
+				<a class="link" href="{{.Repository.Website}}">{{.Repository.Website}}</a>
10
+			</div>
11
+			{{if .RepoSearchEnabled}}
12
+				<div class="ui repo-search">
13
+					<form class="ui form" action="{{.RepoLink}}/search" method="get">
14
+						<div class="field">
15
+							<div class="ui action input">
16
+								<input name="q" value="{{.Keyword}}" placeholder="{{.i18n.Tr "repo.search.search_repo"}}">
17
+								<button class="ui icon button" type="submit">
18
+									<i class="search icon"></i>
19
+								</button>
20
+							</div>
21
+						</div>
22
+					</form>
23
+				</div>
24
+			{{end}}
25
+		</div>
10 26
 		{{template "repo/sub_menu" .}}
11 27
 		<div class="ui secondary menu">
12 28
 			{{if .PullRequestCtx.Allowed}}

+ 49 - 0
templates/repo/search.tmpl

@@ -0,0 +1,49 @@
1
+{{template "base/head" .}}
2
+<div class="repository file list">
3
+	{{template "repo/header" .}}
4
+	<div class="ui container">
5
+		<div class="ui repo-search">
6
+			<form class="ui form" method="get">
7
+				<div class="ui fluid action input">
8
+					<input name="q" value="{{.Keyword}}" placeholder="{{.i18n.Tr "repo.search.search_repo"}}">
9
+					<button class="ui button" type="submit">
10
+						<i class="search icon"></i>
11
+					</button>
12
+				</div>
13
+			</form>
14
+		</div>
15
+		{{if .Keyword}}
16
+			<h3>
17
+				{{.i18n.Tr "repo.search.results" .Keyword .RepoLink .RepoName | Str2html}}
18
+			</h3>
19
+			<div class="repository search">
20
+				{{range $result := .SearchResults}}
21
+					<div class="diff-file-box diff-box file-content non-diff-file-content repo-search-result">
22
+						<h4 class="ui top attached normal header">
23
+							<span class="file">{{.Filename}}</span>
24
+							<a class="ui basic grey tiny button" rel="nofollow" href="{{EscapePound $.SourcePath}}/{{EscapePound .Filename}}">{{$.i18n.Tr "repo.diff.view_file"}}</a>
25
+						</h4>
26
+						<div class="ui attached table segment">
27
+							<div class="file-body file-code code-view">
28
+								<table>
29
+									<tbody>
30
+										<tr>
31
+											<td class="lines-num">
32
+												{{range .LineNumbers}}
33
+													<a href="{{EscapePound $.SourcePath}}/{{EscapePound $result.Filename}}#L{{.}}"><span>{{.}}</span></a>
34
+												{{end}}
35
+											</td>
36
+											<td class="lines-code"><pre><code class="{{.HighlightClass}}"><ol class="linenums">{{.FormattedLines}}</ol></code></pre></td>
37
+										</tr>
38
+									</tbody>
39
+								</table>
40
+							</div>
41
+						</div>
42
+					</div>
43
+				{{end}}
44
+			</div>
45
+			{{template "base/paginate" .}}
46
+		{{end}}
47
+	</div>
48
+</div>
49
+{{template "base/footer" .}}

+ 1 - 1
templates/repo/view_file.tmpl

@@ -1,4 +1,4 @@
1
-<div id="file-content" class="{{TabSizeClass .Editorconfig .FileName}}">
1
+<div class="{{TabSizeClass .Editorconfig .FileName}} non-diff-file-content">
2 2
 	<h4 class="ui top attached header" id="{{if .ReadmeExist}}repo-readme{{else}}repo-read-file{{end}}">
3 3
 		{{if .ReadmeExist}}
4 4
 			<i class="book icon ui left"></i>

+ 78 - 0
vendor/github.com/blevesearch/bleve/analysis/token/camelcase/camelcase.go

@@ -0,0 +1,78 @@
1
+//  Copyright (c) 2016 Couchbase, Inc.
2
+//
3
+// Licensed under the Apache License, Version 2.0 (the "License");
4
+// you may not use this file except in compliance with the License.
5
+// You may obtain a copy of the License at
6
+//
7
+// 		http://www.apache.org/licenses/LICENSE-2.0
8
+//
9
+// Unless required by applicable law or agreed to in writing, software
10
+// distributed under the License is distributed on an "AS IS" BASIS,
11
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+// See the License for the specific language governing permissions and
13
+// limitations under the License.
14
+
15
+package camelcase
16
+
17
+import (
18
+	"bytes"
19
+	"unicode/utf8"
20
+
21
+	"github.com/blevesearch/bleve/analysis"
22
+	"github.com/blevesearch/bleve/registry"
23
+)
24
+
25
+const Name = "camelCase"
26
+
27
+// CamelCaseFilter splits a given token into a set of tokens where each resulting token
28
+// falls into one the following classes:
29
+// 1) Upper case followed by lower case letters.
30
+//		Terminated by a number, an upper case letter, and a non alpha-numeric symbol.
31
+// 2) Upper case followed by upper case letters.
32
+//		Terminated by a number, an upper case followed by a lower case letter, and a non alpha-numeric symbol.
33
+// 3) Lower case followed by lower case letters.
34
+//		Terminated by a number, an upper case letter, and a non alpha-numeric symbol.
35
+// 4) Number followed by numbers.
36
+//		Terminated by a letter, and a non alpha-numeric symbol.
37
+// 5) Non alpha-numeric symbol followed by non alpha-numeric symbols.
38
+//		Terminated by a number, and a letter.
39
+//
40
+// It does a one-time sequential pass over an input token, from left to right.
41
+// The scan is greedy and generates the longest substring that fits into one of the classes.
42
+//
43
+// See the test file for examples of classes and their parsings.
44
+type CamelCaseFilter struct{}
45
+
46
+func NewCamelCaseFilter() *CamelCaseFilter {
47
+	return &CamelCaseFilter{}
48
+}
49
+
50
+func (f *CamelCaseFilter) Filter(input analysis.TokenStream) analysis.TokenStream {
51
+	rv := make(analysis.TokenStream, 0, len(input))
52
+
53
+	nextPosition := 1
54
+	for _, token := range input {
55
+		runeCount := utf8.RuneCount(token.Term)
56
+		runes := bytes.Runes(token.Term)
57
+
58
+		p := NewParser(runeCount, nextPosition, token.Start)
59
+		for i := 0; i < runeCount; i++ {
60
+			if i+1 >= runeCount {
61
+				p.Push(runes[i], nil)
62
+			} else {
63
+				p.Push(runes[i], &runes[i+1])
64
+			}
65
+		}
66
+		rv = append(rv, p.FlushTokens()...)
67
+		nextPosition = p.NextPosition()
68
+	}
69
+	return rv
70
+}
71
+
72
+func CamelCaseFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) {
73
+	return NewCamelCaseFilter(), nil
74
+}
75
+
76
+func init() {
77
+	registry.RegisterTokenFilter(Name, CamelCaseFilterConstructor)
78
+}

+ 109 - 0
vendor/github.com/blevesearch/bleve/analysis/token/camelcase/parser.go

@@ -0,0 +1,109 @@
1
+//  Copyright (c) 2016 Couchbase, Inc.
2
+//
3
+// Licensed under the Apache License, Version 2.0 (the "License");
4
+// you may not use this file except in compliance with the License.
5
+// You may obtain a copy of the License at
6
+//
7
+// 		http://www.apache.org/licenses/LICENSE-2.0
8
+//
9
+// Unless required by applicable law or agreed to in writing, software
10
+// distributed under the License is distributed on an "AS IS" BASIS,
11
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+// See the License for the specific language governing permissions and
13
+// limitations under the License.
14
+
15
+package camelcase
16
+
17
+import (
18
+	"github.com/blevesearch/bleve/analysis"
19
+)
20
+
21
+func (p *Parser) buildTokenFromTerm(buffer []rune) *analysis.Token {
22
+	term := analysis.BuildTermFromRunes(buffer)
23
+	token := &analysis.Token{
24
+		Term:     term,
25
+		Position: p.position,
26
+		Start:    p.index,
27
+		End:      p.index + len(term),
28
+	}
29
+	p.position++
30
+	p.index += len(term)
31
+	return token
32
+}
33
+
34
+// Parser accepts a symbol and passes it to the current state (representing a class).
35
+// The state can accept it (and accumulate it). Otherwise, the parser creates a new state that
36
+// starts with the pushed symbol.
37
+//
38
+// Parser accumulates a new resulting token every time it switches state.
39
+// Use FlushTokens() to get the results after the last symbol was pushed.
40
+type Parser struct {
41
+	bufferLen int
42
+	buffer    []rune
43
+	current   State
44
+	tokens    []*analysis.Token
45
+	position  int
46
+	index     int
47
+}
48
+
49
+func NewParser(len, position, index int) *Parser {
50
+	return &Parser{
51
+		bufferLen: len,
52
+		buffer:    make([]rune, 0, len),
53
+		tokens:    make([]*analysis.Token, 0, len),
54
+		position:  position,
55
+		index:     index,
56
+	}
57
+}
58
+
59
+func (p *Parser) Push(sym rune, peek *rune) {
60
+	if p.current == nil {
61
+		// the start of parsing
62
+		p.current = p.NewState(sym)
63
+		p.buffer = append(p.buffer, sym)
64
+
65
+	} else if p.current.Member(sym, peek) {
66
+		// same state, just accumulate
67
+		p.buffer = append(p.buffer, sym)
68
+
69
+	} else {
70
+		// the old state is no more, thus convert the buffer
71
+		p.tokens = append(p.tokens, p.buildTokenFromTerm(p.buffer))
72
+
73
+		// let the new state begin
74
+		p.current = p.NewState(sym)
75
+		p.buffer = make([]rune, 0, p.bufferLen)
76
+		p.buffer = append(p.buffer, sym)
77
+	}
78
+}
79
+
80
+// Note. States have to have different starting symbols.
81
+func (p *Parser) NewState(sym rune) State {
82
+	var found State
83
+
84
+	found = &LowerCaseState{}
85
+	if found.StartSym(sym) {
86
+		return found
87
+	}
88
+
89
+	found = &UpperCaseState{}
90
+	if found.StartSym(sym) {
91
+		return found
92
+	}
93
+
94
+	found = &NumberCaseState{}
95
+	if found.StartSym(sym) {
96
+		return found
97
+	}
98
+
99
+	return &NonAlphaNumericCaseState{}
100
+}
101
+
102
+func (p *Parser) FlushTokens() []*analysis.Token {
103
+	p.tokens = append(p.tokens, p.buildTokenFromTerm(p.buffer))
104
+	return p.tokens
105
+}
106
+
107
+func (p *Parser) NextPosition() int {
108
+	return p.position
109
+}

+ 87 - 0
vendor/github.com/blevesearch/bleve/analysis/token/camelcase/states.go

@@ -0,0 +1,87 @@
1
+//  Copyright (c) 2016 Couchbase, Inc.
2
+//
3
+// Licensed under the Apache License, Version 2.0 (the "License");
4
+// you may not use this file except in compliance with the License.
5
+// You may obtain a copy of the License at
6
+//
7
+// 		http://www.apache.org/licenses/LICENSE-2.0
8
+//
9
+// Unless required by applicable law or agreed to in writing, software
10
+// distributed under the License is distributed on an "AS IS" BASIS,
11
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+// See the License for the specific language governing permissions and
13
+// limitations under the License.
14
+
15
+package camelcase
16
+
17
+import (
18
+	"unicode"
19
+)
20
+
21
+// States codify the classes that the parser recognizes.
22
+type State interface {
23
+	// is _sym_ the start character
24
+	StartSym(sym rune) bool
25
+
26
+	// is _sym_ a member of a class.
27
+	// peek, the next sym on the tape, can also be used to determine a class.
28
+	Member(sym rune, peek *rune) bool
29
+}
30
+
31
+type LowerCaseState struct{}
32
+
33
+func (s *LowerCaseState) Member(sym rune, peek *rune) bool {
34
+	return unicode.IsLower(sym)
35
+}
36
+
37
+func (s *LowerCaseState) StartSym(sym rune) bool {
38
+	return s.Member(sym, nil)
39
+}
40
+
41
+type UpperCaseState struct {
42
+	startedCollecting bool // denotes that the start character has been read
43
+	collectingUpper   bool // denotes if this is a class of all upper case letters
44
+}
45
+
46
+func (s *UpperCaseState) Member(sym rune, peek *rune) bool {
47
+	if !(unicode.IsLower(sym) || unicode.IsUpper(sym)) {
48
+		return false
49
+	}
50
+
51
+	if peek != nil && unicode.IsUpper(sym) && unicode.IsLower(*peek) {
52
+		return false
53
+	}
54
+
55
+	if !s.startedCollecting {
56
+		// now we have to determine if upper-case letters are collected.
57
+		s.startedCollecting = true
58
+		s.collectingUpper = unicode.IsUpper(sym)
59
+		return true
60
+	}
61
+
62
+	return s.collectingUpper == unicode.IsUpper(sym)
63
+}
64
+
65
+func (s *UpperCaseState) StartSym(sym rune) bool {
66
+	return unicode.IsUpper(sym)
67
+}
68
+
69
+type NumberCaseState struct{}
70
+
71
+func (s *NumberCaseState) Member(sym rune, peek *rune) bool {
72
+	return unicode.IsNumber(sym)
73
+}
74
+
75
+func (s *NumberCaseState) StartSym(sym rune) bool {
76
+	return s.Member(sym, nil)
77
+}
78
+
79
+type NonAlphaNumericCaseState struct{}
80
+
81
+func (s *NonAlphaNumericCaseState) Member(sym rune, peek *rune) bool {
82
+	return !unicode.IsLower(sym) && !unicode.IsUpper(sym) && !unicode.IsNumber(sym)
83
+}
84
+
85
+func (s *NonAlphaNumericCaseState) StartSym(sym rune) bool {
86
+	return s.Member(sym, nil)
87
+}

+ 6 - 0
vendor/vendor.json

@@ -99,6 +99,12 @@
99 99
 			"revisionTime": "2017-06-14T16:31:07Z"
100 100
 		},
101 101
 		{
102
+			"checksumSHA1": "xj8o/nQj59yt+o+RZSa0n9V3vKY=",
103
+			"path": "github.com/blevesearch/bleve/analysis/token/camelcase",
104
+			"revision": "174f8ed44a0bf65e7c8fb228b60b58de62654cd2",
105
+			"revisionTime": "2017-06-28T17:18:15Z"
106
+		},
107
+		{
102 108
 			"checksumSHA1": "3VIPkl12t1ko4y6DkbPcz+MtQjY=",
103 109
 			"path": "github.com/blevesearch/bleve/analysis/token/lowercase",
104 110
 			"revision": "011b168f7b84ffef05aed6716d73d21b1a33e971",