Browse Source

Memory usage improvements (#3013)

* govendor update code.gitea.io/git

Signed-off-by: Duncan Ogilvie <mr.exodia.tpodt@gmail.com>

* Greatly improve memory usage

Signed-off-by: Duncan Ogilvie <mr.exodia.tpodt@gmail.com>
Duncan Ogilvie 1 year ago
parent
commit
551f3cbe42

+ 3 - 0
modules/context/repo.go

@@ -143,6 +143,9 @@ func (r *Repository) GetEditorconfig() (*editorconfig.Editorconfig, error) {
143 143
 	if err != nil {
144 144
 		return nil, err
145 145
 	}
146
+	if treeEntry.Blob().Size() >= setting.UI.MaxDisplayFileSize {
147
+		return nil, git.ErrNotExist{ID: "", RelPath: ".editorconfig"}
148
+	}
146 149
 	reader, err := treeEntry.Blob().Data()
147 150
 	if err != nil {
148 151
 		return nil, err

+ 2 - 1
routers/repo/download.go

@@ -45,10 +45,11 @@ func ServeData(ctx *context.Context, name string, reader io.Reader) error {
45 45
 
46 46
 // ServeBlob download a git.Blob
47 47
 func ServeBlob(ctx *context.Context, blob *git.Blob) error {
48
-	dataRc, err := blob.Data()
48
+	dataRc, err := blob.DataAsync()
49 49
 	if err != nil {
50 50
 		return err
51 51
 	}
52
+	defer dataRc.Close()
52 53
 
53 54
 	return ServeData(ctx, ctx.Repo.TreePath, dataRc)
54 55
 }

+ 7 - 2
routers/repo/editor.go

@@ -73,11 +73,16 @@ func editFile(ctx *context.Context, isNewFile bool) {
73 73
 
74 74
 		// No way to edit a directory online.
75 75
 		if entry.IsDir() {
76
-			ctx.Handle(404, "", nil)
76
+			ctx.Handle(404, "entry.IsDir", nil)
77 77
 			return
78 78
 		}
79 79
 
80 80
 		blob := entry.Blob()
81
+		if blob.Size() >= setting.UI.MaxDisplayFileSize {
82
+			ctx.Handle(404, "blob.Size", err)
83
+			return
84
+		}
85
+
81 86
 		dataRc, err := blob.Data()
82 87
 		if err != nil {
83 88
 			ctx.Handle(404, "blob.Data", err)
@@ -93,7 +98,7 @@ func editFile(ctx *context.Context, isNewFile bool) {
93 98
 
94 99
 		// Only text file are editable online.
95 100
 		if !base.IsTextFile(buf) {
96
-			ctx.Handle(404, "", nil)
101
+			ctx.Handle(404, "base.IsTextFile", nil)
97 102
 			return
98 103
 		}
99 104
 

+ 3 - 0
routers/repo/issue.go

@@ -319,6 +319,9 @@ func getFileContentFromDefaultBranch(ctx *context.Context, filename string) (str
319 319
 	if err != nil {
320 320
 		return "", false
321 321
 	}
322
+	if entry.Blob().Size() >= setting.UI.MaxDisplayFileSize {
323
+		return "", false
324
+	}
322 325
 	r, err = entry.Blob().Data()
323 326
 	if err != nil {
324 327
 		return "", false

+ 19 - 10
routers/repo/view.go

@@ -76,11 +76,12 @@ func renderDirectory(ctx *context.Context, treeLink string) {
76 76
 		ctx.Data["ReadmeInList"] = true
77 77
 		ctx.Data["ReadmeExist"] = true
78 78
 
79
-		dataRc, err := readmeFile.Data()
79
+		dataRc, err := readmeFile.DataAsync()
80 80
 		if err != nil {
81 81
 			ctx.Handle(500, "Data", err)
82 82
 			return
83 83
 		}
84
+		defer dataRc.Close()
84 85
 
85 86
 		buf := make([]byte, 1024)
86 87
 		n, _ := dataRc.Read(buf)
@@ -91,14 +92,21 @@ func renderDirectory(ctx *context.Context, treeLink string) {
91 92
 		ctx.Data["FileName"] = readmeFile.Name()
92 93
 		// FIXME: what happens when README file is an image?
93 94
 		if isTextFile {
94
-			d, _ := ioutil.ReadAll(dataRc)
95
-			buf = append(buf, d...)
96
-			if markup.Type(readmeFile.Name()) != "" {
97
-				ctx.Data["IsMarkup"] = true
98
-				ctx.Data["FileContent"] = string(markup.Render(readmeFile.Name(), buf, treeLink, ctx.Repo.Repository.ComposeMetas()))
95
+			if readmeFile.Size() >= setting.UI.MaxDisplayFileSize {
96
+				// Pretend that this is a normal text file to display 'This file is too large to be shown'
97
+				ctx.Data["IsFileTooLarge"] = true
98
+				ctx.Data["IsTextFile"] = true
99
+				ctx.Data["FileSize"] = readmeFile.Size()
99 100
 			} else {
100
-				ctx.Data["IsRenderedHTML"] = true
101
-				ctx.Data["FileContent"] = string(bytes.Replace(buf, []byte("\n"), []byte(`<br>`), -1))
101
+				d, _ := ioutil.ReadAll(dataRc)
102
+				buf = append(buf, d...)
103
+				if markup.Type(readmeFile.Name()) != "" {
104
+					ctx.Data["IsMarkup"] = true
105
+					ctx.Data["FileContent"] = string(markup.Render(readmeFile.Name(), buf, treeLink, ctx.Repo.Repository.ComposeMetas()))
106
+				} else {
107
+					ctx.Data["IsRenderedHTML"] = true
108
+					ctx.Data["FileContent"] = string(bytes.Replace(buf, []byte("\n"), []byte(`<br>`), -1))
109
+				}
102 110
 			}
103 111
 		}
104 112
 	}
@@ -135,11 +143,12 @@ func renderFile(ctx *context.Context, entry *git.TreeEntry, treeLink, rawLink st
135 143
 	ctx.Data["IsViewFile"] = true
136 144
 
137 145
 	blob := entry.Blob()
138
-	dataRc, err := blob.Data()
146
+	dataRc, err := blob.DataAsync()
139 147
 	if err != nil {
140
-		ctx.Handle(500, "Data", err)
148
+		ctx.Handle(500, "DataAsync", err)
141 149
 		return
142 150
 	}
151
+	defer dataRc.Close()
143 152
 
144 153
 	ctx.Data["FileSize"] = blob.Size()
145 154
 	ctx.Data["FileName"] = blob.Name()

+ 46 - 4
vendor/code.gitea.io/git/blob.go

@@ -6,7 +6,11 @@ package git
6 6
 
7 7
 import (
8 8
 	"bytes"
9
+	"fmt"
9 10
 	"io"
11
+	"io/ioutil"
12
+	"os"
13
+	"os/exec"
10 14
 )
11 15
 
12 16
 // Blob represents a Git object.
@@ -18,14 +22,52 @@ type Blob struct {
18 22
 // Data gets content of blob all at once and wrap it as io.Reader.
19 23
 // This can be very slow and memory consuming for huge content.
20 24
 func (b *Blob) Data() (io.Reader, error) {
21
-	stdout, err := NewCommand("show", b.ID.String()).RunInDirBytes(b.repo.Path)
22
-	if err != nil {
23
-		return nil, err
25
+	stdout := new(bytes.Buffer)
26
+	stderr := new(bytes.Buffer)
27
+
28
+	// Preallocate memory to save ~50% memory usage on big files.
29
+	stdout.Grow(int(b.Size() + 2048))
30
+
31
+	if err := b.DataPipeline(stdout, stderr); err != nil {
32
+		return nil, concatenateError(err, stderr.String())
24 33
 	}
25
-	return bytes.NewBuffer(stdout), nil
34
+	return stdout, nil
26 35
 }
27 36
 
28 37
 // DataPipeline gets content of blob and write the result or error to stdout or stderr
29 38
 func (b *Blob) DataPipeline(stdout, stderr io.Writer) error {
30 39
 	return NewCommand("show", b.ID.String()).RunInDirPipeline(b.repo.Path, stdout, stderr)
31 40
 }
41
+
42
+type cmdReadCloser struct {
43
+	cmd    *exec.Cmd
44
+	stdout io.Reader
45
+}
46
+
47
+func (c cmdReadCloser) Read(p []byte) (int, error) {
48
+	return c.stdout.Read(p)
49
+}
50
+
51
+func (c cmdReadCloser) Close() error {
52
+	io.Copy(ioutil.Discard, c.stdout)
53
+	return c.cmd.Wait()
54
+}
55
+
56
+// DataAsync gets a ReadCloser for the contents of a blob without reading it all.
57
+// Calling the Close function on the result will discard all unread output.
58
+func (b *Blob) DataAsync() (io.ReadCloser, error) {
59
+	cmd := exec.Command("git", "show", b.ID.String())
60
+	cmd.Dir = b.repo.Path
61
+	cmd.Stderr = os.Stderr
62
+
63
+	stdout, err := cmd.StdoutPipe()
64
+	if err != nil {
65
+		return nil, fmt.Errorf("StdoutPipe: %v", err)
66
+	}
67
+
68
+	if err = cmd.Start(); err != nil {
69
+		return nil, fmt.Errorf("Start: %v", err)
70
+	}
71
+
72
+	return cmdReadCloser{stdout: stdout, cmd: cmd}, nil
73
+}

+ 2 - 1
vendor/code.gitea.io/git/commit.go

@@ -98,10 +98,11 @@ func (c *Commit) IsImageFile(name string) bool {
98 98
 		return false
99 99
 	}
100 100
 
101
-	dataRc, err := blob.Data()
101
+	dataRc, err := blob.DataAsync()
102 102
 	if err != nil {
103 103
 		return false
104 104
 	}
105
+	defer dataRc.Close()
105 106
 	buf := make([]byte, 1024)
106 107
 	n, _ := dataRc.Read(buf)
107 108
 	buf = buf[:n]

+ 1 - 1
vendor/code.gitea.io/git/git.go

@@ -25,7 +25,7 @@ var (
25 25
 	// Prefix the log prefix
26 26
 	Prefix = "[git-module] "
27 27
 	// GitVersionRequired is the minimum Git version required
28
-	GitVersionRequired = "1.8.1.6"
28
+	GitVersionRequired = "1.7.2"
29 29
 )
30 30
 
31 31
 func log(format string, args ...interface{}) {

+ 3 - 3
vendor/vendor.json

@@ -3,10 +3,10 @@
3 3
 	"ignore": "test appengine",
4 4
 	"package": [
5 5
 		{
6
-			"checksumSHA1": "JN/re4+x/hCzMLGHmieUcykVDAg=",
6
+			"checksumSHA1": "vAVjAz7Wpjnu7GGba4JLIDTpQEw=",
7 7
 			"path": "code.gitea.io/git",
8
-			"revision": "d47b98c44c9a6472e44ab80efe65235e11c6da2a",
9
-			"revisionTime": "2017-10-23T00:52:09Z"
8
+			"revision": "f9dd6826bbb51c92c6964ce18176c304ea286e54",
9
+			"revisionTime": "2017-11-28T15:25:05Z"
10 10
 		},
11 11
 		{
12 12
 			"checksumSHA1": "QQ7g7B9+EIzGjO14KCGEs9TNEzM=",