Memory usage improvements (#3013) (#3028)

* govendor update code.gitea.io/git

Signed-off-by: Duncan Ogilvie <[email protected]>

* Greatly improve memory usage

Signed-off-by: Duncan Ogilvie <[email protected]>

Author: lafriks

modules/context/repo.go

@@ -143,6 +143,9 @@ func (r *Repository) GetEditorconfig() (*editorconfig.Editorconfig, error) {
 	if err != nil {
 		return nil, err
 	}
+	if treeEntry.Blob().Size() >= setting.UI.MaxDisplayFileSize {
+		return nil, git.ErrNotExist{ID: "", RelPath: ".editorconfig"}
+	}
 	reader, err := treeEntry.Blob().Data()
 	if err != nil {
 		return nil, err

routers/repo/download.go

@@ -45,10 +45,11 @@ func ServeData(ctx *context.Context, name string, reader io.Reader) error {
 
 // ServeBlob download a git.Blob
 func ServeBlob(ctx *context.Context, blob *git.Blob) error {
-	dataRc, err := blob.Data()
+	dataRc, err := blob.DataAsync()
 	if err != nil {
 		return err
 	}
+	defer dataRc.Close()
 
 	return ServeData(ctx, ctx.Repo.TreePath, dataRc)
 }

routers/repo/editor.go

@@ -73,11 +73,16 @@ func editFile(ctx *context.Context, isNewFile bool) {
 
 		// No way to edit a directory online.
 		if entry.IsDir() {
-			ctx.Handle(404, "", nil)
+			ctx.Handle(404, "entry.IsDir", nil)
 			return
 		}
 
 		blob := entry.Blob()
+		if blob.Size() >= setting.UI.MaxDisplayFileSize {
+			ctx.Handle(404, "blob.Size", err)
+			return
+		}
+
 		dataRc, err := blob.Data()
 		if err != nil {
 			ctx.Handle(404, "blob.Data", err)
@@ -93,7 +98,7 @@ func editFile(ctx *context.Context, isNewFile bool) {
 
 		// Only text file are editable online.
 		if !base.IsTextFile(buf) {
-			ctx.Handle(404, "", nil)
+			ctx.Handle(404, "base.IsTextFile", nil)
 			return
 		}
 

routers/repo/issue.go

@@ -319,6 +319,9 @@ func getFileContentFromDefaultBranch(ctx *context.Context, filename string) (str
 	if err != nil {
 		return "", false
 	}
+	if entry.Blob().Size() >= setting.UI.MaxDisplayFileSize {
+		return "", false
+	}
 	r, err = entry.Blob().Data()
 	if err != nil {
 		return "", false

routers/repo/view.go

@@ -76,11 +76,12 @@ func renderDirectory(ctx *context.Context, treeLink string) {
 		ctx.Data["ReadmeInList"] = true
 		ctx.Data["ReadmeExist"] = true
 
-		dataRc, err := readmeFile.Data()
+		dataRc, err := readmeFile.DataAsync()
 		if err != nil {
 			ctx.Handle(500, "Data", err)
 			return
 		}
+		defer dataRc.Close()
 
 		buf := make([]byte, 1024)
 		n, _ := dataRc.Read(buf)
@@ -91,14 +92,21 @@ func renderDirectory(ctx *context.Context, treeLink string) {
 		ctx.Data["FileName"] = readmeFile.Name()
 		// FIXME: what happens when README file is an image?
 		if isTextFile {
-			d, _ := ioutil.ReadAll(dataRc)
-			buf = append(buf, d...)
-			if markup.Type(readmeFile.Name()) != "" {
-				ctx.Data["IsMarkup"] = true
-				ctx.Data["FileContent"] = string(markup.Render(readmeFile.Name(), buf, treeLink, ctx.Repo.Repository.ComposeMetas()))
+			if readmeFile.Size() >= setting.UI.MaxDisplayFileSize {
+				// Pretend that this is a normal text file to display 'This file is too large to be shown'
+				ctx.Data["IsFileTooLarge"] = true
+				ctx.Data["IsTextFile"] = true
+				ctx.Data["FileSize"] = readmeFile.Size()
 			} else {
-				ctx.Data["IsRenderedHTML"] = true
-				ctx.Data["FileContent"] = string(bytes.Replace(buf, []byte("\n"), []byte(`<br>`), -1))
+				d, _ := ioutil.ReadAll(dataRc)
+				buf = append(buf, d...)
+				if markup.Type(readmeFile.Name()) != "" {
+					ctx.Data["IsMarkup"] = true
+					ctx.Data["FileContent"] = string(markup.Render(readmeFile.Name(), buf, treeLink, ctx.Repo.Repository.ComposeMetas()))
+				} else {
+					ctx.Data["IsRenderedHTML"] = true
+					ctx.Data["FileContent"] = string(bytes.Replace(buf, []byte("\n"), []byte(`<br>`), -1))
+				}
 			}
 		}
 	}
@@ -135,11 +143,12 @@ func renderFile(ctx *context.Context, entry *git.TreeEntry, treeLink, rawLink st
 	ctx.Data["IsViewFile"] = true
 
 	blob := entry.Blob()
-	dataRc, err := blob.Data()
+	dataRc, err := blob.DataAsync()
 	if err != nil {
-		ctx.Handle(500, "Data", err)
+		ctx.Handle(500, "DataAsync", err)
 		return
 	}
+	defer dataRc.Close()
 
 	ctx.Data["FileSize"] = blob.Size()
 	ctx.Data["FileName"] = blob.Name()

vendor/code.gitea.io/git/blob.go

@@ -3,10 +3,10 @@
 	"ignore": "test appengine",
 	"package": [
 		{
-			"checksumSHA1": "JN/re4+x/hCzMLGHmieUcykVDAg=",
+			"checksumSHA1": "vAVjAz7Wpjnu7GGba4JLIDTpQEw=",
 			"path": "code.gitea.io/git",
-			"revision": "d47b98c44c9a6472e44ab80efe65235e11c6da2a",
-			"revisionTime": "2017-10-23T00:52:09Z"
+			"revision": "f9dd6826bbb51c92c6964ce18176c304ea286e54",
+			"revisionTime": "2017-11-28T15:25:05Z"
 		},
 		{
 			"checksumSHA1": "OICEgmUefW4L4l/FK/NVFnl/aOM=",