From 0c98d8e9ed7013ce722d53ee094f1adc79b90079 Mon Sep 17 00:00:00 2001 From: Tatsushi Demachi Date: Sun, 16 Aug 2015 12:30:22 +0900 Subject: Use LazyFileReader for reading file contents Simple ioutil.ReadFile is used for reading file contents but it reads all of the file contents and copies them into the memory and is run in a single goroutine. It causes much memory consumption at copying media files in content directory to publish directory and it is not good at performance. This improves the both issue by replacing ReadFile with LazyFileReader. It postpones reading the file contents until it is really needed. As the result, actual file read is run in parallelized goroutine. It improves performance especially in a really big site. In addition, if this reader is called from io.Copy, it does not copy the file contents into the memory but just copies them into destination file. It improves much memory consumption issue when the site has many media files. Fix #1181 --- source/filesystem.go | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) (limited to 'source') diff --git a/source/filesystem.go b/source/filesystem.go index 7b7ebb6b7..05167701b 100644 --- a/source/filesystem.go +++ b/source/filesystem.go @@ -14,10 +14,8 @@ package source import ( - "bytes" "github.com/spf13/viper" "io" - "io/ioutil" "os" "path/filepath" "regexp" @@ -114,11 +112,11 @@ func (f *Filesystem) captureFiles() { if isNonProcessablePath(filePath) { return nil } - data, err := ioutil.ReadFile(filePath) + rd, err := NewLazyFileReader(filePath) if err != nil { return err } - f.add(filePath, bytes.NewBuffer(data)) + f.add(filePath, rd) return nil } -- cgit v1.2.3