Merge pull request #3984 from jianzhiyao/develop

fix bug:static can not real hit cache & memory leak

admin_test.go

@@ -52,6 +52,8 @@ func oldMap() M {
 	m["BConfig.WebConfig.DirectoryIndex"] = BConfig.WebConfig.DirectoryIndex
 	m["BConfig.WebConfig.StaticDir"] = BConfig.WebConfig.StaticDir
 	m["BConfig.WebConfig.StaticExtensionsToGzip"] = BConfig.WebConfig.StaticExtensionsToGzip
+	m["BConfig.WebConfig.StaticCacheFileSize"] = BConfig.WebConfig.StaticCacheFileSize
+	m["BConfig.WebConfig.StaticCacheFileNum"] = BConfig.WebConfig.StaticCacheFileNum
 	m["BConfig.WebConfig.TemplateLeft"] = BConfig.WebConfig.TemplateLeft
 	m["BConfig.WebConfig.TemplateRight"] = BConfig.WebConfig.TemplateRight
 	m["BConfig.WebConfig.ViewsPath"] = BConfig.WebConfig.ViewsPath

config.go

@@ -81,6 +81,8 @@ type WebConfig struct {
 	DirectoryIndex         bool
 	StaticDir              map[string]string
 	StaticExtensionsToGzip []string
+	StaticCacheFileSize    int
+	StaticCacheFileNum     int
 	TemplateLeft           string
 	TemplateRight          string
 	ViewsPath              string
@@ -236,6 +238,8 @@ func newBConfig() *Config {
 			DirectoryIndex:         false,
 			StaticDir:              map[string]string{"/static": "static"},
 			StaticExtensionsToGzip: []string{".css", ".js"},
+			StaticCacheFileSize:    1024 * 100,
+			StaticCacheFileNum:     1000,
 			TemplateLeft:           "{{",
 			TemplateRight:          "}}",
 			ViewsPath:              "views",
@@ -317,6 +321,14 @@ func assignConfig(ac config.Configer) error {
 		}
 	}
 
+	if sfs, err := ac.Int("StaticCacheFileSize"); err == nil {
+		BConfig.WebConfig.StaticCacheFileSize = sfs
+	}
+
+	if sfn, err := ac.Int("StaticCacheFileNum"); err == nil {
+		BConfig.WebConfig.StaticCacheFileNum = sfn
+	}
+
 	if lo := ac.String("LogOutputs"); lo != "" {
 		// if lo is not nil or empty
 		// means user has set his own LogOutputs

config/json.go

@@ -20,6 +20,7 @@ import (
 	"fmt"
 	"io/ioutil"
 	"os"
+	"strconv"
 	"strings"
 	"sync"
 )
@@ -91,13 +92,7 @@ func (c *JSONConfigContainer) DefaultBool(key string, defaultval bool) bool {
 // Int returns the integer value for a given key.
 func (c *JSONConfigContainer) Int(key string) (int, error) {
 	val := c.getData(key)
-	if val != nil {
-		if v, ok := val.(float64); ok {
-			return int(v), nil
-		}
-		return 0, errors.New("not int value")
-	}
-	return 0, errors.New("not exist key:" + key)
+	return strconv.Atoi(val.(string))
 }
 
 // DefaultInt returns the integer value for a given key.

config_test.go

@@ -115,6 +115,8 @@ func TestAssignConfig_03(t *testing.T) {
 	ac.Set("RunMode", "online")
 	ac.Set("StaticDir", "download:down download2:down2")
 	ac.Set("StaticExtensionsToGzip", ".css,.js,.html,.jpg,.png")
+	ac.Set("StaticCacheFileSize", "87456")
+	ac.Set("StaticCacheFileNum", "1254")
 	assignConfig(ac)
 
 	t.Logf("%#v", BConfig)
@@ -132,6 +134,12 @@ func TestAssignConfig_03(t *testing.T) {
 	if BConfig.WebConfig.StaticDir["/download2"] != "down2" {
 		t.FailNow()
 	}
+	if BConfig.WebConfig.StaticCacheFileSize != 87456 {
+		t.FailNow()
+	}
+	if BConfig.WebConfig.StaticCacheFileNum != 1254 {
+		t.FailNow()
+	}
 	if len(BConfig.WebConfig.StaticExtensionsToGzip) != 5 {
 		t.FailNow()
 	}

staticfile.go

@@ -28,6 +28,7 @@ import (
 
 	"github.com/astaxie/beego/context"
 	"github.com/astaxie/beego/logs"
+	"github.com/hashicorp/golang-lru"
 )
 
 var errNotStaticRequest = errors.New("request not a static file request")
@@ -96,6 +97,7 @@ type serveContentHolder struct {
 	data       []byte
 	modTime    time.Time
 	size       int64
+	originSize int64 //original file size:to judge file changed
 	encoding   string
 }
 
@@ -104,22 +106,36 @@ type serveContentReader struct {
 }
 
 var (
-	staticFileMap = make(map[string]*serveContentHolder)
-	mapLock       sync.RWMutex
+	staticFileLruCache *lru.Cache
+	lruLock            sync.RWMutex
 )
 
 func openFile(filePath string, fi os.FileInfo, acceptEncoding string) (bool, string, *serveContentHolder, *serveContentReader, error) {
+	if staticFileLruCache == nil {
+		//avoid lru cache error
+		if BConfig.WebConfig.StaticCacheFileNum >= 1 {
+			staticFileLruCache, _ = lru.New(BConfig.WebConfig.StaticCacheFileNum)
+		} else {
+			staticFileLruCache, _ = lru.New(1)
+		}
+	}
 	mapKey := acceptEncoding + ":" + filePath
-	mapLock.RLock()
-	mapFile := staticFileMap[mapKey]
-	mapLock.RUnlock()
+	lruLock.RLock()
+	var mapFile *serveContentHolder
+	if cacheItem, ok := staticFileLruCache.Get(mapKey); ok {
+		mapFile = cacheItem.(*serveContentHolder)
+	}
+	lruLock.RUnlock()
 	if isOk(mapFile, fi) {
 		reader := &serveContentReader{Reader: bytes.NewReader(mapFile.data)}
 		return mapFile.encoding != "", mapFile.encoding, mapFile, reader, nil
 	}
-	mapLock.Lock()
-	defer mapLock.Unlock()
-	if mapFile = staticFileMap[mapKey]; !isOk(mapFile, fi) {
+	lruLock.Lock()
+	defer lruLock.Unlock()
+	if cacheItem, ok := staticFileLruCache.Get(mapKey); ok {
+		mapFile = cacheItem.(*serveContentHolder)
+	}
+	if !isOk(mapFile, fi) {
 		file, err := os.Open(filePath)
 		if err != nil {
 			return false, "", nil, nil, err
@@ -130,8 +146,10 @@ func openFile(filePath string, fi os.FileInfo, acceptEncoding string) (bool, str
 		if err != nil {
 			return false, "", nil, nil, err
 		}
-		mapFile = &serveContentHolder{data: bufferWriter.Bytes(), modTime: fi.ModTime(), size: int64(bufferWriter.Len()), encoding: n}
-		staticFileMap[mapKey] = mapFile
+		mapFile = &serveContentHolder{data: bufferWriter.Bytes(), modTime: fi.ModTime(), size: int64(bufferWriter.Len()), originSize: fi.Size(), encoding: n}
+		if isOk(mapFile, fi) {
+			staticFileLruCache.Add(mapKey, mapFile)
+		}
 	}
 
 	reader := &serveContentReader{Reader: bytes.NewReader(mapFile.data)}
@@ -141,8 +159,10 @@ func openFile(filePath string, fi os.FileInfo, acceptEncoding string) (bool, str
 func isOk(s *serveContentHolder, fi os.FileInfo) bool {
 	if s == nil {
 		return false
+	} else if s.size > int64(BConfig.WebConfig.StaticCacheFileSize) {
+		return false
 	}
-	return s.modTime == fi.ModTime() && s.size == fi.Size()
+	return s.modTime == fi.ModTime() && s.originSize == fi.Size()
 }
 
 // isStaticCompress detect static files

staticfile_test.go

@@ -4,6 +4,7 @@ import (
 	"bytes"
 	"compress/gzip"
 	"compress/zlib"
+	"fmt"
 	"io"
 	"io/ioutil"
 	"os"
@@ -53,6 +54,31 @@ func TestOpenStaticFileDeflate_1(t *testing.T) {
 	testOpenFile("deflate", content, t)
 }
 
+func TestStaticCacheWork(t *testing.T) {
+	encodings := []string{"", "gzip", "deflate"}
+
+	fi, _ := os.Stat(licenseFile)
+	for _, encoding := range encodings {
+		_, _, first, _, err := openFile(licenseFile, fi, encoding)
+		if err != nil {
+			t.Error(err)
+			continue
+		}
+
+		_, _, second, _, err := openFile(licenseFile, fi, encoding)
+		if err != nil {
+			t.Error(err)
+			continue
+		}
+
+		address1 := fmt.Sprintf("%p", first)
+		address2 := fmt.Sprintf("%p", second)
+		if address1 != address2 {
+			t.Errorf("encoding '%v' can not hit cache", encoding)
+		}
+	}
+}
+
 func assetOpenFileAndContent(sch *serveContentHolder, reader *serveContentReader, content []byte, t *testing.T) {
 	t.Log(sch.size, len(content))
 	if sch.size != int64(len(content)) {
@@ -66,7 +92,7 @@ func assetOpenFileAndContent(sch *serveContentHolder, reader *serveContentReader
 			t.Fail()
 		}
 	}
-	if len(staticFileMap) == 0 {
+	if staticFileLruCache.Len() == 0 {
 		t.Log("men map is empty")
 		t.Fail()
 	}