diff --git a/admin_test.go b/admin_test.go index 539837cf..71cc209e 100644 --- a/admin_test.go +++ b/admin_test.go @@ -52,6 +52,8 @@ func oldMap() M { m["BConfig.WebConfig.DirectoryIndex"] = BConfig.WebConfig.DirectoryIndex m["BConfig.WebConfig.StaticDir"] = BConfig.WebConfig.StaticDir m["BConfig.WebConfig.StaticExtensionsToGzip"] = BConfig.WebConfig.StaticExtensionsToGzip + m["BConfig.WebConfig.StaticCacheFileSize"] = BConfig.WebConfig.StaticCacheFileSize + m["BConfig.WebConfig.StaticCacheFileNum"] = BConfig.WebConfig.StaticCacheFileNum m["BConfig.WebConfig.TemplateLeft"] = BConfig.WebConfig.TemplateLeft m["BConfig.WebConfig.TemplateRight"] = BConfig.WebConfig.TemplateRight m["BConfig.WebConfig.ViewsPath"] = BConfig.WebConfig.ViewsPath diff --git a/config.go b/config.go index 7969dcea..72b8a333 100644 --- a/config.go +++ b/config.go @@ -81,6 +81,8 @@ type WebConfig struct { DirectoryIndex bool StaticDir map[string]string StaticExtensionsToGzip []string + StaticCacheFileSize int + StaticCacheFileNum int TemplateLeft string TemplateRight string ViewsPath string @@ -236,6 +238,8 @@ func newBConfig() *Config { DirectoryIndex: false, StaticDir: map[string]string{"/static": "static"}, StaticExtensionsToGzip: []string{".css", ".js"}, + StaticCacheFileSize: 1024 * 100, + StaticCacheFileNum: 1000, TemplateLeft: "{{", TemplateRight: "}}", ViewsPath: "views", @@ -317,6 +321,14 @@ func assignConfig(ac config.Configer) error { } } + if sfs, err := ac.Int("StaticCacheFileSize"); err == nil { + BConfig.WebConfig.StaticCacheFileSize = sfs + } + + if sfn, err := ac.Int("StaticCacheFileNum"); err == nil { + BConfig.WebConfig.StaticCacheFileNum = sfn + } + if lo := ac.String("LogOutputs"); lo != "" { // if lo is not nil or empty // means user has set his own LogOutputs diff --git a/config/json.go b/config/json.go index 74c18c9c..07084297 100644 --- a/config/json.go +++ b/config/json.go @@ -20,6 +20,7 @@ import ( "fmt" "io/ioutil" "os" + "strconv" "strings" "sync" ) @@ -91,13 +92,7 @@ func (c *JSONConfigContainer) DefaultBool(key string, defaultval bool) bool { // Int returns the integer value for a given key. func (c *JSONConfigContainer) Int(key string) (int, error) { val := c.getData(key) - if val != nil { - if v, ok := val.(float64); ok { - return int(v), nil - } - return 0, errors.New("not int value") - } - return 0, errors.New("not exist key:" + key) + return strconv.Atoi(val.(string)) } // DefaultInt returns the integer value for a given key. diff --git a/config_test.go b/config_test.go index 53411b01..5f71f1c3 100644 --- a/config_test.go +++ b/config_test.go @@ -115,6 +115,8 @@ func TestAssignConfig_03(t *testing.T) { ac.Set("RunMode", "online") ac.Set("StaticDir", "download:down download2:down2") ac.Set("StaticExtensionsToGzip", ".css,.js,.html,.jpg,.png") + ac.Set("StaticCacheFileSize", "87456") + ac.Set("StaticCacheFileNum", "1254") assignConfig(ac) t.Logf("%#v", BConfig) @@ -132,6 +134,12 @@ func TestAssignConfig_03(t *testing.T) { if BConfig.WebConfig.StaticDir["/download2"] != "down2" { t.FailNow() } + if BConfig.WebConfig.StaticCacheFileSize != 87456 { + t.FailNow() + } + if BConfig.WebConfig.StaticCacheFileNum != 1254 { + t.FailNow() + } if len(BConfig.WebConfig.StaticExtensionsToGzip) != 5 { t.FailNow() } diff --git a/staticfile.go b/staticfile.go index 68241a86..464c1175 100644 --- a/staticfile.go +++ b/staticfile.go @@ -28,6 +28,7 @@ import ( "github.com/astaxie/beego/context" "github.com/astaxie/beego/logs" + "github.com/hashicorp/golang-lru" ) var errNotStaticRequest = errors.New("request not a static file request") @@ -93,10 +94,11 @@ func serverStaticRouter(ctx *context.Context) { } type serveContentHolder struct { - data []byte - modTime time.Time - size int64 - encoding string + data []byte + modTime time.Time + size int64 + originSize int64 //original file size:to judge file changed + encoding string } type serveContentReader struct { @@ -104,22 +106,36 @@ type serveContentReader struct { } var ( - staticFileMap = make(map[string]*serveContentHolder) - mapLock sync.RWMutex + staticFileLruCache *lru.Cache + lruLock sync.RWMutex ) func openFile(filePath string, fi os.FileInfo, acceptEncoding string) (bool, string, *serveContentHolder, *serveContentReader, error) { + if staticFileLruCache == nil { + //avoid lru cache error + if BConfig.WebConfig.StaticCacheFileNum >= 1 { + staticFileLruCache, _ = lru.New(BConfig.WebConfig.StaticCacheFileNum) + } else { + staticFileLruCache, _ = lru.New(1) + } + } mapKey := acceptEncoding + ":" + filePath - mapLock.RLock() - mapFile := staticFileMap[mapKey] - mapLock.RUnlock() + lruLock.RLock() + var mapFile *serveContentHolder + if cacheItem, ok := staticFileLruCache.Get(mapKey); ok { + mapFile = cacheItem.(*serveContentHolder) + } + lruLock.RUnlock() if isOk(mapFile, fi) { reader := &serveContentReader{Reader: bytes.NewReader(mapFile.data)} return mapFile.encoding != "", mapFile.encoding, mapFile, reader, nil } - mapLock.Lock() - defer mapLock.Unlock() - if mapFile = staticFileMap[mapKey]; !isOk(mapFile, fi) { + lruLock.Lock() + defer lruLock.Unlock() + if cacheItem, ok := staticFileLruCache.Get(mapKey); ok { + mapFile = cacheItem.(*serveContentHolder) + } + if !isOk(mapFile, fi) { file, err := os.Open(filePath) if err != nil { return false, "", nil, nil, err @@ -130,8 +146,10 @@ func openFile(filePath string, fi os.FileInfo, acceptEncoding string) (bool, str if err != nil { return false, "", nil, nil, err } - mapFile = &serveContentHolder{data: bufferWriter.Bytes(), modTime: fi.ModTime(), size: int64(bufferWriter.Len()), encoding: n} - staticFileMap[mapKey] = mapFile + mapFile = &serveContentHolder{data: bufferWriter.Bytes(), modTime: fi.ModTime(), size: int64(bufferWriter.Len()), originSize: fi.Size(), encoding: n} + if isOk(mapFile, fi) { + staticFileLruCache.Add(mapKey, mapFile) + } } reader := &serveContentReader{Reader: bytes.NewReader(mapFile.data)} @@ -141,8 +159,10 @@ func openFile(filePath string, fi os.FileInfo, acceptEncoding string) (bool, str func isOk(s *serveContentHolder, fi os.FileInfo) bool { if s == nil { return false + } else if s.size > int64(BConfig.WebConfig.StaticCacheFileSize) { + return false } - return s.modTime == fi.ModTime() && s.size == fi.Size() + return s.modTime == fi.ModTime() && s.originSize == fi.Size() } // isStaticCompress detect static files diff --git a/staticfile_test.go b/staticfile_test.go index 69667bf8..e46c13ec 100644 --- a/staticfile_test.go +++ b/staticfile_test.go @@ -4,6 +4,7 @@ import ( "bytes" "compress/gzip" "compress/zlib" + "fmt" "io" "io/ioutil" "os" @@ -53,6 +54,31 @@ func TestOpenStaticFileDeflate_1(t *testing.T) { testOpenFile("deflate", content, t) } +func TestStaticCacheWork(t *testing.T) { + encodings := []string{"", "gzip", "deflate"} + + fi, _ := os.Stat(licenseFile) + for _, encoding := range encodings { + _, _, first, _, err := openFile(licenseFile, fi, encoding) + if err != nil { + t.Error(err) + continue + } + + _, _, second, _, err := openFile(licenseFile, fi, encoding) + if err != nil { + t.Error(err) + continue + } + + address1 := fmt.Sprintf("%p", first) + address2 := fmt.Sprintf("%p", second) + if address1 != address2 { + t.Errorf("encoding '%v' can not hit cache", encoding) + } + } +} + func assetOpenFileAndContent(sch *serveContentHolder, reader *serveContentReader, content []byte, t *testing.T) { t.Log(sch.size, len(content)) if sch.size != int64(len(content)) { @@ -66,7 +92,7 @@ func assetOpenFileAndContent(sch *serveContentHolder, reader *serveContentReader t.Fail() } } - if len(staticFileMap) == 0 { + if staticFileLruCache.Len() == 0 { t.Log("men map is empty") t.Fail() }