From 1dffa20435c6f5fc4f2d4e49923249acd0cdf07e Mon Sep 17 00:00:00 2001 From: Ming Deng Date: Sat, 10 Oct 2020 21:34:02 +0800 Subject: [PATCH] make stmt cache smaller --- build_info.go | 8 ++--- cache/redis/redis.go | 5 ++-- config/yaml/yaml.go | 2 +- context/param/parsers_test.go | 8 +++-- controller_test.go | 3 +- logs/accesslog.go | 2 +- logs/file.go | 30 +++++++++---------- logs/file_test.go | 30 +++++++++---------- metric/prometheus.go | 6 ++-- orm/cmd_utils.go | 6 ++-- orm/db_alias.go | 9 ++++-- orm/orm_log.go | 2 +- plugins/authz/authz.go | 3 +- plugins/authz/authz_test.go | 7 +++-- session/redis_cluster/redis_cluster.go | 16 +++++----- session/redis_sentinel/sess_redis_sentinel.go | 5 ++-- session/sess_file_test.go | 5 ++-- staticfile.go | 2 +- template_test.go | 5 ++-- templatefunc.go | 2 +- testdata/bindata.go | 3 +- validation/util.go | 2 +- validation/validators.go | 3 +- 23 files changed, 89 insertions(+), 75 deletions(-) diff --git a/build_info.go b/build_info.go index 6dc2835e..c31152ea 100644 --- a/build_info.go +++ b/build_info.go @@ -15,11 +15,11 @@ package beego var ( - BuildVersion string + BuildVersion string BuildGitRevision string - BuildStatus string - BuildTag string - BuildTime string + BuildStatus string + BuildTag string + BuildTime string GoVersion string diff --git a/cache/redis/redis.go b/cache/redis/redis.go index 56faf211..dde760ea 100644 --- a/cache/redis/redis.go +++ b/cache/redis/redis.go @@ -38,8 +38,9 @@ import ( "github.com/gomodule/redigo/redis" - "github.com/astaxie/beego/cache" "strings" + + "github.com/astaxie/beego/cache" ) var ( @@ -57,7 +58,7 @@ type Cache struct { maxIdle int //the timeout to a value less than the redis server's timeout. - timeout time.Duration + timeout time.Duration } // NewRedisCache create new redis cache with default collection name. diff --git a/config/yaml/yaml.go b/config/yaml/yaml.go index 5def2da3..a5644c7b 100644 --- a/config/yaml/yaml.go +++ b/config/yaml/yaml.go @@ -296,7 +296,7 @@ func (c *ConfigContainer) getData(key string) (interface{}, error) { case map[string]interface{}: { tmpData = v.(map[string]interface{}) - if idx == len(keys) - 1 { + if idx == len(keys)-1 { return tmpData, nil } } diff --git a/context/param/parsers_test.go b/context/param/parsers_test.go index 7065a28e..81a821f1 100644 --- a/context/param/parsers_test.go +++ b/context/param/parsers_test.go @@ -1,8 +1,10 @@ package param -import "testing" -import "reflect" -import "time" +import ( + "reflect" + "testing" + "time" +) type testDefinition struct { strValue string diff --git a/controller_test.go b/controller_test.go index 215a7411..a5888f62 100644 --- a/controller_test.go +++ b/controller_test.go @@ -19,9 +19,10 @@ import ( "strconv" "testing" - "github.com/astaxie/beego/context" "os" "path/filepath" + + "github.com/astaxie/beego/context" ) func TestGetInt(t *testing.T) { diff --git a/logs/accesslog.go b/logs/accesslog.go index 3ff9e20f..9011b602 100644 --- a/logs/accesslog.go +++ b/logs/accesslog.go @@ -16,9 +16,9 @@ package logs import ( "bytes" - "strings" "encoding/json" "fmt" + "strings" "time" ) diff --git a/logs/file.go b/logs/file.go index 222db989..40a3572a 100644 --- a/logs/file.go +++ b/logs/file.go @@ -373,21 +373,21 @@ func (w *fileLogWriter) deleteOldLog() { if info == nil { return } - if w.Hourly { - if !info.IsDir() && info.ModTime().Add(1 * time.Hour * time.Duration(w.MaxHours)).Before(time.Now()) { - if strings.HasPrefix(filepath.Base(path), filepath.Base(w.fileNameOnly)) && - strings.HasSuffix(filepath.Base(path), w.suffix) { - os.Remove(path) - } - } - } else if w.Daily { - if !info.IsDir() && info.ModTime().Add(24 * time.Hour * time.Duration(w.MaxDays)).Before(time.Now()) { - if strings.HasPrefix(filepath.Base(path), filepath.Base(w.fileNameOnly)) && - strings.HasSuffix(filepath.Base(path), w.suffix) { - os.Remove(path) - } - } - } + if w.Hourly { + if !info.IsDir() && info.ModTime().Add(1*time.Hour*time.Duration(w.MaxHours)).Before(time.Now()) { + if strings.HasPrefix(filepath.Base(path), filepath.Base(w.fileNameOnly)) && + strings.HasSuffix(filepath.Base(path), w.suffix) { + os.Remove(path) + } + } + } else if w.Daily { + if !info.IsDir() && info.ModTime().Add(24*time.Hour*time.Duration(w.MaxDays)).Before(time.Now()) { + if strings.HasPrefix(filepath.Base(path), filepath.Base(w.fileNameOnly)) && + strings.HasSuffix(filepath.Base(path), w.suffix) { + os.Remove(path) + } + } + } return }) } diff --git a/logs/file_test.go b/logs/file_test.go index e7c2ca9a..385eac43 100644 --- a/logs/file_test.go +++ b/logs/file_test.go @@ -186,7 +186,7 @@ func TestFileDailyRotate_06(t *testing.T) { //test file mode func TestFileHourlyRotate_01(t *testing.T) { log := NewLogger(10000) - log.SetLogger("file", `{"filename":"test3.log","hourly":true,"maxlines":4}`) + log.SetLogger("file", `{"filename":"test3.log","hourly":true,"maxlines":4}`) log.Debug("debug") log.Info("info") log.Notice("notice") @@ -237,7 +237,7 @@ func TestFileHourlyRotate_05(t *testing.T) { func TestFileHourlyRotate_06(t *testing.T) { //test file mode log := NewLogger(10000) - log.SetLogger("file", `{"filename":"test3.log", "hourly":true, "maxlines":4}`) + log.SetLogger("file", `{"filename":"test3.log", "hourly":true, "maxlines":4}`) log.Debug("debug") log.Info("info") log.Notice("notice") @@ -269,19 +269,19 @@ func testFileRotate(t *testing.T, fn1, fn2 string, daily, hourly bool) { RotatePerm: "0440", } - if daily { - fw.Init(fmt.Sprintf(`{"filename":"%v","maxdays":1}`, fn1)) - fw.dailyOpenTime = time.Now().Add(-24 * time.Hour) - fw.dailyOpenDate = fw.dailyOpenTime.Day() - } + if daily { + fw.Init(fmt.Sprintf(`{"filename":"%v","maxdays":1}`, fn1)) + fw.dailyOpenTime = time.Now().Add(-24 * time.Hour) + fw.dailyOpenDate = fw.dailyOpenTime.Day() + } - if hourly { - fw.Init(fmt.Sprintf(`{"filename":"%v","maxhours":1}`, fn1)) - fw.hourlyOpenTime = time.Now().Add(-1 * time.Hour) - fw.hourlyOpenDate = fw.hourlyOpenTime.Day() - } + if hourly { + fw.Init(fmt.Sprintf(`{"filename":"%v","maxhours":1}`, fn1)) + fw.hourlyOpenTime = time.Now().Add(-1 * time.Hour) + fw.hourlyOpenDate = fw.hourlyOpenTime.Day() + } - fw.WriteMsg(time.Now(), "this is a msg for test", LevelDebug) + fw.WriteMsg(time.Now(), "this is a msg for test", LevelDebug) for _, file := range []string{fn1, fn2} { _, err := os.Stat(file) @@ -328,8 +328,8 @@ func testFileDailyRotate(t *testing.T, fn1, fn2 string) { func testFileHourlyRotate(t *testing.T, fn1, fn2 string) { fw := &fileLogWriter{ - Hourly: true, - MaxHours: 168, + Hourly: true, + MaxHours: 168, Rotate: true, Level: LevelTrace, Perm: "0660", diff --git a/metric/prometheus.go b/metric/prometheus.go index 7722240b..86e2c1b1 100644 --- a/metric/prometheus.go +++ b/metric/prometheus.go @@ -57,15 +57,15 @@ func registerBuildInfo() { Subsystem: "build_info", Help: "The building information", ConstLabels: map[string]string{ - "appname": beego.BConfig.AppName, + "appname": beego.BConfig.AppName, "build_version": beego.BuildVersion, "build_revision": beego.BuildGitRevision, "build_status": beego.BuildStatus, "build_tag": beego.BuildTag, - "build_time": strings.Replace(beego.BuildTime, "--", " ", 1), + "build_time": strings.Replace(beego.BuildTime, "--", " ", 1), "go_version": beego.GoVersion, "git_branch": beego.GitBranch, - "start_time": time.Now().Format("2006-01-02 15:04:05"), + "start_time": time.Now().Format("2006-01-02 15:04:05"), }, }, []string{}) diff --git a/orm/cmd_utils.go b/orm/cmd_utils.go index 61f17346..692a079f 100644 --- a/orm/cmd_utils.go +++ b/orm/cmd_utils.go @@ -197,9 +197,9 @@ func getDbCreateSQL(al *alias) (sqls []string, tableIndexes map[string][]dbIndex if strings.Contains(column, "%COL%") { column = strings.Replace(column, "%COL%", fi.column, -1) } - - if fi.description != "" && al.Driver!=DRSqlite { - column += " " + fmt.Sprintf("COMMENT '%s'",fi.description) + + if fi.description != "" && al.Driver != DRSqlite { + column += " " + fmt.Sprintf("COMMENT '%s'", fi.description) } columns = append(columns, column) diff --git a/orm/db_alias.go b/orm/db_alias.go index bf6c350c..369802a7 100644 --- a/orm/db_alias.go +++ b/orm/db_alias.go @@ -18,10 +18,11 @@ import ( "context" "database/sql" "fmt" - lru "github.com/hashicorp/golang-lru" "reflect" "sync" "time" + + lru "github.com/hashicorp/golang-lru" ) // DriverType database driver constant int. @@ -424,7 +425,7 @@ func GetDB(aliasNames ...string) (*sql.DB, error) { } type stmtDecorator struct { - wg sync.WaitGroup + wg sync.WaitGroup stmt *sql.Stmt } @@ -459,7 +460,9 @@ func newStmtDecorator(sqlStmt *sql.Stmt) *stmtDecorator { } func newStmtDecoratorLruWithEvict() *lru.Cache { - cache, _ := lru.NewWithEvict(1000, func(key interface{}, value interface{}) { + // temporarily solution + // we fixed this problem in v2.x + cache, _ := lru.NewWithEvict(50, func(key interface{}, value interface{}) { value.(*stmtDecorator).destroy() }) return cache diff --git a/orm/orm_log.go b/orm/orm_log.go index f107bb59..5bb3a24f 100644 --- a/orm/orm_log.go +++ b/orm/orm_log.go @@ -61,7 +61,7 @@ func debugLogQueies(alias *alias, operaton, query string, t time.Time, err error con += " - " + err.Error() } logMap["sql"] = fmt.Sprintf("%s-`%s`", query, strings.Join(cons, "`, `")) - if LogFunc != nil{ + if LogFunc != nil { LogFunc(logMap) } DebugLog.Println(con) diff --git a/plugins/authz/authz.go b/plugins/authz/authz.go index 9dc0db76..879fdea6 100644 --- a/plugins/authz/authz.go +++ b/plugins/authz/authz.go @@ -40,10 +40,11 @@ package authz import ( + "net/http" + "github.com/astaxie/beego" "github.com/astaxie/beego/context" "github.com/casbin/casbin" - "net/http" ) // NewAuthorizer returns the authorizer. diff --git a/plugins/authz/authz_test.go b/plugins/authz/authz_test.go index 49aed84c..e9d397a2 100644 --- a/plugins/authz/authz_test.go +++ b/plugins/authz/authz_test.go @@ -15,13 +15,14 @@ package authz import ( + "net/http" + "net/http/httptest" + "testing" + "github.com/astaxie/beego" "github.com/astaxie/beego/context" "github.com/astaxie/beego/plugins/auth" "github.com/casbin/casbin" - "net/http" - "net/http/httptest" - "testing" ) func testRequest(t *testing.T, handler *beego.ControllerRegister, user string, path string, method string, code int) { diff --git a/session/redis_cluster/redis_cluster.go b/session/redis_cluster/redis_cluster.go index 2fe300df..802c1c39 100644 --- a/session/redis_cluster/redis_cluster.go +++ b/session/redis_cluster/redis_cluster.go @@ -31,14 +31,16 @@ // // more docs: http://beego.me/docs/module/session.md package redis_cluster + import ( "net/http" "strconv" "strings" "sync" + "time" + "github.com/astaxie/beego/session" rediss "github.com/go-redis/redis" - "time" ) var redispder = &Provider{} @@ -101,7 +103,7 @@ func (rs *SessionStore) SessionRelease(w http.ResponseWriter) { return } c := rs.p - c.Set(rs.sid, string(b), time.Duration(rs.maxlifetime) * time.Second) + c.Set(rs.sid, string(b), time.Duration(rs.maxlifetime)*time.Second) } // Provider redis_cluster session provider @@ -146,10 +148,10 @@ func (rp *Provider) SessionInit(maxlifetime int64, savePath string) error { } else { rp.dbNum = 0 } - + rp.poollist = rediss.NewClusterClient(&rediss.ClusterOptions{ Addrs: strings.Split(rp.savePath, ";"), - Password: rp.password, + Password: rp.password, PoolSize: rp.poolsize, }) return rp.poollist.Ping().Err() @@ -186,15 +188,15 @@ func (rp *Provider) SessionExist(sid string) bool { // SessionRegenerate generate new sid for redis_cluster session func (rp *Provider) SessionRegenerate(oldsid, sid string) (session.Store, error) { c := rp.poollist - + if existed, err := c.Exists(oldsid).Result(); err != nil || existed == 0 { // oldsid doesn't exists, set the new sid directly // ignore error here, since if it return error // the existed value will be 0 - c.Set(sid, "", time.Duration(rp.maxlifetime) * time.Second) + c.Set(sid, "", time.Duration(rp.maxlifetime)*time.Second) } else { c.Rename(oldsid, sid) - c.Expire(sid, time.Duration(rp.maxlifetime) * time.Second) + c.Expire(sid, time.Duration(rp.maxlifetime)*time.Second) } return rp.SessionRead(sid) } diff --git a/session/redis_sentinel/sess_redis_sentinel.go b/session/redis_sentinel/sess_redis_sentinel.go index 6ecb2977..29284d03 100644 --- a/session/redis_sentinel/sess_redis_sentinel.go +++ b/session/redis_sentinel/sess_redis_sentinel.go @@ -33,13 +33,14 @@ package redis_sentinel import ( - "github.com/astaxie/beego/session" - "github.com/go-redis/redis" "net/http" "strconv" "strings" "sync" "time" + + "github.com/astaxie/beego/session" + "github.com/go-redis/redis" ) var redispder = &Provider{} diff --git a/session/sess_file_test.go b/session/sess_file_test.go index 0cf021db..021c43fc 100644 --- a/session/sess_file_test.go +++ b/session/sess_file_test.go @@ -369,8 +369,7 @@ func TestFileSessionStore_SessionRelease(t *testing.T) { t.Error(err) } - - s.Set(i,i) + s.Set(i, i) s.SessionRelease(nil) } @@ -384,4 +383,4 @@ func TestFileSessionStore_SessionRelease(t *testing.T) { t.Error() } } -} \ No newline at end of file +} diff --git a/staticfile.go b/staticfile.go index 84e9aa7b..e26776c5 100644 --- a/staticfile.go +++ b/staticfile.go @@ -202,7 +202,7 @@ func searchFile(ctx *context.Context) (string, os.FileInfo, error) { if !strings.Contains(requestPath, prefix) { continue } - if prefix != "/" && len(requestPath) > len(prefix) && requestPath[len(prefix)] != '/' { + if prefix != "/" && len(requestPath) > len(prefix) && requestPath[len(prefix)] != '/' { continue } filePath := path.Join(staticDir, requestPath[len(prefix):]) diff --git a/template_test.go b/template_test.go index 049655db..b688c8a0 100644 --- a/template_test.go +++ b/template_test.go @@ -16,12 +16,13 @@ package beego import ( "bytes" - "github.com/astaxie/beego/testdata" - "github.com/elazarl/go-bindata-assetfs" "net/http" "os" "path/filepath" "testing" + + "github.com/astaxie/beego/testdata" + "github.com/elazarl/go-bindata-assetfs" ) var header = `{{define "header"}} diff --git a/templatefunc.go b/templatefunc.go index ba1ec5eb..6f02b8d6 100644 --- a/templatefunc.go +++ b/templatefunc.go @@ -362,7 +362,7 @@ func parseFormToStruct(form url.Values, objT reflect.Type, objV reflect.Value) e value = value[:25] t, err = time.ParseInLocation(time.RFC3339, value, time.Local) } else if strings.HasSuffix(strings.ToUpper(value), "Z") { - t, err = time.ParseInLocation(time.RFC3339, value, time.Local) + t, err = time.ParseInLocation(time.RFC3339, value, time.Local) } else if len(value) >= 19 { if strings.Contains(value, "T") { value = value[:19] diff --git a/testdata/bindata.go b/testdata/bindata.go index beade103..ccfb51d4 100644 --- a/testdata/bindata.go +++ b/testdata/bindata.go @@ -11,13 +11,14 @@ import ( "bytes" "compress/gzip" "fmt" - "github.com/elazarl/go-bindata-assetfs" "io" "io/ioutil" "os" "path/filepath" "strings" "time" + + "github.com/elazarl/go-bindata-assetfs" ) func bindataRead(data []byte, name string) ([]byte, error) { diff --git a/validation/util.go b/validation/util.go index 82206f4f..918b206c 100644 --- a/validation/util.go +++ b/validation/util.go @@ -213,7 +213,7 @@ func parseFunc(vfunc, key string, label string) (v ValidFunc, err error) { return } - tParams, err := trim(name, key+"."+ name + "." + label, params) + tParams, err := trim(name, key+"."+name+"."+label, params) if err != nil { return } diff --git a/validation/validators.go b/validation/validators.go index 38b6f1aa..a0e4bcbb 100644 --- a/validation/validators.go +++ b/validation/validators.go @@ -16,13 +16,14 @@ package validation import ( "fmt" - "github.com/astaxie/beego/logs" "reflect" "regexp" "strings" "sync" "time" "unicode/utf8" + + "github.com/astaxie/beego/logs" ) // CanSkipFuncs will skip valid if RequiredFirst is true and the struct field's value is empty