1
0
mirror of https://github.com/astaxie/beego.git synced 2024-11-14 22:50:55 +00:00

make stmt cache smaller

This commit is contained in:
Ming Deng 2020-10-10 21:34:02 +08:00
parent 8e37fe3b78
commit 1dffa20435
23 changed files with 89 additions and 75 deletions

View File

@ -15,11 +15,11 @@
package beego package beego
var ( var (
BuildVersion string BuildVersion string
BuildGitRevision string BuildGitRevision string
BuildStatus string BuildStatus string
BuildTag string BuildTag string
BuildTime string BuildTime string
GoVersion string GoVersion string

View File

@ -38,8 +38,9 @@ import (
"github.com/gomodule/redigo/redis" "github.com/gomodule/redigo/redis"
"github.com/astaxie/beego/cache"
"strings" "strings"
"github.com/astaxie/beego/cache"
) )
var ( var (
@ -57,7 +58,7 @@ type Cache struct {
maxIdle int maxIdle int
//the timeout to a value less than the redis server's timeout. //the timeout to a value less than the redis server's timeout.
timeout time.Duration timeout time.Duration
} }
// NewRedisCache create new redis cache with default collection name. // NewRedisCache create new redis cache with default collection name.

View File

@ -296,7 +296,7 @@ func (c *ConfigContainer) getData(key string) (interface{}, error) {
case map[string]interface{}: case map[string]interface{}:
{ {
tmpData = v.(map[string]interface{}) tmpData = v.(map[string]interface{})
if idx == len(keys) - 1 { if idx == len(keys)-1 {
return tmpData, nil return tmpData, nil
} }
} }

View File

@ -1,8 +1,10 @@
package param package param
import "testing" import (
import "reflect" "reflect"
import "time" "testing"
"time"
)
type testDefinition struct { type testDefinition struct {
strValue string strValue string

View File

@ -19,9 +19,10 @@ import (
"strconv" "strconv"
"testing" "testing"
"github.com/astaxie/beego/context"
"os" "os"
"path/filepath" "path/filepath"
"github.com/astaxie/beego/context"
) )
func TestGetInt(t *testing.T) { func TestGetInt(t *testing.T) {

View File

@ -16,9 +16,9 @@ package logs
import ( import (
"bytes" "bytes"
"strings"
"encoding/json" "encoding/json"
"fmt" "fmt"
"strings"
"time" "time"
) )

View File

@ -373,21 +373,21 @@ func (w *fileLogWriter) deleteOldLog() {
if info == nil { if info == nil {
return return
} }
if w.Hourly { if w.Hourly {
if !info.IsDir() && info.ModTime().Add(1 * time.Hour * time.Duration(w.MaxHours)).Before(time.Now()) { if !info.IsDir() && info.ModTime().Add(1*time.Hour*time.Duration(w.MaxHours)).Before(time.Now()) {
if strings.HasPrefix(filepath.Base(path), filepath.Base(w.fileNameOnly)) && if strings.HasPrefix(filepath.Base(path), filepath.Base(w.fileNameOnly)) &&
strings.HasSuffix(filepath.Base(path), w.suffix) { strings.HasSuffix(filepath.Base(path), w.suffix) {
os.Remove(path) os.Remove(path)
} }
} }
} else if w.Daily { } else if w.Daily {
if !info.IsDir() && info.ModTime().Add(24 * time.Hour * time.Duration(w.MaxDays)).Before(time.Now()) { if !info.IsDir() && info.ModTime().Add(24*time.Hour*time.Duration(w.MaxDays)).Before(time.Now()) {
if strings.HasPrefix(filepath.Base(path), filepath.Base(w.fileNameOnly)) && if strings.HasPrefix(filepath.Base(path), filepath.Base(w.fileNameOnly)) &&
strings.HasSuffix(filepath.Base(path), w.suffix) { strings.HasSuffix(filepath.Base(path), w.suffix) {
os.Remove(path) os.Remove(path)
} }
} }
} }
return return
}) })
} }

View File

@ -186,7 +186,7 @@ func TestFileDailyRotate_06(t *testing.T) { //test file mode
func TestFileHourlyRotate_01(t *testing.T) { func TestFileHourlyRotate_01(t *testing.T) {
log := NewLogger(10000) log := NewLogger(10000)
log.SetLogger("file", `{"filename":"test3.log","hourly":true,"maxlines":4}`) log.SetLogger("file", `{"filename":"test3.log","hourly":true,"maxlines":4}`)
log.Debug("debug") log.Debug("debug")
log.Info("info") log.Info("info")
log.Notice("notice") log.Notice("notice")
@ -237,7 +237,7 @@ func TestFileHourlyRotate_05(t *testing.T) {
func TestFileHourlyRotate_06(t *testing.T) { //test file mode func TestFileHourlyRotate_06(t *testing.T) { //test file mode
log := NewLogger(10000) log := NewLogger(10000)
log.SetLogger("file", `{"filename":"test3.log", "hourly":true, "maxlines":4}`) log.SetLogger("file", `{"filename":"test3.log", "hourly":true, "maxlines":4}`)
log.Debug("debug") log.Debug("debug")
log.Info("info") log.Info("info")
log.Notice("notice") log.Notice("notice")
@ -269,19 +269,19 @@ func testFileRotate(t *testing.T, fn1, fn2 string, daily, hourly bool) {
RotatePerm: "0440", RotatePerm: "0440",
} }
if daily { if daily {
fw.Init(fmt.Sprintf(`{"filename":"%v","maxdays":1}`, fn1)) fw.Init(fmt.Sprintf(`{"filename":"%v","maxdays":1}`, fn1))
fw.dailyOpenTime = time.Now().Add(-24 * time.Hour) fw.dailyOpenTime = time.Now().Add(-24 * time.Hour)
fw.dailyOpenDate = fw.dailyOpenTime.Day() fw.dailyOpenDate = fw.dailyOpenTime.Day()
} }
if hourly { if hourly {
fw.Init(fmt.Sprintf(`{"filename":"%v","maxhours":1}`, fn1)) fw.Init(fmt.Sprintf(`{"filename":"%v","maxhours":1}`, fn1))
fw.hourlyOpenTime = time.Now().Add(-1 * time.Hour) fw.hourlyOpenTime = time.Now().Add(-1 * time.Hour)
fw.hourlyOpenDate = fw.hourlyOpenTime.Day() fw.hourlyOpenDate = fw.hourlyOpenTime.Day()
} }
fw.WriteMsg(time.Now(), "this is a msg for test", LevelDebug) fw.WriteMsg(time.Now(), "this is a msg for test", LevelDebug)
for _, file := range []string{fn1, fn2} { for _, file := range []string{fn1, fn2} {
_, err := os.Stat(file) _, err := os.Stat(file)
@ -328,8 +328,8 @@ func testFileDailyRotate(t *testing.T, fn1, fn2 string) {
func testFileHourlyRotate(t *testing.T, fn1, fn2 string) { func testFileHourlyRotate(t *testing.T, fn1, fn2 string) {
fw := &fileLogWriter{ fw := &fileLogWriter{
Hourly: true, Hourly: true,
MaxHours: 168, MaxHours: 168,
Rotate: true, Rotate: true,
Level: LevelTrace, Level: LevelTrace,
Perm: "0660", Perm: "0660",

View File

@ -57,15 +57,15 @@ func registerBuildInfo() {
Subsystem: "build_info", Subsystem: "build_info",
Help: "The building information", Help: "The building information",
ConstLabels: map[string]string{ ConstLabels: map[string]string{
"appname": beego.BConfig.AppName, "appname": beego.BConfig.AppName,
"build_version": beego.BuildVersion, "build_version": beego.BuildVersion,
"build_revision": beego.BuildGitRevision, "build_revision": beego.BuildGitRevision,
"build_status": beego.BuildStatus, "build_status": beego.BuildStatus,
"build_tag": beego.BuildTag, "build_tag": beego.BuildTag,
"build_time": strings.Replace(beego.BuildTime, "--", " ", 1), "build_time": strings.Replace(beego.BuildTime, "--", " ", 1),
"go_version": beego.GoVersion, "go_version": beego.GoVersion,
"git_branch": beego.GitBranch, "git_branch": beego.GitBranch,
"start_time": time.Now().Format("2006-01-02 15:04:05"), "start_time": time.Now().Format("2006-01-02 15:04:05"),
}, },
}, []string{}) }, []string{})

View File

@ -197,9 +197,9 @@ func getDbCreateSQL(al *alias) (sqls []string, tableIndexes map[string][]dbIndex
if strings.Contains(column, "%COL%") { if strings.Contains(column, "%COL%") {
column = strings.Replace(column, "%COL%", fi.column, -1) column = strings.Replace(column, "%COL%", fi.column, -1)
} }
if fi.description != "" && al.Driver!=DRSqlite { if fi.description != "" && al.Driver != DRSqlite {
column += " " + fmt.Sprintf("COMMENT '%s'",fi.description) column += " " + fmt.Sprintf("COMMENT '%s'", fi.description)
} }
columns = append(columns, column) columns = append(columns, column)

View File

@ -18,10 +18,11 @@ import (
"context" "context"
"database/sql" "database/sql"
"fmt" "fmt"
lru "github.com/hashicorp/golang-lru"
"reflect" "reflect"
"sync" "sync"
"time" "time"
lru "github.com/hashicorp/golang-lru"
) )
// DriverType database driver constant int. // DriverType database driver constant int.
@ -424,7 +425,7 @@ func GetDB(aliasNames ...string) (*sql.DB, error) {
} }
type stmtDecorator struct { type stmtDecorator struct {
wg sync.WaitGroup wg sync.WaitGroup
stmt *sql.Stmt stmt *sql.Stmt
} }
@ -459,7 +460,9 @@ func newStmtDecorator(sqlStmt *sql.Stmt) *stmtDecorator {
} }
func newStmtDecoratorLruWithEvict() *lru.Cache { func newStmtDecoratorLruWithEvict() *lru.Cache {
cache, _ := lru.NewWithEvict(1000, func(key interface{}, value interface{}) { // temporarily solution
// we fixed this problem in v2.x
cache, _ := lru.NewWithEvict(50, func(key interface{}, value interface{}) {
value.(*stmtDecorator).destroy() value.(*stmtDecorator).destroy()
}) })
return cache return cache

View File

@ -61,7 +61,7 @@ func debugLogQueies(alias *alias, operaton, query string, t time.Time, err error
con += " - " + err.Error() con += " - " + err.Error()
} }
logMap["sql"] = fmt.Sprintf("%s-`%s`", query, strings.Join(cons, "`, `")) logMap["sql"] = fmt.Sprintf("%s-`%s`", query, strings.Join(cons, "`, `"))
if LogFunc != nil{ if LogFunc != nil {
LogFunc(logMap) LogFunc(logMap)
} }
DebugLog.Println(con) DebugLog.Println(con)

View File

@ -40,10 +40,11 @@
package authz package authz
import ( import (
"net/http"
"github.com/astaxie/beego" "github.com/astaxie/beego"
"github.com/astaxie/beego/context" "github.com/astaxie/beego/context"
"github.com/casbin/casbin" "github.com/casbin/casbin"
"net/http"
) )
// NewAuthorizer returns the authorizer. // NewAuthorizer returns the authorizer.

View File

@ -15,13 +15,14 @@
package authz package authz
import ( import (
"net/http"
"net/http/httptest"
"testing"
"github.com/astaxie/beego" "github.com/astaxie/beego"
"github.com/astaxie/beego/context" "github.com/astaxie/beego/context"
"github.com/astaxie/beego/plugins/auth" "github.com/astaxie/beego/plugins/auth"
"github.com/casbin/casbin" "github.com/casbin/casbin"
"net/http"
"net/http/httptest"
"testing"
) )
func testRequest(t *testing.T, handler *beego.ControllerRegister, user string, path string, method string, code int) { func testRequest(t *testing.T, handler *beego.ControllerRegister, user string, path string, method string, code int) {

View File

@ -31,14 +31,16 @@
// //
// more docs: http://beego.me/docs/module/session.md // more docs: http://beego.me/docs/module/session.md
package redis_cluster package redis_cluster
import ( import (
"net/http" "net/http"
"strconv" "strconv"
"strings" "strings"
"sync" "sync"
"time"
"github.com/astaxie/beego/session" "github.com/astaxie/beego/session"
rediss "github.com/go-redis/redis" rediss "github.com/go-redis/redis"
"time"
) )
var redispder = &Provider{} var redispder = &Provider{}
@ -101,7 +103,7 @@ func (rs *SessionStore) SessionRelease(w http.ResponseWriter) {
return return
} }
c := rs.p c := rs.p
c.Set(rs.sid, string(b), time.Duration(rs.maxlifetime) * time.Second) c.Set(rs.sid, string(b), time.Duration(rs.maxlifetime)*time.Second)
} }
// Provider redis_cluster session provider // Provider redis_cluster session provider
@ -146,10 +148,10 @@ func (rp *Provider) SessionInit(maxlifetime int64, savePath string) error {
} else { } else {
rp.dbNum = 0 rp.dbNum = 0
} }
rp.poollist = rediss.NewClusterClient(&rediss.ClusterOptions{ rp.poollist = rediss.NewClusterClient(&rediss.ClusterOptions{
Addrs: strings.Split(rp.savePath, ";"), Addrs: strings.Split(rp.savePath, ";"),
Password: rp.password, Password: rp.password,
PoolSize: rp.poolsize, PoolSize: rp.poolsize,
}) })
return rp.poollist.Ping().Err() return rp.poollist.Ping().Err()
@ -186,15 +188,15 @@ func (rp *Provider) SessionExist(sid string) bool {
// SessionRegenerate generate new sid for redis_cluster session // SessionRegenerate generate new sid for redis_cluster session
func (rp *Provider) SessionRegenerate(oldsid, sid string) (session.Store, error) { func (rp *Provider) SessionRegenerate(oldsid, sid string) (session.Store, error) {
c := rp.poollist c := rp.poollist
if existed, err := c.Exists(oldsid).Result(); err != nil || existed == 0 { if existed, err := c.Exists(oldsid).Result(); err != nil || existed == 0 {
// oldsid doesn't exists, set the new sid directly // oldsid doesn't exists, set the new sid directly
// ignore error here, since if it return error // ignore error here, since if it return error
// the existed value will be 0 // the existed value will be 0
c.Set(sid, "", time.Duration(rp.maxlifetime) * time.Second) c.Set(sid, "", time.Duration(rp.maxlifetime)*time.Second)
} else { } else {
c.Rename(oldsid, sid) c.Rename(oldsid, sid)
c.Expire(sid, time.Duration(rp.maxlifetime) * time.Second) c.Expire(sid, time.Duration(rp.maxlifetime)*time.Second)
} }
return rp.SessionRead(sid) return rp.SessionRead(sid)
} }

View File

@ -33,13 +33,14 @@
package redis_sentinel package redis_sentinel
import ( import (
"github.com/astaxie/beego/session"
"github.com/go-redis/redis"
"net/http" "net/http"
"strconv" "strconv"
"strings" "strings"
"sync" "sync"
"time" "time"
"github.com/astaxie/beego/session"
"github.com/go-redis/redis"
) )
var redispder = &Provider{} var redispder = &Provider{}

View File

@ -369,8 +369,7 @@ func TestFileSessionStore_SessionRelease(t *testing.T) {
t.Error(err) t.Error(err)
} }
s.Set(i, i)
s.Set(i,i)
s.SessionRelease(nil) s.SessionRelease(nil)
} }
@ -384,4 +383,4 @@ func TestFileSessionStore_SessionRelease(t *testing.T) {
t.Error() t.Error()
} }
} }
} }

View File

@ -202,7 +202,7 @@ func searchFile(ctx *context.Context) (string, os.FileInfo, error) {
if !strings.Contains(requestPath, prefix) { if !strings.Contains(requestPath, prefix) {
continue continue
} }
if prefix != "/" && len(requestPath) > len(prefix) && requestPath[len(prefix)] != '/' { if prefix != "/" && len(requestPath) > len(prefix) && requestPath[len(prefix)] != '/' {
continue continue
} }
filePath := path.Join(staticDir, requestPath[len(prefix):]) filePath := path.Join(staticDir, requestPath[len(prefix):])

View File

@ -16,12 +16,13 @@ package beego
import ( import (
"bytes" "bytes"
"github.com/astaxie/beego/testdata"
"github.com/elazarl/go-bindata-assetfs"
"net/http" "net/http"
"os" "os"
"path/filepath" "path/filepath"
"testing" "testing"
"github.com/astaxie/beego/testdata"
"github.com/elazarl/go-bindata-assetfs"
) )
var header = `{{define "header"}} var header = `{{define "header"}}

View File

@ -362,7 +362,7 @@ func parseFormToStruct(form url.Values, objT reflect.Type, objV reflect.Value) e
value = value[:25] value = value[:25]
t, err = time.ParseInLocation(time.RFC3339, value, time.Local) t, err = time.ParseInLocation(time.RFC3339, value, time.Local)
} else if strings.HasSuffix(strings.ToUpper(value), "Z") { } else if strings.HasSuffix(strings.ToUpper(value), "Z") {
t, err = time.ParseInLocation(time.RFC3339, value, time.Local) t, err = time.ParseInLocation(time.RFC3339, value, time.Local)
} else if len(value) >= 19 { } else if len(value) >= 19 {
if strings.Contains(value, "T") { if strings.Contains(value, "T") {
value = value[:19] value = value[:19]

3
testdata/bindata.go vendored
View File

@ -11,13 +11,14 @@ import (
"bytes" "bytes"
"compress/gzip" "compress/gzip"
"fmt" "fmt"
"github.com/elazarl/go-bindata-assetfs"
"io" "io"
"io/ioutil" "io/ioutil"
"os" "os"
"path/filepath" "path/filepath"
"strings" "strings"
"time" "time"
"github.com/elazarl/go-bindata-assetfs"
) )
func bindataRead(data []byte, name string) ([]byte, error) { func bindataRead(data []byte, name string) ([]byte, error) {

View File

@ -213,7 +213,7 @@ func parseFunc(vfunc, key string, label string) (v ValidFunc, err error) {
return return
} }
tParams, err := trim(name, key+"."+ name + "." + label, params) tParams, err := trim(name, key+"."+name+"."+label, params)
if err != nil { if err != nil {
return return
} }

View File

@ -16,13 +16,14 @@ package validation
import ( import (
"fmt" "fmt"
"github.com/astaxie/beego/logs"
"reflect" "reflect"
"regexp" "regexp"
"strings" "strings"
"sync" "sync"
"time" "time"
"unicode/utf8" "unicode/utf8"
"github.com/astaxie/beego/logs"
) )
// CanSkipFuncs will skip valid if RequiredFirst is true and the struct field's value is empty // CanSkipFuncs will skip valid if RequiredFirst is true and the struct field's value is empty