From a5e8344a0a8518a1b7442d52d6f90e6b4d81dc6f Mon Sep 17 00:00:00 2001 From: jianzhiyao Date: Mon, 22 Jun 2020 23:23:52 +0800 Subject: [PATCH 1/2] add stmt garbage recycle --- orm/db_alias.go | 92 +++++++++++++++++++++++++++++++++++++++---------- orm/orm.go | 6 ++-- 2 files changed, 76 insertions(+), 22 deletions(-) diff --git a/orm/db_alias.go b/orm/db_alias.go index 7d8ece4b..a69259ed 100644 --- a/orm/db_alias.go +++ b/orm/db_alias.go @@ -18,6 +18,7 @@ import ( "context" "database/sql" "fmt" + lru "github.com/hashicorp/golang-lru" "reflect" "sync" "time" @@ -106,8 +107,8 @@ func (ac *_dbCache) getDefault() (al *alias) { type DB struct { *sync.RWMutex - DB *sql.DB - stmts map[string]*sql.Stmt + DB *sql.DB + stmtDecorators *lru.Cache } func (d *DB) Begin() (*sql.Tx, error) { @@ -118,19 +119,22 @@ func (d *DB) BeginTx(ctx context.Context, opts *sql.TxOptions) (*sql.Tx, error) return d.DB.BeginTx(ctx, opts) } -func (d *DB) getStmt(query string) (*sql.Stmt, error) { +//su must call release to release *sql.Stmt after using +func (d *DB) getStmtDecorator(query string) (*stmtDecorator, error) { d.RLock() - c, ok := d.stmts[query] + c, ok := d.stmtDecorators.Get(query) d.RUnlock() if ok { - return c, nil + c.(*stmtDecorator).acquire() + return c.(*stmtDecorator), nil } d.Lock() - c, ok = d.stmts[query] + c, ok = d.stmtDecorators.Get(query) if ok { d.Unlock() - return c, nil + c.(*stmtDecorator).acquire() + return c.(*stmtDecorator), nil } stmt, err := d.Prepare(query) @@ -138,9 +142,12 @@ func (d *DB) getStmt(query string) (*sql.Stmt, error) { d.Unlock() return nil, err } - d.stmts[query] = stmt + sd := newStmtDecorator(stmt) + d.stmtDecorators.Add(query, sd) d.Unlock() - return stmt, nil + + sd.acquire() + return sd, nil } func (d *DB) Prepare(query string) (*sql.Stmt, error) { @@ -152,52 +159,63 @@ func (d *DB) PrepareContext(ctx context.Context, query string) (*sql.Stmt, error } func (d *DB) Exec(query string, args ...interface{}) (sql.Result, error) { - stmt, err := d.getStmt(query) + sd, err := d.getStmtDecorator(query) if err != nil { return nil, err } + stmt := sd.getStmt() + defer sd.release() return stmt.Exec(args...) } func (d *DB) ExecContext(ctx context.Context, query string, args ...interface{}) (sql.Result, error) { - stmt, err := d.getStmt(query) + sd, err := d.getStmtDecorator(query) if err != nil { return nil, err } + stmt := sd.getStmt() + defer sd.release() return stmt.ExecContext(ctx, args...) } func (d *DB) Query(query string, args ...interface{}) (*sql.Rows, error) { - stmt, err := d.getStmt(query) + sd, err := d.getStmtDecorator(query) if err != nil { return nil, err } + stmt := sd.getStmt() + defer sd.release() return stmt.Query(args...) } func (d *DB) QueryContext(ctx context.Context, query string, args ...interface{}) (*sql.Rows, error) { - stmt, err := d.getStmt(query) + sd, err := d.getStmtDecorator(query) if err != nil { return nil, err } + stmt := sd.getStmt() + defer sd.release() return stmt.QueryContext(ctx, args...) } func (d *DB) QueryRow(query string, args ...interface{}) *sql.Row { - stmt, err := d.getStmt(query) + sd, err := d.getStmtDecorator(query) if err != nil { panic(err) } + stmt := sd.getStmt() + defer sd.release() return stmt.QueryRow(args...) } func (d *DB) QueryRowContext(ctx context.Context, query string, args ...interface{}) *sql.Row { - - stmt, err := d.getStmt(query) + sd, err := d.getStmtDecorator(query) if err != nil { panic(err) } + stmt := sd.getStmt() + defer sd.release() return stmt.QueryRowContext(ctx, args) } @@ -275,9 +293,9 @@ func addAliasWthDB(aliasName, driverName string, db *sql.DB) (*alias, error) { al.Name = aliasName al.DriverName = driverName al.DB = &DB{ - RWMutex: new(sync.RWMutex), - DB: db, - stmts: make(map[string]*sql.Stmt), + RWMutex: new(sync.RWMutex), + DB: db, + stmtDecorators: newStmtDecoratorLruWithEvict(), } if dr, ok := drivers[driverName]; ok { @@ -402,3 +420,39 @@ func GetDB(aliasNames ...string) (*sql.DB, error) { } return nil, fmt.Errorf("DataBase of alias name `%s` not found", name) } + +type stmtDecorator struct { + wg sync.WaitGroup + stmt *sql.Stmt +} + +func (s *stmtDecorator) getStmt() *sql.Stmt { + return s.stmt +} + +func (s *stmtDecorator) acquire() { + s.wg.Add(1) +} + +func (s *stmtDecorator) release() { + s.wg.Done() +} + +//garbage recycle for stmt +func (s *stmtDecorator) destroy() { + s.wg.Wait() + _ = s.stmt.Close() +} + +func newStmtDecorator(sqlStmt *sql.Stmt) *stmtDecorator { + return &stmtDecorator{ + stmt: sqlStmt, + } +} + +func newStmtDecoratorLruWithEvict() *lru.Cache { + cache, _ := lru.NewWithEvict(1000, func(key interface{}, value interface{}) { + value.(*stmtDecorator).destroy() + }) + return cache +} diff --git a/orm/orm.go b/orm/orm.go index 11e38fd9..0551b1cd 100644 --- a/orm/orm.go +++ b/orm/orm.go @@ -559,9 +559,9 @@ func NewOrmWithDB(driverName, aliasName string, db *sql.DB) (Ormer, error) { al.Name = aliasName al.DriverName = driverName al.DB = &DB{ - RWMutex: new(sync.RWMutex), - DB: db, - stmts: make(map[string]*sql.Stmt), + RWMutex: new(sync.RWMutex), + DB: db, + stmtDecorators: newStmtDecoratorLruWithEvict(), } detectTZ(al) From a28d294a83699d6fc8cfb5ca019ac33b0f975313 Mon Sep 17 00:00:00 2001 From: jianzhiyao Date: Tue, 23 Jun 2020 13:46:19 +0800 Subject: [PATCH 2/2] upgrade acquire method return *sql.Stmt --- orm/db_alias.go | 25 +++++++++++-------------- 1 file changed, 11 insertions(+), 14 deletions(-) diff --git a/orm/db_alias.go b/orm/db_alias.go index a69259ed..a38f1d60 100644 --- a/orm/db_alias.go +++ b/orm/db_alias.go @@ -125,7 +125,6 @@ func (d *DB) getStmtDecorator(query string) (*stmtDecorator, error) { c, ok := d.stmtDecorators.Get(query) d.RUnlock() if ok { - c.(*stmtDecorator).acquire() return c.(*stmtDecorator), nil } @@ -133,7 +132,6 @@ func (d *DB) getStmtDecorator(query string) (*stmtDecorator, error) { c, ok = d.stmtDecorators.Get(query) if ok { d.Unlock() - c.(*stmtDecorator).acquire() return c.(*stmtDecorator), nil } @@ -146,7 +144,6 @@ func (d *DB) getStmtDecorator(query string) (*stmtDecorator, error) { d.stmtDecorators.Add(query, sd) d.Unlock() - sd.acquire() return sd, nil } @@ -163,7 +160,7 @@ func (d *DB) Exec(query string, args ...interface{}) (sql.Result, error) { if err != nil { return nil, err } - stmt := sd.getStmt() + stmt := sd.acquire() defer sd.release() return stmt.Exec(args...) } @@ -173,7 +170,7 @@ func (d *DB) ExecContext(ctx context.Context, query string, args ...interface{}) if err != nil { return nil, err } - stmt := sd.getStmt() + stmt := sd.acquire() defer sd.release() return stmt.ExecContext(ctx, args...) } @@ -183,7 +180,7 @@ func (d *DB) Query(query string, args ...interface{}) (*sql.Rows, error) { if err != nil { return nil, err } - stmt := sd.getStmt() + stmt := sd.acquire() defer sd.release() return stmt.Query(args...) } @@ -193,7 +190,7 @@ func (d *DB) QueryContext(ctx context.Context, query string, args ...interface{} if err != nil { return nil, err } - stmt := sd.getStmt() + stmt := sd.acquire() defer sd.release() return stmt.QueryContext(ctx, args...) } @@ -203,7 +200,7 @@ func (d *DB) QueryRow(query string, args ...interface{}) *sql.Row { if err != nil { panic(err) } - stmt := sd.getStmt() + stmt := sd.acquire() defer sd.release() return stmt.QueryRow(args...) @@ -214,7 +211,7 @@ func (d *DB) QueryRowContext(ctx context.Context, query string, args ...interfac if err != nil { panic(err) } - stmt := sd.getStmt() + stmt := sd.acquire() defer sd.release() return stmt.QueryRowContext(ctx, args) } @@ -423,15 +420,14 @@ func GetDB(aliasNames ...string) (*sql.DB, error) { type stmtDecorator struct { wg sync.WaitGroup + lastUse int64 stmt *sql.Stmt } -func (s *stmtDecorator) getStmt() *sql.Stmt { - return s.stmt -} - -func (s *stmtDecorator) acquire() { +func (s *stmtDecorator) acquire() *sql.Stmt{ s.wg.Add(1) + s.lastUse = time.Now().Unix() + return s.stmt } func (s *stmtDecorator) release() { @@ -447,6 +443,7 @@ func (s *stmtDecorator) destroy() { func newStmtDecorator(sqlStmt *sql.Stmt) *stmtDecorator { return &stmtDecorator{ stmt: sqlStmt, + lastUse: time.Now().Unix(), } }