Merge branch 'develop' into feature/YAML

.travis.yml

@@ -2,7 +2,7 @@ language: go
 
 go:
   - "1.9.2"
-  - "1.10.2"
+  - "1.10.3"
 services:
   - redis-server
   - mysql
@@ -39,6 +39,7 @@ install:
   - go get -u github.com/mdempsky/unconvert
   - go get -u github.com/gordonklaus/ineffassign
   - go get -u github.com/golang/lint/golint
+  - go get -u github.com/go-redis/redis
 before_script:
   - psql --version
   - sh -c "if [ '$ORM_DRIVER' = 'postgres' ]; then psql -c 'create database orm_test;' -U postgres; fi"

admin.go

@@ -76,6 +76,18 @@ func adminIndex(rw http.ResponseWriter, r *http.Request) {
 func qpsIndex(rw http.ResponseWriter, r *http.Request) {
 	data := make(map[interface{}]interface{})
 	data["Content"] = toolbox.StatisticsMap.GetMap()
+
+	// do html escape before display path, avoid xss
+	if content, ok := (data["Content"]).(map[string]interface{}); ok {
+		if resultLists, ok := (content["Data"]).([][]string); ok {
+			for i := range resultLists {
+				if len(resultLists[i]) > 0 {
+					resultLists[i][0] = template.HTMLEscapeString(resultLists[i][0])
+				}
+			}
+		}
+	}
+
 	execTpl(rw, data, qpsTpl, defaultScriptsTpl)
 }
 

admin_test.go

@@ -67,6 +67,7 @@ func oldMap() map[string]interface{} {
 	m["BConfig.WebConfig.Session.SessionDomain"] = BConfig.WebConfig.Session.SessionDomain
 	m["BConfig.WebConfig.Session.SessionDisableHTTPOnly"] = BConfig.WebConfig.Session.SessionDisableHTTPOnly
 	m["BConfig.Log.AccessLogs"] = BConfig.Log.AccessLogs
+	m["BConfig.Log.EnableStaticLogs"] = BConfig.Log.EnableStaticLogs
 	m["BConfig.Log.AccessLogsFormat"] = BConfig.Log.AccessLogsFormat
 	m["BConfig.Log.FileLineNum"] = BConfig.Log.FileLineNum
 	m["BConfig.Log.Outputs"] = BConfig.Log.Outputs

app.go

@@ -24,8 +24,8 @@ import (
 	"net/http/fcgi"
 	"os"
 	"path"
-	"time"
 	"strings"
+	"time"
 
 	"github.com/astaxie/beego/grace"
 	"github.com/astaxie/beego/logs"
@@ -101,7 +101,7 @@ func (app *App) Run(mws ...MiddleWare) {
 	}
 
 	app.Server.Handler = app.Handlers
-	for i:=len(mws)-1;i>=0;i-- {
+	for i := len(mws) - 1; i >= 0; i-- {
 		if mws[i] == nil {
 			continue
 		}
@@ -117,7 +117,7 @@ func (app *App) Run(mws ...MiddleWare) {
 		app.Server.Addr = httpsAddr
 		if BConfig.Listen.EnableHTTPS || BConfig.Listen.EnableMutualHTTPS {
 			go func() {
-				time.Sleep(20 * time.Microsecond)
+				time.Sleep(1000 * time.Microsecond)
 				if BConfig.Listen.HTTPSPort != 0 {
 					httpsAddr = fmt.Sprintf("%s:%d", BConfig.Listen.HTTPSAddr, BConfig.Listen.HTTPSPort)
 					app.Server.Addr = httpsAddr
@@ -163,7 +163,7 @@ func (app *App) Run(mws ...MiddleWare) {
 	// run normal mode
 	if BConfig.Listen.EnableHTTPS || BConfig.Listen.EnableMutualHTTPS {
 		go func() {
-			time.Sleep(20 * time.Microsecond)
+			time.Sleep(1000 * time.Microsecond)
 			if BConfig.Listen.HTTPSPort != 0 {
 				app.Server.Addr = fmt.Sprintf("%s:%d", BConfig.Listen.HTTPSAddr, BConfig.Listen.HTTPSPort)
 			} else if BConfig.Listen.EnableHTTP {

config.go

@@ -106,6 +106,7 @@ type SessionConfig struct {
 // LogConfig holds Log related config
 type LogConfig struct {
 	AccessLogs       bool
+	EnableStaticLogs bool   //log static files requests default: false
 	AccessLogsFormat string //access log format: JSON_FORMAT, APACHE_FORMAT or empty string
 	FileLineNum      bool
 	Outputs          map[string]string // Store Adaptor : config
@@ -182,6 +183,11 @@ func recoverPanic(ctx *context.Context) {
 		if BConfig.RunMode == DEV && BConfig.EnableErrorsRender {
 			showErr(err, ctx, stack)
 		}
+		if ctx.Output.Status != 0 {
+			ctx.ResponseWriter.WriteHeader(ctx.Output.Status)
+		} else {
+			ctx.ResponseWriter.WriteHeader(500)
+		}
 	}
 }
 
@@ -247,6 +253,7 @@ func newBConfig() *Config {
 		},
 		Log: LogConfig{
 			AccessLogs:       false,
+			EnableStaticLogs: false,
 			AccessLogsFormat: "APACHE_FORMAT",
 			FileLineNum:      true,
 			Outputs:          map[string]string{"console": ""},

config_test.go

@@ -75,7 +75,7 @@ func TestAssignConfig_02(t *testing.T) {
 
 	jcf := &config.JSONConfig{}
 	bs, _ = json.Marshal(configMap)
-	ac, _ := jcf.ParseData([]byte(bs))
+	ac, _ := jcf.ParseData(bs)
 
 	for _, i := range []interface{}{_BConfig, &_BConfig.Listen, &_BConfig.WebConfig, &_BConfig.Log, &_BConfig.WebConfig.Session} {
 		assignSingleConfig(i, ac)

context/output.go

@@ -275,7 +275,7 @@ func (output *BeegoOutput) Download(file string, filename ...string) {
 	} else {
 		fName = filepath.Base(file)
 	}
-	output.Header("Content-Disposition", "attachment; filename="+url.QueryEscape(fName))
+	output.Header("Content-Disposition", "attachment; filename="+url.PathEscape(fName))
 	output.Header("Content-Description", "File Transfer")
 	output.Header("Content-Type", "application/octet-stream")
 	output.Header("Content-Transfer-Encoding", "binary")
@@ -365,6 +365,11 @@ func stringsToJSON(str string) string {
 			jsons.WriteRune(r)
 		} else {
 			jsons.WriteString("\\u")
+			if rint < 0x100 {
+				jsons.WriteString("00")
+			} else if rint < 0x1000 {
+				jsons.WriteString("0")
+			}
 			jsons.WriteString(strconv.FormatInt(int64(rint), 16))
 		}
 	}

controller.go

@@ -273,7 +273,22 @@ func (c *Controller) viewPath() string {
 
 // Redirect sends the redirection response to url with status code.
 func (c *Controller) Redirect(url string, code int) {
+	logAccess(c.Ctx, nil, code)
 	c.Ctx.Redirect(code, url)
+	panic(ErrAbort)
+}
+
+// Set the data depending on the accepted
+func (c *Controller) SetData(data interface{}) {
+	accept := c.Ctx.Input.Header("Accept")
+	switch accept {
+	case applicationJSON:
+		c.Data["json"] = data
+	case applicationXML, textXML:
+		c.Data["xml"] = data
+	default:
+		c.Data["json"] = data
+	}
 }
 
 // Abort stops controller handler and show the error data if code is defined in ErrorMap or code string.

error.go

@@ -93,11 +93,6 @@ func showErr(err interface{}, ctx *context.Context, stack string) {
 		"BeegoVersion":  VERSION,
 		"GoVersion":     runtime.Version(),
 	}
-	if ctx.Output.Status != 0 {
-		ctx.ResponseWriter.WriteHeader(ctx.Output.Status)
-	} else {
-		ctx.ResponseWriter.WriteHeader(500)
-	}
 	t.Execute(ctx.ResponseWriter, data)
 }
 
@@ -439,6 +434,9 @@ func exception(errCode string, ctx *context.Context) {
 }
 
 func executeError(err *errorInfo, ctx *context.Context, code int) {
+	//make sure to log the error in the access log
+	logAccess(ctx, nil, code)
+
 	if err.errorType == errorTypeHandler {
 		ctx.ResponseWriter.WriteHeader(code)
 		err.handler(ctx.ResponseWriter, ctx.Request)

httplib/httplib.go

@@ -318,6 +318,7 @@ func (b *BeegoHTTPRequest) Body(data interface{}) *BeegoHTTPRequest {
 	}
 	return b
 }
+
 // XMLBody adds request raw body encoding by XML.
 func (b *BeegoHTTPRequest) XMLBody(obj interface{}) (*BeegoHTTPRequest, error) {
 	if b.req.Body == nil && obj != nil {
@@ -331,6 +332,7 @@ func (b *BeegoHTTPRequest) XMLBody(obj interface{}) (*BeegoHTTPRequest, error) {
 	}
 	return b, nil
 }
+
 // YAMLBody adds request raw body encoding by YAML.
 func (b *BeegoHTTPRequest) YAMLBody(obj interface{}) (*BeegoHTTPRequest, error) {
 	if b.req.Body == nil && obj != nil {
@@ -344,6 +346,7 @@ func (b *BeegoHTTPRequest) YAMLBody(obj interface{}) (*BeegoHTTPRequest, error)
 	}
 	return b, nil
 }
+
 // JSONBody adds request raw body encoding by JSON.
 func (b *BeegoHTTPRequest) JSONBody(obj interface{}) (*BeegoHTTPRequest, error) {
 	if b.req.Body == nil && obj != nil {
@@ -458,7 +461,7 @@ func (b *BeegoHTTPRequest) DoRequest() (resp *http.Response, err error) {
 			TLSClientConfig:     b.setting.TLSClientConfig,
 			Proxy:               b.setting.Proxy,
 			Dial:                TimeoutDialer(b.setting.ConnectTimeout, b.setting.ReadWriteTimeout),
-			MaxIdleConnsPerHost: -1,
+			MaxIdleConnsPerHost: 100,
 		}
 	} else {
 		// if b.transport is *http.Transport then set the settings.

logs/README.md

@@ -16,48 +16,57 @@ As of now this logs support console, file,smtp and conn.
 
 First you must import it
 
-	import (
+```golang
+import (
 	"github.com/astaxie/beego/logs"
-	)
+)
+```
 
 Then init a Log (example with console adapter)
 
-	log := NewLogger(10000)
+```golang
-	log.SetLogger("console", "")	
+log := logs.NewLogger(10000)
+log.SetLogger("console", "")
+```
 
 > the first params stand for how many channel
 
 Use it like this:
 
-	log.Trace("trace")
+```golang
-	log.Info("info")
+log.Trace("trace")
-	log.Warn("warning")
+log.Info("info")
-	log.Debug("debug")
+log.Warn("warning")
-	log.Critical("critical")
+log.Debug("debug")
-
+log.Critical("critical")
+```
 
 ## File adapter
 
 Configure file adapter like this:
 
-	log := NewLogger(10000)
+```golang
-	log.SetLogger("file", `{"filename":"test.log"}`)
+log := NewLogger(10000)
-
+log.SetLogger("file", `{"filename":"test.log"}`)
+```
 
 ## Conn adapter
 
 Configure like this:
 
-	log := NewLogger(1000)
+```golang
-	log.SetLogger("conn", `{"net":"tcp","addr":":7020"}`)
+log := NewLogger(1000)
-	log.Info("info")
+log.SetLogger("conn", `{"net":"tcp","addr":":7020"}`)
-
+log.Info("info")
+```
 
 ## Smtp adapter
 
 Configure like this:
 
-	log := NewLogger(10000)
+```golang
-	log.SetLogger("smtp", `{"username":"beegotest@gmail.com","password":"xxxxxxxx","host":"smtp.gmail.com:587","sendTos":["xiemengjun@gmail.com"]}`)
+log := NewLogger(10000)
-	log.Critical("sendmail critical")
+log.SetLogger("smtp", `{"username":"beegotest@gmail.com","password":"xxxxxxxx","host":"smtp.gmail.com:587","sendTos":["xiemengjun@gmail.com"]}`)
-	time.Sleep(time.Second * 30)
+log.Critical("sendmail critical")
+time.Sleep(time.Second * 30)
+```

logs/accesslog.go

@@ -16,9 +16,10 @@ package logs
 
 import (
 	"bytes"
+	"strings"
 	"encoding/json"
-	"time"
 	"fmt"
+	"time"
 )
 
 const (
@@ -53,10 +54,9 @@ func (r *AccessLogRecord) json() ([]byte, error) {
 }
 
 func disableEscapeHTML(i interface{}) {
-	e, ok := i.(interface {
+	if e, ok := i.(interface {
 		SetEscapeHTML(bool)
-	});
+	}); ok {
-	if ok {
 		e.SetEscapeHTML(false)
 	}
 }
@@ -64,9 +64,7 @@ func disableEscapeHTML(i interface{}) {
 // AccessLog - Format and print access log.
 func AccessLog(r *AccessLogRecord, format string) {
 	var msg string
-
 	switch format {
-
 	case apacheFormat:
 		timeFormatted := r.RequestTime.Format("02/Jan/2006 03:04:05")
 		msg = fmt.Sprintf(apacheFormatPattern, r.RemoteAddr, timeFormatted, r.Request, r.Status, r.BodyBytesSent,
@@ -81,6 +79,5 @@ func AccessLog(r *AccessLogRecord, format string) {
 			msg = string(jsonData)
 		}
 	}
-
+	beeLogger.writeMsg(levelLoggerImpl, strings.TrimSpace(msg))
-	beeLogger.Debug(msg)
 }

logs/file.go

@@ -21,6 +21,7 @@ import (
 	"fmt"
 	"io"
 	"os"
+	"path"
 	"path/filepath"
 	"strconv"
 	"strings"
@@ -40,6 +41,9 @@ type fileLogWriter struct {
 	MaxLines         int `json:"maxlines"`
 	maxLinesCurLines int
 
+	MaxFiles         int `json:"maxfiles"`
+	MaxFilesCurFiles int
+
 	// Rotate at size
 	MaxSize        int `json:"maxsize"`
 	maxSizeCurSize int
@@ -70,6 +74,9 @@ func newFileWriter() Logger {
 		RotatePerm: "0440",
 		Level:      LevelTrace,
 		Perm:       "0660",
+		MaxLines:   10000000,
+		MaxFiles:   999,
+		MaxSize:    1 << 28,
 	}
 	return w
 }
@@ -161,6 +168,10 @@ func (w *fileLogWriter) createLogFile() (*os.File, error) {
 	if err != nil {
 		return nil, err
 	}
+
+	filepath := path.Dir(w.Filename)
+	os.MkdirAll(filepath, os.FileMode(perm))
+
 	fd, err := os.OpenFile(w.Filename, os.O_WRONLY|os.O_APPEND|os.O_CREATE, os.FileMode(perm))
 	if err == nil {
 		// Make sure file perm is user set perm cause of `os.OpenFile` will obey umask
@@ -238,7 +249,7 @@ func (w *fileLogWriter) lines() (int, error) {
 func (w *fileLogWriter) doRotate(logTime time.Time) error {
 	// file exists
 	// Find the next available number
-	num := 1
+	num := w.MaxFilesCurFiles + 1
 	fName := ""
 	rotatePerm, err := strconv.ParseInt(w.RotatePerm, 8, 64)
 	if err != nil {
@@ -251,18 +262,16 @@ func (w *fileLogWriter) doRotate(logTime time.Time) error {
 		goto RESTART_LOGGER
 	}
 
+	// only when one of them be setted, then the file would be splited
 	if w.MaxLines > 0 || w.MaxSize > 0 {
-		for ; err == nil && num <= 999; num++ {
+		for ; err == nil && num <= w.MaxFiles; num++ {
 			fName = w.fileNameOnly + fmt.Sprintf(".%s.%03d%s", logTime.Format("2006-01-02"), num, w.suffix)
 			_, err = os.Lstat(fName)
 		}
 	} else {
-		fName = fmt.Sprintf("%s.%s%s", w.fileNameOnly, w.dailyOpenTime.Format("2006-01-02"), w.suffix)
-		_, err = os.Lstat(fName)
-		for ; err == nil && num <= 999; num++ {
 		fName = w.fileNameOnly + fmt.Sprintf(".%s.%03d%s", w.dailyOpenTime.Format("2006-01-02"), num, w.suffix)
 		_, err = os.Lstat(fName)
-		}
+		w.MaxFilesCurFiles = num
 	}
 	// return error if the last file checked still existed
 	if err == nil {
@@ -308,7 +317,7 @@ func (w *fileLogWriter) deleteOldLog() {
 			return
 		}
 
-		if !info.IsDir() && info.ModTime().Add(24*time.Hour*time.Duration(w.MaxDays)).Before(time.Now()) {
+		if !info.IsDir() && info.ModTime().Add(24 * time.Hour * time.Duration(w.MaxDays)).Before(time.Now()) {
 			if strings.HasPrefix(filepath.Base(path), filepath.Base(w.fileNameOnly)) &&
 				strings.HasSuffix(filepath.Base(path), w.suffix) {
 				os.Remove(path)

logs/file_test.go

@@ -135,7 +135,7 @@ func TestFileRotate_01(t *testing.T) {
 
 func TestFileRotate_02(t *testing.T) {
 	fn1 := "rotate_day.log"
-	fn2 := "rotate_day." + time.Now().Add(-24*time.Hour).Format("2006-01-02") + ".log"
+	fn2 := "rotate_day." + time.Now().Add(-24*time.Hour).Format("2006-01-02") + ".001.log"
 	testFileRotate(t, fn1, fn2)
 }
 
@@ -150,7 +150,7 @@ func TestFileRotate_03(t *testing.T) {
 
 func TestFileRotate_04(t *testing.T) {
 	fn1 := "rotate_day.log"
-	fn2 := "rotate_day." + time.Now().Add(-24*time.Hour).Format("2006-01-02") + ".log"
+	fn2 := "rotate_day." + time.Now().Add(-24*time.Hour).Format("2006-01-02") + ".001.log"
 	testFileDailyRotate(t, fn1, fn2)
 }
 
@@ -200,6 +200,7 @@ func testFileRotate(t *testing.T, fn1, fn2 string) {
 	for _, file := range []string{fn1, fn2} {
 		_, err := os.Stat(file)
 		if err != nil {
+			t.Log(err)
 			t.FailNow()
 		}
 		os.Remove(file)

logs/logger.go

@@ -93,7 +93,7 @@ const (
 func formatTimeHeader(when time.Time) ([]byte, int) {
 	y, mo, d := when.Date()
 	h, mi, s := when.Clock()
-	ns := when.Nanosecond()/1000000
+	ns := when.Nanosecond() / 1000000
 	//len("2006/01/02 15:04:05.123 ")==24
 	var buf [24]byte
 

orm/cmd_utils.go

@@ -198,6 +198,10 @@ func getDbCreateSQL(al *alias) (sqls []string, tableIndexes map[string][]dbIndex
 				column = strings.Replace(column, "%COL%", fi.column, -1)
 			}
 			
+			if fi.description != "" {
+				column += " " + fmt.Sprintf("COMMENT '%s'",fi.description)
+			}
+
 			columns = append(columns, column)
 		}
 

orm/db.go

@@ -969,6 +969,10 @@ func (d *dbBase) ReadBatch(q dbQuerier, qs *querySet, mi *modelInfo, cond *Condi
 	}
 	query := fmt.Sprintf("%s %s FROM %s%s%s T0 %s%s%s%s%s", sqlSelect, sels, Q, mi.table, Q, join, where, groupBy, orderBy, limit)
 
+	if qs.forupdate {
+		query += " FOR UPDATE"
+	}
+
 	d.ins.ReplaceMarks(&query)
 
 	var rs *sql.Rows

orm/models_fields.go

@@ -86,7 +86,7 @@ func (e *BooleanField) SetRaw(value interface{}) error {
 		e.Set(d)
 	case string:
 		v, err := StrTo(d).Bool()
-		if err != nil {
+		if err == nil {
 			e.Set(v)
 		}
 		return err
@@ -191,7 +191,7 @@ func (e *TimeField) SetRaw(value interface{}) error {
 		e.Set(d)
 	case string:
 		v, err := timeParse(d, formatTime)
-		if err != nil {
+		if err == nil {
 			e.Set(v)
 		}
 		return err
@@ -250,7 +250,7 @@ func (e *DateField) SetRaw(value interface{}) error {
 		e.Set(d)
 	case string:
 		v, err := timeParse(d, formatDate)
-		if err != nil {
+		if err == nil {
 			e.Set(v)
 		}
 		return err
@@ -300,7 +300,7 @@ func (e *DateTimeField) SetRaw(value interface{}) error {
 		e.Set(d)
 	case string:
 		v, err := timeParse(d, formatDateTime)
-		if err != nil {
+		if err == nil {
 			e.Set(v)
 		}
 		return err
@@ -350,9 +350,10 @@ func (e *FloatField) SetRaw(value interface{}) error {
 		e.Set(d)
 	case string:
 		v, err := StrTo(d).Float64()
-		if err != nil {
+		if err == nil {
 			e.Set(v)
 		}
+		return err
 	default:
 		return fmt.Errorf("<FloatField.SetRaw> unknown value `%s`", value)
 	}
@@ -397,9 +398,10 @@ func (e *SmallIntegerField) SetRaw(value interface{}) error {
 		e.Set(d)
 	case string:
 		v, err := StrTo(d).Int16()
-		if err != nil {
+		if err == nil {
 			e.Set(v)
 		}
+		return err
 	default:
 		return fmt.Errorf("<SmallIntegerField.SetRaw> unknown value `%s`", value)
 	}
@@ -444,9 +446,10 @@ func (e *IntegerField) SetRaw(value interface{}) error {
 		e.Set(d)
 	case string:
 		v, err := StrTo(d).Int32()
-		if err != nil {
+		if err == nil {
 			e.Set(v)
 		}
+		return err
 	default:
 		return fmt.Errorf("<IntegerField.SetRaw> unknown value `%s`", value)
 	}
@@ -491,9 +494,10 @@ func (e *BigIntegerField) SetRaw(value interface{}) error {
 		e.Set(d)
 	case string:
 		v, err := StrTo(d).Int64()
-		if err != nil {
+		if err == nil {
 			e.Set(v)
 		}
+		return err
 	default:
 		return fmt.Errorf("<BigIntegerField.SetRaw> unknown value `%s`", value)
 	}
@@ -538,9 +542,10 @@ func (e *PositiveSmallIntegerField) SetRaw(value interface{}) error {
 		e.Set(d)
 	case string:
 		v, err := StrTo(d).Uint16()
-		if err != nil {
+		if err == nil {
 			e.Set(v)
 		}
+		return err
 	default:
 		return fmt.Errorf("<PositiveSmallIntegerField.SetRaw> unknown value `%s`", value)
 	}
@@ -585,9 +590,10 @@ func (e *PositiveIntegerField) SetRaw(value interface{}) error {
 		e.Set(d)
 	case string:
 		v, err := StrTo(d).Uint32()
-		if err != nil {
+		if err == nil {
 			e.Set(v)
 		}
+		return err
 	default:
 		return fmt.Errorf("<PositiveIntegerField.SetRaw> unknown value `%s`", value)
 	}
@@ -632,9 +638,10 @@ func (e *PositiveBigIntegerField) SetRaw(value interface{}) error {
 		e.Set(d)
 	case string:
 		v, err := StrTo(d).Uint64()
-		if err != nil {
+		if err == nil {
 			e.Set(v)
 		}
+		return err
 	default:
 		return fmt.Errorf("<PositiveBigIntegerField.SetRaw> unknown value `%s`", value)
 	}

orm/models_info_f.go

@@ -136,6 +136,7 @@ type fieldInfo struct {
 	decimals            int
 	isFielder           bool // implement Fielder interface
 	onDelete            string
+	description         string
 }
 
 // new field info
@@ -300,6 +301,7 @@ checkType:
 	fi.sf = sf
 	fi.fullName = mi.fullName + mName + "." + sf.Name
 
+	fi.description = sf.Tag.Get("description")
 	fi.null = attrs["null"]
 	fi.index = attrs["index"]
 	fi.auto = attrs["auto"]

orm/models_info_m.go

@@ -75,7 +75,8 @@ func addModelFields(mi *modelInfo, ind reflect.Value, mName string, index []int)
 			break
 		}
 		//record current field index
-		fi.fieldIndex = append(index, i)
+		fi.fieldIndex = append(fi.fieldIndex, index...)
+		fi.fieldIndex = append(fi.fieldIndex, i)
 		fi.mi = mi
 		fi.inModel = true
 		if !mi.fields.Add(fi) {

orm/models_test.go

@@ -433,53 +433,57 @@ var (
 	dDbBaser dbBaser
 )
 
-func init() {
+var (
-	Debug, _ = StrTo(DBARGS.Debug).Bool()
+	helpinfo = `need driver and source!
 
-	if DBARGS.Driver == "" || DBARGS.Source == "" {
+	Default DB Drivers.
-		fmt.Println(`need driver and source!
-
-Default DB Drivers.
 	
 	  driver: url
 	   mysql: https://github.com/go-sql-driver/mysql
 	 sqlite3: https://github.com/mattn/go-sqlite3
-postgres: https://github.com/lib/pq
+	postgres: https://github.com/lib/pq
-tidb: https://github.com/pingcap/tidb
+	tidb: https://github.com/pingcap/tidb
 	
-usage:
+	usage:
 	
-go get -u github.com/astaxie/beego/orm
+	go get -u github.com/astaxie/beego/orm
-go get -u github.com/go-sql-driver/mysql
+	go get -u github.com/go-sql-driver/mysql
-go get -u github.com/mattn/go-sqlite3
+	go get -u github.com/mattn/go-sqlite3
-go get -u github.com/lib/pq
+	go get -u github.com/lib/pq
-go get -u github.com/pingcap/tidb
+	go get -u github.com/pingcap/tidb
 	
-#### MySQL
+	#### MySQL
-mysql -u root -e 'create database orm_test;'
+	mysql -u root -e 'create database orm_test;'
-export ORM_DRIVER=mysql
+	export ORM_DRIVER=mysql
-export ORM_SOURCE="root:@/orm_test?charset=utf8"
+	export ORM_SOURCE="root:@/orm_test?charset=utf8"
-go test -v github.com/astaxie/beego/orm
+	go test -v github.com/astaxie/beego/orm
 	
 	
-#### Sqlite3
+	#### Sqlite3
-export ORM_DRIVER=sqlite3
+	export ORM_DRIVER=sqlite3
-export ORM_SOURCE='file:memory_test?mode=memory'
+	export ORM_SOURCE='file:memory_test?mode=memory'
-go test -v github.com/astaxie/beego/orm
+	go test -v github.com/astaxie/beego/orm
 	
 	
-#### PostgreSQL
+	#### PostgreSQL
-psql -c 'create database orm_test;' -U postgres
+	psql -c 'create database orm_test;' -U postgres
-export ORM_DRIVER=postgres
+	export ORM_DRIVER=postgres
-export ORM_SOURCE="user=postgres dbname=orm_test sslmode=disable"
+	export ORM_SOURCE="user=postgres dbname=orm_test sslmode=disable"
-go test -v github.com/astaxie/beego/orm
+	go test -v github.com/astaxie/beego/orm
 	
-#### TiDB
+	#### TiDB
-export ORM_DRIVER=tidb
+	export ORM_DRIVER=tidb
-export ORM_SOURCE='memory://test/test'
+	export ORM_SOURCE='memory://test/test'
-go test -v github.com/astaxie/beego/orm
+	go test -v github.com/astaxie/beego/orm
 	
-`)
+	`
+)
+
+func init() {
+	Debug, _ = StrTo(DBARGS.Debug).Bool()
+
+	if DBARGS.Driver == "" || DBARGS.Source == "" {
+		fmt.Println(helpinfo)
 		os.Exit(2)
 	}
 

orm/orm_queryset.go

@@ -64,6 +64,7 @@ type querySet struct {
 	groups    []string
 	orders    []string
 	distinct  bool
+	forupdate bool
 	orm       *orm
 }
 
@@ -127,6 +128,12 @@ func (o querySet) Distinct() QuerySeter {
 	return &o
 }
 
+// add FOR UPDATE to SELECT
+func (o querySet) ForUpdate() QuerySeter {
+	o.forupdate = true
+	return &o
+}
+
 // set relation model to query together.
 // it will query relation models and assign to parent model.
 func (o querySet) RelatedSel(params ...interface{}) QuerySeter {
@@ -191,7 +198,11 @@ func (o *querySet) PrepareInsert() (Inserter, error) {
 // query all data and map to containers.
 // cols means the columns when querying.
 func (o *querySet) All(container interface{}, cols ...string) (int64, error) {
-	return o.orm.alias.DbBaser.ReadBatch(o.orm.db, o, o.mi, o.cond, container, o.orm.alias.TZ, cols)
+	num, err := o.orm.alias.DbBaser.ReadBatch(o.orm.db, o, o.mi, o.cond, container, o.orm.alias.TZ, cols)
+	if num == 0 {
+		return 0, ErrNoRows
+	}
+	return num, err
 }
 
 // query one row data and map to containers.

orm/orm_test.go

@@ -1008,13 +1008,13 @@ func TestAll(t *testing.T) {
 
 	qs = dORM.QueryTable("user")
 	num, err = qs.Filter("user_name", "nothing").All(&users)
-	throwFailNow(t, err)
+	throwFailNow(t, AssertIs(err, ErrNoRows))
 	throwFailNow(t, AssertIs(num, 0))
 
 	var users3 []*User
 	qs = dORM.QueryTable("user")
 	num, err = qs.Filter("user_name", "nothing").All(&users3)
-	throwFailNow(t, err)
+	throwFailNow(t, AssertIs(err, ErrNoRows))
 	throwFailNow(t, AssertIs(num, 0))
 	throwFailNow(t, AssertIs(users3 == nil, false))
 }

orm/types.go

@@ -190,6 +190,10 @@ type QuerySeter interface {
 	//    Distinct().
 	//    All(&permissions)
 	Distinct() QuerySeter
+	// set FOR UPDATE to query.
+	// for example:
+	//  o.QueryTable("user").Filter("uid", uid).ForUpdate().All(&users)
+	ForUpdate() QuerySeter
 	// return QuerySeter execution result number
 	// for example:
 	//	num, err = qs.Filter("profile__age__gt", 28).Count()

parser.go

@@ -114,10 +114,11 @@ type parsedParam struct {
 
 func parserComments(f *ast.FuncDecl, controllerName, pkgpath string) error {
 	if f.Doc != nil {
-		parsedComment, err := parseComment(f.Doc.List)
+		parsedComments, err := parseComment(f.Doc.List)
 		if err != nil {
 			return err
 		}
+		for _, parsedComment := range parsedComments {
 			if parsedComment.routerPath != "" {
 				key := pkgpath + ":" + controllerName
 				cc := ControllerComments{}
@@ -127,7 +128,7 @@ func parserComments(f *ast.FuncDecl, controllerName, pkgpath string) error {
 				cc.MethodParams = buildMethodParams(f.Type.Params.List, parsedComment)
 				genInfoList[key] = append(genInfoList[key], cc)
 			}
-
+		}
 	}
 	return nil
 }
@@ -177,26 +178,13 @@ func paramInPath(name, route string) bool {
 
 var routeRegex = regexp.MustCompile(`@router\s+(\S+)(?:\s+\[(\S+)\])?`)
 
-func parseComment(lines []*ast.Comment) (pc *parsedComment, err error) {
+func parseComment(lines []*ast.Comment) (pcs []*parsedComment, err error) {
-	pc = &parsedComment{}
+	pcs = []*parsedComment{}
+	params := map[string]parsedParam{}
+
 	for _, c := range lines {
 		t := strings.TrimSpace(strings.TrimLeft(c.Text, "//"))
-		if strings.HasPrefix(t, "@router") {
+		if strings.HasPrefix(t, "@Param") {
-			matches := routeRegex.FindStringSubmatch(t)
-			if len(matches) == 3 {
-				pc.routerPath = matches[1]
-				methods := matches[2]
-				if methods == "" {
-					pc.methods = []string{"get"}
-					//pc.hasGet = true
-				} else {
-					pc.methods = strings.Split(methods, ",")
-					//pc.hasGet = strings.Contains(methods, "get")
-				}
-			} else {
-				return nil, errors.New("Router information is missing")
-			}
-		} else if strings.HasPrefix(t, "@Param") {
 			pv := getparams(strings.TrimSpace(strings.TrimLeft(t, "@Param")))
 			if len(pv) < 4 {
 				logs.Error("Invalid @Param format. Needs at least 4 parameters")
@@ -217,10 +205,32 @@ func parseComment(lines []*ast.Comment) (pc *parsedComment, err error) {
 				p.defValue = pv[3]
 				p.required, _ = strconv.ParseBool(pv[4])
 			}
-			if pc.params == nil {
+			params[funcParamName] = p
-				pc.params = map[string]parsedParam{}
+		}
+	}
+
+	for _, c := range lines {
+		var pc = &parsedComment{}
+		pc.params = params
+
+		t := strings.TrimSpace(strings.TrimLeft(c.Text, "//"))
+		if strings.HasPrefix(t, "@router") {
+			t := strings.TrimSpace(strings.TrimLeft(c.Text, "//"))
+			matches := routeRegex.FindStringSubmatch(t)
+			if len(matches) == 3 {
+				pc.routerPath = matches[1]
+				methods := matches[2]
+				if methods == "" {
+					pc.methods = []string{"get"}
+					//pc.hasGet = true
+				} else {
+					pc.methods = strings.Split(methods, ",")
+					//pc.hasGet = strings.Contains(methods, "get")
+				}
+				pcs = append(pcs, pc)
+			} else {
+				return nil, errors.New("Router information is missing")
 			}
-			pc.params[funcParamName] = p
 		}
 	}
 	return

router.go

@@ -877,13 +877,15 @@ func (p *ControllerRegister) ServeHTTP(rw http.ResponseWriter, r *http.Request)
 	}
 
 Admin:
-	//admin module record QPS
+//admin module record QPS
 
 	statusCode := context.ResponseWriter.Status
 	if statusCode == 0 {
 		statusCode = 200
 	}
 
+	logAccess(context, &startTime, statusCode)
+
 	if BConfig.Listen.EnableAdmin {
 		timeDur := time.Since(startTime)
 		pattern := ""
@@ -900,31 +902,13 @@ Admin:
 		}
 	}
 
-	if BConfig.RunMode == DEV || BConfig.Log.AccessLogs {
+	if BConfig.RunMode == DEV && !BConfig.Log.AccessLogs {
-		timeDur := time.Since(startTime)
 		var devInfo string
-
+		timeDur := time.Since(startTime)
 		iswin := (runtime.GOOS == "windows")
 		statusColor := logs.ColorByStatus(iswin, statusCode)
 		methodColor := logs.ColorByMethod(iswin, r.Method)
 		resetColor := logs.ColorByMethod(iswin, "")
-		if BConfig.Log.AccessLogsFormat != "" {
-			record := &logs.AccessLogRecord{
-				RemoteAddr:     context.Input.IP(),
-				RequestTime:    startTime,
-				RequestMethod:  r.Method,
-				Request:        fmt.Sprintf("%s %s %s", r.Method, r.RequestURI, r.Proto),
-				ServerProtocol: r.Proto,
-				Host:           r.Host,
-				Status:         statusCode,
-				ElapsedTime:    timeDur,
-				HTTPReferrer:   r.Header.Get("Referer"),
-				HTTPUserAgent:  r.Header.Get("User-Agent"),
-				RemoteUser:     r.Header.Get("Remote-User"),
-				BodyBytesSent:  0, //@todo this one is missing!
-			}
-			logs.AccessLog(record, BConfig.Log.AccessLogsFormat)
-		} else {
 		if findRouter {
 			if routerInfo != nil {
 				devInfo = fmt.Sprintf("|%15s|%s %3d %s|%13s|%8s|%s %-7s %s %-3s   r:%s", context.Input.IP(), statusColor, statusCode,
@@ -944,7 +928,6 @@ Admin:
 			logs.Debug(devInfo)
 		}
 	}
-	}
 	// Call WriteHeader if status code has been set changed
 	if context.Output.Status != 0 {
 		context.ResponseWriter.WriteHeader(context.Output.Status)
@@ -960,7 +943,7 @@ func (p *ControllerRegister) handleParamResponse(context *beecontext.Context, ex
 			context.RenderMethodResult(resultValue)
 		}
 	}
-	if !context.ResponseWriter.Started && context.Output.Status == 0 {
+	if !context.ResponseWriter.Started && len(results) > 0 && context.Output.Status == 0 {
 		context.Output.SetStatus(200)
 	}
 }
@@ -991,3 +974,38 @@ func toURL(params map[string]string) string {
 	}
 	return strings.TrimRight(u, "&")
 }
+
+func logAccess(ctx *beecontext.Context, startTime *time.Time, statusCode int) {
+	//Skip logging if AccessLogs config is false
+	if !BConfig.Log.AccessLogs {
+		return
+	}
+	//Skip logging static requests unless EnableStaticLogs config is true
+	if !BConfig.Log.EnableStaticLogs && DefaultAccessLogFilter.Filter(ctx) {
+		return
+	}
+	var (
+		requestTime time.Time
+		elapsedTime time.Duration
+		r           = ctx.Request
+	)
+	if startTime != nil {
+		requestTime = *startTime
+		elapsedTime = time.Since(*startTime)
+	}
+	record := &logs.AccessLogRecord{
+		RemoteAddr:     ctx.Input.IP(),
+		RequestTime:    requestTime,
+		RequestMethod:  r.Method,
+		Request:        fmt.Sprintf("%s %s %s", r.Method, r.RequestURI, r.Proto),
+		ServerProtocol: r.Proto,
+		Host:           r.Host,
+		Status:         statusCode,
+		ElapsedTime:    elapsedTime,
+		HTTPReferrer:   r.Header.Get("Referer"),
+		HTTPUserAgent:  r.Header.Get("User-Agent"),
+		RemoteUser:     r.Header.Get("Remote-User"),
+		BodyBytesSent:  0, //@todo this one is missing!
+	}
+	logs.AccessLog(record, BConfig.Log.AccessLogsFormat)
+}

session/redis/sess_redis.go

@@ -37,6 +37,7 @@ import (
 	"strconv"
 	"strings"
 	"sync"
+	"time"
 
 	"github.com/astaxie/beego/session"
 
@@ -118,8 +119,8 @@ type Provider struct {
 }
 
 // SessionInit init redis session
-// savepath like redis server addr,pool size,password,dbnum
+// savepath like redis server addr,pool size,password,dbnum,IdleTimeout second
-// e.g. 127.0.0.1:6379,100,astaxie,0
+// e.g. 127.0.0.1:6379,100,astaxie,0,30
 func (rp *Provider) SessionInit(maxlifetime int64, savePath string) error {
 	rp.maxlifetime = maxlifetime
 	configs := strings.Split(savePath, ",")
@@ -149,6 +150,13 @@ func (rp *Provider) SessionInit(maxlifetime int64, savePath string) error {
 	} else {
 		rp.dbNum = 0
 	}
+	var idleTimeout time.Duration = 0
+	if len(configs) > 4 {
+		timeout, err := strconv.Atoi(configs[4])
+		if err == nil && timeout > 0 {
+			idleTimeout = time.Duration(timeout) * time.Second
+		}
+	}
 	rp.poollist = &redis.Pool{
 		Dial: func() (redis.Conn, error) {
 			c, err := redis.Dial("tcp", rp.savePath)
@@ -174,6 +182,8 @@ func (rp *Provider) SessionInit(maxlifetime int64, savePath string) error {
 		MaxIdle: rp.poolsize,
 	}
 
+	rp.poollist.IdleTimeout = idleTimeout
+
 	return rp.poollist.Get().Err()
 }
 

staticfile.go

@@ -74,7 +74,7 @@ func serverStaticRouter(ctx *context.Context) {
 	if enableCompress {
 		acceptEncoding = context.ParseEncoding(ctx.Request)
 	}
-	b, n, sch, err := openFile(filePath, fileInfo, acceptEncoding)
+	b, n, sch, reader, err := openFile(filePath, fileInfo, acceptEncoding)
 	if err != nil {
 		if BConfig.RunMode == DEV {
 			logs.Warn("Can't compress the file:", filePath, err)
@@ -89,47 +89,53 @@ func serverStaticRouter(ctx *context.Context) {
 		ctx.Output.Header("Content-Length", strconv.FormatInt(sch.size, 10))
 	}
 
-	http.ServeContent(ctx.ResponseWriter, ctx.Request, filePath, sch.modTime, sch)
+	http.ServeContent(ctx.ResponseWriter, ctx.Request, filePath, sch.modTime, reader)
 }
 
 type serveContentHolder struct {
-	*bytes.Reader
+	data     []byte
 	modTime  time.Time
 	size     int64
 	encoding string
 }
 
+type serveContentReader struct {
+	*bytes.Reader
+}
+
 var (
 	staticFileMap = make(map[string]*serveContentHolder)
 	mapLock       sync.RWMutex
 )
 
-func openFile(filePath string, fi os.FileInfo, acceptEncoding string) (bool, string, *serveContentHolder, error) {
+func openFile(filePath string, fi os.FileInfo, acceptEncoding string) (bool, string, *serveContentHolder, *serveContentReader, error) {
 	mapKey := acceptEncoding + ":" + filePath
 	mapLock.RLock()
 	mapFile := staticFileMap[mapKey]
 	mapLock.RUnlock()
 	if isOk(mapFile, fi) {
-		return mapFile.encoding != "", mapFile.encoding, mapFile, nil
+		reader := &serveContentReader{Reader: bytes.NewReader(mapFile.data)}
+		return mapFile.encoding != "", mapFile.encoding, mapFile, reader, nil
 	}
 	mapLock.Lock()
 	defer mapLock.Unlock()
 	if mapFile = staticFileMap[mapKey]; !isOk(mapFile, fi) {
 		file, err := os.Open(filePath)
 		if err != nil {
-			return false, "", nil, err
+			return false, "", nil, nil, err
 		}
 		defer file.Close()
 		var bufferWriter bytes.Buffer
 		_, n, err := context.WriteFile(acceptEncoding, &bufferWriter, file)
 		if err != nil {
-			return false, "", nil, err
+			return false, "", nil, nil, err
 		}
-		mapFile = &serveContentHolder{Reader: bytes.NewReader(bufferWriter.Bytes()), modTime: fi.ModTime(), size: int64(bufferWriter.Len()), encoding: n}
+		mapFile = &serveContentHolder{data: bufferWriter.Bytes(), modTime: fi.ModTime(), size: int64(bufferWriter.Len()), encoding: n}
 		staticFileMap[mapKey] = mapFile
 	}
 
-	return mapFile.encoding != "", mapFile.encoding, mapFile, nil
+	reader := &serveContentReader{Reader: bytes.NewReader(mapFile.data)}
+	return mapFile.encoding != "", mapFile.encoding, mapFile, reader, nil
 }
 
 func isOk(s *serveContentHolder, fi os.FileInfo) bool {

staticfile_test.go

@@ -16,7 +16,7 @@ var licenseFile = filepath.Join(currentWorkDir, "LICENSE")
 
 func testOpenFile(encoding string, content []byte, t *testing.T) {
 	fi, _ := os.Stat(licenseFile)
-	b, n, sch, err := openFile(licenseFile, fi, encoding)
+	b, n, sch, reader, err := openFile(licenseFile, fi, encoding)
 	if err != nil {
 		t.Log(err)
 		t.Fail()
@@ -24,7 +24,7 @@ func testOpenFile(encoding string, content []byte, t *testing.T) {
 
 	t.Log("open static file encoding "+n, b)
 
-	assetOpenFileAndContent(sch, content, t)
+	assetOpenFileAndContent(sch, reader, content, t)
 }
 func TestOpenStaticFile_1(t *testing.T) {
 	file, _ := os.Open(licenseFile)
@@ -53,13 +53,13 @@ func TestOpenStaticFileDeflate_1(t *testing.T) {
 	testOpenFile("deflate", content, t)
 }
 
-func assetOpenFileAndContent(sch *serveContentHolder, content []byte, t *testing.T) {
+func assetOpenFileAndContent(sch *serveContentHolder, reader *serveContentReader, content []byte, t *testing.T) {
 	t.Log(sch.size, len(content))
 	if sch.size != int64(len(content)) {
 		t.Log("static content file size not same")
 		t.Fail()
 	}
-	bs, _ := ioutil.ReadAll(sch)
+	bs, _ := ioutil.ReadAll(reader)
 	for i, v := range content {
 		if v != bs[i] {
 			t.Log("content not same")

templatefunc.go

@@ -17,6 +17,7 @@ package beego
 import (
 	"errors"
 	"fmt"
+	"html"
 	"html/template"
 	"net/url"
 	"reflect"
@@ -207,14 +208,12 @@ func Htmlquote(text string) string {
 	       '<'&">'
 	*/
 
-	text = strings.Replace(text, "&", "&", -1) // Must be done first!
+	text = html.EscapeString(text)
-	text = strings.Replace(text, "<", "&lt;", -1)
+	text = strings.NewReplacer(
-	text = strings.Replace(text, ">", "&gt;", -1)
+		`“`, "&ldquo;",
-	text = strings.Replace(text, "'", "&#39;", -1)
+		`”`, "&rdquo;",
-	text = strings.Replace(text, "\"", "&quot;", -1)
+		` `, "&nbsp;",
-	text = strings.Replace(text, "“", "&ldquo;", -1)
+	).Replace(text)
-	text = strings.Replace(text, "”", "&rdquo;", -1)
-	text = strings.Replace(text, " ", "&nbsp;", -1)
 
 	return strings.TrimSpace(text)
 }
@@ -228,17 +227,7 @@ func Htmlunquote(text string) string {
 	       '<\\'&">'
 	*/
 
-	// strings.Replace(s, old, new, n)
+	text = html.UnescapeString(text)
-	// 在s字符串中，把old字符串替换为new字符串，n表示替换的次数，小于0表示全部替换
-
-	text = strings.Replace(text, "&nbsp;", " ", -1)
-	text = strings.Replace(text, "&rdquo;", "”", -1)
-	text = strings.Replace(text, "&ldquo;", "“", -1)
-	text = strings.Replace(text, "&quot;", "\"", -1)
-	text = strings.Replace(text, "&#39;", "'", -1)
-	text = strings.Replace(text, "&gt;", ">", -1)
-	text = strings.Replace(text, "&lt;", "<", -1)
-	text = strings.Replace(text, "&amp;", "&", -1) // Must be done last!
 
 	return strings.TrimSpace(text)
 }

templatefunc_test.go

@@ -94,7 +94,7 @@ func TestCompareRelated(t *testing.T) {
 }
 
 func TestHtmlquote(t *testing.T) {
-	h := `<' ”“&">`
+	h := `<' ”“&">`
 	s := `<' ”“&">`
 	if Htmlquote(s) != h {
 		t.Error("should be equal")
@@ -102,8 +102,8 @@ func TestHtmlquote(t *testing.T) {
 }
 
 func TestHtmlunquote(t *testing.T) {
-	h := `&lt;&#39;&nbsp;&rdquo;&ldquo;&amp;&quot;&gt;`
+	h := `&lt;&#39;&nbsp;&rdquo;&ldquo;&amp;&#34;&gt;`
-	s := `<' ”“&">`
+	s := `<' ”“&">`
 	if Htmlunquote(h) != s {
 		t.Error("should be equal")
 	}

validation/validation.go

@@ -365,10 +365,10 @@ func (v *Validation) Valid(obj interface{}) (b bool, err error) {
 			return
 		}
 
-		var hasReuired bool
+		var hasRequired bool
 		for _, vf := range vfs {
 			if vf.Name == "Required" {
-				hasReuired = true
+				hasRequired = true
 			}
 
 			currentField := objV.Field(i).Interface()
@@ -382,7 +382,7 @@ func (v *Validation) Valid(obj interface{}) (b bool, err error) {
 
 
 			chk := Required{""}.IsSatisfied(currentField)
-			if !hasReuired && v.RequiredFirst && !chk {
+			if !hasRequired && v.RequiredFirst && !chk {
 				if _, ok := CanSkipFuncs[vf.Name]; ok {
 					continue
 				}