From 2670a860059fbc7d931b8fd07697577ec2050cec Mon Sep 17 00:00:00 2001 From: TankTheFrank Date: Thu, 14 Dec 2017 17:14:53 +0200 Subject: [PATCH 01/32] fix #2979 --- router.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/router.go b/router.go index 2f5d2eae..b9d720b1 100644 --- a/router.go +++ b/router.go @@ -957,7 +957,7 @@ func (p *ControllerRegister) handleParamResponse(context *beecontext.Context, ex context.RenderMethodResult(resultValue) } } - if !context.ResponseWriter.Started && context.Output.Status == 0 { + if !context.ResponseWriter.Started && len(results) > 0 && context.Output.Status == 0 { context.Output.SetStatus(200) } } From 94fba0b2aa69e7ec94aa19b22430dda77ff31928 Mon Sep 17 00:00:00 2001 From: Terry Ding Date: Wed, 20 Dec 2017 14:53:00 +0800 Subject: [PATCH 02/32] fix orm fields SetRaw function error judge problem --- orm/models_fields.go | 29 ++++++++++++++++++----------- 1 file changed, 18 insertions(+), 11 deletions(-) diff --git a/orm/models_fields.go b/orm/models_fields.go index d23c49fa..b4fad94f 100644 --- a/orm/models_fields.go +++ b/orm/models_fields.go @@ -86,7 +86,7 @@ func (e *BooleanField) SetRaw(value interface{}) error { e.Set(d) case string: v, err := StrTo(d).Bool() - if err != nil { + if err == nil { e.Set(v) } return err @@ -191,7 +191,7 @@ func (e *TimeField) SetRaw(value interface{}) error { e.Set(d) case string: v, err := timeParse(d, formatTime) - if err != nil { + if err == nil { e.Set(v) } return err @@ -250,7 +250,7 @@ func (e *DateField) SetRaw(value interface{}) error { e.Set(d) case string: v, err := timeParse(d, formatDate) - if err != nil { + if err == nil { e.Set(v) } return err @@ -300,7 +300,7 @@ func (e *DateTimeField) SetRaw(value interface{}) error { e.Set(d) case string: v, err := timeParse(d, formatDateTime) - if err != nil { + if err == nil { e.Set(v) } return err @@ -350,9 +350,10 @@ func (e *FloatField) SetRaw(value interface{}) error { e.Set(d) case string: v, err := StrTo(d).Float64() - if err != nil { + if err == nil { e.Set(v) } + return err default: return fmt.Errorf(" unknown value `%s`", value) } @@ -397,9 +398,10 @@ func (e *SmallIntegerField) SetRaw(value interface{}) error { e.Set(d) case string: v, err := StrTo(d).Int16() - if err != nil { + if err == nil { e.Set(v) } + return err default: return fmt.Errorf(" unknown value `%s`", value) } @@ -444,9 +446,10 @@ func (e *IntegerField) SetRaw(value interface{}) error { e.Set(d) case string: v, err := StrTo(d).Int32() - if err != nil { + if err == nil { e.Set(v) } + return err default: return fmt.Errorf(" unknown value `%s`", value) } @@ -491,9 +494,10 @@ func (e *BigIntegerField) SetRaw(value interface{}) error { e.Set(d) case string: v, err := StrTo(d).Int64() - if err != nil { + if err == nil { e.Set(v) } + return err default: return fmt.Errorf(" unknown value `%s`", value) } @@ -538,9 +542,10 @@ func (e *PositiveSmallIntegerField) SetRaw(value interface{}) error { e.Set(d) case string: v, err := StrTo(d).Uint16() - if err != nil { + if err == nil { e.Set(v) } + return err default: return fmt.Errorf(" unknown value `%s`", value) } @@ -585,9 +590,10 @@ func (e *PositiveIntegerField) SetRaw(value interface{}) error { e.Set(d) case string: v, err := StrTo(d).Uint32() - if err != nil { + if err == nil { e.Set(v) } + return err default: return fmt.Errorf(" unknown value `%s`", value) } @@ -632,9 +638,10 @@ func (e *PositiveBigIntegerField) SetRaw(value interface{}) error { e.Set(d) case string: v, err := StrTo(d).Uint64() - if err != nil { + if err == nil { e.Set(v) } + return err default: return fmt.Errorf(" unknown value `%s`", value) } From 0bde9cbd91296e71577c01e54678ca82a1c72202 Mon Sep 17 00:00:00 2001 From: dennismao Date: Fri, 22 Dec 2017 16:21:23 +0800 Subject: [PATCH 03/32] fix the issue #2995 --- orm/models_info_m.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/orm/models_info_m.go b/orm/models_info_m.go index 4a3a37f9..a4d733b6 100644 --- a/orm/models_info_m.go +++ b/orm/models_info_m.go @@ -75,7 +75,8 @@ func addModelFields(mi *modelInfo, ind reflect.Value, mName string, index []int) break } //record current field index - fi.fieldIndex = append(index, i) + fi.fieldIndex = append(fi.fieldIndex, index...) + fi.fieldIndex = append(fi.fieldIndex, i) fi.mi = mi fi.inModel = true if !mi.fields.Add(fi) { From 47c1072b787081304e56f8d014d64810d1023fe6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=99=88=E5=9F=B9=E8=BF=9C?= Date: Mon, 8 Jan 2018 19:35:53 +0800 Subject: [PATCH 04/32] do html escape before display path, avoid xss --- admin.go | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/admin.go b/admin.go index 0688dcbc..73d4f9f2 100644 --- a/admin.go +++ b/admin.go @@ -76,6 +76,18 @@ func adminIndex(rw http.ResponseWriter, r *http.Request) { func qpsIndex(rw http.ResponseWriter, r *http.Request) { data := make(map[interface{}]interface{}) data["Content"] = toolbox.StatisticsMap.GetMap() + + // do html escape before display path, avoid xss + if content, ok := (data["Content"]).(map[string]interface{}); ok { + if resultLists, ok := (content["Data"]).([][]string); ok { + for i := range resultLists { + if len(resultLists[i]) > 0 { + resultLists[i][0] = template.HTMLEscapeString(resultLists[i][0]) + } + } + } + } + execTpl(rw, data, qpsTpl, defaultScriptsTpl) } From 6db9ad70021b718b685e6724e0d76df6eda31ceb Mon Sep 17 00:00:00 2001 From: chenkaihui Date: Wed, 4 Apr 2018 15:59:52 +0800 Subject: [PATCH 05/32] auto create log dir --- logs/file.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/logs/file.go b/logs/file.go index 8e5117d2..e11278ea 100644 --- a/logs/file.go +++ b/logs/file.go @@ -21,6 +21,7 @@ import ( "fmt" "io" "os" + "path" "path/filepath" "strconv" "strings" @@ -161,6 +162,10 @@ func (w *fileLogWriter) createLogFile() (*os.File, error) { if err != nil { return nil, err } + + filepath := path.Dir(w.Filename) + os.MkdirAll(filepath, os.FileMode(perm)) + fd, err := os.OpenFile(w.Filename, os.O_WRONLY|os.O_APPEND|os.O_CREATE, os.FileMode(perm)) if err == nil { // Make sure file perm is user set perm cause of `os.OpenFile` will obey umask From 8e61a6a6de6a53e2854af88c6f768c9a500abf07 Mon Sep 17 00:00:00 2001 From: Waleed Gadelkareem Date: Sat, 28 Apr 2018 17:15:19 +0200 Subject: [PATCH 06/32] Allow access log regardless of the log level --- logs/accesslog.go | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/logs/accesslog.go b/logs/accesslog.go index cf799dc1..3f37879a 100644 --- a/logs/accesslog.go +++ b/logs/accesslog.go @@ -16,9 +16,10 @@ package logs import ( "bytes" + "strings" "encoding/json" - "time" "fmt" + "time" ) const ( @@ -53,10 +54,9 @@ func (r *AccessLogRecord) json() ([]byte, error) { } func disableEscapeHTML(i interface{}) { - e, ok := i.(interface { + if e, ok := i.(interface { SetEscapeHTML(bool) - }); - if ok { + }); ok { e.SetEscapeHTML(false) } } @@ -81,6 +81,5 @@ func AccessLog(r *AccessLogRecord, format string) { msg = string(jsonData) } } - - beeLogger.Debug(msg) + beeLogger.writeMsg(levelLoggerImpl, strings.TrimSpace(msg)) } From d5c1c0e9a4a819dd7d3a2969f3261b7f90ddb2fb Mon Sep 17 00:00:00 2001 From: Waleed Gadelkareem Date: Sat, 28 Apr 2018 20:03:39 +0200 Subject: [PATCH 07/32] log errors in access log and make static request logging optional --- admin_test.go | 1 + config.go | 2 ++ error.go | 5 ++- router.go | 92 ++++++++++++++++++++++++++++++--------------------- 4 files changed, 62 insertions(+), 38 deletions(-) diff --git a/admin_test.go b/admin_test.go index a99da6fc..03a3c044 100644 --- a/admin_test.go +++ b/admin_test.go @@ -67,6 +67,7 @@ func oldMap() map[string]interface{} { m["BConfig.WebConfig.Session.SessionDomain"] = BConfig.WebConfig.Session.SessionDomain m["BConfig.WebConfig.Session.SessionDisableHTTPOnly"] = BConfig.WebConfig.Session.SessionDisableHTTPOnly m["BConfig.Log.AccessLogs"] = BConfig.Log.AccessLogs + m["BConfig.Log.EnableStaticLogs"] = BConfig.Log.EnableStaticLogs m["BConfig.Log.AccessLogsFormat"] = BConfig.Log.AccessLogsFormat m["BConfig.Log.FileLineNum"] = BConfig.Log.FileLineNum m["BConfig.Log.Outputs"] = BConfig.Log.Outputs diff --git a/config.go b/config.go index eeeac8ee..0dc1bfbf 100644 --- a/config.go +++ b/config.go @@ -106,6 +106,7 @@ type SessionConfig struct { // LogConfig holds Log related config type LogConfig struct { AccessLogs bool + EnableStaticLogs bool //log static files requests default: false AccessLogsFormat string //access log format: JSON_FORMAT, APACHE_FORMAT or empty string FileLineNum bool Outputs map[string]string // Store Adaptor : config @@ -247,6 +248,7 @@ func newBConfig() *Config { }, Log: LogConfig{ AccessLogs: false, + EnableStaticLogs: false, AccessLogsFormat: "APACHE_FORMAT", FileLineNum: true, Outputs: map[string]string{"console": ""}, diff --git a/error.go b/error.go index b913db39..7958e367 100644 --- a/error.go +++ b/error.go @@ -28,7 +28,7 @@ import ( ) const ( - errorTypeHandler = iota + errorTypeHandler = iota errorTypeController ) @@ -439,6 +439,9 @@ func exception(errCode string, ctx *context.Context) { } func executeError(err *errorInfo, ctx *context.Context, code int) { + //make sure to log the error in the access log + logAccess(ctx, nil, code) + if err.errorType == errorTypeHandler { ctx.ResponseWriter.WriteHeader(code) err.handler(ctx.ResponseWriter, ctx.Request) diff --git a/router.go b/router.go index 11cb4aa8..fa15c35f 100644 --- a/router.go +++ b/router.go @@ -43,7 +43,7 @@ const ( ) const ( - routerTypeBeego = iota + routerTypeBeego = iota routerTypeRESTFul routerTypeHandler ) @@ -877,13 +877,15 @@ func (p *ControllerRegister) ServeHTTP(rw http.ResponseWriter, r *http.Request) } Admin: - //admin module record QPS +//admin module record QPS statusCode := context.ResponseWriter.Status if statusCode == 0 { statusCode = 200 } + logAccess(context, &startTime, statusCode) + if BConfig.Listen.EnableAdmin { timeDur := time.Since(startTime) pattern := "" @@ -900,49 +902,30 @@ Admin: } } - if BConfig.RunMode == DEV || BConfig.Log.AccessLogs { - timeDur := time.Since(startTime) + if BConfig.RunMode == DEV && !BConfig.Log.AccessLogs { var devInfo string - + timeDur := time.Since(startTime) iswin := (runtime.GOOS == "windows") statusColor := logs.ColorByStatus(iswin, statusCode) methodColor := logs.ColorByMethod(iswin, r.Method) resetColor := logs.ColorByMethod(iswin, "") - if BConfig.Log.AccessLogsFormat != "" { - record := &logs.AccessLogRecord{ - RemoteAddr: context.Input.IP(), - RequestTime: startTime, - RequestMethod: r.Method, - Request: fmt.Sprintf("%s %s %s", r.Method, r.RequestURI, r.Proto), - ServerProtocol: r.Proto, - Host: r.Host, - Status: statusCode, - ElapsedTime: timeDur, - HTTPReferrer: r.Header.Get("Referer"), - HTTPUserAgent: r.Header.Get("User-Agent"), - RemoteUser: r.Header.Get("Remote-User"), - BodyBytesSent: 0, //@todo this one is missing! - } - logs.AccessLog(record, BConfig.Log.AccessLogsFormat) - } else { - if findRouter { - if routerInfo != nil { - devInfo = fmt.Sprintf("|%15s|%s %3d %s|%13s|%8s|%s %-7s %s %-3s r:%s", context.Input.IP(), statusColor, statusCode, - resetColor, timeDur.String(), "match", methodColor, r.Method, resetColor, r.URL.Path, - routerInfo.pattern) - } else { - devInfo = fmt.Sprintf("|%15s|%s %3d %s|%13s|%8s|%s %-7s %s %-3s", context.Input.IP(), statusColor, statusCode, resetColor, - timeDur.String(), "match", methodColor, r.Method, resetColor, r.URL.Path) - } + if findRouter { + if routerInfo != nil { + devInfo = fmt.Sprintf("|%15s|%s %3d %s|%13s|%8s|%s %-7s %s %-3s r:%s", context.Input.IP(), statusColor, statusCode, + resetColor, timeDur.String(), "match", methodColor, r.Method, resetColor, r.URL.Path, + routerInfo.pattern) } else { devInfo = fmt.Sprintf("|%15s|%s %3d %s|%13s|%8s|%s %-7s %s %-3s", context.Input.IP(), statusColor, statusCode, resetColor, - timeDur.String(), "nomatch", methodColor, r.Method, resetColor, r.URL.Path) - } - if iswin { - logs.W32Debug(devInfo) - } else { - logs.Debug(devInfo) + timeDur.String(), "match", methodColor, r.Method, resetColor, r.URL.Path) } + } else { + devInfo = fmt.Sprintf("|%15s|%s %3d %s|%13s|%8s|%s %-7s %s %-3s", context.Input.IP(), statusColor, statusCode, resetColor, + timeDur.String(), "nomatch", methodColor, r.Method, resetColor, r.URL.Path) + } + if iswin { + logs.W32Debug(devInfo) + } else { + logs.Debug(devInfo) } } // Call WriteHeader if status code has been set changed @@ -991,3 +974,38 @@ func toURL(params map[string]string) string { } return strings.TrimRight(u, "&") } + +func logAccess(ctx *beecontext.Context, startTime *time.Time, statusCode int) { + //Skip logging if AccessLogs config is false + if !BConfig.Log.AccessLogs { + return + } + //Skip logging static requests unless EnableStaticLogs config is true + if !BConfig.Log.EnableStaticLogs && DefaultAccessLogFilter.Filter(ctx) { + return + } + var ( + requestTime time.Time + elapsedTime time.Duration + r = ctx.Request + ) + if startTime != nil { + requestTime = *startTime + elapsedTime = time.Since(*startTime) + } + record := &logs.AccessLogRecord{ + RemoteAddr: ctx.Input.IP(), + RequestTime: requestTime, + RequestMethod: r.Method, + Request: fmt.Sprintf("%s %s %s", r.Method, r.RequestURI, r.Proto), + ServerProtocol: r.Proto, + Host: r.Host, + Status: statusCode, + ElapsedTime: elapsedTime, + HTTPReferrer: r.Header.Get("Referer"), + HTTPUserAgent: r.Header.Get("User-Agent"), + RemoteUser: r.Header.Get("Remote-User"), + BodyBytesSent: 0, //@todo this one is missing! + } + logs.AccessLog(record, BConfig.Log.AccessLogsFormat) +} From 6df42d63e2d9c44845379d838ca00ad39c15fdae Mon Sep 17 00:00:00 2001 From: Amit Ashkenazi Date: Sun, 29 Apr 2018 15:12:32 +0300 Subject: [PATCH 08/32] Fix response http code --- config.go | 5 +++++ error.go | 5 ----- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/config.go b/config.go index eeeac8ee..0725445b 100644 --- a/config.go +++ b/config.go @@ -182,6 +182,11 @@ func recoverPanic(ctx *context.Context) { if BConfig.RunMode == DEV && BConfig.EnableErrorsRender { showErr(err, ctx, stack) } + if ctx.Output.Status != 0 { + ctx.ResponseWriter.WriteHeader(ctx.Output.Status) + } else { + ctx.ResponseWriter.WriteHeader(500) + } } } diff --git a/error.go b/error.go index b913db39..77ddec26 100644 --- a/error.go +++ b/error.go @@ -93,11 +93,6 @@ func showErr(err interface{}, ctx *context.Context, stack string) { "BeegoVersion": VERSION, "GoVersion": runtime.Version(), } - if ctx.Output.Status != 0 { - ctx.ResponseWriter.WriteHeader(ctx.Output.Status) - } else { - ctx.ResponseWriter.WriteHeader(500) - } t.Execute(ctx.ResponseWriter, data) } From 896c258e449aedf77a4c342d136847dfbee4b957 Mon Sep 17 00:00:00 2001 From: Waleed Gadelkareem Date: Mon, 30 Apr 2018 17:48:01 +0200 Subject: [PATCH 09/32] Log redirects and abort after redirect --- controller.go | 2 ++ logs/accesslog.go | 2 -- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/controller.go b/controller.go index c104eb2a..c66c7d88 100644 --- a/controller.go +++ b/controller.go @@ -272,7 +272,9 @@ func (c *Controller) viewPath() string { // Redirect sends the redirection response to url with status code. func (c *Controller) Redirect(url string, code int) { + logAccess(c.Ctx, nil, code) c.Ctx.Redirect(code, url) + panic(ErrAbort) } // Abort stops controller handler and show the error data if code is defined in ErrorMap or code string. diff --git a/logs/accesslog.go b/logs/accesslog.go index 3f37879a..5790f97e 100644 --- a/logs/accesslog.go +++ b/logs/accesslog.go @@ -64,9 +64,7 @@ func disableEscapeHTML(i interface{}) { // AccessLog - Format and print access log. func AccessLog(r *AccessLogRecord, format string) { var msg string - switch format { - case apacheFormat: timeFormatted := r.RequestTime.Format("02/Jan/2006 03:04:05") msg = fmt.Sprintf(apacheFormatPattern, r.RemoteAddr, timeFormatted, r.Request, r.Status, r.BodyBytesSent, From 0f730505674bd4569efbefef4189d010d74cdcc9 Mon Sep 17 00:00:00 2001 From: Wang-Kai Date: Thu, 3 May 2018 12:05:59 +0800 Subject: [PATCH 10/32] add code style for logs README --- logs/README.md | 57 +++++++++++++++++++++++++++++--------------------- 1 file changed, 33 insertions(+), 24 deletions(-) diff --git a/logs/README.md b/logs/README.md index 57d7abc3..d326cfa1 100644 --- a/logs/README.md +++ b/logs/README.md @@ -16,48 +16,57 @@ As of now this logs support console, file,smtp and conn. First you must import it - import ( - "github.com/astaxie/beego/logs" - ) +```golang +import ( + "github.com/astaxie/beego/logs" +) +``` Then init a Log (example with console adapter) - log := NewLogger(10000) - log.SetLogger("console", "") +```golang +log := NewLogger(10000) +log.SetLogger("console", "") +``` > the first params stand for how many channel -Use it like this: +Use it like this: - log.Trace("trace") - log.Info("info") - log.Warn("warning") - log.Debug("debug") - log.Critical("critical") - +```golang +log.Trace("trace") +log.Info("info") +log.Warn("warning") +log.Debug("debug") +log.Critical("critical") +``` ## File adapter Configure file adapter like this: - - log := NewLogger(10000) - log.SetLogger("file", `{"filename":"test.log"}`) - + +```golang +log := NewLogger(10000) +log.SetLogger("file", `{"filename":"test.log"}`) +``` ## Conn adapter Configure like this: - log := NewLogger(1000) - log.SetLogger("conn", `{"net":"tcp","addr":":7020"}`) - log.Info("info") - +```golang +log := NewLogger(1000) +log.SetLogger("conn", `{"net":"tcp","addr":":7020"}`) +log.Info("info") +``` ## Smtp adapter Configure like this: - log := NewLogger(10000) - log.SetLogger("smtp", `{"username":"beegotest@gmail.com","password":"xxxxxxxx","host":"smtp.gmail.com:587","sendTos":["xiemengjun@gmail.com"]}`) - log.Critical("sendmail critical") - time.Sleep(time.Second * 30) +```golang +log := NewLogger(10000) +log.SetLogger("smtp", `{"username":"beegotest@gmail.com","password":"xxxxxxxx","host":"smtp.gmail.com:587","sendTos":["xiemengjun@gmail.com"]}`) +log.Critical("sendmail critical") +time.Sleep(time.Second * 30) +``` From 1fd7fa5df7be08e15ee989a5951148f278a3476e Mon Sep 17 00:00:00 2001 From: Wang-Kai Date: Thu, 3 May 2018 22:04:49 +0800 Subject: [PATCH 11/32] make example runable --- logs/README.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/logs/README.md b/logs/README.md index d326cfa1..c05bcc04 100644 --- a/logs/README.md +++ b/logs/README.md @@ -25,15 +25,15 @@ import ( Then init a Log (example with console adapter) ```golang -log := NewLogger(10000) -log.SetLogger("console", "") +log := logs.NewLogger(10000) +log.SetLogger("console", "") ``` > the first params stand for how many channel Use it like this: - -```golang + +```golang log.Trace("trace") log.Info("info") log.Warn("warning") @@ -44,8 +44,8 @@ log.Critical("critical") ## File adapter Configure file adapter like this: - -```golang + +```golang log := NewLogger(10000) log.SetLogger("file", `{"filename":"test.log"}`) ``` From 98a3cda2602b57dffb8b8c9aebfb4817cdad5b4a Mon Sep 17 00:00:00 2001 From: Amit Yadav <154998+ayadav@users.noreply.github.com> Date: Mon, 7 May 2018 12:57:13 +0530 Subject: [PATCH 12/32] Fix Unexpected EOF bug in staticfile --- staticfile.go | 24 +++++++++++++++--------- staticfile_test.go | 8 ++++---- 2 files changed, 19 insertions(+), 13 deletions(-) diff --git a/staticfile.go b/staticfile.go index bbb2a1fb..30ee3dfb 100644 --- a/staticfile.go +++ b/staticfile.go @@ -74,7 +74,7 @@ func serverStaticRouter(ctx *context.Context) { if enableCompress { acceptEncoding = context.ParseEncoding(ctx.Request) } - b, n, sch, err := openFile(filePath, fileInfo, acceptEncoding) + b, n, sch, reader, err := openFile(filePath, fileInfo, acceptEncoding) if err != nil { if BConfig.RunMode == DEV { logs.Warn("Can't compress the file:", filePath, err) @@ -89,47 +89,53 @@ func serverStaticRouter(ctx *context.Context) { ctx.Output.Header("Content-Length", strconv.FormatInt(sch.size, 10)) } - http.ServeContent(ctx.ResponseWriter, ctx.Request, filePath, sch.modTime, sch) + http.ServeContent(ctx.ResponseWriter, ctx.Request, filePath, sch.modTime, reader) } type serveContentHolder struct { - *bytes.Reader + data []byte modTime time.Time size int64 encoding string } +type serveContentReader struct { + *bytes.Reader +} + var ( staticFileMap = make(map[string]*serveContentHolder) mapLock sync.RWMutex ) -func openFile(filePath string, fi os.FileInfo, acceptEncoding string) (bool, string, *serveContentHolder, error) { +func openFile(filePath string, fi os.FileInfo, acceptEncoding string) (bool, string, *serveContentHolder, *serveContentReader, error) { mapKey := acceptEncoding + ":" + filePath mapLock.RLock() mapFile := staticFileMap[mapKey] mapLock.RUnlock() if isOk(mapFile, fi) { - return mapFile.encoding != "", mapFile.encoding, mapFile, nil + reader := &serveContentReader{Reader: bytes.NewReader(mapFile.data)} + return mapFile.encoding != "", mapFile.encoding, mapFile, reader, nil } mapLock.Lock() defer mapLock.Unlock() if mapFile = staticFileMap[mapKey]; !isOk(mapFile, fi) { file, err := os.Open(filePath) if err != nil { - return false, "", nil, err + return false, "", nil, nil, err } defer file.Close() var bufferWriter bytes.Buffer _, n, err := context.WriteFile(acceptEncoding, &bufferWriter, file) if err != nil { - return false, "", nil, err + return false, "", nil, nil, err } - mapFile = &serveContentHolder{Reader: bytes.NewReader(bufferWriter.Bytes()), modTime: fi.ModTime(), size: int64(bufferWriter.Len()), encoding: n} + mapFile = &serveContentHolder{data: bufferWriter.Bytes(), modTime: fi.ModTime(), size: int64(bufferWriter.Len()), encoding: n} staticFileMap[mapKey] = mapFile } - return mapFile.encoding != "", mapFile.encoding, mapFile, nil + reader := &serveContentReader{Reader: bytes.NewReader(mapFile.data)} + return mapFile.encoding != "", mapFile.encoding, mapFile, reader, nil } func isOk(s *serveContentHolder, fi os.FileInfo) bool { diff --git a/staticfile_test.go b/staticfile_test.go index a043b4fd..69667bf8 100644 --- a/staticfile_test.go +++ b/staticfile_test.go @@ -16,7 +16,7 @@ var licenseFile = filepath.Join(currentWorkDir, "LICENSE") func testOpenFile(encoding string, content []byte, t *testing.T) { fi, _ := os.Stat(licenseFile) - b, n, sch, err := openFile(licenseFile, fi, encoding) + b, n, sch, reader, err := openFile(licenseFile, fi, encoding) if err != nil { t.Log(err) t.Fail() @@ -24,7 +24,7 @@ func testOpenFile(encoding string, content []byte, t *testing.T) { t.Log("open static file encoding "+n, b) - assetOpenFileAndContent(sch, content, t) + assetOpenFileAndContent(sch, reader, content, t) } func TestOpenStaticFile_1(t *testing.T) { file, _ := os.Open(licenseFile) @@ -53,13 +53,13 @@ func TestOpenStaticFileDeflate_1(t *testing.T) { testOpenFile("deflate", content, t) } -func assetOpenFileAndContent(sch *serveContentHolder, content []byte, t *testing.T) { +func assetOpenFileAndContent(sch *serveContentHolder, reader *serveContentReader, content []byte, t *testing.T) { t.Log(sch.size, len(content)) if sch.size != int64(len(content)) { t.Log("static content file size not same") t.Fail() } - bs, _ := ioutil.ReadAll(sch) + bs, _ := ioutil.ReadAll(reader) for i, v := range content { if v != bs[i] { t.Log("content not same") From e96ae0c24a101efb29eeb0f6d0ec4c0bb8a15673 Mon Sep 17 00:00:00 2001 From: whomm Date: Mon, 21 May 2018 15:18:18 +0800 Subject: [PATCH 13/32] debug stringsToJSON json char: \u four-hex-digits number(http://json.org/) --- context/output.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/context/output.go b/context/output.go index 61ce8cc7..4bf135fd 100644 --- a/context/output.go +++ b/context/output.go @@ -350,6 +350,11 @@ func stringsToJSON(str string) string { jsons.WriteRune(r) } else { jsons.WriteString("\\u") + if rint < 0x100 { + jsons.WriteString("00") + } else if rint < 0x1000 { + jsons.WriteString("0") + } jsons.WriteString(strconv.FormatInt(int64(rint), 16)) } } From 45b68d444d2e7f659643a9b3899ac11c7f87de54 Mon Sep 17 00:00:00 2001 From: Ruben Cid Date: Fri, 1 Jun 2018 19:11:01 +0200 Subject: [PATCH 14/32] Add method to set the data depending on the accepted --- controller.go | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/controller.go b/controller.go index c104eb2a..92d2188b 100644 --- a/controller.go +++ b/controller.go @@ -275,6 +275,19 @@ func (c *Controller) Redirect(url string, code int) { c.Ctx.Redirect(code, url) } +// Set the data depending on the accepted +func (c *Controller) SetData(data interface{}) { + accept := c.Ctx.Input.Header("Accept") + switch accept { + case applicationJSON: + c.Data["json"] = data + case applicationXML, textXML: + c.Data["xml"] = data + default: + c.Data["json"] = data + } +} + // Abort stops controller handler and show the error data if code is defined in ErrorMap or code string. func (c *Controller) Abort(code string) { status, err := strconv.Atoi(code) From f979050a45f1f7214ba63aecb9fc01775a389bec Mon Sep 17 00:00:00 2001 From: umasuo Date: Sun, 3 Jun 2018 23:08:03 +0800 Subject: [PATCH 15/32] Fix defaut value bug, and add default maxfiles 1. add default value for maxlines(100000), maxsize(1 << 28) 2. add maxfiles for configure, just like `maxlines` & maxsize --- logs/file.go | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/logs/file.go b/logs/file.go index 8e5117d2..f085e646 100644 --- a/logs/file.go +++ b/logs/file.go @@ -40,6 +40,9 @@ type fileLogWriter struct { MaxLines int `json:"maxlines"` maxLinesCurLines int + MaxFiles int `json:"maxfiles"` + MaxFilesCurFiles int + // Rotate at size MaxSize int `json:"maxsize"` maxSizeCurSize int @@ -70,6 +73,9 @@ func newFileWriter() Logger { RotatePerm: "0440", Level: LevelTrace, Perm: "0660", + MaxLines: 10000000, + MaxFiles: 999, + MaxSize: 1 << 28, } return w } @@ -238,7 +244,7 @@ func (w *fileLogWriter) lines() (int, error) { func (w *fileLogWriter) doRotate(logTime time.Time) error { // file exists // Find the next available number - num := 1 + num := w.MaxFilesCurFiles + 1 fName := "" rotatePerm, err := strconv.ParseInt(w.RotatePerm, 8, 64) if err != nil { @@ -251,18 +257,16 @@ func (w *fileLogWriter) doRotate(logTime time.Time) error { goto RESTART_LOGGER } + // only when one of them be setted, then the file would be splited if w.MaxLines > 0 || w.MaxSize > 0 { - for ; err == nil && num <= 999; num++ { + for ; err == nil && num <= w.MaxFiles; num++ { fName = w.fileNameOnly + fmt.Sprintf(".%s.%03d%s", logTime.Format("2006-01-02"), num, w.suffix) _, err = os.Lstat(fName) } } else { - fName = fmt.Sprintf("%s.%s%s", w.fileNameOnly, w.dailyOpenTime.Format("2006-01-02"), w.suffix) + fName = w.fileNameOnly + fmt.Sprintf(".%s.%03d%s", w.dailyOpenTime.Format("2006-01-02"), num, w.suffix) _, err = os.Lstat(fName) - for ; err == nil && num <= 999; num++ { - fName = w.fileNameOnly + fmt.Sprintf(".%s.%03d%s", w.dailyOpenTime.Format("2006-01-02"), num, w.suffix) - _, err = os.Lstat(fName) - } + w.MaxFilesCurFiles = num } // return error if the last file checked still existed if err == nil { @@ -308,7 +312,7 @@ func (w *fileLogWriter) deleteOldLog() { return } - if !info.IsDir() && info.ModTime().Add(24*time.Hour*time.Duration(w.MaxDays)).Before(time.Now()) { + if !info.IsDir() && info.ModTime().Add(24 * time.Hour * time.Duration(w.MaxDays)).Before(time.Now()) { if strings.HasPrefix(filepath.Base(path), filepath.Base(w.fileNameOnly)) && strings.HasSuffix(filepath.Base(path), w.suffix) { os.Remove(path) From 1df2662924951e62ed1b8b939af52c70e76cc7b4 Mon Sep 17 00:00:00 2001 From: lintao Date: Wed, 6 Jun 2018 12:33:28 +0800 Subject: [PATCH 16/32] add field comment on create table --- orm/cmd_utils.go | 4 ++++ orm/models_info_f.go | 2 ++ 2 files changed, 6 insertions(+) diff --git a/orm/cmd_utils.go b/orm/cmd_utils.go index ba7afb53..7c9aa51e 100644 --- a/orm/cmd_utils.go +++ b/orm/cmd_utils.go @@ -197,6 +197,10 @@ func getDbCreateSQL(al *alias) (sqls []string, tableIndexes map[string][]dbIndex if strings.Contains(column, "%COL%") { column = strings.Replace(column, "%COL%", fi.column, -1) } + + if fi.description != "" { + column += " " + fmt.Sprintf("COMMENT '%s'",fi.description) + } columns = append(columns, column) } diff --git a/orm/models_info_f.go b/orm/models_info_f.go index 646e2273..479f5ae6 100644 --- a/orm/models_info_f.go +++ b/orm/models_info_f.go @@ -136,6 +136,7 @@ type fieldInfo struct { decimals int isFielder bool // implement Fielder interface onDelete string + description string } // new field info @@ -300,6 +301,7 @@ checkType: fi.sf = sf fi.fullName = mi.fullName + mName + "." + sf.Name + fi.description = sf.Tag.Get("description") fi.null = attrs["null"] fi.index = attrs["index"] fi.auto = attrs["auto"] From bf915c32801ef7349a52570c18aba90cf35fb6e3 Mon Sep 17 00:00:00 2001 From: Sandy Date: Tue, 12 Jun 2018 16:15:20 +0800 Subject: [PATCH 17/32] Update: use PathEscape replace QueryEscape If filename contain space(" "), QueryEscape use "+" instead. --- context/output.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/context/output.go b/context/output.go index 61ce8cc7..a63e3fd4 100644 --- a/context/output.go +++ b/context/output.go @@ -260,7 +260,7 @@ func (output *BeegoOutput) Download(file string, filename ...string) { } else { fName = filepath.Base(file) } - output.Header("Content-Disposition", "attachment; filename="+url.QueryEscape(fName)) + output.Header("Content-Disposition", "attachment; filename="+url.PathEscape(fName)) output.Header("Content-Description", "File Transfer") output.Header("Content-Type", "application/octet-stream") output.Header("Content-Transfer-Encoding", "binary") From ad6c97ec1b60f332c88770c6c56c376c03d22892 Mon Sep 17 00:00:00 2001 From: Openset Date: Wed, 13 Jun 2018 15:43:01 +0800 Subject: [PATCH 18/32] Update: Htmlquote Htmlunquote --- templatefunc.go | 41 +++++++++++++++-------------------------- 1 file changed, 15 insertions(+), 26 deletions(-) diff --git a/templatefunc.go b/templatefunc.go index a104fd24..e4d4667a 100644 --- a/templatefunc.go +++ b/templatefunc.go @@ -17,6 +17,7 @@ package beego import ( "errors" "fmt" + "html" "html/template" "net/url" "reflect" @@ -84,24 +85,24 @@ func DateFormat(t time.Time, layout string) (datestring string) { var datePatterns = []string{ // year "Y", "2006", // A full numeric representation of a year, 4 digits Examples: 1999 or 2003 - "y", "06", //A two digit representation of a year Examples: 99 or 03 + "y", "06", //A two digit representation of a year Examples: 99 or 03 // month - "m", "01", // Numeric representation of a month, with leading zeros 01 through 12 - "n", "1", // Numeric representation of a month, without leading zeros 1 through 12 - "M", "Jan", // A short textual representation of a month, three letters Jan through Dec + "m", "01", // Numeric representation of a month, with leading zeros 01 through 12 + "n", "1", // Numeric representation of a month, without leading zeros 1 through 12 + "M", "Jan", // A short textual representation of a month, three letters Jan through Dec "F", "January", // A full textual representation of a month, such as January or March January through December // day "d", "02", // Day of the month, 2 digits with leading zeros 01 to 31 - "j", "2", // Day of the month without leading zeros 1 to 31 + "j", "2", // Day of the month without leading zeros 1 to 31 // week - "D", "Mon", // A textual representation of a day, three letters Mon through Sun + "D", "Mon", // A textual representation of a day, three letters Mon through Sun "l", "Monday", // A full textual representation of the day of the week Sunday through Saturday // time - "g", "3", // 12-hour format of an hour without leading zeros 1 through 12 + "g", "3", // 12-hour format of an hour without leading zeros 1 through 12 "G", "15", // 24-hour format of an hour without leading zeros 0 through 23 "h", "03", // 12-hour format of an hour with leading zeros 01 through 12 "H", "15", // 24-hour format of an hour with leading zeros 00 through 23 @@ -207,14 +208,12 @@ func Htmlquote(text string) string { '<'&">' */ - text = strings.Replace(text, "&", "&", -1) // Must be done first! - text = strings.Replace(text, "<", "<", -1) - text = strings.Replace(text, ">", ">", -1) - text = strings.Replace(text, "'", "'", -1) - text = strings.Replace(text, "\"", """, -1) - text = strings.Replace(text, "“", "“", -1) - text = strings.Replace(text, "”", "”", -1) - text = strings.Replace(text, " ", " ", -1) + text = html.EscapeString(text) + text = strings.NewReplacer( + `“`, "“", + `”`, "”", + ` `, " ", + ).Replace(text) return strings.TrimSpace(text) } @@ -228,17 +227,7 @@ func Htmlunquote(text string) string { '<\\'&">' */ - // strings.Replace(s, old, new, n) - // 在s字符串中，把old字符串替换为new字符串，n表示替换的次数，小于0表示全部替换 - - text = strings.Replace(text, " ", " ", -1) - text = strings.Replace(text, "”", "”", -1) - text = strings.Replace(text, "“", "“", -1) - text = strings.Replace(text, """, "\"", -1) - text = strings.Replace(text, "'", "'", -1) - text = strings.Replace(text, ">", ">", -1) - text = strings.Replace(text, "<", "<", -1) - text = strings.Replace(text, "&", "&", -1) // Must be done last! + text = html.UnescapeString(text) return strings.TrimSpace(text) } From b80b7b06fc908cb2c921349f035419893ecb62eb Mon Sep 17 00:00:00 2001 From: Openset Date: Thu, 14 Jun 2018 11:55:07 +0800 Subject: [PATCH 19/32] Update: Redundant semicolon disableEscapeHTML --- logs/accesslog.go | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/logs/accesslog.go b/logs/accesslog.go index 461db55b..d046d70f 100644 --- a/logs/accesslog.go +++ b/logs/accesslog.go @@ -17,8 +17,8 @@ package logs import ( "bytes" "encoding/json" - "time" "fmt" + "time" ) const ( @@ -53,10 +53,9 @@ func (r *AccessLogRecord) json() ([]byte, error) { } func disableEscapeHTML(i interface{}) { - e, ok := i.(interface { + if e, ok := i.(interface { SetEscapeHTML(bool) - }); - if ok { + }); ok { e.SetEscapeHTML(false) } } From 87ba3f3cd3e0890af254676b40087bca1f8ebece Mon Sep 17 00:00:00 2001 From: ia Date: Sun, 17 Jun 2018 00:47:51 +0200 Subject: [PATCH 20/32] all: gofmt Run standard gofmt command on project root. - go version go1.10.3 darwin/amd64 Signed-off-by: ia --- app.go | 4 ++-- httplib/httplib.go | 2 ++ logs/accesslog.go | 4 ++-- logs/logger.go | 2 +- 4 files changed, 7 insertions(+), 5 deletions(-) diff --git a/app.go b/app.go index 0c07117a..04fac65e 100644 --- a/app.go +++ b/app.go @@ -24,8 +24,8 @@ import ( "net/http/fcgi" "os" "path" - "time" "strings" + "time" "github.com/astaxie/beego/grace" "github.com/astaxie/beego/logs" @@ -101,7 +101,7 @@ func (app *App) Run(mws ...MiddleWare) { } app.Server.Handler = app.Handlers - for i:=len(mws)-1;i>=0;i-- { + for i := len(mws) - 1; i >= 0; i-- { if mws[i] == nil { continue } diff --git a/httplib/httplib.go b/httplib/httplib.go index 5d82ab33..8a9bbea7 100644 --- a/httplib/httplib.go +++ b/httplib/httplib.go @@ -317,6 +317,7 @@ func (b *BeegoHTTPRequest) Body(data interface{}) *BeegoHTTPRequest { } return b } + // XMLBody adds request raw body encoding by XML. func (b *BeegoHTTPRequest) XMLBody(obj interface{}) (*BeegoHTTPRequest, error) { if b.req.Body == nil && obj != nil { @@ -330,6 +331,7 @@ func (b *BeegoHTTPRequest) XMLBody(obj interface{}) (*BeegoHTTPRequest, error) { } return b, nil } + // JSONBody adds request raw body encoding by JSON. func (b *BeegoHTTPRequest) JSONBody(obj interface{}) (*BeegoHTTPRequest, error) { if b.req.Body == nil && obj != nil { diff --git a/logs/accesslog.go b/logs/accesslog.go index cf799dc1..08c64280 100644 --- a/logs/accesslog.go +++ b/logs/accesslog.go @@ -17,8 +17,8 @@ package logs import ( "bytes" "encoding/json" - "time" "fmt" + "time" ) const ( @@ -55,7 +55,7 @@ func (r *AccessLogRecord) json() ([]byte, error) { func disableEscapeHTML(i interface{}) { e, ok := i.(interface { SetEscapeHTML(bool) - }); + }) if ok { e.SetEscapeHTML(false) } diff --git a/logs/logger.go b/logs/logger.go index 1700901f..6c2dd3f9 100644 --- a/logs/logger.go +++ b/logs/logger.go @@ -93,7 +93,7 @@ const ( func formatTimeHeader(when time.Time) ([]byte, int) { y, mo, d := when.Date() h, mi, s := when.Clock() - ns := when.Nanosecond()/1000000 + ns := when.Nanosecond() / 1000000 //len("2006/01/02 15:04:05.123 ")==24 var buf [24]byte From fc2c96a177e94a40f54c103246a0a8c32bd6a88e Mon Sep 17 00:00:00 2001 From: hurisheng Date: Sat, 23 Jun 2018 22:25:05 +0800 Subject: [PATCH 21/32] add 'FOR UPDATE' support for querySet --- orm/db.go | 4 ++++ orm/orm_queryset.go | 27 +++++++++++++++++---------- orm/types.go | 4 ++++ 3 files changed, 25 insertions(+), 10 deletions(-) diff --git a/orm/db.go b/orm/db.go index 5862d003..6b749425 100644 --- a/orm/db.go +++ b/orm/db.go @@ -969,6 +969,10 @@ func (d *dbBase) ReadBatch(q dbQuerier, qs *querySet, mi *modelInfo, cond *Condi } query := fmt.Sprintf("%s %s FROM %s%s%s T0 %s%s%s%s%s", sqlSelect, sels, Q, mi.table, Q, join, where, groupBy, orderBy, limit) + if qs.forupdate { + query += " FOR UPDATE" + } + d.ins.ReplaceMarks(&query) var rs *sql.Rows diff --git a/orm/orm_queryset.go b/orm/orm_queryset.go index 4e33646d..4bab1d98 100644 --- a/orm/orm_queryset.go +++ b/orm/orm_queryset.go @@ -55,16 +55,17 @@ func ColValue(opt operator, value interface{}) interface{} { // real query struct type querySet struct { - mi *modelInfo - cond *Condition - related []string - relDepth int - limit int64 - offset int64 - groups []string - orders []string - distinct bool - orm *orm + mi *modelInfo + cond *Condition + related []string + relDepth int + limit int64 + offset int64 + groups []string + orders []string + distinct bool + forupdate bool + orm *orm } var _ QuerySeter = new(querySet) @@ -127,6 +128,12 @@ func (o querySet) Distinct() QuerySeter { return &o } +// add FOR UPDATE to SELECT +func (o querySet) ForUpdate() QuerySeter { + o.forupdate = true + return &o +} + // set relation model to query together. // it will query relation models and assign to parent model. func (o querySet) RelatedSel(params ...interface{}) QuerySeter { diff --git a/orm/types.go b/orm/types.go index 3e6a9e87..e3373096 100644 --- a/orm/types.go +++ b/orm/types.go @@ -190,6 +190,10 @@ type QuerySeter interface { // Distinct(). // All(&permissions) Distinct() QuerySeter + // set FOR UPDATE to query. + // for example: + // o.QueryTable("user").Filter("uid", uid).ForUpdate().All(&users) + ForUpdate() QuerySeter // return QuerySeter execution result number // for example: // num, err = qs.Filter("profile__age__gt", 28).Count() From 227c04c9e63d62f045b85960f76fd1ee928ffeb4 Mon Sep 17 00:00:00 2001 From: jinxjinxagain Date: Thu, 28 Jun 2018 15:54:17 +0800 Subject: [PATCH 22/32] fix: When multiply comment routers on one func, only generates the last one controller --- parser.go | 72 +++++++++++++++++++++++++++++++------------------------ 1 file changed, 41 insertions(+), 31 deletions(-) diff --git a/parser.go b/parser.go index 1933c6c6..2ac48b85 100644 --- a/parser.go +++ b/parser.go @@ -114,20 +114,21 @@ type parsedParam struct { func parserComments(f *ast.FuncDecl, controllerName, pkgpath string) error { if f.Doc != nil { - parsedComment, err := parseComment(f.Doc.List) + parsedComments, err := parseComment(f.Doc.List) if err != nil { return err } - if parsedComment.routerPath != "" { - key := pkgpath + ":" + controllerName - cc := ControllerComments{} - cc.Method = f.Name.String() - cc.Router = parsedComment.routerPath - cc.AllowHTTPMethods = parsedComment.methods - cc.MethodParams = buildMethodParams(f.Type.Params.List, parsedComment) - genInfoList[key] = append(genInfoList[key], cc) + for _, parsedComment := range parsedComments { + if parsedComment.routerPath != "" { + key := pkgpath + ":" + controllerName + cc := ControllerComments{} + cc.Method = f.Name.String() + cc.Router = parsedComment.routerPath + cc.AllowHTTPMethods = parsedComment.methods + cc.MethodParams = buildMethodParams(f.Type.Params.List, parsedComment) + genInfoList[key] = append(genInfoList[key], cc) + } } - } return nil } @@ -177,26 +178,13 @@ func paramInPath(name, route string) bool { var routeRegex = regexp.MustCompile(`@router\s+(\S+)(?:\s+\[(\S+)\])?`) -func parseComment(lines []*ast.Comment) (pc *parsedComment, err error) { - pc = &parsedComment{} +func parseComment(lines []*ast.Comment) (pcs []*parsedComment, err error) { + pcs = []*parsedComment{} + params := map[string]parsedParam{} + for _, c := range lines { t := strings.TrimSpace(strings.TrimLeft(c.Text, "//")) - if strings.HasPrefix(t, "@router") { - matches := routeRegex.FindStringSubmatch(t) - if len(matches) == 3 { - pc.routerPath = matches[1] - methods := matches[2] - if methods == "" { - pc.methods = []string{"get"} - //pc.hasGet = true - } else { - pc.methods = strings.Split(methods, ",") - //pc.hasGet = strings.Contains(methods, "get") - } - } else { - return nil, errors.New("Router information is missing") - } - } else if strings.HasPrefix(t, "@Param") { + if strings.HasPrefix(t, "@Param") { pv := getparams(strings.TrimSpace(strings.TrimLeft(t, "@Param"))) if len(pv) < 4 { logs.Error("Invalid @Param format. Needs at least 4 parameters") @@ -217,10 +205,32 @@ func parseComment(lines []*ast.Comment) (pc *parsedComment, err error) { p.defValue = pv[3] p.required, _ = strconv.ParseBool(pv[4]) } - if pc.params == nil { - pc.params = map[string]parsedParam{} + params[funcParamName] = p + } + } + + for _, c := range lines { + var pc = &parsedComment{} + pc.params = params + + t := strings.TrimSpace(strings.TrimLeft(c.Text, "//")) + if strings.HasPrefix(t, "@router") { + t := strings.TrimSpace(strings.TrimLeft(c.Text, "//")) + matches := routeRegex.FindStringSubmatch(t) + if len(matches) == 3 { + pc.routerPath = matches[1] + methods := matches[2] + if methods == "" { + pc.methods = []string{"get"} + //pc.hasGet = true + } else { + pc.methods = strings.Split(methods, ",") + //pc.hasGet = strings.Contains(methods, "get") + } + pcs = append(pcs, pc) + } else { + return nil, errors.New("Router information is missing") } - pc.params[funcParamName] = p } } return From 464d080518e165b143763089e11c7c4571d86936 Mon Sep 17 00:00:00 2001 From: zhujianjian Date: Mon, 2 Jul 2018 11:21:06 +0800 Subject: [PATCH 23/32] fix httpcode in prod --- config.go | 1 + 1 file changed, 1 insertion(+) diff --git a/config.go b/config.go index eeeac8ee..c24709ff 100644 --- a/config.go +++ b/config.go @@ -182,6 +182,7 @@ func recoverPanic(ctx *context.Context) { if BConfig.RunMode == DEV && BConfig.EnableErrorsRender { showErr(err, ctx, stack) } + ctx.ResponseWriter.WriteHeader(500) } } From b3c46a87ac33027436cbf1b95ede475f0c638283 Mon Sep 17 00:00:00 2001 From: Colstuwjx Date: Thu, 5 Jul 2018 19:15:42 +0800 Subject: [PATCH 24/32] Fix: correct MaxIdleConnsPerHost value to net/http default 100. --- httplib/httplib.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/httplib/httplib.go b/httplib/httplib.go index 5d82ab33..47464e13 100644 --- a/httplib/httplib.go +++ b/httplib/httplib.go @@ -317,6 +317,7 @@ func (b *BeegoHTTPRequest) Body(data interface{}) *BeegoHTTPRequest { } return b } + // XMLBody adds request raw body encoding by XML. func (b *BeegoHTTPRequest) XMLBody(obj interface{}) (*BeegoHTTPRequest, error) { if b.req.Body == nil && obj != nil { @@ -330,6 +331,7 @@ func (b *BeegoHTTPRequest) XMLBody(obj interface{}) (*BeegoHTTPRequest, error) { } return b, nil } + // JSONBody adds request raw body encoding by JSON. func (b *BeegoHTTPRequest) JSONBody(obj interface{}) (*BeegoHTTPRequest, error) { if b.req.Body == nil && obj != nil { @@ -444,7 +446,7 @@ func (b *BeegoHTTPRequest) DoRequest() (resp *http.Response, err error) { TLSClientConfig: b.setting.TLSClientConfig, Proxy: b.setting.Proxy, Dial: TimeoutDialer(b.setting.ConnectTimeout, b.setting.ReadWriteTimeout), - MaxIdleConnsPerHost: -1, + MaxIdleConnsPerHost: 100, } } else { // if b.transport is *http.Transport then set the settings. From 6fec0a7831b4c05c9a583db04cb0ff233f26b15f Mon Sep 17 00:00:00 2001 From: guoshaowei Date: Thu, 12 Jul 2018 10:48:50 +0800 Subject: [PATCH 25/32] add session redis IdleTimeout config --- session/redis/sess_redis.go | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/session/redis/sess_redis.go b/session/redis/sess_redis.go index 54583152..5c382d61 100644 --- a/session/redis/sess_redis.go +++ b/session/redis/sess_redis.go @@ -37,6 +37,7 @@ import ( "strconv" "strings" "sync" + "time" "github.com/astaxie/beego/session" @@ -118,8 +119,8 @@ type Provider struct { } // SessionInit init redis session -// savepath like redis server addr,pool size,password,dbnum -// e.g. 127.0.0.1:6379,100,astaxie,0 +// savepath like redis server addr,pool size,password,dbnum,IdleTimeout second +// e.g. 127.0.0.1:6379,100,astaxie,0,30 func (rp *Provider) SessionInit(maxlifetime int64, savePath string) error { rp.maxlifetime = maxlifetime configs := strings.Split(savePath, ",") @@ -149,6 +150,13 @@ func (rp *Provider) SessionInit(maxlifetime int64, savePath string) error { } else { rp.dbNum = 0 } + var idleTimeout time.Duration = 0 + if len(configs) > 4 { + timeout, err := strconv.Atoi(configs[4]) + if err == nil && timeout > 0 { + idleTimeout = time.Duration(timeout) * time.Second + } + } rp.poollist = &redis.Pool{ Dial: func() (redis.Conn, error) { c, err := redis.Dial("tcp", rp.savePath) @@ -171,9 +179,11 @@ func (rp *Provider) SessionInit(maxlifetime int64, savePath string) error { } return c, err }, - MaxIdle: rp.poolsize, + MaxIdle: rp.poolsize, } + rp.poollist.IdleTimeout = idleTimeout + return rp.poollist.Get().Err() } From 0e0718d110268bb68652a087088c9054f50d97d4 Mon Sep 17 00:00:00 2001 From: Xingang Zhang <0x0400@users.noreply.github.com> Date: Tue, 17 Jul 2018 23:32:11 +0800 Subject: [PATCH 26/32] fix typo hasReuired --> hasRequired --- validation/validation.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/validation/validation.go b/validation/validation.go index 2781a72e..ca1e211f 100644 --- a/validation/validation.go +++ b/validation/validation.go @@ -365,10 +365,10 @@ func (v *Validation) Valid(obj interface{}) (b bool, err error) { return } - var hasReuired bool + var hasRequired bool for _, vf := range vfs { if vf.Name == "Required" { - hasReuired = true + hasRequired = true } currentField := objV.Field(i).Interface() @@ -382,7 +382,7 @@ func (v *Validation) Valid(obj interface{}) (b bool, err error) { chk := Required{""}.IsSatisfied(currentField) - if !hasReuired && v.RequiredFirst && !chk { + if !hasRequired && v.RequiredFirst && !chk { if _, ok := CanSkipFuncs[vf.Name]; ok { continue } From 293b54192f2bb0627c5b0f7f5de75411f409ed2c Mon Sep 17 00:00:00 2001 From: mohan2808 Date: Thu, 19 Jul 2018 18:51:16 +0530 Subject: [PATCH 27/32] send ErrNoRows if the query returns zero rows ... in method orm_queryset.All() --- orm/orm_queryset.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/orm/orm_queryset.go b/orm/orm_queryset.go index 4bab1d98..8b5d6fd2 100644 --- a/orm/orm_queryset.go +++ b/orm/orm_queryset.go @@ -198,7 +198,11 @@ func (o *querySet) PrepareInsert() (Inserter, error) { // query all data and map to containers. // cols means the columns when querying. func (o *querySet) All(container interface{}, cols ...string) (int64, error) { - return o.orm.alias.DbBaser.ReadBatch(o.orm.db, o, o.mi, o.cond, container, o.orm.alias.TZ, cols) + num, err := o.orm.alias.DbBaser.ReadBatch(o.orm.db, o, o.mi, o.cond, container, o.orm.alias.TZ, cols) + if num == 0 { + return 0, ErrNoRows + } + return num, err } // query one row data and map to containers. From be75f93d43d67dfd65e568e89f4a589b4ad8b13b Mon Sep 17 00:00:00 2001 From: astaxie Date: Fri, 20 Jul 2018 12:14:27 +0800 Subject: [PATCH 28/32] add miss dep --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index b514f5ec..c0e923d0 100644 --- a/.travis.yml +++ b/.travis.yml @@ -37,6 +37,7 @@ install: - go get -u github.com/mdempsky/unconvert - go get -u github.com/gordonklaus/ineffassign - go get -u github.com/golang/lint/golint + - go get -u github.com/go-redis/redis before_script: - psql --version - sh -c "if [ '$ORM_DRIVER' = 'postgres' ]; then psql -c 'create database orm_test;' -U postgres; fi" From 8f6bce3b87c50a0a49823ae5c4891399927472af Mon Sep 17 00:00:00 2001 From: astaxie Date: Fri, 20 Jul 2018 14:26:43 +0800 Subject: [PATCH 29/32] fix test case --- logs/file_test.go | 5 +++-- templatefunc_test.go | 6 +++--- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/logs/file_test.go b/logs/file_test.go index 626521b9..4216a497 100644 --- a/logs/file_test.go +++ b/logs/file_test.go @@ -135,7 +135,7 @@ func TestFileRotate_01(t *testing.T) { func TestFileRotate_02(t *testing.T) { fn1 := "rotate_day.log" - fn2 := "rotate_day." + time.Now().Add(-24*time.Hour).Format("2006-01-02") + ".log" + fn2 := "rotate_day." + time.Now().Add(-24*time.Hour).Format("2006-01-02") + ".001.log" testFileRotate(t, fn1, fn2) } @@ -150,7 +150,7 @@ func TestFileRotate_03(t *testing.T) { func TestFileRotate_04(t *testing.T) { fn1 := "rotate_day.log" - fn2 := "rotate_day." + time.Now().Add(-24*time.Hour).Format("2006-01-02") + ".log" + fn2 := "rotate_day." + time.Now().Add(-24*time.Hour).Format("2006-01-02") + ".001.log" testFileDailyRotate(t, fn1, fn2) } @@ -200,6 +200,7 @@ func testFileRotate(t *testing.T, fn1, fn2 string) { for _, file := range []string{fn1, fn2} { _, err := os.Stat(file) if err != nil { + t.Log(err) t.FailNow() } os.Remove(file) diff --git a/templatefunc_test.go b/templatefunc_test.go index 9df61125..aa82c26f 100644 --- a/templatefunc_test.go +++ b/templatefunc_test.go @@ -94,7 +94,7 @@ func TestCompareRelated(t *testing.T) { } func TestHtmlquote(t *testing.T) { - h := `<' ”“&">` + h := `<' ”“&">` s := `<' ”“&">` if Htmlquote(s) != h { t.Error("should be equal") @@ -102,8 +102,8 @@ func TestHtmlquote(t *testing.T) { } func TestHtmlunquote(t *testing.T) { - h := `<' ”“&">` - s := `<' ”“&">` + h := `<' ”“&">` + s := `<' ”“&">` if Htmlunquote(h) != s { t.Error("should be equal") } From b61c91d93d9d20fcbda284dabc98b8b63e8967b1 Mon Sep 17 00:00:00 2001 From: astaxie Date: Fri, 20 Jul 2018 14:47:11 +0800 Subject: [PATCH 30/32] remove unnecessary conversion --- config_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config_test.go b/config_test.go index c1973f7b..379fe1c6 100644 --- a/config_test.go +++ b/config_test.go @@ -75,7 +75,7 @@ func TestAssignConfig_02(t *testing.T) { jcf := &config.JSONConfig{} bs, _ = json.Marshal(configMap) - ac, _ := jcf.ParseData([]byte(bs)) + ac, _ := jcf.ParseData(bs) for _, i := range []interface{}{_BConfig, &_BConfig.Listen, &_BConfig.WebConfig, &_BConfig.Log, &_BConfig.WebConfig.Session} { assignSingleConfig(i, ac) From 0711c3289feb2705dea86eeb19495583fc859752 Mon Sep 17 00:00:00 2001 From: astaxie Date: Fri, 20 Jul 2018 19:58:56 +0800 Subject: [PATCH 31/32] fix the orm test --- app.go | 4 ++-- orm/orm_test.go | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/app.go b/app.go index 3a71d175..f286fb7a 100644 --- a/app.go +++ b/app.go @@ -117,7 +117,7 @@ func (app *App) Run(mws ...MiddleWare) { app.Server.Addr = httpsAddr if BConfig.Listen.EnableHTTPS || BConfig.Listen.EnableMutualHTTPS { go func() { - time.Sleep(20 * time.Microsecond) + time.Sleep(1000 * time.Microsecond) if BConfig.Listen.HTTPSPort != 0 { httpsAddr = fmt.Sprintf("%s:%d", BConfig.Listen.HTTPSAddr, BConfig.Listen.HTTPSPort) app.Server.Addr = httpsAddr @@ -163,7 +163,7 @@ func (app *App) Run(mws ...MiddleWare) { // run normal mode if BConfig.Listen.EnableHTTPS || BConfig.Listen.EnableMutualHTTPS { go func() { - time.Sleep(20 * time.Microsecond) + time.Sleep(1000 * time.Microsecond) if BConfig.Listen.HTTPSPort != 0 { app.Server.Addr = fmt.Sprintf("%s:%d", BConfig.Listen.HTTPSAddr, BConfig.Listen.HTTPSPort) } else if BConfig.Listen.EnableHTTP { diff --git a/orm/orm_test.go b/orm/orm_test.go index f1f2d85e..23bf4cde 100644 --- a/orm/orm_test.go +++ b/orm/orm_test.go @@ -1008,13 +1008,13 @@ func TestAll(t *testing.T) { qs = dORM.QueryTable("user") num, err = qs.Filter("user_name", "nothing").All(&users) - throwFailNow(t, err) + throwFailNow(t, AssertIs(err, ErrNoRows)) throwFailNow(t, AssertIs(num, 0)) var users3 []*User qs = dORM.QueryTable("user") num, err = qs.Filter("user_name", "nothing").All(&users3) - throwFailNow(t, err) + throwFailNow(t, AssertIs(err, ErrNoRows)) throwFailNow(t, AssertIs(num, 0)) throwFailNow(t, AssertIs(users3 == nil, false)) } From 868fc2a29f7ca61e5be78877b79df4e8cf5caa17 Mon Sep 17 00:00:00 2001 From: astaxie Date: Fri, 20 Jul 2018 22:45:44 +0800 Subject: [PATCH 32/32] fix go1.10.3 orm test failed --- .travis.yml | 2 +- orm/models_test.go | 90 ++++++++++++++++++++++++---------------------- 2 files changed, 48 insertions(+), 44 deletions(-) diff --git a/.travis.yml b/.travis.yml index c0e923d0..03a9d4fb 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,7 +2,7 @@ language: go go: - "1.9.2" - - "1.10.2" + - "1.10.3" services: - redis-server - mysql diff --git a/orm/models_test.go b/orm/models_test.go index d6c2b581..e3a635f2 100644 --- a/orm/models_test.go +++ b/orm/models_test.go @@ -433,53 +433,57 @@ var ( dDbBaser dbBaser ) +var ( + helpinfo = `need driver and source! + + Default DB Drivers. + + driver: url + mysql: https://github.com/go-sql-driver/mysql + sqlite3: https://github.com/mattn/go-sqlite3 + postgres: https://github.com/lib/pq + tidb: https://github.com/pingcap/tidb + + usage: + + go get -u github.com/astaxie/beego/orm + go get -u github.com/go-sql-driver/mysql + go get -u github.com/mattn/go-sqlite3 + go get -u github.com/lib/pq + go get -u github.com/pingcap/tidb + + #### MySQL + mysql -u root -e 'create database orm_test;' + export ORM_DRIVER=mysql + export ORM_SOURCE="root:@/orm_test?charset=utf8" + go test -v github.com/astaxie/beego/orm + + + #### Sqlite3 + export ORM_DRIVER=sqlite3 + export ORM_SOURCE='file:memory_test?mode=memory' + go test -v github.com/astaxie/beego/orm + + + #### PostgreSQL + psql -c 'create database orm_test;' -U postgres + export ORM_DRIVER=postgres + export ORM_SOURCE="user=postgres dbname=orm_test sslmode=disable" + go test -v github.com/astaxie/beego/orm + + #### TiDB + export ORM_DRIVER=tidb + export ORM_SOURCE='memory://test/test' + go test -v github.com/astaxie/beego/orm + + ` +) + func init() { Debug, _ = StrTo(DBARGS.Debug).Bool() if DBARGS.Driver == "" || DBARGS.Source == "" { - fmt.Println(`need driver and source! - -Default DB Drivers. - - driver: url - mysql: https://github.com/go-sql-driver/mysql - sqlite3: https://github.com/mattn/go-sqlite3 -postgres: https://github.com/lib/pq -tidb: https://github.com/pingcap/tidb - -usage: - -go get -u github.com/astaxie/beego/orm -go get -u github.com/go-sql-driver/mysql -go get -u github.com/mattn/go-sqlite3 -go get -u github.com/lib/pq -go get -u github.com/pingcap/tidb - -#### MySQL -mysql -u root -e 'create database orm_test;' -export ORM_DRIVER=mysql -export ORM_SOURCE="root:@/orm_test?charset=utf8" -go test -v github.com/astaxie/beego/orm - - -#### Sqlite3 -export ORM_DRIVER=sqlite3 -export ORM_SOURCE='file:memory_test?mode=memory' -go test -v github.com/astaxie/beego/orm - - -#### PostgreSQL -psql -c 'create database orm_test;' -U postgres -export ORM_DRIVER=postgres -export ORM_SOURCE="user=postgres dbname=orm_test sslmode=disable" -go test -v github.com/astaxie/beego/orm - -#### TiDB -export ORM_DRIVER=tidb -export ORM_SOURCE='memory://test/test' -go test -v github.com/astaxie/beego/orm - -`) + fmt.Println(helpinfo) os.Exit(2) }