From 48147f50d8e65838938adeffb239de45da4f91aa Mon Sep 17 00:00:00 2001 From: JessonChan Date: Thu, 17 Mar 2016 19:09:21 +0800 Subject: [PATCH 01/10] add some gzip future --- context/acceptencoder.go | 93 ++++++++++++++++++++++++++-------------- 1 file changed, 62 insertions(+), 31 deletions(-) diff --git a/context/acceptencoder.go b/context/acceptencoder.go index 033d9ca8..2e5af83a 100644 --- a/context/acceptencoder.go +++ b/context/acceptencoder.go @@ -25,8 +25,35 @@ import ( "strconv" "strings" "sync" + + "github.com/astaxie/beego/config" ) +var ( +//Content will only be compressed if content length is either unknown or greater than minGzipSize. + gzipMinLength int +//Default size==20B like nginx + defaultGzipMinLength=20 +//The compression level used for deflate compression. (0-9). + gzipCompressLevel int +//List of HTTP methods to compress. If not set, only GET requests are compressed. + includedMethods map[string]bool + getMethodOnly bool +) + +func InitGzip(cf config.Configer) { + gzipMinLength = cf.DefaultInt("gzipMinLength", defaultGzipMinLength) + gzipCompressLevel = cf.DefaultInt("gzipCompressLevel", flate.DefaultCompression) + if gzipCompressLevel < flate.DefaultCompression || gzipCompressLevel > flate.BestCompression { + gzipCompressLevel = flate.BestSpeed + } + methods := cf.DefaultStrings("includedMethods", []string{"GET"}) + getMethodOnly = len(methods) == 1 && strings.ToUpper(methods[0]) == "GET" + for _, v := range methods { + includedMethods[strings.ToUpper(v)] = true + } +} + type resetWriter interface { io.Writer Reset(w io.Writer) @@ -41,20 +68,20 @@ func (n nopResetWriter) Reset(w io.Writer) { } type acceptEncoder struct { - name string - levelEncode func(int) resetWriter - bestSpeedPool *sync.Pool - bestCompressionPool *sync.Pool + name string + levelEncode func(int) resetWriter + customCompressLevelPool *sync.Pool + bestCompressionPool *sync.Pool } func (ac acceptEncoder) encode(wr io.Writer, level int) resetWriter { - if ac.bestSpeedPool == nil || ac.bestCompressionPool == nil { + if ac.customCompressLevelPool == nil || ac.bestCompressionPool == nil { return nopResetWriter{wr} } var rwr resetWriter switch level { case flate.BestSpeed: - rwr = ac.bestSpeedPool.Get().(resetWriter) + rwr = ac.customCompressLevelPool.Get().(resetWriter) case flate.BestCompression: rwr = ac.bestCompressionPool.Get().(resetWriter) default: @@ -65,13 +92,16 @@ func (ac acceptEncoder) encode(wr io.Writer, level int) resetWriter { } func (ac acceptEncoder) put(wr resetWriter, level int) { - if ac.bestSpeedPool == nil || ac.bestCompressionPool == nil { + if ac.customCompressLevelPool == nil || ac.bestCompressionPool == nil { return } wr.Reset(nil) + //notice + //compressionLevel==BestCompression DOES NOT MATTER + //sync.Pool will not memory leak switch level { - case flate.BestSpeed: - ac.bestSpeedPool.Put(wr) + case gzipCompressLevel: + ac.customCompressLevelPool.Put(wr) case flate.BestCompression: ac.bestCompressionPool.Put(wr) } @@ -79,28 +109,22 @@ func (ac acceptEncoder) put(wr resetWriter, level int) { var ( noneCompressEncoder = acceptEncoder{"", nil, nil, nil} - gzipCompressEncoder = acceptEncoder{"gzip", - func(level int) resetWriter { wr, _ := gzip.NewWriterLevel(nil, level); return wr }, - &sync.Pool{ - New: func() interface{} { wr, _ := gzip.NewWriterLevel(nil, flate.BestSpeed); return wr }, - }, - &sync.Pool{ - New: func() interface{} { wr, _ := gzip.NewWriterLevel(nil, flate.BestCompression); return wr }, - }, + gzipCompressEncoder = acceptEncoder{ + name: "gzip", + levelEncode: func(level int) resetWriter { wr, _ := gzip.NewWriterLevel(nil, level); return wr }, + customCompressLevelPool: &sync.Pool{New: func() interface{} { wr, _ := gzip.NewWriterLevel(nil, gzipCompressLevel); return wr }}, + bestCompressionPool: &sync.Pool{New: func() interface{} { wr, _ := gzip.NewWriterLevel(nil, flate.BestCompression); return wr }}, } - //according to the sec :http://tools.ietf.org/html/rfc2616#section-3.5 ,the deflate compress in http is zlib indeed - //deflate - //The "zlib" format defined in RFC 1950 [31] in combination with - //the "deflate" compression mechanism described in RFC 1951 [29]. - deflateCompressEncoder = acceptEncoder{"deflate", - func(level int) resetWriter { wr, _ := zlib.NewWriterLevel(nil, level); return wr }, - &sync.Pool{ - New: func() interface{} { wr, _ := zlib.NewWriterLevel(nil, flate.BestSpeed); return wr }, - }, - &sync.Pool{ - New: func() interface{} { wr, _ := zlib.NewWriterLevel(nil, flate.BestCompression); return wr }, - }, +//according to the sec :http://tools.ietf.org/html/rfc2616#section-3.5 ,the deflate compress in http is zlib indeed +//deflate +//The "zlib" format defined in RFC 1950 [31] in combination with +//the "deflate" compression mechanism described in RFC 1951 [29]. + deflateCompressEncoder = acceptEncoder{ + name: "deflate", + levelEncode: func(level int) resetWriter { wr, _ := zlib.NewWriterLevel(nil, level); return wr }, + customCompressLevelPool: &sync.Pool{New: func() interface{} { wr, _ := zlib.NewWriterLevel(nil, gzipCompressLevel); return wr }}, + bestCompressionPool: &sync.Pool{New: func() interface{} { wr, _ := zlib.NewWriterLevel(nil, flate.BestCompression); return wr }}, } ) @@ -120,7 +144,11 @@ func WriteFile(encoding string, writer io.Writer, file *os.File) (bool, string, // WriteBody reads writes content to writer by the specific encoding(gzip/deflate) func WriteBody(encoding string, writer io.Writer, content []byte) (bool, string, error) { - return writeLevel(encoding, writer, bytes.NewReader(content), flate.BestSpeed) + if encoding == "" || len(content) < gzipMinLength { + _, err := writer.Write(content) + return false, "", err + } + return writeLevel(encoding, writer, bytes.NewReader(content), gzipCompressLevel) } // writeLevel reads from reader,writes to writer by specific encoding and compress level @@ -156,7 +184,10 @@ func ParseEncoding(r *http.Request) string { if r == nil { return "" } - return parseEncoding(r) + if (getMethodOnly && r.Method == "GET") || includedMethods[r.Method] { + return parseEncoding(r) + } + return "" } type q struct { From 5a9bff2000701fd565027c184cfa010c7c820592 Mon Sep 17 00:00:00 2001 From: JessonChan Date: Thu, 17 Mar 2016 19:09:38 +0800 Subject: [PATCH 02/10] init gzip level --- beego.go | 8 +++++--- hooks.go | 8 ++++++++ 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/beego.go b/beego.go index 8f82cdcf..89e4f671 100644 --- a/beego.go +++ b/beego.go @@ -22,12 +22,12 @@ import ( ) const ( - // VERSION represent beego web framework version. +// VERSION represent beego web framework version. VERSION = "1.6.1" - // DEV is for develop +// DEV is for develop DEV = "dev" - // PROD is for production +// PROD is for production PROD = "prod" ) @@ -51,6 +51,7 @@ func AddAPPStartHook(hf hookfunc) { // beego.Run(":8089") // beego.Run("127.0.0.1:8089") func Run(params ...string) { + initBeforeHTTPRun() if len(params) > 0 && params[0] != "" { @@ -74,6 +75,7 @@ func initBeforeHTTPRun() { AddAPPStartHook(registerDocs) AddAPPStartHook(registerTemplate) AddAPPStartHook(registerAdmin) + AddAPPStartHook(registerGzip) for _, hk := range hooks { if err := hk(); err != nil { diff --git a/hooks.go b/hooks.go index 59b10b32..1a7937b5 100644 --- a/hooks.go +++ b/hooks.go @@ -6,6 +6,7 @@ import ( "net/http" "path/filepath" + "github.com/astaxie/beego/context" "github.com/astaxie/beego/session" ) @@ -91,3 +92,10 @@ func registerAdmin() error { } return nil } + +func registerGzip() error { + if BConfig.EnableGzip { + context.InitGzip(AppConfig) + } + return nil +} From 35e34261abd45205becb700e302c7a5972c98fa1 Mon Sep 17 00:00:00 2001 From: JessonChan Date: Thu, 17 Mar 2016 19:40:29 +0800 Subject: [PATCH 03/10] gzip method support --- context/acceptencoder.go | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/context/acceptencoder.go b/context/acceptencoder.go index 2e5af83a..21ede1a5 100644 --- a/context/acceptencoder.go +++ b/context/acceptencoder.go @@ -30,13 +30,13 @@ import ( ) var ( -//Content will only be compressed if content length is either unknown or greater than minGzipSize. + //Content will only be compressed if content length is either unknown or greater than minGzipSize. gzipMinLength int -//Default size==20B like nginx - defaultGzipMinLength=20 -//The compression level used for deflate compression. (0-9). + //Default size==20B like nginx + defaultGzipMinLength = 20 + //The compression level used for deflate compression. (0-9). gzipCompressLevel int -//List of HTTP methods to compress. If not set, only GET requests are compressed. + //List of HTTP methods to compress. If not set, only GET requests are compressed. includedMethods map[string]bool getMethodOnly bool ) @@ -96,9 +96,11 @@ func (ac acceptEncoder) put(wr resetWriter, level int) { return } wr.Reset(nil) + //notice //compressionLevel==BestCompression DOES NOT MATTER //sync.Pool will not memory leak + switch level { case gzipCompressLevel: ac.customCompressLevelPool.Put(wr) @@ -116,10 +118,10 @@ var ( bestCompressionPool: &sync.Pool{New: func() interface{} { wr, _ := gzip.NewWriterLevel(nil, flate.BestCompression); return wr }}, } -//according to the sec :http://tools.ietf.org/html/rfc2616#section-3.5 ,the deflate compress in http is zlib indeed -//deflate -//The "zlib" format defined in RFC 1950 [31] in combination with -//the "deflate" compression mechanism described in RFC 1951 [29]. + //according to the sec :http://tools.ietf.org/html/rfc2616#section-3.5 ,the deflate compress in http is zlib indeed + //deflate + //The "zlib" format defined in RFC 1950 [31] in combination with + //the "deflate" compression mechanism described in RFC 1951 [29]. deflateCompressEncoder = acceptEncoder{ name: "deflate", levelEncode: func(level int) resetWriter { wr, _ := zlib.NewWriterLevel(nil, level); return wr }, From 57eace07a79f80aaeab7b0d06058a8e81ee05af3 Mon Sep 17 00:00:00 2001 From: JessonChan Date: Thu, 17 Mar 2016 19:52:09 +0800 Subject: [PATCH 04/10] comment update --- context/acceptencoder.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/context/acceptencoder.go b/context/acceptencoder.go index 21ede1a5..bc67bb5e 100644 --- a/context/acceptencoder.go +++ b/context/acceptencoder.go @@ -30,7 +30,7 @@ import ( ) var ( - //Content will only be compressed if content length is either unknown or greater than minGzipSize. + //Content will only be compressed if content length is either unknown or greater than gzipMinLength. gzipMinLength int //Default size==20B like nginx defaultGzipMinLength = 20 From 0b401481ef1c9abc920996d6cbe67818970984bf Mon Sep 17 00:00:00 2001 From: JessonChan Date: Thu, 17 Mar 2016 19:59:48 +0800 Subject: [PATCH 05/10] go fmt the comment --- beego.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/beego.go b/beego.go index 89e4f671..65dd79cb 100644 --- a/beego.go +++ b/beego.go @@ -22,12 +22,12 @@ import ( ) const ( -// VERSION represent beego web framework version. + // VERSION represent beego web framework version. VERSION = "1.6.1" -// DEV is for develop + // DEV is for develop DEV = "dev" -// PROD is for production + // PROD is for production PROD = "prod" ) From 9f21928a90e6782e92cfc26462d66223557b680d Mon Sep 17 00:00:00 2001 From: JessonChan Date: Thu, 17 Mar 2016 20:07:24 +0800 Subject: [PATCH 06/10] some typo fixed --- context/acceptencoder.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/context/acceptencoder.go b/context/acceptencoder.go index bc67bb5e..bc048e77 100644 --- a/context/acceptencoder.go +++ b/context/acceptencoder.go @@ -32,7 +32,7 @@ import ( var ( //Content will only be compressed if content length is either unknown or greater than gzipMinLength. gzipMinLength int - //Default size==20B like nginx + //Default size==20B same as nginx defaultGzipMinLength = 20 //The compression level used for deflate compression. (0-9). gzipCompressLevel int @@ -43,7 +43,7 @@ var ( func InitGzip(cf config.Configer) { gzipMinLength = cf.DefaultInt("gzipMinLength", defaultGzipMinLength) - gzipCompressLevel = cf.DefaultInt("gzipCompressLevel", flate.DefaultCompression) + gzipCompressLevel = cf.DefaultInt("gzipCompressLevel", flate.BestSpeed) if gzipCompressLevel < flate.DefaultCompression || gzipCompressLevel > flate.BestCompression { gzipCompressLevel = flate.BestSpeed } From 4caf044be2e7b18d52a6c525c7d5d209ed7d987a Mon Sep 17 00:00:00 2001 From: JessonChan Date: Fri, 18 Mar 2016 15:18:00 +0800 Subject: [PATCH 07/10] getMethodOnly assign fixed --- context/acceptencoder.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/context/acceptencoder.go b/context/acceptencoder.go index bc048e77..e73744f5 100644 --- a/context/acceptencoder.go +++ b/context/acceptencoder.go @@ -48,7 +48,7 @@ func InitGzip(cf config.Configer) { gzipCompressLevel = flate.BestSpeed } methods := cf.DefaultStrings("includedMethods", []string{"GET"}) - getMethodOnly = len(methods) == 1 && strings.ToUpper(methods[0]) == "GET" + getMethodOnly = (len(methods) == 0) || (len(methods) == 1 && strings.ToUpper(methods[0]) == "GET") for _, v := range methods { includedMethods[strings.ToUpper(v)] = true } From 959b9a5a580e2e232b92680f640b1865bf3af24d Mon Sep 17 00:00:00 2001 From: JessonChan Date: Mon, 21 Mar 2016 09:32:41 +0800 Subject: [PATCH 08/10] config index out of range bug fixed --- config.go | 2 +- context/acceptencoder.go | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/config.go b/config.go index 2761e7cb..15d11386 100644 --- a/config.go +++ b/config.go @@ -353,7 +353,7 @@ func (b *beegoAppConfig) String(key string) string { } func (b *beegoAppConfig) Strings(key string) []string { - if v := b.innerConfig.Strings(BConfig.RunMode + "::" + key); v[0] != "" { + if v := b.innerConfig.Strings(BConfig.RunMode + "::" + key); len(v) > 0 { return v } return b.innerConfig.Strings(key) diff --git a/context/acceptencoder.go b/context/acceptencoder.go index e73744f5..fc2775ce 100644 --- a/context/acceptencoder.go +++ b/context/acceptencoder.go @@ -49,6 +49,7 @@ func InitGzip(cf config.Configer) { } methods := cf.DefaultStrings("includedMethods", []string{"GET"}) getMethodOnly = (len(methods) == 0) || (len(methods) == 1 && strings.ToUpper(methods[0]) == "GET") + includedMethods = make(map[string]bool, len(methods)) for _, v := range methods { includedMethods[strings.ToUpper(v)] = true } From 4db78f243e90b1eab3c4b78b6fb186ad0d18cbec Mon Sep 17 00:00:00 2001 From: JessonChan Date: Tue, 22 Mar 2016 16:42:42 +0800 Subject: [PATCH 09/10] change the function args of init gzip method --- context/acceptencoder.go | 15 +++++++-------- hooks.go | 6 +++++- 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/context/acceptencoder.go b/context/acceptencoder.go index fc2775ce..1c3cf1d5 100644 --- a/context/acceptencoder.go +++ b/context/acceptencoder.go @@ -25,15 +25,13 @@ import ( "strconv" "strings" "sync" - - "github.com/astaxie/beego/config" ) var ( - //Content will only be compressed if content length is either unknown or greater than gzipMinLength. - gzipMinLength int //Default size==20B same as nginx defaultGzipMinLength = 20 + //Content will only be compressed if content length is either unknown or greater than gzipMinLength. + gzipMinLength = defaultGzipMinLength //The compression level used for deflate compression. (0-9). gzipCompressLevel int //List of HTTP methods to compress. If not set, only GET requests are compressed. @@ -41,13 +39,14 @@ var ( getMethodOnly bool ) -func InitGzip(cf config.Configer) { - gzipMinLength = cf.DefaultInt("gzipMinLength", defaultGzipMinLength) - gzipCompressLevel = cf.DefaultInt("gzipCompressLevel", flate.BestSpeed) +func InitGzip(minLength, compressLevel int, methods []string) { + if minLength >= 0 { + gzipMinLength = minLength + } + gzipCompressLevel = compressLevel if gzipCompressLevel < flate.DefaultCompression || gzipCompressLevel > flate.BestCompression { gzipCompressLevel = flate.BestSpeed } - methods := cf.DefaultStrings("includedMethods", []string{"GET"}) getMethodOnly = (len(methods) == 0) || (len(methods) == 1 && strings.ToUpper(methods[0]) == "GET") includedMethods = make(map[string]bool, len(methods)) for _, v := range methods { diff --git a/hooks.go b/hooks.go index 1a7937b5..674f2858 100644 --- a/hooks.go +++ b/hooks.go @@ -95,7 +95,11 @@ func registerAdmin() error { func registerGzip() error { if BConfig.EnableGzip { - context.InitGzip(AppConfig) + context.InitGzip( + AppConfig.DefaultInt("gzipMinLength", -1), + AppConfig.DefaultInt("gzipCompressLevel", -1), + AppConfig.DefaultStrings("includedMethods", []string{"GET"}), + ) } return nil } From 7bad3d1c67f8d3da1ef12071e48e31c2d82f8095 Mon Sep 17 00:00:00 2001 From: JessonChan Date: Tue, 22 Mar 2016 16:47:11 +0800 Subject: [PATCH 10/10] change the compress leve to [0~9] --- context/acceptencoder.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/context/acceptencoder.go b/context/acceptencoder.go index 1c3cf1d5..cb735445 100644 --- a/context/acceptencoder.go +++ b/context/acceptencoder.go @@ -44,7 +44,7 @@ func InitGzip(minLength, compressLevel int, methods []string) { gzipMinLength = minLength } gzipCompressLevel = compressLevel - if gzipCompressLevel < flate.DefaultCompression || gzipCompressLevel > flate.BestCompression { + if gzipCompressLevel < flate.NoCompression || gzipCompressLevel > flate.BestCompression { gzipCompressLevel = flate.BestSpeed } getMethodOnly = (len(methods) == 0) || (len(methods) == 1 && strings.ToUpper(methods[0]) == "GET")