From 6a2ee371a5d76b0bf34307d8ec850024a029c6dc Mon Sep 17 00:00:00 2001 From: fugr Date: Mon, 9 Jan 2017 21:04:11 +0800 Subject: [PATCH 01/15] avoid creating new file to implements Config There is no need to create new file in ParseData(data []byte) (Configer, error).Tet's make code simply. --- config/ini.go | 43 +++++++++++++++++++++++-------------------- config/xml/xml.go | 28 ++++++++-------------------- config/yaml/yaml.go | 24 +++++++++++++----------- 3 files changed, 44 insertions(+), 51 deletions(-) diff --git a/config/ini.go b/config/ini.go index b3332bd8..6b78f02a 100644 --- a/config/ini.go +++ b/config/ini.go @@ -18,16 +18,13 @@ import ( "bufio" "bytes" "errors" - "fmt" "io" "io/ioutil" "os" - "path" "path/filepath" "strconv" "strings" "sync" - "time" ) var ( @@ -52,24 +49,26 @@ func (ini *IniConfig) Parse(name string) (Configer, error) { } func (ini *IniConfig) parseFile(name string) (*IniConfigContainer, error) { - file, err := os.Open(name) + data, err := ioutil.ReadFile(name) if err != nil { return nil, err } + return ini.parseData(data) +} + +func (ini *IniConfig) parseData(data []byte) (*IniConfigContainer, error) { cfg := &IniConfigContainer{ - file.Name(), - make(map[string]map[string]string), - make(map[string]string), - make(map[string]string), - sync.RWMutex{}, + data: make(map[string]map[string]string), + sectionComment: make(map[string]string), + keyComment: make(map[string]string), + RWMutex: sync.RWMutex{}, } cfg.Lock() defer cfg.Unlock() - defer file.Close() var comment bytes.Buffer - buf := bufio.NewReader(file) + buf := bufio.NewReader(bytes.NewBuffer(data)) // check the BOM head, err := buf.Peek(3) if err == nil && head[0] == 239 && head[1] == 187 && head[2] == 191 { @@ -130,16 +129,24 @@ func (ini *IniConfig) parseFile(name string) (*IniConfigContainer, error) { // handle include "other.conf" if len(keyValue) == 1 && strings.HasPrefix(key, "include") { + includefiles := strings.Fields(key) if includefiles[0] == "include" && len(includefiles) == 2 { + otherfile := strings.Trim(includefiles[1], "\"") if !filepath.IsAbs(otherfile) { - otherfile = filepath.Join(filepath.Dir(name), otherfile) + dir, err := os.Getwd() + if err != nil { + return nil, err + } + otherfile = filepath.Join(dir, otherfile) } + i, err := ini.parseFile(otherfile) if err != nil { return nil, err } + for sec, dt := range i.data { if _, ok := cfg.data[sec]; !ok { cfg.data[sec] = make(map[string]string) @@ -148,12 +155,15 @@ func (ini *IniConfig) parseFile(name string) (*IniConfigContainer, error) { cfg.data[sec][k] = v } } + for sec, comm := range i.sectionComment { cfg.sectionComment[sec] = comm } + for k, comm := range i.keyComment { cfg.keyComment[k] = comm } + continue } } @@ -178,19 +188,12 @@ func (ini *IniConfig) parseFile(name string) (*IniConfigContainer, error) { // ParseData parse ini the data func (ini *IniConfig) ParseData(data []byte) (Configer, error) { - // Save memory data to temporary file - tmpName := path.Join(os.TempDir(), "beego", fmt.Sprintf("%d", time.Now().Nanosecond())) - os.MkdirAll(path.Dir(tmpName), os.ModePerm) - if err := ioutil.WriteFile(tmpName, data, 0655); err != nil { - return nil, err - } - return ini.Parse(tmpName) + return ini.parseData(data) } // IniConfigContainer A Config represents the ini configuration. // When set and get value, support key as section:name type. type IniConfigContainer struct { - filename string data map[string]map[string]string // section=> key:val sectionComment map[string]string // section : comment keyComment map[string]string // id: []{comment, key...}; id 1 is for main comment. diff --git a/config/xml/xml.go b/config/xml/xml.go index 66115714..b82bf403 100644 --- a/config/xml/xml.go +++ b/config/xml/xml.go @@ -35,11 +35,9 @@ import ( "fmt" "io/ioutil" "os" - "path" "strconv" "strings" "sync" - "time" "github.com/astaxie/beego/config" "github.com/beego/x2j" @@ -52,36 +50,26 @@ type Config struct{} // Parse returns a ConfigContainer with parsed xml config map. func (xc *Config) Parse(filename string) (config.Configer, error) { - file, err := os.Open(filename) + context, err := ioutil.ReadFile(filename) if err != nil { return nil, err } - defer file.Close() + return xc.ParseData(context) +} + +// ParseData xml data +func (xc *Config) ParseData(data []byte) (config.Configer, error) { x := &ConfigContainer{data: make(map[string]interface{})} - content, err := ioutil.ReadAll(file) - if err != nil { - return nil, err - } - d, err := x2j.DocToMap(string(content)) + d, err := x2j.DocToMap(string(data)) if err != nil { return nil, err } x.data = config.ExpandValueEnvForMap(d["config"].(map[string]interface{})) - return x, nil -} -// ParseData xml data -func (xc *Config) ParseData(data []byte) (config.Configer, error) { - // Save memory data to temporary file - tmpName := path.Join(os.TempDir(), "beego", fmt.Sprintf("%d", time.Now().Nanosecond())) - os.MkdirAll(path.Dir(tmpName), os.ModePerm) - if err := ioutil.WriteFile(tmpName, data, 0655); err != nil { - return nil, err - } - return xc.Parse(tmpName) + return x, nil } // ConfigContainer A Config represents the xml configuration. diff --git a/config/yaml/yaml.go b/config/yaml/yaml.go index e3260215..bcef4a20 100644 --- a/config/yaml/yaml.go +++ b/config/yaml/yaml.go @@ -37,10 +37,8 @@ import ( "io/ioutil" "log" "os" - "path" "strings" "sync" - "time" "github.com/astaxie/beego/config" "github.com/beego/goyaml2" @@ -63,26 +61,30 @@ func (yaml *Config) Parse(filename string) (y config.Configer, err error) { // ParseData parse yaml data func (yaml *Config) ParseData(data []byte) (config.Configer, error) { - // Save memory data to temporary file - tmpName := path.Join(os.TempDir(), "beego", fmt.Sprintf("%d", time.Now().Nanosecond())) - os.MkdirAll(path.Dir(tmpName), os.ModePerm) - if err := ioutil.WriteFile(tmpName, data, 0655); err != nil { + cnf, err := parseYML(data) + if err != nil { return nil, err } - return yaml.Parse(tmpName) + + return &ConfigContainer{ + data: cnf, + }, nil } // ReadYmlReader Read yaml file to map. // if json like, use json package, unless goyaml2 package. func ReadYmlReader(path string) (cnf map[string]interface{}, err error) { - f, err := os.Open(path) + buf, err := ioutil.ReadFile(path) if err != nil { return } - defer f.Close() - buf, err := ioutil.ReadAll(f) - if err != nil || len(buf) < 3 { + return parseYML(buf) +} + +// parseYML parse yaml formatted []byte to map. +func parseYML(buf []byte) (cnf map[string]interface{}, err error) { + if len(buf) < 3 { return } From fbc9f8e6409dd0122d49ccf8a8e7696c80de442c Mon Sep 17 00:00:00 2001 From: Faissal Elamraoui Date: Fri, 13 Jan 2017 17:10:54 +0100 Subject: [PATCH 02/15] Package env for working with env variables inside Beego The package env makes it trivial to work with environment variables. It allows getting values with defaults as a fallback. New ENV variables can be set safely on the current process environment. --- env/env.go | 89 +++++++++++++++++++++++++++++++++++++++++++++++++ env/env_test.go | 75 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 164 insertions(+) create mode 100644 env/env.go create mode 100644 env/env_test.go diff --git a/env/env.go b/env/env.go new file mode 100644 index 00000000..014501f5 --- /dev/null +++ b/env/env.go @@ -0,0 +1,89 @@ +// Copyright 2014 beego Author. All Rights Reserved. +// Copyright 2017 Faissal Elamraoui. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package env + +import ( + "fmt" + "os" + "strings" + "sync" +) + +var env struct { + data map[string]string + lock *sync.RWMutex +} + +func init() { + env.data = make(map[string]string) + env.lock = &sync.RWMutex{} + for _, e := range os.Environ() { + splits := strings.Split(e, "=") + env.data[splits[0]] = os.Getenv(splits[0]) + } +} + +// Get returns a value by key. +// If the key does not exist, the default value will be returned. +func Get(key string, defVal string) string { + env.lock.RLock() + defer env.lock.RUnlock() + + if val, ok := env.data[key]; ok { + return val + } + return defVal +} + +// MustGet returns a value by key. +// If the key does not exist, it will return an error. +func MustGet(key string) (string, error) { + env.lock.RLock() + defer env.lock.RUnlock() + + if val, ok := env.data[key]; ok { + return val, nil + } + return "", fmt.Errorf("no env variable with %s", key) +} + +// Set sets a value in the ENV copy. +// This does not affect the child process environment. +func Set(key string, value string) { + env.lock.Lock() + defer env.lock.Unlock() + env.data[key] = value +} + +// MustSet sets a value in the ENV copy and the child process environment. +// It returns an error in case the set operation failed. +func MustSet(key string, value string) error { + env.lock.Lock() + defer env.lock.Unlock() + + err := os.Setenv(key, value) + if err != nil { + return err + } + env.data[key] = value + return nil +} + +// GetAll returns all keys/values in the current child process environment. +func GetAll() map[string]string { + env.lock.RLock() + defer env.lock.RUnlock() + return env.data +} diff --git a/env/env_test.go b/env/env_test.go new file mode 100644 index 00000000..3f1d4dba --- /dev/null +++ b/env/env_test.go @@ -0,0 +1,75 @@ +// Copyright 2014 beego Author. All Rights Reserved. +// Copyright 2017 Faissal Elamraoui. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package env + +import ( + "os" + "testing" +) + +func TestEnvGet(t *testing.T) { + gopath := Get("GOPATH", "") + if gopath != os.Getenv("GOPATH") { + t.Error("expected GOPATH not empty.") + } + + noExistVar := Get("NOEXISTVAR", "foo") + if noExistVar != "foo" { + t.Errorf("expected NOEXISTVAR to equal foo, got %s.", noExistVar) + } +} + +func TestEnvMustGet(t *testing.T) { + gopath, err := MustGet("GOPATH") + if err != nil { + t.Error(err) + } + + if gopath != os.Getenv("GOPATH") { + t.Errorf("expected GOPATH to be the same, got %s.", gopath) + } + + _, err = MustGet("NOEXISTVAR") + if err == nil { + t.Error("expected error to be non-nil") + } +} + +func TestEnvSet(t *testing.T) { + Set("MYVAR", "foo") + myVar := Get("MYVAR", "bar") + if myVar != "foo" { + t.Errorf("expected MYVAR to equal foo, got %s.", myVar) + } +} + +func TestEnvMustSet(t *testing.T) { + err := MustSet("FOO", "bar") + if err != nil { + t.Error(err) + } + + fooVar := os.Getenv("FOO") + if fooVar != "bar" { + t.Errorf("expected FOO variable to equal bar, got %s.", fooVar) + } +} + +func TestEnvGetAll(t *testing.T) { + envMap := GetAll() + if len(envMap) == 0 { + t.Error("expected environment not empty.") + } +} From e32d173b0d43c3987d0f2906467f999be5ba4d7a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=8D=9C=E6=9C=A8?= Date: Sat, 14 Jan 2017 15:03:49 +0800 Subject: [PATCH 03/15] fix the bug to prevent rewrite t the t have paresed in line 212. --- template.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/template.go b/template.go index 5415f5f0..343ef223 100644 --- a/template.go +++ b/template.go @@ -224,7 +224,7 @@ func getTplDeep(root, file, parent string, t *template.Template) (*template.Temp if !HasTemplateExt(m[1]) { continue } - t, _, err = getTplDeep(root, m[1], file, t) + _, _, err = getTplDeep(root, m[1], file, t) if err != nil { return nil, [][]string{}, err } From 957c0630c0296e7628fdc08f7da720e597b1aa09 Mon Sep 17 00:00:00 2001 From: Faissal Elamraoui Date: Sat, 14 Jan 2017 10:15:02 +0100 Subject: [PATCH 04/15] moved the env package to config/ --- {env => config/env}/env.go | 0 {env => config/env}/env_test.go | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename {env => config/env}/env.go (100%) rename {env => config/env}/env_test.go (100%) diff --git a/env/env.go b/config/env/env.go similarity index 100% rename from env/env.go rename to config/env/env.go diff --git a/env/env_test.go b/config/env/env_test.go similarity index 100% rename from env/env_test.go rename to config/env/env_test.go From 24d8290a3f5c9e21ad585d75ee142cb8c3582e57 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9D=9C=E4=BA=9A=E5=93=B2?= Date: Mon, 16 Jan 2017 10:32:33 +0800 Subject: [PATCH 05/15] fix filter with __ne bug --- orm/db.go | 1 + 1 file changed, 1 insertion(+) diff --git a/orm/db.go b/orm/db.go index 30d8ae4e..bca6071d 100644 --- a/orm/db.go +++ b/orm/db.go @@ -48,6 +48,7 @@ var ( "lte": true, "eq": true, "nq": true, + "ne": true, "startswith": true, "endswith": true, "istartswith": true, From 126dbdae2f8a4daa2bd5ddb37a07e4d6bd71281f Mon Sep 17 00:00:00 2001 From: Faissal Elamraoui Date: Mon, 16 Jan 2017 10:08:32 +0100 Subject: [PATCH 06/15] use BeeMap instead of a regular map --- config/env/env.go | 52 ++++++++++++++++++++++------------------------- 1 file changed, 24 insertions(+), 28 deletions(-) diff --git a/config/env/env.go b/config/env/env.go index 014501f5..a819e51a 100644 --- a/config/env/env.go +++ b/config/env/env.go @@ -18,31 +18,25 @@ import ( "fmt" "os" "strings" - "sync" + + "github.com/astaxie/beego/utils" ) -var env struct { - data map[string]string - lock *sync.RWMutex -} +var env *utils.BeeMap func init() { - env.data = make(map[string]string) - env.lock = &sync.RWMutex{} + env = utils.NewBeeMap() for _, e := range os.Environ() { splits := strings.Split(e, "=") - env.data[splits[0]] = os.Getenv(splits[0]) + env.Set(splits[0], os.Getenv(splits[0])) } } // Get returns a value by key. // If the key does not exist, the default value will be returned. func Get(key string, defVal string) string { - env.lock.RLock() - defer env.lock.RUnlock() - - if val, ok := env.data[key]; ok { - return val + if val := env.Get(key); val != nil { + return val.(string) } return defVal } @@ -50,11 +44,8 @@ func Get(key string, defVal string) string { // MustGet returns a value by key. // If the key does not exist, it will return an error. func MustGet(key string) (string, error) { - env.lock.RLock() - defer env.lock.RUnlock() - - if val, ok := env.data[key]; ok { - return val, nil + if val := env.Get(key); val != nil { + return val.(string), nil } return "", fmt.Errorf("no env variable with %s", key) } @@ -62,28 +53,33 @@ func MustGet(key string) (string, error) { // Set sets a value in the ENV copy. // This does not affect the child process environment. func Set(key string, value string) { - env.lock.Lock() - defer env.lock.Unlock() - env.data[key] = value + env.Set(key, value) } // MustSet sets a value in the ENV copy and the child process environment. // It returns an error in case the set operation failed. func MustSet(key string, value string) error { - env.lock.Lock() - defer env.lock.Unlock() - err := os.Setenv(key, value) if err != nil { return err } - env.data[key] = value + env.Set(key, value) return nil } // GetAll returns all keys/values in the current child process environment. func GetAll() map[string]string { - env.lock.RLock() - defer env.lock.RUnlock() - return env.data + items := env.Items() + envs := make(map[string]string, env.Count()) + + for key, val := range items { + switch key := key.(type) { + case string: + switch val := val.(type) { + case string: + envs[key] = val + } + } + } + return envs } From 5c76f621034043bbff4b8160bbed37f1c9aa4200 Mon Sep 17 00:00:00 2001 From: kerwin Date: Wed, 18 Jan 2017 17:04:23 +0800 Subject: [PATCH 07/15] Add GetCond func to querySet --- orm/orm_queryset.go | 5 +++++ orm/types.go | 10 ++++++++++ 2 files changed, 15 insertions(+) diff --git a/orm/orm_queryset.go b/orm/orm_queryset.go index 575f62ae..4e33646d 100644 --- a/orm/orm_queryset.go +++ b/orm/orm_queryset.go @@ -153,6 +153,11 @@ func (o querySet) SetCond(cond *Condition) QuerySeter { return &o } +// get condition from QuerySeter +func (o querySet) GetCond() *Condition { + return o.cond +} + // return QuerySeter execution result number func (o *querySet) Count() (int64, error) { return o.orm.alias.DbBaser.Count(o.orm.db, o, o.mi, o.cond, o.orm.alias.TZ) diff --git a/orm/types.go b/orm/types.go index fd3062ab..3e6a9e87 100644 --- a/orm/types.go +++ b/orm/types.go @@ -145,6 +145,16 @@ type QuerySeter interface { // //sql-> WHERE T0.`profile_id` IS NOT NULL AND NOT T0.`Status` IN (?) OR T1.`age` > 2000 // num, err := qs.SetCond(cond1).Count() SetCond(*Condition) QuerySeter + // get condition from QuerySeter. + // sql's where condition + // cond := orm.NewCondition() + // cond = cond.And("profile__isnull", false).AndNot("status__in", 1) + // qs = qs.SetCond(cond) + // cond = qs.GetCond() + // cond := cond.Or("profile__age__gt", 2000) + // //sql-> WHERE T0.`profile_id` IS NOT NULL AND NOT T0.`Status` IN (?) OR T1.`age` > 2000 + // num, err := qs.SetCond(cond).Count() + GetCond() *Condition // add LIMIT value. // args[0] means offset, e.g. LIMIT num,offset. // if Limit <= 0 then Limit will be set to default limit ,eg 1000 From 9b714a75187e4ef066db2a6cf79c284723606c26 Mon Sep 17 00:00:00 2001 From: xhzhang Date: Mon, 6 Feb 2017 12:33:15 +0800 Subject: [PATCH 08/15] Add config field EnableErrorsRender Add config field EnableErrorsRender to disable errors output with the template data, sometimes we do not want errors output with it even in dev mode, especially in API projects. --- config.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/config.go b/config.go index 36bf445c..8d4b3c09 100644 --- a/config.go +++ b/config.go @@ -41,6 +41,7 @@ type Config struct { EnableGzip bool MaxMemory int64 EnableErrorsShow bool + EnableErrorsRender bool Listen Listen WebConfig WebConfig Log LogConfig @@ -174,7 +175,7 @@ func recoverPanic(ctx *context.Context) { logs.Critical(fmt.Sprintf("%s:%d", file, line)) stack = stack + fmt.Sprintln(fmt.Sprintf("%s:%d", file, line)) } - if BConfig.RunMode == DEV { + if BConfig.RunMode == DEV && BConfig.EnableErrorsRender { showErr(err, ctx, stack) } } @@ -192,6 +193,7 @@ func newBConfig() *Config { EnableGzip: false, MaxMemory: 1 << 26, //64MB EnableErrorsShow: true, + EnableErrorsRender: true, Listen: Listen{ Graceful: false, ServerTimeOut: 0, From dfa74faf431c627c44b943356a96bf00e43a4d6e Mon Sep 17 00:00:00 2001 From: Eyal Post Date: Tue, 7 Feb 2017 18:28:03 +0200 Subject: [PATCH 09/15] support for multiple view paths --- controller.go | 16 +++++++++---- controller_test.go | 58 ++++++++++++++++++++++++++++++++++++++++++++++ hooks.go | 2 +- template.go | 42 ++++++++++++++++++++++----------- template_test.go | 10 +++++++- 5 files changed, 109 insertions(+), 19 deletions(-) diff --git a/controller.go b/controller.go index a6fbdd23..488ffcda 100644 --- a/controller.go +++ b/controller.go @@ -69,6 +69,7 @@ type Controller struct { // template data TplName string + ViewPath string Layout string LayoutSections map[string]string // the key is the section name and the value is the template name TplPrefix string @@ -213,7 +214,7 @@ func (c *Controller) RenderBytes() ([]byte, error) { continue } buf.Reset() - err = ExecuteTemplate(&buf, sectionTpl, c.Data) + err = ExecuteViewPathTemplate(&buf, sectionTpl, c.viewPath(), c.Data) if err != nil { return nil, err } @@ -222,7 +223,7 @@ func (c *Controller) RenderBytes() ([]byte, error) { } buf.Reset() - ExecuteTemplate(&buf, c.Layout, c.Data) + ExecuteViewPathTemplate(&buf, c.Layout, c.viewPath() ,c.Data) } return buf.Bytes(), err } @@ -248,9 +249,16 @@ func (c *Controller) renderTemplate() (bytes.Buffer, error) { } } } - BuildTemplate(BConfig.WebConfig.ViewsPath, buildFiles...) + BuildTemplate(c.viewPath() , buildFiles...) } - return buf, ExecuteTemplate(&buf, c.TplName, c.Data) + return buf, ExecuteViewPathTemplate(&buf, c.TplName, c.viewPath(), c.Data) +} + +func (c *Controller) viewPath() string { + if c.ViewPath == "" { + return BConfig.WebConfig.ViewsPath + } + return c.ViewPath } // Redirect sends the redirection response to url with status code. diff --git a/controller_test.go b/controller_test.go index 132971a1..c2025860 100644 --- a/controller_test.go +++ b/controller_test.go @@ -20,6 +20,8 @@ import ( "testing" "github.com/astaxie/beego/context" + "os" + "path/filepath" ) func TestGetInt(t *testing.T) { @@ -121,3 +123,59 @@ func TestGetUint64(t *testing.T) { t.Errorf("TestGetUint64 expect %v,get %T,%v", uint64(math.MaxUint64), val, val) } } + +func TestAdditionalViewPaths(t *testing.T) { + dir1 := "_beeTmp" + dir2 := "_beeTmp2" + defer os.RemoveAll(dir1) + defer os.RemoveAll(dir2) + + dir1file := "file1.tpl" + dir2file := "file2.tpl" + + genFile := func(dir string, name string, content string) { + os.MkdirAll(filepath.Dir(filepath.Join(dir, name)), 0777) + if f, err := os.Create(filepath.Join(dir, name)); err != nil { + t.Fatal(err) + } else { + defer f.Close() + f.WriteString(content) + f.Close() + } + + } + genFile(dir1, dir1file, `
{{.Content}}
`) + genFile(dir2, dir2file, `{{.Content}}`) + + AddViewPath(dir1) + AddViewPath(dir2) + + ctrl := Controller{ + TplName: "file1.tpl", + ViewPath: dir1, + } + ctrl.Data = map[interface{}]interface{}{ + "Content": "value2", + } + if result, err := ctrl.RenderString(); err != nil { + t.Fatal(err) + } else { + if result != "
value2
" { + t.Fatalf("TestAdditionalViewPaths expect %s got %s", "
value2
", result) + } + } + + func() { + ctrl.TplName = "file2.tpl" + defer func() { + if r := recover(); r == nil { + t.Fatal("TestAdditionalViewPaths expected error") + } + }() + ctrl.RenderString(); + }() + + ctrl.TplName = "file2.tpl" + ctrl.ViewPath = dir2 + ctrl.RenderString(); +} diff --git a/hooks.go b/hooks.go index b5a5e6c5..528b58cc 100644 --- a/hooks.go +++ b/hooks.go @@ -72,7 +72,7 @@ func registerSession() error { } func registerTemplate() error { - if err := BuildTemplate(BConfig.WebConfig.ViewsPath); err != nil { + if err := AddViewPath(BConfig.WebConfig.ViewsPath); err != nil { if BConfig.RunMode == DEV { logs.Warn(err) } diff --git a/template.go b/template.go index 5415f5f0..d4d0ed12 100644 --- a/template.go +++ b/template.go @@ -32,8 +32,8 @@ import ( var ( beegoTplFuncMap = make(template.FuncMap) - // beeTemplates caching map and supported template file extensions. - beeTemplates = make(map[string]*template.Template) + // beeViewPathTemplates caching map and supported template file extensions. + beeViewPathTemplates = make(map[string]map[string]*template.Template) templatesLock sync.RWMutex // beeTemplateExt stores the template extension which will build beeTemplateExt = []string{"tpl", "html"} @@ -45,23 +45,30 @@ var ( // writing the output to wr. // A template will be executed safely in parallel. func ExecuteTemplate(wr io.Writer, name string, data interface{}) error { + return ExecuteViewPathTemplate(wr,name, BConfig.WebConfig.ViewsPath, data) +} + +func ExecuteViewPathTemplate(wr io.Writer, name string, viewPath string, data interface{}) error { if BConfig.RunMode == DEV { templatesLock.RLock() defer templatesLock.RUnlock() } - if t, ok := beeTemplates[name]; ok { - var err error - if t.Lookup(name) != nil { - err = t.ExecuteTemplate(wr, name, data) - } else { - err = t.Execute(wr, data) + if beeTemplates,ok := beeViewPathTemplates[viewPath]; ok { + if t, ok := beeTemplates[name]; ok { + var err error + if t.Lookup(name) != nil { + err = t.ExecuteTemplate(wr, name, data) + } else { + err = t.Execute(wr, data) + } + if err != nil { + logs.Trace("template Execute err:", err) + } + return err } - if err != nil { - logs.Trace("template Execute err:", err) - } - return err + panic("can't find templatefile in the path:" + viewPath + "/" + name) } - panic("can't find templatefile in the path:" + name) + panic("Uknown view path:" + viewPath) } func init() { @@ -149,6 +156,11 @@ func AddTemplateExt(ext string) { beeTemplateExt = append(beeTemplateExt, ext) } +func AddViewPath(viewPath string) error { + beeViewPathTemplates[viewPath] = make(map[string]*template.Template) + return BuildTemplate(viewPath) +} + // BuildTemplate will build all template files in a directory. // it makes beego can render any template file in view directory. func BuildTemplate(dir string, files ...string) error { @@ -158,6 +170,10 @@ func BuildTemplate(dir string, files ...string) error { } return errors.New("dir open err") } + beeTemplates,ok := beeViewPathTemplates[dir]; + if !ok { + panic("Unknown view path: " + dir) + } self := &templateFile{ root: dir, files: make(map[string][]string), diff --git a/template_test.go b/template_test.go index 4f13736c..17690965 100644 --- a/template_test.go +++ b/template_test.go @@ -67,9 +67,10 @@ func TestTemplate(t *testing.T) { f.Close() } } - if err := BuildTemplate(dir); err != nil { + if err := AddViewPath(dir); err != nil { t.Fatal(err) } + beeTemplates := beeViewPathTemplates[dir] if len(beeTemplates) != 3 { t.Fatalf("should be 3 but got %v", len(beeTemplates)) } @@ -103,6 +104,12 @@ var user = ` func TestRelativeTemplate(t *testing.T) { dir := "_beeTmp" + + //Just add dir to known viewPaths + if err := AddViewPath(dir); err != nil { + t.Fatal(err) + } + files := []string{ "easyui/public/menu.tpl", "easyui/rbac/user.tpl", @@ -126,6 +133,7 @@ func TestRelativeTemplate(t *testing.T) { if err := BuildTemplate(dir, files[1]); err != nil { t.Fatal(err) } + beeTemplates := beeViewPathTemplates[dir] if err := beeTemplates["easyui/rbac/user.tpl"].ExecuteTemplate(os.Stdout, "easyui/rbac/user.tpl", nil); err != nil { t.Fatal(err) } From dade92d98b1164682931a8eddcbddd24c6b49357 Mon Sep 17 00:00:00 2001 From: liming <327135508@qq.com> Date: Thu, 9 Feb 2017 14:16:23 +0800 Subject: [PATCH 10/15] close mysql connection --- session/mysql/sess_mysql.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/session/mysql/sess_mysql.go b/session/mysql/sess_mysql.go index 838ec669..7683ee1f 100644 --- a/session/mysql/sess_mysql.go +++ b/session/mysql/sess_mysql.go @@ -143,6 +143,7 @@ func (mp *Provider) SessionInit(maxlifetime int64, savePath string) error { // SessionRead get mysql session by sid func (mp *Provider) SessionRead(sid string) (session.Store, error) { c := mp.connectInit() + defer c.Close() row := c.QueryRow("select session_data from "+TableName+" where session_key=?", sid) var sessiondata []byte err := row.Scan(&sessiondata) @@ -179,6 +180,7 @@ func (mp *Provider) SessionExist(sid string) bool { // SessionRegenerate generate new sid for mysql session func (mp *Provider) SessionRegenerate(oldsid, sid string) (session.Store, error) { c := mp.connectInit() + defer c.Close() row := c.QueryRow("select session_data from "+TableName+" where session_key=?", oldsid) var sessiondata []byte err := row.Scan(&sessiondata) From 872a964145e3fec20d723437941f857cc3733c0e Mon Sep 17 00:00:00 2001 From: liming <327135508@qq.com> Date: Thu, 9 Feb 2017 13:47:24 +0800 Subject: [PATCH 11/15] 1.add "defer f.close()" in SessionRead to fix file handle leak if DecodeGob failed 2.rewrite SessionRegenerate --- session/sess_file.go | 85 +++++++++++++++++++++++++------------------- 1 file changed, 48 insertions(+), 37 deletions(-) diff --git a/session/sess_file.go b/session/sess_file.go index 132f5a00..50687c9e 100644 --- a/session/sess_file.go +++ b/session/sess_file.go @@ -15,8 +15,7 @@ package session import ( - "errors" - "io" + "fmt" "io/ioutil" "net/http" "os" @@ -135,6 +134,9 @@ func (fp *FileProvider) SessionRead(sid string) (Store, error) { } else { return nil, err } + + defer f.Close() + os.Chtimes(path.Join(fp.savePath, string(sid[0]), string(sid[1]), sid), time.Now(), time.Now()) var kv map[interface{}]interface{} b, err := ioutil.ReadAll(f) @@ -149,7 +151,7 @@ func (fp *FileProvider) SessionRead(sid string) (Store, error) { return nil, err } } - f.Close() + ss := &FileSessionStore{sid: sid, values: kv} return ss, nil } @@ -204,49 +206,58 @@ func (fp *FileProvider) SessionRegenerate(oldsid, sid string) (Store, error) { filepder.lock.Lock() defer filepder.lock.Unlock() - err := os.MkdirAll(path.Join(fp.savePath, string(oldsid[0]), string(oldsid[1])), 0777) - if err != nil { - SLogger.Println(err.Error()) - } - err = os.MkdirAll(path.Join(fp.savePath, string(sid[0]), string(sid[1])), 0777) - if err != nil { - SLogger.Println(err.Error()) - } - _, err = os.Stat(path.Join(fp.savePath, string(sid[0]), string(sid[1]), sid)) - var newf *os.File + oldPath := path.Join(fp.savePath, string(oldsid[0]), string(oldsid[1])) + oldSidFile := path.Join(oldPath, oldsid) + newPath := path.Join(fp.savePath, string(sid[0]), string(sid[1])) + newSidFile := path.Join(newPath, sid) + + // new sid file is exist + _, err := os.Stat(newSidFile) if err == nil { - return nil, errors.New("newsid exist") - } else if os.IsNotExist(err) { - newf, err = os.Create(path.Join(fp.savePath, string(sid[0]), string(sid[1]), sid)) + return nil, fmt.Errorf("newsid %s exist", newSidFile) } - _, err = os.Stat(path.Join(fp.savePath, string(oldsid[0]), string(oldsid[1]), oldsid)) - var f *os.File - if err == nil { - f, err = os.OpenFile(path.Join(fp.savePath, string(oldsid[0]), string(oldsid[1]), oldsid), os.O_RDWR, 0777) - io.Copy(newf, f) - } else if os.IsNotExist(err) { - newf, err = os.Create(path.Join(fp.savePath, string(sid[0]), string(sid[1]), sid)) - } else { - return nil, err - } - f.Close() - os.Remove(path.Join(fp.savePath, string(oldsid[0]), string(oldsid[1]))) - os.Chtimes(path.Join(fp.savePath, string(sid[0]), string(sid[1]), sid), time.Now(), time.Now()) - var kv map[interface{}]interface{} - b, err := ioutil.ReadAll(newf) + err = os.MkdirAll(newPath, 0777) if err != nil { - return nil, err + SLogger.Println(err.Error()) } - if len(b) == 0 { - kv = make(map[interface{}]interface{}) - } else { - kv, err = DecodeGob(b) + + // if old sid file exist + // 1.read and parse file content + // 2.write content to new sid file + // 3.remove old sid file, change new sid file atime and ctime + // 4.return FileSessionStore + _, err = os.Stat(oldSidFile) + if err == nil { + b, err := ioutil.ReadFile(oldSidFile) if err != nil { return nil, err } + + var kv map[interface{}]interface{} + if len(b) == 0 { + kv = make(map[interface{}]interface{}) + } else { + kv, err = DecodeGob(b) + if err != nil { + return nil, err + } + } + + ioutil.WriteFile(newSidFile, b, 0777) + os.Remove(oldSidFile) + os.Chtimes(newSidFile, time.Now(), time.Now()) + ss := &FileSessionStore{sid: sid, values: kv} + return ss, nil } - ss := &FileSessionStore{sid: sid, values: kv} + + // if old sid file not exist, just create new sid file and return + newf, err := os.Create(newSidFile) + if err != nil { + return nil, err + } + newf.Close() + ss := &FileSessionStore{sid: sid, values: make(map[interface{}]interface{})} return ss, nil } From db67ffbb94af848025403b4ca10b476e792cfb86 Mon Sep 17 00:00:00 2001 From: liming <327135508@qq.com> Date: Fri, 10 Feb 2017 09:35:23 +0800 Subject: [PATCH 12/15] 1.simplify reading and writing file code 2.add apiauth test --- cache/file.go | 25 ++-------------- plugins/apiauth/apiauth.go | 52 ++++++++++----------------------- plugins/apiauth/apiauth_test.go | 20 +++++++++++++ 3 files changed, 39 insertions(+), 58 deletions(-) create mode 100644 plugins/apiauth/apiauth_test.go diff --git a/cache/file.go b/cache/file.go index 4b030980..691ce7cd 100644 --- a/cache/file.go +++ b/cache/file.go @@ -22,6 +22,7 @@ import ( "encoding/json" "fmt" "io" + "io/ioutil" "os" "path/filepath" "reflect" @@ -222,33 +223,13 @@ func exists(path string) (bool, error) { // FileGetContents Get bytes to file. // if non-exist, create this file. func FileGetContents(filename string) (data []byte, e error) { - f, e := os.OpenFile(filename, os.O_RDWR|os.O_CREATE, os.ModePerm) - if e != nil { - return - } - defer f.Close() - stat, e := f.Stat() - if e != nil { - return - } - data = make([]byte, stat.Size()) - result, e := f.Read(data) - if e != nil || int64(result) != stat.Size() { - return nil, e - } - return + return ioutil.ReadFile(filename) } // FilePutContents Put bytes to file. // if non-exist, create this file. func FilePutContents(filename string, content []byte) error { - fp, err := os.OpenFile(filename, os.O_RDWR|os.O_CREATE, os.ModePerm) - if err != nil { - return err - } - defer fp.Close() - _, err = fp.Write(content) - return err + return ioutil.WriteFile(filename, content, os.ModePerm) } // GobEncode Gob encodes file cache item. diff --git a/plugins/apiauth/apiauth.go b/plugins/apiauth/apiauth.go index 10636d1c..f816029c 100644 --- a/plugins/apiauth/apiauth.go +++ b/plugins/apiauth/apiauth.go @@ -56,6 +56,7 @@ package apiauth import ( + "bytes" "crypto/hmac" "crypto/sha256" "encoding/base64" @@ -128,53 +129,32 @@ func APISecretAuth(f AppIDToAppSecret, timeout int) beego.FilterFunc { // Signature used to generate signature with the appsecret/method/params/RequestURI func Signature(appsecret, method string, params url.Values, RequestURL string) (result string) { - var query string + var b bytes.Buffer + keys := make([]string, len(params)) pa := make(map[string]string) for k, v := range params { pa[k] = v[0] + keys = append(keys, k) } - vs := mapSorter(pa) - vs.Sort() - for i := 0; i < vs.Len(); i++ { - if vs.Keys[i] == "signature" { + + sort.Strings(keys) + + for _, key := range keys { + if key == "signature" { continue } - if vs.Keys[i] != "" && vs.Vals[i] != "" { - query = fmt.Sprintf("%v%v%v", query, vs.Keys[i], vs.Vals[i]) + + val := pa[key] + if key != "" && val != "" { + b.WriteString(key) + b.WriteString(val) } } - stringToSign := fmt.Sprintf("%v\n%v\n%v\n", method, query, RequestURL) + + stringToSign := fmt.Sprintf("%v\n%v\n%v\n", method, b.String(), RequestURL) sha256 := sha256.New hash := hmac.New(sha256, []byte(appsecret)) hash.Write([]byte(stringToSign)) return base64.StdEncoding.EncodeToString(hash.Sum(nil)) } - -type valSorter struct { - Keys []string - Vals []string -} - -func mapSorter(m map[string]string) *valSorter { - vs := &valSorter{ - Keys: make([]string, 0, len(m)), - Vals: make([]string, 0, len(m)), - } - for k, v := range m { - vs.Keys = append(vs.Keys, k) - vs.Vals = append(vs.Vals, v) - } - return vs -} - -func (vs *valSorter) Sort() { - sort.Sort(vs) -} - -func (vs *valSorter) Len() int { return len(vs.Keys) } -func (vs *valSorter) Less(i, j int) bool { return vs.Keys[i] < vs.Keys[j] } -func (vs *valSorter) Swap(i, j int) { - vs.Vals[i], vs.Vals[j] = vs.Vals[j], vs.Vals[i] - vs.Keys[i], vs.Keys[j] = vs.Keys[j], vs.Keys[i] -} diff --git a/plugins/apiauth/apiauth_test.go b/plugins/apiauth/apiauth_test.go new file mode 100644 index 00000000..1f56cb0f --- /dev/null +++ b/plugins/apiauth/apiauth_test.go @@ -0,0 +1,20 @@ +package apiauth + +import ( + "net/url" + "testing" +) + +func TestSignature(t *testing.T) { + appsecret := "beego secret" + method := "GET" + RequestURL := "http://localhost/test/url" + params := make(url.Values) + params.Add("arg1", "hello") + params.Add("arg2", "beego") + + signature := "mFdpvLh48ca4mDVEItE9++AKKQ/IVca7O/ZyyB8hR58=" + if Signature(appsecret, method, params, RequestURL) != signature { + t.Error("Signature error") + } +} From 8a2b6976251fba8a319cf421d54e456f1469ac5e Mon Sep 17 00:00:00 2001 From: awengo Date: Fri, 10 Feb 2017 17:45:47 +0900 Subject: [PATCH 13/15] Add http methods --- router.go | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/router.go b/router.go index 74cf02a1..9f573f26 100644 --- a/router.go +++ b/router.go @@ -51,15 +51,22 @@ const ( var ( // HTTPMETHOD list the supported http methods. HTTPMETHOD = map[string]string{ - "GET": "GET", - "POST": "POST", - "PUT": "PUT", - "DELETE": "DELETE", - "PATCH": "PATCH", - "OPTIONS": "OPTIONS", - "HEAD": "HEAD", - "TRACE": "TRACE", - "CONNECT": "CONNECT", + "GET": "GET", + "POST": "POST", + "PUT": "PUT", + "DELETE": "DELETE", + "PATCH": "PATCH", + "OPTIONS": "OPTIONS", + "HEAD": "HEAD", + "TRACE": "TRACE", + "CONNECT": "CONNECT", + "MKCOL": "MKCOL", + "COPY": "COPY", + "MOVE": "MOVE", + "PROPFIND": "PROPFIND", + "PROPPATCH": "PROPPATCH", + "LOCK": "LOCK", + "UNLOCK": "UNLOCK", } // these beego.Controller's methods shouldn't reflect to AutoRouter exceptMethod = []string{"Init", "Prepare", "Finish", "Render", "RenderString", From fc2c0f4fbac53d3df06c706c7d79c77fc61e0ee1 Mon Sep 17 00:00:00 2001 From: Eyal Post Date: Sat, 11 Feb 2017 22:00:30 +0200 Subject: [PATCH 14/15] Don't allow AddViewPath after beego run --- hooks.go | 1 + template.go | 16 +++++++++++++++- 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/hooks.go b/hooks.go index 528b58cc..091ecbc7 100644 --- a/hooks.go +++ b/hooks.go @@ -72,6 +72,7 @@ func registerSession() error { } func registerTemplate() error { + defer lockViewPaths() if err := AddViewPath(BConfig.WebConfig.ViewsPath); err != nil { if BConfig.RunMode == DEV { logs.Warn(err) diff --git a/template.go b/template.go index d4d0ed12..2dba9d1f 100644 --- a/template.go +++ b/template.go @@ -32,7 +32,8 @@ import ( var ( beegoTplFuncMap = make(template.FuncMap) - // beeViewPathTemplates caching map and supported template file extensions. + beeViewPathTemplateLocked = false + // beeViewPathTemplates caching map and supported template file extensions per view beeViewPathTemplates = make(map[string]map[string]*template.Template) templatesLock sync.RWMutex // beeTemplateExt stores the template extension which will build @@ -48,6 +49,9 @@ func ExecuteTemplate(wr io.Writer, name string, data interface{}) error { return ExecuteViewPathTemplate(wr,name, BConfig.WebConfig.ViewsPath, data) } +// ExecuteViewPathTemplate applies the template with name and from specific viewPath to the specified data object, +// writing the output to wr. +// A template will be executed safely in parallel. func ExecuteViewPathTemplate(wr io.Writer, name string, viewPath string, data interface{}) error { if BConfig.RunMode == DEV { templatesLock.RLock() @@ -156,11 +160,21 @@ func AddTemplateExt(ext string) { beeTemplateExt = append(beeTemplateExt, ext) } +// AddViewPath adds a new path to the supported view paths. +//Can later be used by setting a controller ViewPath to this folder +//will panic if called after beego.Run() func AddViewPath(viewPath string) error { + if beeViewPathTemplateLocked { + panic("Can not add new view paths after beego.Run()") + } beeViewPathTemplates[viewPath] = make(map[string]*template.Template) return BuildTemplate(viewPath) } +func lockViewPaths() { + beeViewPathTemplateLocked = true +} + // BuildTemplate will build all template files in a directory. // it makes beego can render any template file in view directory. func BuildTemplate(dir string, files ...string) error { From 393e4c4969f62c71ce4d65a68fca96ea60e84b0e Mon Sep 17 00:00:00 2001 From: jiayukun Date: Wed, 22 Feb 2017 17:38:26 +0800 Subject: [PATCH 15/15] Improve json coding performance --- context/output.go | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/context/output.go b/context/output.go index 4b513dd8..564ef96d 100644 --- a/context/output.go +++ b/context/output.go @@ -331,16 +331,17 @@ func (output *BeegoOutput) IsServerError() bool { func stringsToJSON(str string) string { rs := []rune(str) - jsons := "" + var jsons bytes.Buffer for _, r := range rs { rint := int(r) if rint < 128 { - jsons += string(r) + jsons.WriteRune(r) } else { - jsons += "\\u" + strconv.FormatInt(int64(rint), 16) // json + jsons.WriteString("\\u") + jsons.WriteString(strconv.FormatInt(int64(rint), 16)) } } - return jsons + return jsons.String() } // Session sets session item value with given key.