From b788d74fd1bd1f6300b7923d3e4455bc165339cf Mon Sep 17 00:00:00 2001 From: codejuan Date: Tue, 6 Dec 2016 12:44:00 +0800 Subject: [PATCH 01/40] set perm of rotated log to 440 --- logs/file.go | 1 + 1 file changed, 1 insertion(+) diff --git a/logs/file.go b/logs/file.go index 42146dae..bd3c22a9 100644 --- a/logs/file.go +++ b/logs/file.go @@ -270,6 +270,7 @@ func (w *fileLogWriter) doRotate(logTime time.Time) error { // Rename the file to its new found name // even if occurs error,we MUST guarantee to restart new logger err = os.Rename(w.Filename, fName) + err = os.Chmod(fName, os.FileMode(440)) // re-start logger RESTART_LOGGER: From c0c113036b5443092e21dbf92d9bbab418cdb505 Mon Sep 17 00:00:00 2001 From: mengskysama Date: Tue, 6 Dec 2016 14:57:15 +0800 Subject: [PATCH 02/40] statistics lock --- toolbox/statistics.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/toolbox/statistics.go b/toolbox/statistics.go index 69b88772..c6a9489f 100644 --- a/toolbox/statistics.go +++ b/toolbox/statistics.go @@ -117,7 +117,9 @@ func (m *URLMap) GetMap() map[string]interface{} { // GetMapData return all mapdata func (m *URLMap) GetMapData() []map[string]interface{} { - + m.lock.Lock() + defer m.lock.Unlock() + var resultLists []map[string]interface{} for k, v := range m.urlmap { From fc4801494dde1bfc2c6f00cbbbefbb8e9711335e Mon Sep 17 00:00:00 2001 From: Faissal Elamraoui Date: Tue, 6 Dec 2016 13:36:59 +0100 Subject: [PATCH 03/40] Added hookable signals --- grace/grace.go | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/grace/grace.go b/grace/grace.go index af4e9068..6ebf8455 100644 --- a/grace/grace.go +++ b/grace/grace.go @@ -85,23 +85,31 @@ var ( isChild bool socketOrder string - once sync.Once + + hookableSignals []os.Signal ) -func onceInit() { - regLock = &sync.Mutex{} +func init() { flag.BoolVar(&isChild, "graceful", false, "listen on open fd (after forking)") flag.StringVar(&socketOrder, "socketorder", "", "previous initialization order - used when more than one listener was started") + + regLock = &sync.Mutex{} runningServers = make(map[string]*Server) runningServersOrder = []string{} socketPtrOffsetMap = make(map[string]uint) + + hookableSignals = []os.Signal{ + syscall.SIGHUP, + syscall.SIGINT, + syscall.SIGTERM, + } } // NewServer returns a new graceServer. func NewServer(addr string, handler http.Handler) (srv *Server) { - once.Do(onceInit) regLock.Lock() defer regLock.Unlock() + if !flag.Parsed() { flag.Parse() } From eb50221a154fe3732c23364a944063a07215cdba Mon Sep 17 00:00:00 2001 From: Faissal Elamraoui Date: Tue, 6 Dec 2016 13:38:18 +0100 Subject: [PATCH 04/40] Added method to register Pre/Post signal handlers --- grace/server.go | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/grace/server.go b/grace/server.go index 101bda56..cc985552 100644 --- a/grace/server.go +++ b/grace/server.go @@ -162,9 +162,7 @@ func (srv *Server) handleSignals() { signal.Notify( srv.sigChan, - syscall.SIGHUP, - syscall.SIGINT, - syscall.SIGTERM, + hookableSignals..., ) pid := syscall.Getpid() @@ -290,3 +288,19 @@ func (srv *Server) fork() (err error) { return } + +// RegisterSignalHook registers a function to be run PreSignal or PostSignal for a given signal. +func (srv *Server) RegisterSignalHook(ppFlag int, sig os.Signal, f func()) (err error) { + if ppFlag != PreSignal && ppFlag != PostSignal { + err = fmt.Errorf("Invalid ppFlag argument. Must be either grace.PreSignal or grace.PostSignal.") + return + } + for _, s := range hookableSignals { + if s == sig { + srv.SignalHooks[ppFlag][sig] = append(srv.SignalHooks[ppFlag][sig], f) + return + } + } + err = fmt.Errorf("Signal '%v' is not supported.", sig) + return +} From e90f4bee1a33646edee0dcb88546f2173543a5f3 Mon Sep 17 00:00:00 2001 From: mlgd Date: Fri, 9 Dec 2016 09:37:10 +0100 Subject: [PATCH 05/40] Remove a regression on AppPath The application path is incorrect on Windows with the command line "go run". AppPath is assigned to the temp directory instead the folder project --- config.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/config.go b/config.go index 36bf445c..52e647db 100644 --- a/config.go +++ b/config.go @@ -144,9 +144,6 @@ func init() { if err = parseConfig(appConfigPath); err != nil { panic(err) } - if err = os.Chdir(AppPath); err != nil { - panic(err) - } } func recoverPanic(ctx *context.Context) { From 8fac2d8d58474812affd5ab10c088a4d0b90f4c7 Mon Sep 17 00:00:00 2001 From: Amine KABAB Date: Wed, 14 Dec 2016 17:19:31 +0000 Subject: [PATCH 06/40] Don't rewrite content-type --- controller.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/controller.go b/controller.go index e484ce49..a6fbdd23 100644 --- a/controller.go +++ b/controller.go @@ -185,7 +185,11 @@ func (c *Controller) Render() error { if err != nil { return err } - c.Ctx.Output.Header("Content-Type", "text/html; charset=utf-8") + + if c.Ctx.ResponseWriter.Header().Get("Content-Type") == "" { + c.Ctx.Output.Header("Content-Type", "text/html; charset=utf-8") + } + return c.Ctx.Output.Body(rb) } From c9c284be27b372be59f203367ea01f1d93d97ca2 Mon Sep 17 00:00:00 2001 From: legendtkl Date: Sun, 25 Dec 2016 21:09:06 +0800 Subject: [PATCH 07/40] Modify func camelString to be more robust 1. In previous edition, for case "pic_url_1", the func will return "PicUrl_1", but "PicUrl1" seems to be more reasonable. 2. More test cases please refer to utils_test.go --- orm/utils.go | 21 ++++++++------------- orm/utils_test.go | 36 ++++++++++++++++++++++++++++++++++++ 2 files changed, 44 insertions(+), 13 deletions(-) create mode 100644 orm/utils_test.go diff --git a/orm/utils.go b/orm/utils.go index bf43ceb0..6aac8e5d 100644 --- a/orm/utils.go +++ b/orm/utils.go @@ -219,22 +219,17 @@ func snakeString(s string) string { // camel string, xx_yy to XxYy func camelString(s string) string { data := make([]byte, 0, len(s)) - j := false - k := false - num := len(s) - 1 + flag, num := true, len(s)-1 for i := 0; i <= num; i++ { d := s[i] - if k == false && d >= 'A' && d <= 'Z' { - k = true - } - if d >= 'a' && d <= 'z' && (j || k == false) { - d = d - 32 - j = false - k = true - } - if k && d == '_' && num > i && s[i+1] >= 'a' && s[i+1] <= 'z' { - j = true + if d == '_' { + flag = true continue + } else if flag == true { + if d >= 'a' && d <= 'z' { + d = d - 32 + } + flag = false } data = append(data, d) } diff --git a/orm/utils_test.go b/orm/utils_test.go new file mode 100644 index 00000000..8c7c5008 --- /dev/null +++ b/orm/utils_test.go @@ -0,0 +1,36 @@ +// Copyright 2014 beego Author. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package orm + +import ( + "testing" +) + +func TestCamelString(t *testing.T) { + snake := []string{"pic_url", "hello_world_", "hello__World", "_HelLO_Word", "pic_url_1", "pic_url__1"} + camel := []string{"PicUrl", "HelloWorld", "HelloWorld", "HelLOWord", "PicUrl1", "PicUrl1"} + + answer := make(map[string]string) + for i, v := range snake { + answer[v] = camel[i] + } + + for _, v := range snake { + res := camelString(v) + if res != answer[v] { + t.Error("Unit Test Fail:", v, res, answer[v]) + } + } +} From d736d0ca87a0e86d05324f4d3f642cec275f140e Mon Sep 17 00:00:00 2001 From: Faissal Elamraoui Date: Wed, 28 Dec 2016 12:36:39 +0100 Subject: [PATCH 08/40] Adds Count method to BeeMap struct This adds a Count() method to BeeMap struct that returns the number of items within the safe map. --- utils/safemap.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/utils/safemap.go b/utils/safemap.go index 2e438f2c..a834a55a 100644 --- a/utils/safemap.go +++ b/utils/safemap.go @@ -84,3 +84,10 @@ func (m *BeeMap) Items() map[interface{}]interface{} { } return r } + +// Count returns the number of items within the map. +func (m *BeeMap) Count() int { + m.lock.RLock() + defer m.lock.RUnlock() + return len(m.bm) +} From 75ec8d33a289a99f248e2104742dc97f7fc78cde Mon Sep 17 00:00:00 2001 From: Faissal Elamraoui Date: Wed, 28 Dec 2016 12:39:21 +0100 Subject: [PATCH 09/40] Rewrite safemap_test suite --- utils/safemap_test.go | 55 +++++++++++++++++++++++++++++-------------- 1 file changed, 37 insertions(+), 18 deletions(-) diff --git a/utils/safemap_test.go b/utils/safemap_test.go index fb271d18..1bfe8699 100644 --- a/utils/safemap_test.go +++ b/utils/safemap_test.go @@ -14,25 +14,44 @@ package utils -import ( - "testing" -) +import "testing" -func Test_beemap(t *testing.T) { - bm := NewBeeMap() - if !bm.Set("astaxie", 1) { - t.Error("set Error") - } - if !bm.Check("astaxie") { - t.Error("check err") - } +var safeMap *BeeMap - if v := bm.Get("astaxie"); v.(int) != 1 { - t.Error("get err") - } - - bm.Delete("astaxie") - if bm.Check("astaxie") { - t.Error("delete err") +func TestNewBeeMap(t *testing.T) { + safeMap = NewBeeMap() + if safeMap == nil { + t.Fatal("expected to return non-nil BeeMap", "got", safeMap) + } +} + +func TestSet(t *testing.T) { + if ok := safeMap.Set("astaxie", 1); !ok { + t.Error("expected", true, "got", false) + } +} + +func TestCheck(t *testing.T) { + if exists := safeMap.Check("astaxie"); !exists { + t.Error("expected", true, "got", false) + } +} + +func TestGet(t *testing.T) { + if val := safeMap.Get("astaxie"); val.(int) != 1 { + t.Error("expected value", 1, "got", val) + } +} + +func TestDelete(t *testing.T) { + safeMap.Delete("astaxie") + if exists := safeMap.Check("astaxie"); exists { + t.Error("expected element to be deleted") + } +} + +func TestCount(t *testing.T) { + if count := safeMap.Count(); count != 0 { + t.Error("expected count to be", 0, "got", count) } } From fe21305bb3eae3ec585a7ae8d7955112d3fe94e8 Mon Sep 17 00:00:00 2001 From: Faissal Elamraoui Date: Thu, 29 Dec 2016 11:05:35 +0100 Subject: [PATCH 10/40] Removes redundant check if key exists in BeeMap --- utils/safemap.go | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/utils/safemap.go b/utils/safemap.go index a834a55a..1793030a 100644 --- a/utils/safemap.go +++ b/utils/safemap.go @@ -61,10 +61,8 @@ func (m *BeeMap) Set(k interface{}, v interface{}) bool { func (m *BeeMap) Check(k interface{}) bool { m.lock.RLock() defer m.lock.RUnlock() - if _, ok := m.bm[k]; !ok { - return false - } - return true + _, ok := m.bm[k] + return ok } // Delete the given key and value. From caca5e37ba81e71e33131b3698c15ec7d65ba343 Mon Sep 17 00:00:00 2001 From: "Kariuki, Stanley (Contractor)" Date: Thu, 29 Dec 2016 12:24:27 -0500 Subject: [PATCH 11/40] fixed typo in models_boot --- orm/models_boot.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/orm/models_boot.go b/orm/models_boot.go index 319785ce..4ba5affd 100644 --- a/orm/models_boot.go +++ b/orm/models_boot.go @@ -117,7 +117,7 @@ func bootStrap() { name := getFullName(elm) mii, ok := modelCache.getByFullName(name) if !ok || mii.pkg != elm.PkgPath() { - err = fmt.Errorf("can not found rel in field `%s`, `%s` may be miss register", fi.fullName, elm.String()) + err = fmt.Errorf("can not find rel in field `%s`, `%s` may be miss register", fi.fullName, elm.String()) goto end } fi.relModelInfo = mii From d77160dafeac9b8a44d3f3450afa14cb801bb3a8 Mon Sep 17 00:00:00 2001 From: Faissal Elamraoui Date: Thu, 29 Dec 2016 22:30:56 +0100 Subject: [PATCH 12/40] ignore .vscode folder --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 9806457b..e1b65291 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ .idea +.vscode .DS_Store *.swp *.swo From 96387e9a9b9e30824637c7a153311c51cc9cbc07 Mon Sep 17 00:00:00 2001 From: kbynd Date: Sat, 31 Dec 2016 16:04:34 +0530 Subject: [PATCH 13/40] EnableGZip=true,then content-length header missing This results in responses with Content-Type as gzip as opposed to original content type. This affects ServeJSON() function. --- context/output.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/context/output.go b/context/output.go index c09b9d19..7b238ab8 100644 --- a/context/output.go +++ b/context/output.go @@ -67,9 +67,9 @@ func (output *BeegoOutput) Body(content []byte) error { } if b, n, _ := WriteBody(encoding, buf, content); b { output.Header("Content-Encoding", n) - } else { - output.Header("Content-Length", strconv.Itoa(len(content))) - } + } + output.Header("Content-Length", strconv.Itoa(len(content))) + // Write status code if it has been set manually // Set it to 0 afterwards to prevent "multiple response.WriteHeader calls" if output.Status != 0 { From f0d1d7149bfc2e4618cf5a388f795c901d9c74d0 Mon Sep 17 00:00:00 2001 From: kbynd Date: Sat, 31 Dec 2016 16:14:38 +0530 Subject: [PATCH 14/40] Update output.go --- context/output.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/context/output.go b/context/output.go index 7b238ab8..76b523d3 100644 --- a/context/output.go +++ b/context/output.go @@ -67,8 +67,11 @@ func (output *BeegoOutput) Body(content []byte) error { } if b, n, _ := WriteBody(encoding, buf, content); b { output.Header("Content-Encoding", n) - } - output.Header("Content-Length", strconv.Itoa(len(content))) + output.Header("Content-Length", strconv.Itoa(buf.Len())) + } else { + output.Header("Content-Length", strconv.Itoa(len(content))) + } + // Write status code if it has been set manually // Set it to 0 afterwards to prevent "multiple response.WriteHeader calls" From 2f6da122fdcef15780daff6c491997fde5211c1e Mon Sep 17 00:00:00 2001 From: kbynd Date: Mon, 2 Jan 2017 09:17:17 +0530 Subject: [PATCH 15/40] Update output.go --- context/output.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/context/output.go b/context/output.go index 76b523d3..4b513dd8 100644 --- a/context/output.go +++ b/context/output.go @@ -71,8 +71,6 @@ func (output *BeegoOutput) Body(content []byte) error { } else { output.Header("Content-Length", strconv.Itoa(len(content))) } - - // Write status code if it has been set manually // Set it to 0 afterwards to prevent "multiple response.WriteHeader calls" if output.Status != 0 { From 61e694f388c9ad3b81585a17c8fbdef18859411e Mon Sep 17 00:00:00 2001 From: astaxie Date: Tue, 3 Jan 2017 22:50:45 +0800 Subject: [PATCH 16/40] add retry --- httplib/httplib.go | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/httplib/httplib.go b/httplib/httplib.go index 510ad75e..87513c94 100644 --- a/httplib/httplib.go +++ b/httplib/httplib.go @@ -140,6 +140,7 @@ type BeegoHTTPSettings struct { EnableCookie bool Gzip bool DumpBody bool + Retries int // if set to -1 means will retry forever } // BeegoHTTPRequest provides more useful methods for requesting one url than http.Request. @@ -189,6 +190,15 @@ func (b *BeegoHTTPRequest) Debug(isdebug bool) *BeegoHTTPRequest { return b } +// Retries sets Retries times. +// default is 0 means no retried. +// -1 means retried forever. +// others means retried times. +func (b *BeegoHTTPRequest) Retries(times int) *BeegoHTTPRequest { + b.setting.Retries = times + return b +} + // DumpBody setting whether need to Dump the Body. func (b *BeegoHTTPRequest) DumpBody(isdump bool) *BeegoHTTPRequest { b.setting.DumpBody = isdump @@ -390,7 +400,7 @@ func (b *BeegoHTTPRequest) getResponse() (*http.Response, error) { } // DoRequest will do the client.Do -func (b *BeegoHTTPRequest) DoRequest() (*http.Response, error) { +func (b *BeegoHTTPRequest) DoRequest() (resp *http.Response, err error) { var paramBody string if len(b.params) > 0 { var buf bytes.Buffer @@ -467,7 +477,16 @@ func (b *BeegoHTTPRequest) DoRequest() (*http.Response, error) { } b.dump = dump } - return client.Do(b.req) + // retries default value is 0, it will run once. + // retries equal to -1, it will run forever until success + // retries is setted, it will retries fixed times. + for i := 0; i == 0 || b.setting.Retries == -1 || i < b.setting.Retries; i++ { + resp, err = client.Do(b.req) + if err == nil { + break + } + } + return resp, err } // String returns the body string in response. From 09c405990c8aa60f16903dfd467e603511a420af Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B4=AA=E4=BA=AE?= Date: Tue, 27 Dec 2016 09:58:42 +0800 Subject: [PATCH 17/40] Add Aliyun Logger --- .travis.yml | 2 + logs/alils/alils.go | 192 +++++++ logs/alils/config.go | 13 + logs/alils/log.pb.go | 984 ++++++++++++++++++++++++++++++++++++ logs/alils/log_config.go | 39 ++ logs/alils/log_project.go | 818 ++++++++++++++++++++++++++++++ logs/alils/log_store.go | 269 ++++++++++ logs/alils/machine_group.go | 87 ++++ logs/alils/request.go | 62 +++ logs/alils/signature.go | 112 ++++ logs/log.go | 1 + 11 files changed, 2579 insertions(+) create mode 100644 logs/alils/alils.go create mode 100755 logs/alils/config.go create mode 100755 logs/alils/log.pb.go create mode 100755 logs/alils/log_config.go create mode 100755 logs/alils/log_project.go create mode 100755 logs/alils/log_store.go create mode 100755 logs/alils/machine_group.go create mode 100755 logs/alils/request.go create mode 100755 logs/alils/signature.go diff --git a/.travis.yml b/.travis.yml index 93536488..df3e923f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -31,6 +31,8 @@ install: - go get github.com/siddontang/ledisdb/config - go get github.com/siddontang/ledisdb/ledis - go get github.com/ssdb/gossdb/ssdb + - go get github.com/cloudflare/golz4 + - go get github.com/gogo/protobuf/proto before_script: - psql --version - sh -c "if [ '$ORM_DRIVER' = 'postgres' ]; then psql -c 'create database orm_test;' -U postgres; fi" diff --git a/logs/alils/alils.go b/logs/alils/alils.go new file mode 100644 index 00000000..30a09243 --- /dev/null +++ b/logs/alils/alils.go @@ -0,0 +1,192 @@ +package alils + +import ( + "encoding/json" + "github.com/astaxie/beego/logs" + "github.com/gogo/protobuf/proto" + "strings" + "sync" + "time" +) + +const ( + CacheSize int = 64 + Delimiter string = "##" +) + +type AliLSConfig struct { + Project string `json:"project"` + Endpoint string `json:"endpoint"` + KeyID string `json:"key_id"` + KeySecret string `json:"key_secret"` + LogStore string `json:"log_store"` + Topics []string `json:"topics"` + Source string `json:"source"` + Level int `json:"level"` + FlushWhen int `json:"flush_when"` +} + +// aliLSWriter implements LoggerInterface. +// it writes messages in keep-live tcp connection. +type aliLSWriter struct { + store *LogStore + group []*LogGroup + withMap bool + groupMap map[string]*LogGroup + lock *sync.Mutex + AliLSConfig +} + +// 创建提供Logger接口的日志服务 +func NewAliLS() logs.Logger { + alils := new(aliLSWriter) + alils.Level = logs.LevelTrace + return alils +} + +// 读取配置 +// 初始化必要的数据结构 +func (c *aliLSWriter) Init(jsonConfig string) (err error) { + + json.Unmarshal([]byte(jsonConfig), c) + + if c.FlushWhen > CacheSize { + c.FlushWhen = CacheSize + } + + // 初始化Project + prj := &LogProject{ + Name: c.Project, + Endpoint: c.Endpoint, + AccessKeyId: c.KeyID, + AccessKeySecret: c.KeySecret, + } + + // 获取logstore + c.store, err = prj.GetLogStore(c.LogStore) + if err != nil { + return err + } + + // 创建默认Log Group + c.group = append(c.group, &LogGroup{ + Topic: proto.String(""), + Source: proto.String(c.Source), + Logs: make([]*Log, 0, c.FlushWhen), + }) + + // 创建其它Log Group + c.groupMap = make(map[string]*LogGroup) + for _, topic := range c.Topics { + + lg := &LogGroup{ + Topic: proto.String(topic), + Source: proto.String(c.Source), + Logs: make([]*Log, 0, c.FlushWhen), + } + + c.group = append(c.group, lg) + c.groupMap[topic] = lg + } + + if len(c.group) == 1 { + c.withMap = false + } else { + c.withMap = true + } + + c.lock = &sync.Mutex{} + + return nil +} + +// WriteMsg write message in connection. +// if connection is down, try to re-connect. +func (c *aliLSWriter) WriteMsg(when time.Time, msg string, level int) (err error) { + + if level > c.Level { + return nil + } + + var topic string + var content string + var lg *LogGroup + if c.withMap { + + // 解析出Topic，并匹配LogGroup + strs := strings.SplitN(msg, Delimiter, 2) + if len(strs) == 2 { + pos := strings.LastIndex(strs[0], " ") + topic = strs[0][pos+1 : len(strs[0])] + content = strs[0][0:pos] + strs[1] + lg = c.groupMap[topic] + } + + // 默认发到空Topic + if lg == nil { + topic = "" + content = msg + lg = c.group[0] + } + } else { + topic = "" + content = msg + lg = c.group[0] + } + + // 生成日志 + c1 := &Log_Content{ + Key: proto.String("msg"), + Value: proto.String(content), + } + + l := &Log{ + Time: proto.Uint32(uint32(when.Unix())), // 填写日志时间 + Contents: []*Log_Content{ + c1, + }, + } + + c.lock.Lock() + lg.Logs = append(lg.Logs, l) + c.lock.Unlock() + + // 满足条件则Flush + if len(lg.Logs) >= c.FlushWhen { + c.flush(lg) + } + + return nil +} + +// Flush implementing method. empty. +func (c *aliLSWriter) Flush() { + + // flush所有group + for _, lg := range c.group { + c.flush(lg) + } +} + +// Destroy destroy connection writer and close tcp listener. +func (c *aliLSWriter) Destroy() { +} + +func (c *aliLSWriter) flush(lg *LogGroup) { + + c.lock.Lock() + defer c.lock.Unlock() + + // 把以上的LogGroup推送到SLS服务器， + // SLS服务器会根据该logstore的shard个数自动进行负载均衡。 + err := c.store.PutLogs(lg) + if err != nil { + return + } + + lg.Logs = make([]*Log, 0, c.FlushWhen) +} + +func init() { + logs.Register(logs.AdapterAliLS, NewAliLS) +} diff --git a/logs/alils/config.go b/logs/alils/config.go new file mode 100755 index 00000000..e8c24448 --- /dev/null +++ b/logs/alils/config.go @@ -0,0 +1,13 @@ +package alils + +const ( + version = "0.5.0" // SDK version + signatureMethod = "hmac-sha1" // Signature method + + // OffsetNewest stands for the log head offset, i.e. the offset that will be + // assigned to the next message that will be produced to the shard. + OffsetNewest = "end" + // OffsetOldest stands for the oldest offset available on the logstore for a + // shard. + OffsetOldest = "begin" +) diff --git a/logs/alils/log.pb.go b/logs/alils/log.pb.go new file mode 100755 index 00000000..42f7e892 --- /dev/null +++ b/logs/alils/log.pb.go @@ -0,0 +1,984 @@ +package alils + +import "github.com/gogo/protobuf/proto" +import "fmt" +import "math" + +// discarding unused import gogoproto "." + +import github_com_gogo_protobuf_proto "github.com/gogo/protobuf/proto" + +import "io" + +// Reference imports to suppress errors if they are not otherwise used. +var _ = proto.Marshal +var _ = fmt.Errorf +var _ = math.Inf + +type Log struct { + Time *uint32 `protobuf:"varint,1,req,name=Time" json:"Time,omitempty"` + Contents []*Log_Content `protobuf:"bytes,2,rep,name=Contents" json:"Contents,omitempty"` + XXX_unrecognized []byte `json:"-"` +} + +func (m *Log) Reset() { *m = Log{} } +func (m *Log) String() string { return proto.CompactTextString(m) } +func (*Log) ProtoMessage() {} + +func (m *Log) GetTime() uint32 { + if m != nil && m.Time != nil { + return *m.Time + } + return 0 +} + +func (m *Log) GetContents() []*Log_Content { + if m != nil { + return m.Contents + } + return nil +} + +type Log_Content struct { + Key *string `protobuf:"bytes,1,req,name=Key" json:"Key,omitempty"` + Value *string `protobuf:"bytes,2,req,name=Value" json:"Value,omitempty"` + XXX_unrecognized []byte `json:"-"` +} + +func (m *Log_Content) Reset() { *m = Log_Content{} } +func (m *Log_Content) String() string { return proto.CompactTextString(m) } +func (*Log_Content) ProtoMessage() {} + +func (m *Log_Content) GetKey() string { + if m != nil && m.Key != nil { + return *m.Key + } + return "" +} + +func (m *Log_Content) GetValue() string { + if m != nil && m.Value != nil { + return *m.Value + } + return "" +} + +type LogGroup struct { + Logs []*Log `protobuf:"bytes,1,rep,name=Logs" json:"Logs,omitempty"` + Reserved *string `protobuf:"bytes,2,opt,name=Reserved" json:"Reserved,omitempty"` + Topic *string `protobuf:"bytes,3,opt,name=Topic" json:"Topic,omitempty"` + Source *string `protobuf:"bytes,4,opt,name=Source" json:"Source,omitempty"` + XXX_unrecognized []byte `json:"-"` +} + +func (m *LogGroup) Reset() { *m = LogGroup{} } +func (m *LogGroup) String() string { return proto.CompactTextString(m) } +func (*LogGroup) ProtoMessage() {} + +func (m *LogGroup) GetLogs() []*Log { + if m != nil { + return m.Logs + } + return nil +} + +func (m *LogGroup) GetReserved() string { + if m != nil && m.Reserved != nil { + return *m.Reserved + } + return "" +} + +func (m *LogGroup) GetTopic() string { + if m != nil && m.Topic != nil { + return *m.Topic + } + return "" +} + +func (m *LogGroup) GetSource() string { + if m != nil && m.Source != nil { + return *m.Source + } + return "" +} + +type LogGroupList struct { + LogGroups []*LogGroup `protobuf:"bytes,1,rep,name=logGroups" json:"logGroups,omitempty"` + XXX_unrecognized []byte `json:"-"` +} + +func (m *LogGroupList) Reset() { *m = LogGroupList{} } +func (m *LogGroupList) String() string { return proto.CompactTextString(m) } +func (*LogGroupList) ProtoMessage() {} + +func (m *LogGroupList) GetLogGroups() []*LogGroup { + if m != nil { + return m.LogGroups + } + return nil +} + +func (m *Log) Marshal() (data []byte, err error) { + size := m.Size() + data = make([]byte, size) + n, err := m.MarshalTo(data) + if err != nil { + return nil, err + } + return data[:n], nil +} + +func (m *Log) MarshalTo(data []byte) (int, error) { + var i int + _ = i + var l int + _ = l + if m.Time == nil { + return 0, github_com_gogo_protobuf_proto.NewRequiredNotSetError("Time") + } else { + data[i] = 0x8 + i++ + i = encodeVarintLog(data, i, uint64(*m.Time)) + } + if len(m.Contents) > 0 { + for _, msg := range m.Contents { + data[i] = 0x12 + i++ + i = encodeVarintLog(data, i, uint64(msg.Size())) + n, err := msg.MarshalTo(data[i:]) + if err != nil { + return 0, err + } + i += n + } + } + if m.XXX_unrecognized != nil { + i += copy(data[i:], m.XXX_unrecognized) + } + return i, nil +} + +func (m *Log_Content) Marshal() (data []byte, err error) { + size := m.Size() + data = make([]byte, size) + n, err := m.MarshalTo(data) + if err != nil { + return nil, err + } + return data[:n], nil +} + +func (m *Log_Content) MarshalTo(data []byte) (int, error) { + var i int + _ = i + var l int + _ = l + if m.Key == nil { + return 0, github_com_gogo_protobuf_proto.NewRequiredNotSetError("Key") + } else { + data[i] = 0xa + i++ + i = encodeVarintLog(data, i, uint64(len(*m.Key))) + i += copy(data[i:], *m.Key) + } + if m.Value == nil { + return 0, github_com_gogo_protobuf_proto.NewRequiredNotSetError("Value") + } else { + data[i] = 0x12 + i++ + i = encodeVarintLog(data, i, uint64(len(*m.Value))) + i += copy(data[i:], *m.Value) + } + if m.XXX_unrecognized != nil { + i += copy(data[i:], m.XXX_unrecognized) + } + return i, nil +} + +func (m *LogGroup) Marshal() (data []byte, err error) { + size := m.Size() + data = make([]byte, size) + n, err := m.MarshalTo(data) + if err != nil { + return nil, err + } + return data[:n], nil +} + +func (m *LogGroup) MarshalTo(data []byte) (int, error) { + var i int + _ = i + var l int + _ = l + if len(m.Logs) > 0 { + for _, msg := range m.Logs { + data[i] = 0xa + i++ + i = encodeVarintLog(data, i, uint64(msg.Size())) + n, err := msg.MarshalTo(data[i:]) + if err != nil { + return 0, err + } + i += n + } + } + if m.Reserved != nil { + data[i] = 0x12 + i++ + i = encodeVarintLog(data, i, uint64(len(*m.Reserved))) + i += copy(data[i:], *m.Reserved) + } + if m.Topic != nil { + data[i] = 0x1a + i++ + i = encodeVarintLog(data, i, uint64(len(*m.Topic))) + i += copy(data[i:], *m.Topic) + } + if m.Source != nil { + data[i] = 0x22 + i++ + i = encodeVarintLog(data, i, uint64(len(*m.Source))) + i += copy(data[i:], *m.Source) + } + if m.XXX_unrecognized != nil { + i += copy(data[i:], m.XXX_unrecognized) + } + return i, nil +} + +func (m *LogGroupList) Marshal() (data []byte, err error) { + size := m.Size() + data = make([]byte, size) + n, err := m.MarshalTo(data) + if err != nil { + return nil, err + } + return data[:n], nil +} + +func (m *LogGroupList) MarshalTo(data []byte) (int, error) { + var i int + _ = i + var l int + _ = l + if len(m.LogGroups) > 0 { + for _, msg := range m.LogGroups { + data[i] = 0xa + i++ + i = encodeVarintLog(data, i, uint64(msg.Size())) + n, err := msg.MarshalTo(data[i:]) + if err != nil { + return 0, err + } + i += n + } + } + if m.XXX_unrecognized != nil { + i += copy(data[i:], m.XXX_unrecognized) + } + return i, nil +} + +func encodeFixed64Log(data []byte, offset int, v uint64) int { + data[offset] = uint8(v) + data[offset+1] = uint8(v >> 8) + data[offset+2] = uint8(v >> 16) + data[offset+3] = uint8(v >> 24) + data[offset+4] = uint8(v >> 32) + data[offset+5] = uint8(v >> 40) + data[offset+6] = uint8(v >> 48) + data[offset+7] = uint8(v >> 56) + return offset + 8 +} +func encodeFixed32Log(data []byte, offset int, v uint32) int { + data[offset] = uint8(v) + data[offset+1] = uint8(v >> 8) + data[offset+2] = uint8(v >> 16) + data[offset+3] = uint8(v >> 24) + return offset + 4 +} +func encodeVarintLog(data []byte, offset int, v uint64) int { + for v >= 1<<7 { + data[offset] = uint8(v&0x7f | 0x80) + v >>= 7 + offset++ + } + data[offset] = uint8(v) + return offset + 1 +} +func (m *Log) Size() (n int) { + var l int + _ = l + if m.Time != nil { + n += 1 + sovLog(uint64(*m.Time)) + } + if len(m.Contents) > 0 { + for _, e := range m.Contents { + l = e.Size() + n += 1 + l + sovLog(uint64(l)) + } + } + if m.XXX_unrecognized != nil { + n += len(m.XXX_unrecognized) + } + return n +} + +func (m *Log_Content) Size() (n int) { + var l int + _ = l + if m.Key != nil { + l = len(*m.Key) + n += 1 + l + sovLog(uint64(l)) + } + if m.Value != nil { + l = len(*m.Value) + n += 1 + l + sovLog(uint64(l)) + } + if m.XXX_unrecognized != nil { + n += len(m.XXX_unrecognized) + } + return n +} + +func (m *LogGroup) Size() (n int) { + var l int + _ = l + if len(m.Logs) > 0 { + for _, e := range m.Logs { + l = e.Size() + n += 1 + l + sovLog(uint64(l)) + } + } + if m.Reserved != nil { + l = len(*m.Reserved) + n += 1 + l + sovLog(uint64(l)) + } + if m.Topic != nil { + l = len(*m.Topic) + n += 1 + l + sovLog(uint64(l)) + } + if m.Source != nil { + l = len(*m.Source) + n += 1 + l + sovLog(uint64(l)) + } + if m.XXX_unrecognized != nil { + n += len(m.XXX_unrecognized) + } + return n +} + +func (m *LogGroupList) Size() (n int) { + var l int + _ = l + if len(m.LogGroups) > 0 { + for _, e := range m.LogGroups { + l = e.Size() + n += 1 + l + sovLog(uint64(l)) + } + } + if m.XXX_unrecognized != nil { + n += len(m.XXX_unrecognized) + } + return n +} + +func sovLog(x uint64) (n int) { + for { + n++ + x >>= 7 + if x == 0 { + break + } + } + return n +} +func sozLog(x uint64) (n int) { + return sovLog(uint64((x << 1) ^ uint64((int64(x) >> 63)))) +} +func (m *Log) Unmarshal(data []byte) error { + var hasFields [1]uint64 + l := len(data) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowLog + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := data[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: Log: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: Log: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Time", wireType) + } + var v uint32 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowLog + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := data[iNdEx] + iNdEx++ + v |= (uint32(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + m.Time = &v + hasFields[0] |= uint64(0x00000001) + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Contents", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowLog + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := data[iNdEx] + iNdEx++ + msglen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthLog + } + postIndex := iNdEx + msglen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Contents = append(m.Contents, &Log_Content{}) + if err := m.Contents[len(m.Contents)-1].Unmarshal(data[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipLog(data[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthLog + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.XXX_unrecognized = append(m.XXX_unrecognized, data[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + if hasFields[0]&uint64(0x00000001) == 0 { + return github_com_gogo_protobuf_proto.NewRequiredNotSetError("Time") + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *Log_Content) Unmarshal(data []byte) error { + var hasFields [1]uint64 + l := len(data) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowLog + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := data[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: Content: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: Content: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Key", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowLog + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := data[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthLog + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + s := string(data[iNdEx:postIndex]) + m.Key = &s + iNdEx = postIndex + hasFields[0] |= uint64(0x00000001) + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Value", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowLog + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := data[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthLog + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + s := string(data[iNdEx:postIndex]) + m.Value = &s + iNdEx = postIndex + hasFields[0] |= uint64(0x00000002) + default: + iNdEx = preIndex + skippy, err := skipLog(data[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthLog + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.XXX_unrecognized = append(m.XXX_unrecognized, data[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + if hasFields[0]&uint64(0x00000001) == 0 { + return github_com_gogo_protobuf_proto.NewRequiredNotSetError("Key") + } + if hasFields[0]&uint64(0x00000002) == 0 { + return github_com_gogo_protobuf_proto.NewRequiredNotSetError("Value") + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *LogGroup) Unmarshal(data []byte) error { + l := len(data) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowLog + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := data[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: LogGroup: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: LogGroup: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Logs", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowLog + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := data[iNdEx] + iNdEx++ + msglen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthLog + } + postIndex := iNdEx + msglen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Logs = append(m.Logs, &Log{}) + if err := m.Logs[len(m.Logs)-1].Unmarshal(data[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Reserved", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowLog + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := data[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthLog + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + s := string(data[iNdEx:postIndex]) + m.Reserved = &s + iNdEx = postIndex + case 3: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Topic", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowLog + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := data[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthLog + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + s := string(data[iNdEx:postIndex]) + m.Topic = &s + iNdEx = postIndex + case 4: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Source", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowLog + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := data[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthLog + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + s := string(data[iNdEx:postIndex]) + m.Source = &s + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipLog(data[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthLog + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.XXX_unrecognized = append(m.XXX_unrecognized, data[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *LogGroupList) Unmarshal(data []byte) error { + l := len(data) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowLog + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := data[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: LogGroupList: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: LogGroupList: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field LogGroups", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowLog + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := data[iNdEx] + iNdEx++ + msglen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthLog + } + postIndex := iNdEx + msglen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.LogGroups = append(m.LogGroups, &LogGroup{}) + if err := m.LogGroups[len(m.LogGroups)-1].Unmarshal(data[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipLog(data[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthLog + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.XXX_unrecognized = append(m.XXX_unrecognized, data[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func skipLog(data []byte) (n int, err error) { + l := len(data) + iNdEx := 0 + for iNdEx < l { + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return 0, ErrIntOverflowLog + } + if iNdEx >= l { + return 0, io.ErrUnexpectedEOF + } + b := data[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + wireType := int(wire & 0x7) + switch wireType { + case 0: + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return 0, ErrIntOverflowLog + } + if iNdEx >= l { + return 0, io.ErrUnexpectedEOF + } + iNdEx++ + if data[iNdEx-1] < 0x80 { + break + } + } + return iNdEx, nil + case 1: + iNdEx += 8 + return iNdEx, nil + case 2: + var length int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return 0, ErrIntOverflowLog + } + if iNdEx >= l { + return 0, io.ErrUnexpectedEOF + } + b := data[iNdEx] + iNdEx++ + length |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + iNdEx += length + if length < 0 { + return 0, ErrInvalidLengthLog + } + return iNdEx, nil + case 3: + for { + var innerWire uint64 + var start int = iNdEx + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return 0, ErrIntOverflowLog + } + if iNdEx >= l { + return 0, io.ErrUnexpectedEOF + } + b := data[iNdEx] + iNdEx++ + innerWire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + innerWireType := int(innerWire & 0x7) + if innerWireType == 4 { + break + } + next, err := skipLog(data[start:]) + if err != nil { + return 0, err + } + iNdEx = start + next + } + return iNdEx, nil + case 4: + return iNdEx, nil + case 5: + iNdEx += 4 + return iNdEx, nil + default: + return 0, fmt.Errorf("proto: illegal wireType %d", wireType) + } + } + panic("unreachable") +} + +var ( + ErrInvalidLengthLog = fmt.Errorf("proto: negative length found during unmarshaling") + ErrIntOverflowLog = fmt.Errorf("proto: integer overflow") +) diff --git a/logs/alils/log_config.go b/logs/alils/log_config.go new file mode 100755 index 00000000..41fa0959 --- /dev/null +++ b/logs/alils/log_config.go @@ -0,0 +1,39 @@ +package alils + +type InputDetail struct { + LogType string `json:"logType"` + LogPath string `json:"logPath"` + FilePattern string `json:"filePattern"` + LocalStorage bool `json:"localStorage"` + TimeFormat string `json:"timeFormat"` + LogBeginRegex string `json:"logBeginRegex"` + Regex string `json:"regex"` + Keys []string `json:"key"` + FilterKeys []string `json:"filterKey"` + FilterRegex []string `json:"filterRegex"` + TopicFormat string `json:"topicFormat"` +} + +type OutputDetail struct { + Endpoint string `json:"endpoint"` + LogStoreName string `json:"logstoreName"` +} + +type LogConfig struct { + Name string `json:"configName"` + InputType string `json:"inputType"` + InputDetail InputDetail `json:"inputDetail"` + OutputType string `json:"outputType"` + OutputDetail OutputDetail `json:"outputDetail"` + + CreateTime uint32 + LastModifyTime uint32 + + project *LogProject +} + +// GetAppliedMachineGroup returns applied machine group of this config. +func (c *LogConfig) GetAppliedMachineGroup(confName string) (groupNames []string, err error) { + groupNames, err = c.project.GetAppliedMachineGroups(c.Name) + return +} diff --git a/logs/alils/log_project.go b/logs/alils/log_project.go new file mode 100755 index 00000000..63ab07f8 --- /dev/null +++ b/logs/alils/log_project.go @@ -0,0 +1,818 @@ +/* +Package sls implements the SDK(v0.5.0) of Simple Log Service(abbr. SLS). + +For more description about SLS, please read this article: +http://gitlab.alibaba-inc.com/sls/doc. +*/ +package alils + +import ( + "encoding/json" + "fmt" + "io/ioutil" + "net/http" + "net/http/httputil" +) + +// Error message in SLS HTTP response. +type errorMessage struct { + Code string `json:"errorCode"` + Message string `json:"errorMessage"` +} + +type LogProject struct { + Name string // Project name + Endpoint string // IP or hostname of SLS endpoint + AccessKeyId string + AccessKeySecret string +} + +// NewLogProject creates a new SLS project. +func NewLogProject(name, endpoint, accessKeyId, accessKeySecret string) (p *LogProject, err error) { + p = &LogProject{ + Name: name, + Endpoint: endpoint, + AccessKeyId: accessKeyId, + AccessKeySecret: accessKeySecret, + } + return p, nil +} + +// ListLogStore returns all logstore names of project p. +func (p *LogProject) ListLogStore() (storeNames []string, err error) { + h := map[string]string{ + "x-sls-bodyrawsize": "0", + } + + uri := fmt.Sprintf("/logstores") + r, err := request(p, "GET", uri, h, nil) + if err != nil { + return + } + + buf, err := ioutil.ReadAll(r.Body) + if err != nil { + return + } + + if r.StatusCode != http.StatusOK { + errMsg := &errorMessage{} + err = json.Unmarshal(buf, errMsg) + if err != nil { + err = fmt.Errorf("failed to list logstore") + dump, _ := httputil.DumpResponse(r, true) + fmt.Printf("%s\n", dump) + return + } + err = fmt.Errorf("%v:%v", errMsg.Code, errMsg.Message) + return + } + + type Body struct { + Count int + LogStores []string + } + body := &Body{} + + err = json.Unmarshal(buf, body) + if err != nil { + return + } + + storeNames = body.LogStores + + return +} + +// GetLogStore returns logstore according by logstore name. +func (p *LogProject) GetLogStore(name string) (s *LogStore, err error) { + h := map[string]string{ + "x-sls-bodyrawsize": "0", + } + + r, err := request(p, "GET", "/logstores/"+name, h, nil) + if err != nil { + return + } + + buf, err := ioutil.ReadAll(r.Body) + if err != nil { + return + } + + if r.StatusCode != http.StatusOK { + errMsg := &errorMessage{} + err = json.Unmarshal(buf, errMsg) + if err != nil { + err = fmt.Errorf("failed to get logstore") + dump, _ := httputil.DumpResponse(r, true) + fmt.Printf("%s\n", dump) + return + } + err = fmt.Errorf("%v:%v", errMsg.Code, errMsg.Message) + return + } + + s = &LogStore{} + err = json.Unmarshal(buf, s) + if err != nil { + return + } + s.project = p + return +} + +// CreateLogStore creates a new logstore in SLS, +// where name is logstore name, +// and ttl is time-to-live(in day) of logs, +// and shardCnt is the number of shards. +func (p *LogProject) CreateLogStore(name string, ttl, shardCnt int) (err error) { + + type Body struct { + Name string `json:"logstoreName"` + TTL int `json:"ttl"` + ShardCount int `json:"shardCount"` + } + + store := &Body{ + Name: name, + TTL: ttl, + ShardCount: shardCnt, + } + + body, err := json.Marshal(store) + if err != nil { + return + } + + h := map[string]string{ + "x-sls-bodyrawsize": fmt.Sprintf("%v", len(body)), + "Content-Type": "application/json", + "Accept-Encoding": "deflate", // TODO: support lz4 + } + + r, err := request(p, "POST", "/logstores", h, body) + if err != nil { + return + } + + body, err = ioutil.ReadAll(r.Body) + if err != nil { + return + } + + if r.StatusCode != http.StatusOK { + errMsg := &errorMessage{} + err = json.Unmarshal(body, errMsg) + if err != nil { + err = fmt.Errorf("failed to create logstore") + dump, _ := httputil.DumpResponse(r, true) + fmt.Printf("%s\n", dump) + return + } + err = fmt.Errorf("%v:%v", errMsg.Code, errMsg.Message) + return + } + + return +} + +// DeleteLogStore deletes a logstore according by logstore name. +func (p *LogProject) DeleteLogStore(name string) (err error) { + h := map[string]string{ + "x-sls-bodyrawsize": "0", + } + + r, err := request(p, "DELETE", "/logstores/"+name, h, nil) + if err != nil { + return + } + + body, err := ioutil.ReadAll(r.Body) + if err != nil { + return + } + + if r.StatusCode != http.StatusOK { + errMsg := &errorMessage{} + err = json.Unmarshal(body, errMsg) + if err != nil { + err = fmt.Errorf("failed to delete logstore") + dump, _ := httputil.DumpResponse(r, true) + fmt.Printf("%s\n", dump) + return + } + err = fmt.Errorf("%v:%v", errMsg.Code, errMsg.Message) + return + } + return +} + +// UpdateLogStore updates a logstore according by logstore name, +// obviously we can't modify the logstore name itself. +func (p *LogProject) UpdateLogStore(name string, ttl, shardCnt int) (err error) { + + type Body struct { + Name string `json:"logstoreName"` + TTL int `json:"ttl"` + ShardCount int `json:"shardCount"` + } + + store := &Body{ + Name: name, + TTL: ttl, + ShardCount: shardCnt, + } + + body, err := json.Marshal(store) + if err != nil { + return + } + + h := map[string]string{ + "x-sls-bodyrawsize": fmt.Sprintf("%v", len(body)), + "Content-Type": "application/json", + "Accept-Encoding": "deflate", // TODO: support lz4 + } + + r, err := request(p, "PUT", "/logstores", h, body) + if err != nil { + return + } + + body, err = ioutil.ReadAll(r.Body) + if err != nil { + return + } + + if r.StatusCode != http.StatusOK { + errMsg := &errorMessage{} + err = json.Unmarshal(body, errMsg) + if err != nil { + err = fmt.Errorf("failed to update logstore") + dump, _ := httputil.DumpResponse(r, true) + fmt.Printf("%s\n", dump) + return + } + err = fmt.Errorf("%v:%v", errMsg.Code, errMsg.Message) + return + } + + return +} + +// ListMachineGroup returns machine group name list and the total number of machine groups. +// The offset starts from 0 and the size is the max number of machine groups could be returned. +func (p *LogProject) ListMachineGroup(offset, size int) (m []string, total int, err error) { + h := map[string]string{ + "x-sls-bodyrawsize": "0", + } + + if size <= 0 { + size = 500 + } + + uri := fmt.Sprintf("/machinegroups?offset=%v&size=%v", offset, size) + r, err := request(p, "GET", uri, h, nil) + if err != nil { + return + } + + buf, err := ioutil.ReadAll(r.Body) + if err != nil { + return + } + + if r.StatusCode != http.StatusOK { + errMsg := &errorMessage{} + err = json.Unmarshal(buf, errMsg) + if err != nil { + err = fmt.Errorf("failed to list machine group") + dump, _ := httputil.DumpResponse(r, true) + fmt.Printf("%s\n", dump) + return + } + err = fmt.Errorf("%v:%v", errMsg.Code, errMsg.Message) + return + } + + type Body struct { + MachineGroups []string + Count int + Total int + } + body := &Body{} + + err = json.Unmarshal(buf, body) + if err != nil { + return + } + + m = body.MachineGroups + total = body.Total + + return +} + +// GetMachineGroup retruns machine group according by machine group name. +func (p *LogProject) GetMachineGroup(name string) (m *MachineGroup, err error) { + h := map[string]string{ + "x-sls-bodyrawsize": "0", + } + + r, err := request(p, "GET", "/machinegroups/"+name, h, nil) + if err != nil { + return + } + + buf, err := ioutil.ReadAll(r.Body) + if err != nil { + return + } + + if r.StatusCode != http.StatusOK { + errMsg := &errorMessage{} + err = json.Unmarshal(buf, errMsg) + if err != nil { + err = fmt.Errorf("failed to get machine group:%v", name) + dump, _ := httputil.DumpResponse(r, true) + fmt.Printf("%s\n", dump) + return + } + err = fmt.Errorf("%v:%v", errMsg.Code, errMsg.Message) + return + } + + m = &MachineGroup{} + err = json.Unmarshal(buf, m) + if err != nil { + return + } + m.project = p + return +} + +// CreateMachineGroup creates a new machine group in SLS. +func (p *LogProject) CreateMachineGroup(m *MachineGroup) (err error) { + + body, err := json.Marshal(m) + if err != nil { + return + } + + h := map[string]string{ + "x-sls-bodyrawsize": fmt.Sprintf("%v", len(body)), + "Content-Type": "application/json", + "Accept-Encoding": "deflate", // TODO: support lz4 + } + + r, err := request(p, "POST", "/machinegroups", h, body) + if err != nil { + return + } + + body, err = ioutil.ReadAll(r.Body) + if err != nil { + return + } + + if r.StatusCode != http.StatusOK { + errMsg := &errorMessage{} + err = json.Unmarshal(body, errMsg) + if err != nil { + err = fmt.Errorf("failed to create machine group") + dump, _ := httputil.DumpResponse(r, true) + fmt.Printf("%s\n", dump) + return + } + err = fmt.Errorf("%v:%v", errMsg.Code, errMsg.Message) + return + } + + return +} + +// UpdateMachineGroup updates a machine group. +func (p *LogProject) UpdateMachineGroup(m *MachineGroup) (err error) { + + body, err := json.Marshal(m) + if err != nil { + return + } + + h := map[string]string{ + "x-sls-bodyrawsize": fmt.Sprintf("%v", len(body)), + "Content-Type": "application/json", + "Accept-Encoding": "deflate", // TODO: support lz4 + } + + r, err := request(p, "PUT", "/machinegroups/"+m.Name, h, body) + if err != nil { + return + } + + body, err = ioutil.ReadAll(r.Body) + if err != nil { + return + } + + if r.StatusCode != http.StatusOK { + errMsg := &errorMessage{} + err = json.Unmarshal(body, errMsg) + if err != nil { + err = fmt.Errorf("failed to update machine group") + dump, _ := httputil.DumpResponse(r, true) + fmt.Printf("%s\n", dump) + return + } + err = fmt.Errorf("%v:%v", errMsg.Code, errMsg.Message) + return + } + + return +} + +// DeleteMachineGroup deletes machine group according machine group name. +func (p *LogProject) DeleteMachineGroup(name string) (err error) { + h := map[string]string{ + "x-sls-bodyrawsize": "0", + } + + r, err := request(p, "DELETE", "/machinegroups/"+name, h, nil) + if err != nil { + return + } + + body, err := ioutil.ReadAll(r.Body) + if err != nil { + return + } + + if r.StatusCode != http.StatusOK { + errMsg := &errorMessage{} + err = json.Unmarshal(body, errMsg) + if err != nil { + err = fmt.Errorf("failed to delete machine group") + dump, _ := httputil.DumpResponse(r, true) + fmt.Printf("%s\n", dump) + return + } + err = fmt.Errorf("%v:%v", errMsg.Code, errMsg.Message) + return + } + return +} + +// ListConfig returns config names list and the total number of configs. +// The offset starts from 0 and the size is the max number of configs could be returned. +func (p *LogProject) ListConfig(offset, size int) (cfgNames []string, total int, err error) { + h := map[string]string{ + "x-sls-bodyrawsize": "0", + } + + if size <= 0 { + size = 100 + } + + uri := fmt.Sprintf("/configs?offset=%v&size=%v", offset, size) + r, err := request(p, "GET", uri, h, nil) + if err != nil { + return + } + + buf, err := ioutil.ReadAll(r.Body) + if err != nil { + return + } + + if r.StatusCode != http.StatusOK { + errMsg := &errorMessage{} + err = json.Unmarshal(buf, errMsg) + if err != nil { + err = fmt.Errorf("failed to delete machine group") + dump, _ := httputil.DumpResponse(r, true) + fmt.Printf("%s\n", dump) + return + } + err = fmt.Errorf("%v:%v", errMsg.Code, errMsg.Message) + return + } + + type Body struct { + Total int + Configs []string + } + body := &Body{} + + err = json.Unmarshal(buf, body) + if err != nil { + return + } + + cfgNames = body.Configs + total = body.Total + return +} + +// GetConfig returns config according by config name. +func (p *LogProject) GetConfig(name string) (c *LogConfig, err error) { + h := map[string]string{ + "x-sls-bodyrawsize": "0", + } + + r, err := request(p, "GET", "/configs/"+name, h, nil) + if err != nil { + return + } + + buf, err := ioutil.ReadAll(r.Body) + if err != nil { + return + } + + if r.StatusCode != http.StatusOK { + errMsg := &errorMessage{} + err = json.Unmarshal(buf, errMsg) + if err != nil { + err = fmt.Errorf("failed to delete config") + dump, _ := httputil.DumpResponse(r, true) + fmt.Printf("%s\n", dump) + return + } + err = fmt.Errorf("%v:%v", errMsg.Code, errMsg.Message) + return + } + + c = &LogConfig{} + err = json.Unmarshal(buf, c) + if err != nil { + return + } + c.project = p + return +} + +// UpdateConfig updates a config. +func (p *LogProject) UpdateConfig(c *LogConfig) (err error) { + + body, err := json.Marshal(c) + if err != nil { + return + } + + h := map[string]string{ + "x-sls-bodyrawsize": fmt.Sprintf("%v", len(body)), + "Content-Type": "application/json", + "Accept-Encoding": "deflate", // TODO: support lz4 + } + + r, err := request(p, "PUT", "/configs/"+c.Name, h, body) + if err != nil { + return + } + + body, err = ioutil.ReadAll(r.Body) + if err != nil { + return + } + + if r.StatusCode != http.StatusOK { + errMsg := &errorMessage{} + err = json.Unmarshal(body, errMsg) + if err != nil { + err = fmt.Errorf("failed to update config") + dump, _ := httputil.DumpResponse(r, true) + fmt.Printf("%s\n", dump) + return + } + err = fmt.Errorf("%v:%v", errMsg.Code, errMsg.Message) + return + } + + return +} + +// CreateConfig creates a new config in SLS. +func (p *LogProject) CreateConfig(c *LogConfig) (err error) { + + body, err := json.Marshal(c) + if err != nil { + return + } + + h := map[string]string{ + "x-sls-bodyrawsize": fmt.Sprintf("%v", len(body)), + "Content-Type": "application/json", + "Accept-Encoding": "deflate", // TODO: support lz4 + } + + r, err := request(p, "POST", "/configs", h, body) + if err != nil { + return + } + + body, err = ioutil.ReadAll(r.Body) + if err != nil { + return + } + + if r.StatusCode != http.StatusOK { + errMsg := &errorMessage{} + err = json.Unmarshal(body, errMsg) + if err != nil { + err = fmt.Errorf("failed to update config") + dump, _ := httputil.DumpResponse(r, true) + fmt.Printf("%s\n", dump) + return + } + err = fmt.Errorf("%v:%v", errMsg.Code, errMsg.Message) + return + } + + return +} + +// DeleteConfig deletes a config according by config name. +func (p *LogProject) DeleteConfig(name string) (err error) { + h := map[string]string{ + "x-sls-bodyrawsize": "0", + } + + r, err := request(p, "DELETE", "/configs/"+name, h, nil) + if err != nil { + return + } + + body, err := ioutil.ReadAll(r.Body) + if err != nil { + return + } + + if r.StatusCode != http.StatusOK { + errMsg := &errorMessage{} + err = json.Unmarshal(body, errMsg) + if err != nil { + err = fmt.Errorf("failed to delete config") + dump, _ := httputil.DumpResponse(r, true) + fmt.Printf("%s\n", dump) + return + } + err = fmt.Errorf("%v:%v", errMsg.Code, errMsg.Message) + return + } + return +} + +// GetAppliedMachineGroups returns applied machine group names list according config name. +func (p *LogProject) GetAppliedMachineGroups(confName string) (groupNames []string, err error) { + h := map[string]string{ + "x-sls-bodyrawsize": "0", + } + + uri := fmt.Sprintf("/configs/%v/machinegroups", confName) + r, err := request(p, "GET", uri, h, nil) + if err != nil { + return + } + + buf, err := ioutil.ReadAll(r.Body) + if err != nil { + return + } + + if r.StatusCode != http.StatusOK { + errMsg := &errorMessage{} + err = json.Unmarshal(buf, errMsg) + if err != nil { + err = fmt.Errorf("failed to get applied machine groups") + dump, _ := httputil.DumpResponse(r, true) + fmt.Printf("%s\n", dump) + return + } + err = fmt.Errorf("%v:%v", errMsg.Code, errMsg.Message) + return + } + + type Body struct { + Count int + Machinegroups []string + } + + body := &Body{} + err = json.Unmarshal(buf, body) + if err != nil { + return + } + + groupNames = body.Machinegroups + return +} + +// GetAppliedConfigs returns applied config names list according machine group name groupName. +func (p *LogProject) GetAppliedConfigs(groupName string) (confNames []string, err error) { + h := map[string]string{ + "x-sls-bodyrawsize": "0", + } + + uri := fmt.Sprintf("/machinegroups/%v/configs", groupName) + r, err := request(p, "GET", uri, h, nil) + if err != nil { + return + } + + buf, err := ioutil.ReadAll(r.Body) + if err != nil { + return + } + + if r.StatusCode != http.StatusOK { + errMsg := &errorMessage{} + err = json.Unmarshal(buf, errMsg) + if err != nil { + err = fmt.Errorf("failed to applied configs") + dump, _ := httputil.DumpResponse(r, true) + fmt.Printf("%s\n", dump) + return + } + err = fmt.Errorf("%v:%v", errMsg.Code, errMsg.Message) + return + } + + type Cfg struct { + Count int `json:"count"` + Configs []string `json:"configs"` + } + + body := &Cfg{} + err = json.Unmarshal(buf, body) + if err != nil { + return + } + + confNames = body.Configs + return +} + +// ApplyConfigToMachineGroup applies config to machine group. +func (p *LogProject) ApplyConfigToMachineGroup(confName, groupName string) (err error) { + h := map[string]string{ + "x-sls-bodyrawsize": "0", + } + + uri := fmt.Sprintf("/machinegroups/%v/configs/%v", groupName, confName) + r, err := request(p, "PUT", uri, h, nil) + if err != nil { + return + } + + buf, err := ioutil.ReadAll(r.Body) + if err != nil { + return + } + + if r.StatusCode != http.StatusOK { + errMsg := &errorMessage{} + err = json.Unmarshal(buf, errMsg) + if err != nil { + err = fmt.Errorf("failed to apply config to machine group") + dump, _ := httputil.DumpResponse(r, true) + fmt.Printf("%s\n", dump) + return + } + err = fmt.Errorf("%v:%v", errMsg.Code, errMsg.Message) + return + } + return +} + +// RemoveConfigFromMachineGroup removes config from machine group. +func (p *LogProject) RemoveConfigFromMachineGroup(confName, groupName string) (err error) { + h := map[string]string{ + "x-sls-bodyrawsize": "0", + } + + uri := fmt.Sprintf("/machinegroups/%v/configs/%v", groupName, confName) + r, err := request(p, "DELETE", uri, h, nil) + if err != nil { + return + } + + buf, err := ioutil.ReadAll(r.Body) + if err != nil { + return + } + + if r.StatusCode != http.StatusOK { + errMsg := &errorMessage{} + err = json.Unmarshal(buf, errMsg) + if err != nil { + err = fmt.Errorf("failed to remove config from machine group") + dump, _ := httputil.DumpResponse(r, true) + fmt.Printf("%s\n", dump) + return + } + err = fmt.Errorf("%v:%v", errMsg.Code, errMsg.Message) + return + } + return +} diff --git a/logs/alils/log_store.go b/logs/alils/log_store.go new file mode 100755 index 00000000..009e39c4 --- /dev/null +++ b/logs/alils/log_store.go @@ -0,0 +1,269 @@ +package alils + +import ( + "encoding/json" + "fmt" + "io/ioutil" + "net/http" + "net/http/httputil" + "strconv" + + lz4 "github.com/cloudflare/golz4" + "github.com/gogo/protobuf/proto" +) + +type LogStore struct { + Name string `json:"logstoreName"` + TTL int + ShardCount int + + CreateTime uint32 + LastModifyTime uint32 + + project *LogProject +} + +type Shard struct { + ShardID int `json:"shardID"` +} + +// ListShards returns shard id list of this logstore. +func (s *LogStore) ListShards() (shardIDs []int, err error) { + h := map[string]string{ + "x-sls-bodyrawsize": "0", + } + + uri := fmt.Sprintf("/logstores/%v/shards", s.Name) + r, err := request(s.project, "GET", uri, h, nil) + if err != nil { + return + } + + buf, err := ioutil.ReadAll(r.Body) + if err != nil { + return + } + + if r.StatusCode != http.StatusOK { + errMsg := &errorMessage{} + err = json.Unmarshal(buf, errMsg) + if err != nil { + err = fmt.Errorf("failed to list logstore") + dump, _ := httputil.DumpResponse(r, true) + fmt.Println(dump) + return + } + err = fmt.Errorf("%v:%v", errMsg.Code, errMsg.Message) + return + } + + var shards []*Shard + err = json.Unmarshal(buf, &shards) + if err != nil { + return + } + + for _, v := range shards { + shardIDs = append(shardIDs, v.ShardID) + } + return +} + +// PutLogs put logs into logstore. +// The callers should transform user logs into LogGroup. +func (s *LogStore) PutLogs(lg *LogGroup) (err error) { + body, err := proto.Marshal(lg) + if err != nil { + return + } + + // Compresse body with lz4 + out := make([]byte, lz4.CompressBound(body)) + n, err := lz4.Compress(body, out) + if err != nil { + return + } + + h := map[string]string{ + "x-sls-compresstype": "lz4", + "x-sls-bodyrawsize": fmt.Sprintf("%v", len(body)), + "Content-Type": "application/x-protobuf", + } + + uri := fmt.Sprintf("/logstores/%v", s.Name) + r, err := request(s.project, "POST", uri, h, out[:n]) + if err != nil { + return + } + + buf, err := ioutil.ReadAll(r.Body) + if err != nil { + return + } + + if r.StatusCode != http.StatusOK { + errMsg := &errorMessage{} + err = json.Unmarshal(buf, errMsg) + if err != nil { + err = fmt.Errorf("failed to put logs") + dump, _ := httputil.DumpResponse(r, true) + fmt.Println(dump) + return + } + err = fmt.Errorf("%v:%v", errMsg.Code, errMsg.Message) + return + } + return +} + +// GetCursor gets log cursor of one shard specified by shardId. +// The from can be in three form: a) unix timestamp in seccond, b) "begin", c) "end". +// For more detail please read: http://gitlab.alibaba-inc.com/sls/doc/blob/master/api/shard.md#logstore +func (s *LogStore) GetCursor(shardId int, from string) (cursor string, err error) { + h := map[string]string{ + "x-sls-bodyrawsize": "0", + } + + uri := fmt.Sprintf("/logstores/%v/shards/%v?type=cursor&from=%v", + s.Name, shardId, from) + + r, err := request(s.project, "GET", uri, h, nil) + if err != nil { + return + } + + buf, err := ioutil.ReadAll(r.Body) + if err != nil { + return + } + + if r.StatusCode != http.StatusOK { + errMsg := &errorMessage{} + err = json.Unmarshal(buf, errMsg) + if err != nil { + err = fmt.Errorf("failed to get cursor") + dump, _ := httputil.DumpResponse(r, true) + fmt.Println(dump) + return + } + err = fmt.Errorf("%v:%v", errMsg.Code, errMsg.Message) + return + } + + type Body struct { + Cursor string + } + body := &Body{} + + err = json.Unmarshal(buf, body) + if err != nil { + return + } + cursor = body.Cursor + return +} + +// GetLogsBytes gets logs binary data from shard specified by shardId according cursor. +// The logGroupMaxCount is the max number of logGroup could be returned. +// The nextCursor is the next curosr can be used to read logs at next time. +func (s *LogStore) GetLogsBytes(shardId int, cursor string, + logGroupMaxCount int) (out []byte, nextCursor string, err error) { + + h := map[string]string{ + "x-sls-bodyrawsize": "0", + "Accept": "application/x-protobuf", + "Accept-Encoding": "lz4", + } + + uri := fmt.Sprintf("/logstores/%v/shards/%v?type=logs&cursor=%v&count=%v", + s.Name, shardId, cursor, logGroupMaxCount) + + r, err := request(s.project, "GET", uri, h, nil) + if err != nil { + return + } + + buf, err := ioutil.ReadAll(r.Body) + if err != nil { + return + } + + if r.StatusCode != http.StatusOK { + errMsg := &errorMessage{} + err = json.Unmarshal(buf, errMsg) + if err != nil { + err = fmt.Errorf("failed to get cursor") + dump, _ := httputil.DumpResponse(r, true) + fmt.Println(dump) + return + } + err = fmt.Errorf("%v:%v", errMsg.Code, errMsg.Message) + return + } + + v, ok := r.Header["X-Sls-Compresstype"] + if !ok || len(v) == 0 { + err = fmt.Errorf("can't find 'x-sls-compresstype' header") + return + } + if v[0] != "lz4" { + err = fmt.Errorf("unexpected compress type:%v", v[0]) + return + } + + v, ok = r.Header["X-Sls-Cursor"] + if !ok || len(v) == 0 { + err = fmt.Errorf("can't find 'x-sls-cursor' header") + return + } + nextCursor = v[0] + + v, ok = r.Header["X-Sls-Bodyrawsize"] + if !ok || len(v) == 0 { + err = fmt.Errorf("can't find 'x-sls-bodyrawsize' header") + return + } + bodyRawSize, err := strconv.Atoi(v[0]) + if err != nil { + return + } + + out = make([]byte, bodyRawSize) + err = lz4.Uncompress(buf, out) + if err != nil { + return + } + + return +} + +// LogsBytesDecode decodes logs binary data retruned by GetLogsBytes API +func LogsBytesDecode(data []byte) (gl *LogGroupList, err error) { + + gl = &LogGroupList{} + err = proto.Unmarshal(data, gl) + if err != nil { + return + } + + return +} + +// GetLogs gets logs from shard specified by shardId according cursor. +// The logGroupMaxCount is the max number of logGroup could be returned. +// The nextCursor is the next curosr can be used to read logs at next time. +func (s *LogStore) GetLogs(shardId int, cursor string, + logGroupMaxCount int) (gl *LogGroupList, nextCursor string, err error) { + + out, nextCursor, err := s.GetLogsBytes(shardId, cursor, logGroupMaxCount) + if err != nil { + return + } + + gl, err = LogsBytesDecode(out) + if err != nil { + return + } + + return +} diff --git a/logs/alils/machine_group.go b/logs/alils/machine_group.go new file mode 100755 index 00000000..7a0aace1 --- /dev/null +++ b/logs/alils/machine_group.go @@ -0,0 +1,87 @@ +package alils + +import ( + "encoding/json" + "fmt" + "io/ioutil" + "net/http" + "net/http/httputil" +) + +type MachinGroupAttribute struct { + ExternalName string `json:"externalName"` + TopicName string `json:"groupTopic"` +} + +type MachineGroup struct { + Name string `json:"groupName"` + Type string `json:"groupType"` + MachineIdType string `json:"machineIdentifyType"` + MachineIdList []string `json:"machineList"` + + Attribute MachinGroupAttribute `json:"groupAttribute"` + + CreateTime uint32 + LastModifyTime uint32 + + project *LogProject +} + +type Machine struct { + IP string + UniqueId string `json:"machine-uniqueid"` + UserdefinedId string `json:"userdefined-id"` +} + +type MachineList struct { + Total int + Machines []*Machine +} + +// ListMachines returns machine list of this machine group. +func (m *MachineGroup) ListMachines() (ms []*Machine, total int, err error) { + h := map[string]string{ + "x-sls-bodyrawsize": "0", + } + + uri := fmt.Sprintf("/machinegroups/%v/machines", m.Name) + r, err := request(m.project, "GET", uri, h, nil) + if err != nil { + return + } + + buf, err := ioutil.ReadAll(r.Body) + if err != nil { + return + } + + if r.StatusCode != http.StatusOK { + errMsg := &errorMessage{} + err = json.Unmarshal(buf, errMsg) + if err != nil { + err = fmt.Errorf("failed to remove config from machine group") + dump, _ := httputil.DumpResponse(r, true) + fmt.Println(dump) + return + } + err = fmt.Errorf("%v:%v", errMsg.Code, errMsg.Message) + return + } + + body := &MachineList{} + err = json.Unmarshal(buf, body) + if err != nil { + return + } + + ms = body.Machines + total = body.Total + + return +} + +// GetAppliedConfigs returns applied configs of this machine group. +func (m *MachineGroup) GetAppliedConfigs() (confNames []string, err error) { + confNames, err = m.project.GetAppliedConfigs(m.Name) + return +} diff --git a/logs/alils/request.go b/logs/alils/request.go new file mode 100755 index 00000000..20df45b4 --- /dev/null +++ b/logs/alils/request.go @@ -0,0 +1,62 @@ +package alils + +import ( + "bytes" + "crypto/md5" + "fmt" + "net/http" +) + +// request sends a request to SLS. +func request(project *LogProject, method, uri string, headers map[string]string, + body []byte) (resp *http.Response, err error) { + + // The caller should provide 'x-sls-bodyrawsize' header + if _, ok := headers["x-sls-bodyrawsize"]; !ok { + err = fmt.Errorf("Can't find 'x-sls-bodyrawsize' header") + return + } + + // SLS public request headers + headers["Host"] = project.Name + "." + project.Endpoint + headers["Date"] = nowRFC1123() + headers["x-sls-apiversion"] = version + headers["x-sls-signaturemethod"] = signatureMethod + if body != nil { + bodyMD5 := fmt.Sprintf("%X", md5.Sum(body)) + headers["Content-MD5"] = bodyMD5 + + if _, ok := headers["Content-Type"]; !ok { + err = fmt.Errorf("Can't find 'Content-Type' header") + return + } + } + + // Calc Authorization + // Authorization = "SLS :" + digest, err := signature(project, method, uri, headers) + if err != nil { + return + } + auth := fmt.Sprintf("SLS %v:%v", project.AccessKeyId, digest) + headers["Authorization"] = auth + + // Initialize http request + reader := bytes.NewReader(body) + urlStr := fmt.Sprintf("http://%v.%v%v", project.Name, project.Endpoint, uri) + req, err := http.NewRequest(method, urlStr, reader) + if err != nil { + return + } + for k, v := range headers { + req.Header.Add(k, v) + } + + // Get ready to do request + resp, err = http.DefaultClient.Do(req) + if err != nil { + return + } + + return +} diff --git a/logs/alils/signature.go b/logs/alils/signature.go new file mode 100755 index 00000000..e0e4b3f7 --- /dev/null +++ b/logs/alils/signature.go @@ -0,0 +1,112 @@ +package alils + +import ( + "crypto/hmac" + "crypto/sha1" + "encoding/base64" + "fmt" + "net/url" + "sort" + "strings" + "time" +) + +// GMT location +var gmtLoc = time.FixedZone("GMT", 0) + +// NowRFC1123 returns now time in RFC1123 format with GMT timezone, +// eg. "Mon, 02 Jan 2006 15:04:05 GMT". +func nowRFC1123() string { + return time.Now().In(gmtLoc).Format(time.RFC1123) +} + +// signature calculates a request's signature digest. +func signature(project *LogProject, method, uri string, + headers map[string]string) (digest string, err error) { + var contentMD5, contentType, date, canoHeaders, canoResource string + var slsHeaderKeys sort.StringSlice + + // SignString = VERB + "\n" + // + CONTENT-MD5 + "\n" + // + CONTENT-TYPE + "\n" + // + DATE + "\n" + // + CanonicalizedSLSHeaders + "\n" + // + CanonicalizedResource + + if val, ok := headers["Content-MD5"]; ok { + contentMD5 = val + } + + if val, ok := headers["Content-Type"]; ok { + contentType = val + } + + date, ok := headers["Date"] + if !ok { + err = fmt.Errorf("Can't find 'Date' header") + return + } + + // Calc CanonicalizedSLSHeaders + slsHeaders := make(map[string]string, len(headers)) + for k, v := range headers { + l := strings.TrimSpace(strings.ToLower(k)) + if strings.HasPrefix(l, "x-sls-") { + slsHeaders[l] = strings.TrimSpace(v) + slsHeaderKeys = append(slsHeaderKeys, l) + } + } + + sort.Sort(slsHeaderKeys) + for i, k := range slsHeaderKeys { + canoHeaders += k + ":" + slsHeaders[k] + if i+1 < len(slsHeaderKeys) { + canoHeaders += "\n" + } + } + + // Calc CanonicalizedResource + u, err := url.Parse(uri) + if err != nil { + return + } + + canoResource += url.QueryEscape(u.Path) + if u.RawQuery != "" { + var keys sort.StringSlice + + vals := u.Query() + for k, _ := range vals { + keys = append(keys, k) + } + + sort.Sort(keys) + canoResource += "?" + for i, k := range keys { + if i > 0 { + canoResource += "&" + } + + for _, v := range vals[k] { + canoResource += k + "=" + v + } + } + } + + signStr := method + "\n" + + contentMD5 + "\n" + + contentType + "\n" + + date + "\n" + + canoHeaders + "\n" + + canoResource + + // Signature = base64(hmac-sha1(UTF8-Encoding-Of(SignString)，AccessKeySecret)) + mac := hmac.New(sha1.New, []byte(project.AccessKeySecret)) + _, err = mac.Write([]byte(signStr)) + if err != nil { + return + } + digest = base64.StdEncoding.EncodeToString(mac.Sum(nil)) + return +} + diff --git a/logs/log.go b/logs/log.go index 806ebaa0..c351c473 100644 --- a/logs/log.go +++ b/logs/log.go @@ -71,6 +71,7 @@ const ( AdapterEs = "es" AdapterJianLiao = "jianliao" AdapterSlack = "slack" + AdapterAliLS = "alils" ) // Legacy log level constants to ensure backwards compatibility. From 9266ece7a408aa00c918db7789a6713b11695065 Mon Sep 17 00:00:00 2001 From: astaxie Date: Thu, 5 Jan 2017 18:27:23 +0800 Subject: [PATCH 18/40] fix the retried --- httplib/httplib.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/httplib/httplib.go b/httplib/httplib.go index 87513c94..39480469 100644 --- a/httplib/httplib.go +++ b/httplib/httplib.go @@ -480,7 +480,7 @@ func (b *BeegoHTTPRequest) DoRequest() (resp *http.Response, err error) { // retries default value is 0, it will run once. // retries equal to -1, it will run forever until success // retries is setted, it will retries fixed times. - for i := 0; i == 0 || b.setting.Retries == -1 || i < b.setting.Retries; i++ { + for i := 0; b.setting.Retries == -1 || i <= b.setting.Retries; i++ { resp, err = client.Do(b.req) if err == nil { break From 6a2ee371a5d76b0bf34307d8ec850024a029c6dc Mon Sep 17 00:00:00 2001 From: fugr Date: Mon, 9 Jan 2017 21:04:11 +0800 Subject: [PATCH 19/40] avoid creating new file to implements Config There is no need to create new file in ParseData(data []byte) (Configer, error).Tet's make code simply. --- config/ini.go | 43 +++++++++++++++++++++++-------------------- config/xml/xml.go | 28 ++++++++-------------------- config/yaml/yaml.go | 24 +++++++++++++----------- 3 files changed, 44 insertions(+), 51 deletions(-) diff --git a/config/ini.go b/config/ini.go index b3332bd8..6b78f02a 100644 --- a/config/ini.go +++ b/config/ini.go @@ -18,16 +18,13 @@ import ( "bufio" "bytes" "errors" - "fmt" "io" "io/ioutil" "os" - "path" "path/filepath" "strconv" "strings" "sync" - "time" ) var ( @@ -52,24 +49,26 @@ func (ini *IniConfig) Parse(name string) (Configer, error) { } func (ini *IniConfig) parseFile(name string) (*IniConfigContainer, error) { - file, err := os.Open(name) + data, err := ioutil.ReadFile(name) if err != nil { return nil, err } + return ini.parseData(data) +} + +func (ini *IniConfig) parseData(data []byte) (*IniConfigContainer, error) { cfg := &IniConfigContainer{ - file.Name(), - make(map[string]map[string]string), - make(map[string]string), - make(map[string]string), - sync.RWMutex{}, + data: make(map[string]map[string]string), + sectionComment: make(map[string]string), + keyComment: make(map[string]string), + RWMutex: sync.RWMutex{}, } cfg.Lock() defer cfg.Unlock() - defer file.Close() var comment bytes.Buffer - buf := bufio.NewReader(file) + buf := bufio.NewReader(bytes.NewBuffer(data)) // check the BOM head, err := buf.Peek(3) if err == nil && head[0] == 239 && head[1] == 187 && head[2] == 191 { @@ -130,16 +129,24 @@ func (ini *IniConfig) parseFile(name string) (*IniConfigContainer, error) { // handle include "other.conf" if len(keyValue) == 1 && strings.HasPrefix(key, "include") { + includefiles := strings.Fields(key) if includefiles[0] == "include" && len(includefiles) == 2 { + otherfile := strings.Trim(includefiles[1], "\"") if !filepath.IsAbs(otherfile) { - otherfile = filepath.Join(filepath.Dir(name), otherfile) + dir, err := os.Getwd() + if err != nil { + return nil, err + } + otherfile = filepath.Join(dir, otherfile) } + i, err := ini.parseFile(otherfile) if err != nil { return nil, err } + for sec, dt := range i.data { if _, ok := cfg.data[sec]; !ok { cfg.data[sec] = make(map[string]string) @@ -148,12 +155,15 @@ func (ini *IniConfig) parseFile(name string) (*IniConfigContainer, error) { cfg.data[sec][k] = v } } + for sec, comm := range i.sectionComment { cfg.sectionComment[sec] = comm } + for k, comm := range i.keyComment { cfg.keyComment[k] = comm } + continue } } @@ -178,19 +188,12 @@ func (ini *IniConfig) parseFile(name string) (*IniConfigContainer, error) { // ParseData parse ini the data func (ini *IniConfig) ParseData(data []byte) (Configer, error) { - // Save memory data to temporary file - tmpName := path.Join(os.TempDir(), "beego", fmt.Sprintf("%d", time.Now().Nanosecond())) - os.MkdirAll(path.Dir(tmpName), os.ModePerm) - if err := ioutil.WriteFile(tmpName, data, 0655); err != nil { - return nil, err - } - return ini.Parse(tmpName) + return ini.parseData(data) } // IniConfigContainer A Config represents the ini configuration. // When set and get value, support key as section:name type. type IniConfigContainer struct { - filename string data map[string]map[string]string // section=> key:val sectionComment map[string]string // section : comment keyComment map[string]string // id: []{comment, key...}; id 1 is for main comment. diff --git a/config/xml/xml.go b/config/xml/xml.go index 66115714..b82bf403 100644 --- a/config/xml/xml.go +++ b/config/xml/xml.go @@ -35,11 +35,9 @@ import ( "fmt" "io/ioutil" "os" - "path" "strconv" "strings" "sync" - "time" "github.com/astaxie/beego/config" "github.com/beego/x2j" @@ -52,36 +50,26 @@ type Config struct{} // Parse returns a ConfigContainer with parsed xml config map. func (xc *Config) Parse(filename string) (config.Configer, error) { - file, err := os.Open(filename) + context, err := ioutil.ReadFile(filename) if err != nil { return nil, err } - defer file.Close() + return xc.ParseData(context) +} + +// ParseData xml data +func (xc *Config) ParseData(data []byte) (config.Configer, error) { x := &ConfigContainer{data: make(map[string]interface{})} - content, err := ioutil.ReadAll(file) - if err != nil { - return nil, err - } - d, err := x2j.DocToMap(string(content)) + d, err := x2j.DocToMap(string(data)) if err != nil { return nil, err } x.data = config.ExpandValueEnvForMap(d["config"].(map[string]interface{})) - return x, nil -} -// ParseData xml data -func (xc *Config) ParseData(data []byte) (config.Configer, error) { - // Save memory data to temporary file - tmpName := path.Join(os.TempDir(), "beego", fmt.Sprintf("%d", time.Now().Nanosecond())) - os.MkdirAll(path.Dir(tmpName), os.ModePerm) - if err := ioutil.WriteFile(tmpName, data, 0655); err != nil { - return nil, err - } - return xc.Parse(tmpName) + return x, nil } // ConfigContainer A Config represents the xml configuration. diff --git a/config/yaml/yaml.go b/config/yaml/yaml.go index e3260215..bcef4a20 100644 --- a/config/yaml/yaml.go +++ b/config/yaml/yaml.go @@ -37,10 +37,8 @@ import ( "io/ioutil" "log" "os" - "path" "strings" "sync" - "time" "github.com/astaxie/beego/config" "github.com/beego/goyaml2" @@ -63,26 +61,30 @@ func (yaml *Config) Parse(filename string) (y config.Configer, err error) { // ParseData parse yaml data func (yaml *Config) ParseData(data []byte) (config.Configer, error) { - // Save memory data to temporary file - tmpName := path.Join(os.TempDir(), "beego", fmt.Sprintf("%d", time.Now().Nanosecond())) - os.MkdirAll(path.Dir(tmpName), os.ModePerm) - if err := ioutil.WriteFile(tmpName, data, 0655); err != nil { + cnf, err := parseYML(data) + if err != nil { return nil, err } - return yaml.Parse(tmpName) + + return &ConfigContainer{ + data: cnf, + }, nil } // ReadYmlReader Read yaml file to map. // if json like, use json package, unless goyaml2 package. func ReadYmlReader(path string) (cnf map[string]interface{}, err error) { - f, err := os.Open(path) + buf, err := ioutil.ReadFile(path) if err != nil { return } - defer f.Close() - buf, err := ioutil.ReadAll(f) - if err != nil || len(buf) < 3 { + return parseYML(buf) +} + +// parseYML parse yaml formatted []byte to map. +func parseYML(buf []byte) (cnf map[string]interface{}, err error) { + if len(buf) < 3 { return } From 3fa7fc6e419eb426f2b111f6e6a62dfed929e9d3 Mon Sep 17 00:00:00 2001 From: fugr Date: Wed, 11 Jan 2017 18:55:53 +0800 Subject: [PATCH 20/40] config:fix handle include other.conf When include other.conf,other.conf is either absolute directory or under beego in default temporary directory(/tmp/beego). maybe replace by current directory is better. --- config/ini.go | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/config/ini.go b/config/ini.go index 6b78f02a..c09bcf5c 100644 --- a/config/ini.go +++ b/config/ini.go @@ -54,10 +54,10 @@ func (ini *IniConfig) parseFile(name string) (*IniConfigContainer, error) { return nil, err } - return ini.parseData(data) + return ini.parseData(filepath.Dir(name), data) } -func (ini *IniConfig) parseData(data []byte) (*IniConfigContainer, error) { +func (ini *IniConfig) parseData(dir string, data []byte) (*IniConfigContainer, error) { cfg := &IniConfigContainer{ data: make(map[string]map[string]string), sectionComment: make(map[string]string), @@ -135,10 +135,6 @@ func (ini *IniConfig) parseData(data []byte) (*IniConfigContainer, error) { otherfile := strings.Trim(includefiles[1], "\"") if !filepath.IsAbs(otherfile) { - dir, err := os.Getwd() - if err != nil { - return nil, err - } otherfile = filepath.Join(dir, otherfile) } @@ -187,8 +183,13 @@ func (ini *IniConfig) parseData(data []byte) (*IniConfigContainer, error) { } // ParseData parse ini the data +// When include other.conf,other.conf is either absolute directory +// or under beego in default temporary directory(/tmp/beego). func (ini *IniConfig) ParseData(data []byte) (Configer, error) { - return ini.parseData(data) + dir := filepath.Join(os.TempDir(), "beego") + os.MkdirAll(dir, os.ModePerm) + + return ini.parseData(dir, data) } // IniConfigContainer A Config represents the ini configuration. From 82d2ace3bd35f472fef85d824275c4c383a5dc35 Mon Sep 17 00:00:00 2001 From: chesedo Date: Wed, 11 Jan 2017 12:51:32 +0200 Subject: [PATCH 21/40] Add strong relationship support to orm --- orm/db_utils.go | 2 ++ orm/models_test.go | 5 +++++ orm/orm.go | 2 ++ orm/orm_test.go | 44 ++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 53 insertions(+) diff --git a/orm/db_utils.go b/orm/db_utils.go index 923917ec..7ae10ca5 100644 --- a/orm/db_utils.go +++ b/orm/db_utils.go @@ -41,6 +41,8 @@ func getExistPk(mi *modelInfo, ind reflect.Value) (column string, value interfac vu := v.Int() exist = true value = vu + } else if fi.fieldType&IsRelField > 0 { + _, value, exist = getExistPk(fi.relModelInfo, reflect.Indirect(v)) } else { vu := v.String() exist = vu != "" diff --git a/orm/models_test.go b/orm/models_test.go index 462370b2..9843a87d 100644 --- a/orm/models_test.go +++ b/orm/models_test.go @@ -406,6 +406,11 @@ type UintPk struct { Name string } +type PtrPk struct { + ID *IntegerPk `orm:"pk;rel(one)"` + Positive bool +} + var DBARGS = struct { Driver string Source string diff --git a/orm/orm.go b/orm/orm.go index 538916e4..d9d1cd77 100644 --- a/orm/orm.go +++ b/orm/orm.go @@ -153,6 +153,8 @@ func (o *orm) ReadOrCreate(md interface{}, col1 string, cols ...string) (bool, i id, vid := int64(0), ind.FieldByIndex(mi.fields.pk.fieldIndex) if mi.fields.pk.fieldType&IsPositiveIntegerField > 0 { id = int64(vid.Uint()) + } else if mi.fields.pk.rel { + return o.ReadOrCreate(vid.Interface(), mi.fields.pk.relModelInfo.fields.pk.name) } else { id = vid.Int() } diff --git a/orm/orm_test.go b/orm/orm_test.go index adfe0066..8738952b 100644 --- a/orm/orm_test.go +++ b/orm/orm_test.go @@ -193,6 +193,7 @@ func TestSyncDb(t *testing.T) { RegisterModel(new(InLineOneToOne)) RegisterModel(new(IntegerPk)) RegisterModel(new(UintPk)) + RegisterModel(new(PtrPk)) err := RunSyncdb("default", true, Debug) throwFail(t, err) @@ -216,6 +217,7 @@ func TestRegisterModels(t *testing.T) { RegisterModel(new(InLineOneToOne)) RegisterModel(new(IntegerPk)) RegisterModel(new(UintPk)) + RegisterModel(new(PtrPk)) BootStrap() @@ -2144,6 +2146,48 @@ func TestUintPk(t *testing.T) { dORM.Delete(u) } +func TestPtrPk(t *testing.T) { + parent := &IntegerPk{ID: 10, Value: "10"} + + id, _ := dORM.Insert(parent) + if !IsMysql { + // MySql does not support last_insert_id in this case: see #2382 + throwFail(t, AssertIs(id, 10)) + } + + ptr := PtrPk{ID: parent, Positive: true} + num, err := dORM.InsertMulti(2, []PtrPk{ptr}) + throwFail(t, err) + throwFail(t, AssertIs(num, 1)) + throwFail(t, AssertIs(ptr.ID, parent)) + + nptr := &PtrPk{ID: parent} + created, pk, err := dORM.ReadOrCreate(nptr, "ID") + throwFail(t, err) + throwFail(t, AssertIs(created, false)) + throwFail(t, AssertIs(pk, 10)) + throwFail(t, AssertIs(nptr.ID, parent)) + throwFail(t, AssertIs(nptr.Positive, true)) + + nptr = &PtrPk{Positive: true} + created, pk, err = dORM.ReadOrCreate(nptr, "Positive") + throwFail(t, err) + throwFail(t, AssertIs(created, false)) + throwFail(t, AssertIs(pk, 10)) + throwFail(t, AssertIs(nptr.ID, parent)) + + nptr.Positive = false + num, err = dORM.Update(nptr) + throwFail(t, err) + throwFail(t, AssertIs(num, 1)) + throwFail(t, AssertIs(nptr.ID, parent)) + throwFail(t, AssertIs(nptr.Positive, false)) + + num, err = dORM.Delete(nptr) + throwFail(t, err) + throwFail(t, AssertIs(num, 1)) +} + func TestSnake(t *testing.T) { cases := map[string]string{ "i": "i", From fbc9f8e6409dd0122d49ccf8a8e7696c80de442c Mon Sep 17 00:00:00 2001 From: Faissal Elamraoui Date: Fri, 13 Jan 2017 17:10:54 +0100 Subject: [PATCH 22/40] Package env for working with env variables inside Beego The package env makes it trivial to work with environment variables. It allows getting values with defaults as a fallback. New ENV variables can be set safely on the current process environment. --- env/env.go | 89 +++++++++++++++++++++++++++++++++++++++++++++++++ env/env_test.go | 75 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 164 insertions(+) create mode 100644 env/env.go create mode 100644 env/env_test.go diff --git a/env/env.go b/env/env.go new file mode 100644 index 00000000..014501f5 --- /dev/null +++ b/env/env.go @@ -0,0 +1,89 @@ +// Copyright 2014 beego Author. All Rights Reserved. +// Copyright 2017 Faissal Elamraoui. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package env + +import ( + "fmt" + "os" + "strings" + "sync" +) + +var env struct { + data map[string]string + lock *sync.RWMutex +} + +func init() { + env.data = make(map[string]string) + env.lock = &sync.RWMutex{} + for _, e := range os.Environ() { + splits := strings.Split(e, "=") + env.data[splits[0]] = os.Getenv(splits[0]) + } +} + +// Get returns a value by key. +// If the key does not exist, the default value will be returned. +func Get(key string, defVal string) string { + env.lock.RLock() + defer env.lock.RUnlock() + + if val, ok := env.data[key]; ok { + return val + } + return defVal +} + +// MustGet returns a value by key. +// If the key does not exist, it will return an error. +func MustGet(key string) (string, error) { + env.lock.RLock() + defer env.lock.RUnlock() + + if val, ok := env.data[key]; ok { + return val, nil + } + return "", fmt.Errorf("no env variable with %s", key) +} + +// Set sets a value in the ENV copy. +// This does not affect the child process environment. +func Set(key string, value string) { + env.lock.Lock() + defer env.lock.Unlock() + env.data[key] = value +} + +// MustSet sets a value in the ENV copy and the child process environment. +// It returns an error in case the set operation failed. +func MustSet(key string, value string) error { + env.lock.Lock() + defer env.lock.Unlock() + + err := os.Setenv(key, value) + if err != nil { + return err + } + env.data[key] = value + return nil +} + +// GetAll returns all keys/values in the current child process environment. +func GetAll() map[string]string { + env.lock.RLock() + defer env.lock.RUnlock() + return env.data +} diff --git a/env/env_test.go b/env/env_test.go new file mode 100644 index 00000000..3f1d4dba --- /dev/null +++ b/env/env_test.go @@ -0,0 +1,75 @@ +// Copyright 2014 beego Author. All Rights Reserved. +// Copyright 2017 Faissal Elamraoui. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package env + +import ( + "os" + "testing" +) + +func TestEnvGet(t *testing.T) { + gopath := Get("GOPATH", "") + if gopath != os.Getenv("GOPATH") { + t.Error("expected GOPATH not empty.") + } + + noExistVar := Get("NOEXISTVAR", "foo") + if noExistVar != "foo" { + t.Errorf("expected NOEXISTVAR to equal foo, got %s.", noExistVar) + } +} + +func TestEnvMustGet(t *testing.T) { + gopath, err := MustGet("GOPATH") + if err != nil { + t.Error(err) + } + + if gopath != os.Getenv("GOPATH") { + t.Errorf("expected GOPATH to be the same, got %s.", gopath) + } + + _, err = MustGet("NOEXISTVAR") + if err == nil { + t.Error("expected error to be non-nil") + } +} + +func TestEnvSet(t *testing.T) { + Set("MYVAR", "foo") + myVar := Get("MYVAR", "bar") + if myVar != "foo" { + t.Errorf("expected MYVAR to equal foo, got %s.", myVar) + } +} + +func TestEnvMustSet(t *testing.T) { + err := MustSet("FOO", "bar") + if err != nil { + t.Error(err) + } + + fooVar := os.Getenv("FOO") + if fooVar != "bar" { + t.Errorf("expected FOO variable to equal bar, got %s.", fooVar) + } +} + +func TestEnvGetAll(t *testing.T) { + envMap := GetAll() + if len(envMap) == 0 { + t.Error("expected environment not empty.") + } +} From e32d173b0d43c3987d0f2906467f999be5ba4d7a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=8D=9C=E6=9C=A8?= Date: Sat, 14 Jan 2017 15:03:49 +0800 Subject: [PATCH 23/40] fix the bug to prevent rewrite t the t have paresed in line 212. --- template.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/template.go b/template.go index 5415f5f0..343ef223 100644 --- a/template.go +++ b/template.go @@ -224,7 +224,7 @@ func getTplDeep(root, file, parent string, t *template.Template) (*template.Temp if !HasTemplateExt(m[1]) { continue } - t, _, err = getTplDeep(root, m[1], file, t) + _, _, err = getTplDeep(root, m[1], file, t) if err != nil { return nil, [][]string{}, err } From 957c0630c0296e7628fdc08f7da720e597b1aa09 Mon Sep 17 00:00:00 2001 From: Faissal Elamraoui Date: Sat, 14 Jan 2017 10:15:02 +0100 Subject: [PATCH 24/40] moved the env package to config/ --- {env => config/env}/env.go | 0 {env => config/env}/env_test.go | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename {env => config/env}/env.go (100%) rename {env => config/env}/env_test.go (100%) diff --git a/env/env.go b/config/env/env.go similarity index 100% rename from env/env.go rename to config/env/env.go diff --git a/env/env_test.go b/config/env/env_test.go similarity index 100% rename from env/env_test.go rename to config/env/env_test.go From 126dbdae2f8a4daa2bd5ddb37a07e4d6bd71281f Mon Sep 17 00:00:00 2001 From: Faissal Elamraoui Date: Mon, 16 Jan 2017 10:08:32 +0100 Subject: [PATCH 25/40] use BeeMap instead of a regular map --- config/env/env.go | 52 ++++++++++++++++++++++------------------------- 1 file changed, 24 insertions(+), 28 deletions(-) diff --git a/config/env/env.go b/config/env/env.go index 014501f5..a819e51a 100644 --- a/config/env/env.go +++ b/config/env/env.go @@ -18,31 +18,25 @@ import ( "fmt" "os" "strings" - "sync" + + "github.com/astaxie/beego/utils" ) -var env struct { - data map[string]string - lock *sync.RWMutex -} +var env *utils.BeeMap func init() { - env.data = make(map[string]string) - env.lock = &sync.RWMutex{} + env = utils.NewBeeMap() for _, e := range os.Environ() { splits := strings.Split(e, "=") - env.data[splits[0]] = os.Getenv(splits[0]) + env.Set(splits[0], os.Getenv(splits[0])) } } // Get returns a value by key. // If the key does not exist, the default value will be returned. func Get(key string, defVal string) string { - env.lock.RLock() - defer env.lock.RUnlock() - - if val, ok := env.data[key]; ok { - return val + if val := env.Get(key); val != nil { + return val.(string) } return defVal } @@ -50,11 +44,8 @@ func Get(key string, defVal string) string { // MustGet returns a value by key. // If the key does not exist, it will return an error. func MustGet(key string) (string, error) { - env.lock.RLock() - defer env.lock.RUnlock() - - if val, ok := env.data[key]; ok { - return val, nil + if val := env.Get(key); val != nil { + return val.(string), nil } return "", fmt.Errorf("no env variable with %s", key) } @@ -62,28 +53,33 @@ func MustGet(key string) (string, error) { // Set sets a value in the ENV copy. // This does not affect the child process environment. func Set(key string, value string) { - env.lock.Lock() - defer env.lock.Unlock() - env.data[key] = value + env.Set(key, value) } // MustSet sets a value in the ENV copy and the child process environment. // It returns an error in case the set operation failed. func MustSet(key string, value string) error { - env.lock.Lock() - defer env.lock.Unlock() - err := os.Setenv(key, value) if err != nil { return err } - env.data[key] = value + env.Set(key, value) return nil } // GetAll returns all keys/values in the current child process environment. func GetAll() map[string]string { - env.lock.RLock() - defer env.lock.RUnlock() - return env.data + items := env.Items() + envs := make(map[string]string, env.Count()) + + for key, val := range items { + switch key := key.(type) { + case string: + switch val := val.(type) { + case string: + envs[key] = val + } + } + } + return envs } From 5c76f621034043bbff4b8160bbed37f1c9aa4200 Mon Sep 17 00:00:00 2001 From: kerwin Date: Wed, 18 Jan 2017 17:04:23 +0800 Subject: [PATCH 26/40] Add GetCond func to querySet --- orm/orm_queryset.go | 5 +++++ orm/types.go | 10 ++++++++++ 2 files changed, 15 insertions(+) diff --git a/orm/orm_queryset.go b/orm/orm_queryset.go index 575f62ae..4e33646d 100644 --- a/orm/orm_queryset.go +++ b/orm/orm_queryset.go @@ -153,6 +153,11 @@ func (o querySet) SetCond(cond *Condition) QuerySeter { return &o } +// get condition from QuerySeter +func (o querySet) GetCond() *Condition { + return o.cond +} + // return QuerySeter execution result number func (o *querySet) Count() (int64, error) { return o.orm.alias.DbBaser.Count(o.orm.db, o, o.mi, o.cond, o.orm.alias.TZ) diff --git a/orm/types.go b/orm/types.go index fd3062ab..3e6a9e87 100644 --- a/orm/types.go +++ b/orm/types.go @@ -145,6 +145,16 @@ type QuerySeter interface { // //sql-> WHERE T0.`profile_id` IS NOT NULL AND NOT T0.`Status` IN (?) OR T1.`age` > 2000 // num, err := qs.SetCond(cond1).Count() SetCond(*Condition) QuerySeter + // get condition from QuerySeter. + // sql's where condition + // cond := orm.NewCondition() + // cond = cond.And("profile__isnull", false).AndNot("status__in", 1) + // qs = qs.SetCond(cond) + // cond = qs.GetCond() + // cond := cond.Or("profile__age__gt", 2000) + // //sql-> WHERE T0.`profile_id` IS NOT NULL AND NOT T0.`Status` IN (?) OR T1.`age` > 2000 + // num, err := qs.SetCond(cond).Count() + GetCond() *Condition // add LIMIT value. // args[0] means offset, e.g. LIMIT num,offset. // if Limit <= 0 then Limit will be set to default limit ,eg 1000 From 9b714a75187e4ef066db2a6cf79c284723606c26 Mon Sep 17 00:00:00 2001 From: xhzhang Date: Mon, 6 Feb 2017 12:33:15 +0800 Subject: [PATCH 27/40] Add config field EnableErrorsRender Add config field EnableErrorsRender to disable errors output with the template data, sometimes we do not want errors output with it even in dev mode, especially in API projects. --- config.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/config.go b/config.go index 36bf445c..8d4b3c09 100644 --- a/config.go +++ b/config.go @@ -41,6 +41,7 @@ type Config struct { EnableGzip bool MaxMemory int64 EnableErrorsShow bool + EnableErrorsRender bool Listen Listen WebConfig WebConfig Log LogConfig @@ -174,7 +175,7 @@ func recoverPanic(ctx *context.Context) { logs.Critical(fmt.Sprintf("%s:%d", file, line)) stack = stack + fmt.Sprintln(fmt.Sprintf("%s:%d", file, line)) } - if BConfig.RunMode == DEV { + if BConfig.RunMode == DEV && BConfig.EnableErrorsRender { showErr(err, ctx, stack) } } @@ -192,6 +193,7 @@ func newBConfig() *Config { EnableGzip: false, MaxMemory: 1 << 26, //64MB EnableErrorsShow: true, + EnableErrorsRender: true, Listen: Listen{ Graceful: false, ServerTimeOut: 0, From dfa74faf431c627c44b943356a96bf00e43a4d6e Mon Sep 17 00:00:00 2001 From: Eyal Post Date: Tue, 7 Feb 2017 18:28:03 +0200 Subject: [PATCH 28/40] support for multiple view paths --- controller.go | 16 +++++++++---- controller_test.go | 58 ++++++++++++++++++++++++++++++++++++++++++++++ hooks.go | 2 +- template.go | 42 ++++++++++++++++++++++----------- template_test.go | 10 +++++++- 5 files changed, 109 insertions(+), 19 deletions(-) diff --git a/controller.go b/controller.go index a6fbdd23..488ffcda 100644 --- a/controller.go +++ b/controller.go @@ -69,6 +69,7 @@ type Controller struct { // template data TplName string + ViewPath string Layout string LayoutSections map[string]string // the key is the section name and the value is the template name TplPrefix string @@ -213,7 +214,7 @@ func (c *Controller) RenderBytes() ([]byte, error) { continue } buf.Reset() - err = ExecuteTemplate(&buf, sectionTpl, c.Data) + err = ExecuteViewPathTemplate(&buf, sectionTpl, c.viewPath(), c.Data) if err != nil { return nil, err } @@ -222,7 +223,7 @@ func (c *Controller) RenderBytes() ([]byte, error) { } buf.Reset() - ExecuteTemplate(&buf, c.Layout, c.Data) + ExecuteViewPathTemplate(&buf, c.Layout, c.viewPath() ,c.Data) } return buf.Bytes(), err } @@ -248,9 +249,16 @@ func (c *Controller) renderTemplate() (bytes.Buffer, error) { } } } - BuildTemplate(BConfig.WebConfig.ViewsPath, buildFiles...) + BuildTemplate(c.viewPath() , buildFiles...) } - return buf, ExecuteTemplate(&buf, c.TplName, c.Data) + return buf, ExecuteViewPathTemplate(&buf, c.TplName, c.viewPath(), c.Data) +} + +func (c *Controller) viewPath() string { + if c.ViewPath == "" { + return BConfig.WebConfig.ViewsPath + } + return c.ViewPath } // Redirect sends the redirection response to url with status code. diff --git a/controller_test.go b/controller_test.go index 132971a1..c2025860 100644 --- a/controller_test.go +++ b/controller_test.go @@ -20,6 +20,8 @@ import ( "testing" "github.com/astaxie/beego/context" + "os" + "path/filepath" ) func TestGetInt(t *testing.T) { @@ -121,3 +123,59 @@ func TestGetUint64(t *testing.T) { t.Errorf("TestGetUint64 expect %v,get %T,%v", uint64(math.MaxUint64), val, val) } } + +func TestAdditionalViewPaths(t *testing.T) { + dir1 := "_beeTmp" + dir2 := "_beeTmp2" + defer os.RemoveAll(dir1) + defer os.RemoveAll(dir2) + + dir1file := "file1.tpl" + dir2file := "file2.tpl" + + genFile := func(dir string, name string, content string) { + os.MkdirAll(filepath.Dir(filepath.Join(dir, name)), 0777) + if f, err := os.Create(filepath.Join(dir, name)); err != nil { + t.Fatal(err) + } else { + defer f.Close() + f.WriteString(content) + f.Close() + } + + } + genFile(dir1, dir1file, `
{{.Content}}
`) + genFile(dir2, dir2file, `{{.Content}}`) + + AddViewPath(dir1) + AddViewPath(dir2) + + ctrl := Controller{ + TplName: "file1.tpl", + ViewPath: dir1, + } + ctrl.Data = map[interface{}]interface{}{ + "Content": "value2", + } + if result, err := ctrl.RenderString(); err != nil { + t.Fatal(err) + } else { + if result != "
value2
" { + t.Fatalf("TestAdditionalViewPaths expect %s got %s", "
value2
", result) + } + } + + func() { + ctrl.TplName = "file2.tpl" + defer func() { + if r := recover(); r == nil { + t.Fatal("TestAdditionalViewPaths expected error") + } + }() + ctrl.RenderString(); + }() + + ctrl.TplName = "file2.tpl" + ctrl.ViewPath = dir2 + ctrl.RenderString(); +} diff --git a/hooks.go b/hooks.go index b5a5e6c5..528b58cc 100644 --- a/hooks.go +++ b/hooks.go @@ -72,7 +72,7 @@ func registerSession() error { } func registerTemplate() error { - if err := BuildTemplate(BConfig.WebConfig.ViewsPath); err != nil { + if err := AddViewPath(BConfig.WebConfig.ViewsPath); err != nil { if BConfig.RunMode == DEV { logs.Warn(err) } diff --git a/template.go b/template.go index 5415f5f0..d4d0ed12 100644 --- a/template.go +++ b/template.go @@ -32,8 +32,8 @@ import ( var ( beegoTplFuncMap = make(template.FuncMap) - // beeTemplates caching map and supported template file extensions. - beeTemplates = make(map[string]*template.Template) + // beeViewPathTemplates caching map and supported template file extensions. + beeViewPathTemplates = make(map[string]map[string]*template.Template) templatesLock sync.RWMutex // beeTemplateExt stores the template extension which will build beeTemplateExt = []string{"tpl", "html"} @@ -45,23 +45,30 @@ var ( // writing the output to wr. // A template will be executed safely in parallel. func ExecuteTemplate(wr io.Writer, name string, data interface{}) error { + return ExecuteViewPathTemplate(wr,name, BConfig.WebConfig.ViewsPath, data) +} + +func ExecuteViewPathTemplate(wr io.Writer, name string, viewPath string, data interface{}) error { if BConfig.RunMode == DEV { templatesLock.RLock() defer templatesLock.RUnlock() } - if t, ok := beeTemplates[name]; ok { - var err error - if t.Lookup(name) != nil { - err = t.ExecuteTemplate(wr, name, data) - } else { - err = t.Execute(wr, data) + if beeTemplates,ok := beeViewPathTemplates[viewPath]; ok { + if t, ok := beeTemplates[name]; ok { + var err error + if t.Lookup(name) != nil { + err = t.ExecuteTemplate(wr, name, data) + } else { + err = t.Execute(wr, data) + } + if err != nil { + logs.Trace("template Execute err:", err) + } + return err } - if err != nil { - logs.Trace("template Execute err:", err) - } - return err + panic("can't find templatefile in the path:" + viewPath + "/" + name) } - panic("can't find templatefile in the path:" + name) + panic("Uknown view path:" + viewPath) } func init() { @@ -149,6 +156,11 @@ func AddTemplateExt(ext string) { beeTemplateExt = append(beeTemplateExt, ext) } +func AddViewPath(viewPath string) error { + beeViewPathTemplates[viewPath] = make(map[string]*template.Template) + return BuildTemplate(viewPath) +} + // BuildTemplate will build all template files in a directory. // it makes beego can render any template file in view directory. func BuildTemplate(dir string, files ...string) error { @@ -158,6 +170,10 @@ func BuildTemplate(dir string, files ...string) error { } return errors.New("dir open err") } + beeTemplates,ok := beeViewPathTemplates[dir]; + if !ok { + panic("Unknown view path: " + dir) + } self := &templateFile{ root: dir, files: make(map[string][]string), diff --git a/template_test.go b/template_test.go index 4f13736c..17690965 100644 --- a/template_test.go +++ b/template_test.go @@ -67,9 +67,10 @@ func TestTemplate(t *testing.T) { f.Close() } } - if err := BuildTemplate(dir); err != nil { + if err := AddViewPath(dir); err != nil { t.Fatal(err) } + beeTemplates := beeViewPathTemplates[dir] if len(beeTemplates) != 3 { t.Fatalf("should be 3 but got %v", len(beeTemplates)) } @@ -103,6 +104,12 @@ var user = ` func TestRelativeTemplate(t *testing.T) { dir := "_beeTmp" + + //Just add dir to known viewPaths + if err := AddViewPath(dir); err != nil { + t.Fatal(err) + } + files := []string{ "easyui/public/menu.tpl", "easyui/rbac/user.tpl", @@ -126,6 +133,7 @@ func TestRelativeTemplate(t *testing.T) { if err := BuildTemplate(dir, files[1]); err != nil { t.Fatal(err) } + beeTemplates := beeViewPathTemplates[dir] if err := beeTemplates["easyui/rbac/user.tpl"].ExecuteTemplate(os.Stdout, "easyui/rbac/user.tpl", nil); err != nil { t.Fatal(err) } From dade92d98b1164682931a8eddcbddd24c6b49357 Mon Sep 17 00:00:00 2001 From: liming <327135508@qq.com> Date: Thu, 9 Feb 2017 14:16:23 +0800 Subject: [PATCH 29/40] close mysql connection --- session/mysql/sess_mysql.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/session/mysql/sess_mysql.go b/session/mysql/sess_mysql.go index 838ec669..7683ee1f 100644 --- a/session/mysql/sess_mysql.go +++ b/session/mysql/sess_mysql.go @@ -143,6 +143,7 @@ func (mp *Provider) SessionInit(maxlifetime int64, savePath string) error { // SessionRead get mysql session by sid func (mp *Provider) SessionRead(sid string) (session.Store, error) { c := mp.connectInit() + defer c.Close() row := c.QueryRow("select session_data from "+TableName+" where session_key=?", sid) var sessiondata []byte err := row.Scan(&sessiondata) @@ -179,6 +180,7 @@ func (mp *Provider) SessionExist(sid string) bool { // SessionRegenerate generate new sid for mysql session func (mp *Provider) SessionRegenerate(oldsid, sid string) (session.Store, error) { c := mp.connectInit() + defer c.Close() row := c.QueryRow("select session_data from "+TableName+" where session_key=?", oldsid) var sessiondata []byte err := row.Scan(&sessiondata) From 872a964145e3fec20d723437941f857cc3733c0e Mon Sep 17 00:00:00 2001 From: liming <327135508@qq.com> Date: Thu, 9 Feb 2017 13:47:24 +0800 Subject: [PATCH 30/40] 1.add "defer f.close()" in SessionRead to fix file handle leak if DecodeGob failed 2.rewrite SessionRegenerate --- session/sess_file.go | 85 +++++++++++++++++++++++++------------------- 1 file changed, 48 insertions(+), 37 deletions(-) diff --git a/session/sess_file.go b/session/sess_file.go index 132f5a00..50687c9e 100644 --- a/session/sess_file.go +++ b/session/sess_file.go @@ -15,8 +15,7 @@ package session import ( - "errors" - "io" + "fmt" "io/ioutil" "net/http" "os" @@ -135,6 +134,9 @@ func (fp *FileProvider) SessionRead(sid string) (Store, error) { } else { return nil, err } + + defer f.Close() + os.Chtimes(path.Join(fp.savePath, string(sid[0]), string(sid[1]), sid), time.Now(), time.Now()) var kv map[interface{}]interface{} b, err := ioutil.ReadAll(f) @@ -149,7 +151,7 @@ func (fp *FileProvider) SessionRead(sid string) (Store, error) { return nil, err } } - f.Close() + ss := &FileSessionStore{sid: sid, values: kv} return ss, nil } @@ -204,49 +206,58 @@ func (fp *FileProvider) SessionRegenerate(oldsid, sid string) (Store, error) { filepder.lock.Lock() defer filepder.lock.Unlock() - err := os.MkdirAll(path.Join(fp.savePath, string(oldsid[0]), string(oldsid[1])), 0777) - if err != nil { - SLogger.Println(err.Error()) - } - err = os.MkdirAll(path.Join(fp.savePath, string(sid[0]), string(sid[1])), 0777) - if err != nil { - SLogger.Println(err.Error()) - } - _, err = os.Stat(path.Join(fp.savePath, string(sid[0]), string(sid[1]), sid)) - var newf *os.File + oldPath := path.Join(fp.savePath, string(oldsid[0]), string(oldsid[1])) + oldSidFile := path.Join(oldPath, oldsid) + newPath := path.Join(fp.savePath, string(sid[0]), string(sid[1])) + newSidFile := path.Join(newPath, sid) + + // new sid file is exist + _, err := os.Stat(newSidFile) if err == nil { - return nil, errors.New("newsid exist") - } else if os.IsNotExist(err) { - newf, err = os.Create(path.Join(fp.savePath, string(sid[0]), string(sid[1]), sid)) + return nil, fmt.Errorf("newsid %s exist", newSidFile) } - _, err = os.Stat(path.Join(fp.savePath, string(oldsid[0]), string(oldsid[1]), oldsid)) - var f *os.File - if err == nil { - f, err = os.OpenFile(path.Join(fp.savePath, string(oldsid[0]), string(oldsid[1]), oldsid), os.O_RDWR, 0777) - io.Copy(newf, f) - } else if os.IsNotExist(err) { - newf, err = os.Create(path.Join(fp.savePath, string(sid[0]), string(sid[1]), sid)) - } else { - return nil, err - } - f.Close() - os.Remove(path.Join(fp.savePath, string(oldsid[0]), string(oldsid[1]))) - os.Chtimes(path.Join(fp.savePath, string(sid[0]), string(sid[1]), sid), time.Now(), time.Now()) - var kv map[interface{}]interface{} - b, err := ioutil.ReadAll(newf) + err = os.MkdirAll(newPath, 0777) if err != nil { - return nil, err + SLogger.Println(err.Error()) } - if len(b) == 0 { - kv = make(map[interface{}]interface{}) - } else { - kv, err = DecodeGob(b) + + // if old sid file exist + // 1.read and parse file content + // 2.write content to new sid file + // 3.remove old sid file, change new sid file atime and ctime + // 4.return FileSessionStore + _, err = os.Stat(oldSidFile) + if err == nil { + b, err := ioutil.ReadFile(oldSidFile) if err != nil { return nil, err } + + var kv map[interface{}]interface{} + if len(b) == 0 { + kv = make(map[interface{}]interface{}) + } else { + kv, err = DecodeGob(b) + if err != nil { + return nil, err + } + } + + ioutil.WriteFile(newSidFile, b, 0777) + os.Remove(oldSidFile) + os.Chtimes(newSidFile, time.Now(), time.Now()) + ss := &FileSessionStore{sid: sid, values: kv} + return ss, nil } - ss := &FileSessionStore{sid: sid, values: kv} + + // if old sid file not exist, just create new sid file and return + newf, err := os.Create(newSidFile) + if err != nil { + return nil, err + } + newf.Close() + ss := &FileSessionStore{sid: sid, values: make(map[interface{}]interface{})} return ss, nil } From db67ffbb94af848025403b4ca10b476e792cfb86 Mon Sep 17 00:00:00 2001 From: liming <327135508@qq.com> Date: Fri, 10 Feb 2017 09:35:23 +0800 Subject: [PATCH 31/40] 1.simplify reading and writing file code 2.add apiauth test --- cache/file.go | 25 ++-------------- plugins/apiauth/apiauth.go | 52 ++++++++++----------------------- plugins/apiauth/apiauth_test.go | 20 +++++++++++++ 3 files changed, 39 insertions(+), 58 deletions(-) create mode 100644 plugins/apiauth/apiauth_test.go diff --git a/cache/file.go b/cache/file.go index 4b030980..691ce7cd 100644 --- a/cache/file.go +++ b/cache/file.go @@ -22,6 +22,7 @@ import ( "encoding/json" "fmt" "io" + "io/ioutil" "os" "path/filepath" "reflect" @@ -222,33 +223,13 @@ func exists(path string) (bool, error) { // FileGetContents Get bytes to file. // if non-exist, create this file. func FileGetContents(filename string) (data []byte, e error) { - f, e := os.OpenFile(filename, os.O_RDWR|os.O_CREATE, os.ModePerm) - if e != nil { - return - } - defer f.Close() - stat, e := f.Stat() - if e != nil { - return - } - data = make([]byte, stat.Size()) - result, e := f.Read(data) - if e != nil || int64(result) != stat.Size() { - return nil, e - } - return + return ioutil.ReadFile(filename) } // FilePutContents Put bytes to file. // if non-exist, create this file. func FilePutContents(filename string, content []byte) error { - fp, err := os.OpenFile(filename, os.O_RDWR|os.O_CREATE, os.ModePerm) - if err != nil { - return err - } - defer fp.Close() - _, err = fp.Write(content) - return err + return ioutil.WriteFile(filename, content, os.ModePerm) } // GobEncode Gob encodes file cache item. diff --git a/plugins/apiauth/apiauth.go b/plugins/apiauth/apiauth.go index 10636d1c..f816029c 100644 --- a/plugins/apiauth/apiauth.go +++ b/plugins/apiauth/apiauth.go @@ -56,6 +56,7 @@ package apiauth import ( + "bytes" "crypto/hmac" "crypto/sha256" "encoding/base64" @@ -128,53 +129,32 @@ func APISecretAuth(f AppIDToAppSecret, timeout int) beego.FilterFunc { // Signature used to generate signature with the appsecret/method/params/RequestURI func Signature(appsecret, method string, params url.Values, RequestURL string) (result string) { - var query string + var b bytes.Buffer + keys := make([]string, len(params)) pa := make(map[string]string) for k, v := range params { pa[k] = v[0] + keys = append(keys, k) } - vs := mapSorter(pa) - vs.Sort() - for i := 0; i < vs.Len(); i++ { - if vs.Keys[i] == "signature" { + + sort.Strings(keys) + + for _, key := range keys { + if key == "signature" { continue } - if vs.Keys[i] != "" && vs.Vals[i] != "" { - query = fmt.Sprintf("%v%v%v", query, vs.Keys[i], vs.Vals[i]) + + val := pa[key] + if key != "" && val != "" { + b.WriteString(key) + b.WriteString(val) } } - stringToSign := fmt.Sprintf("%v\n%v\n%v\n", method, query, RequestURL) + + stringToSign := fmt.Sprintf("%v\n%v\n%v\n", method, b.String(), RequestURL) sha256 := sha256.New hash := hmac.New(sha256, []byte(appsecret)) hash.Write([]byte(stringToSign)) return base64.StdEncoding.EncodeToString(hash.Sum(nil)) } - -type valSorter struct { - Keys []string - Vals []string -} - -func mapSorter(m map[string]string) *valSorter { - vs := &valSorter{ - Keys: make([]string, 0, len(m)), - Vals: make([]string, 0, len(m)), - } - for k, v := range m { - vs.Keys = append(vs.Keys, k) - vs.Vals = append(vs.Vals, v) - } - return vs -} - -func (vs *valSorter) Sort() { - sort.Sort(vs) -} - -func (vs *valSorter) Len() int { return len(vs.Keys) } -func (vs *valSorter) Less(i, j int) bool { return vs.Keys[i] < vs.Keys[j] } -func (vs *valSorter) Swap(i, j int) { - vs.Vals[i], vs.Vals[j] = vs.Vals[j], vs.Vals[i] - vs.Keys[i], vs.Keys[j] = vs.Keys[j], vs.Keys[i] -} diff --git a/plugins/apiauth/apiauth_test.go b/plugins/apiauth/apiauth_test.go new file mode 100644 index 00000000..1f56cb0f --- /dev/null +++ b/plugins/apiauth/apiauth_test.go @@ -0,0 +1,20 @@ +package apiauth + +import ( + "net/url" + "testing" +) + +func TestSignature(t *testing.T) { + appsecret := "beego secret" + method := "GET" + RequestURL := "http://localhost/test/url" + params := make(url.Values) + params.Add("arg1", "hello") + params.Add("arg2", "beego") + + signature := "mFdpvLh48ca4mDVEItE9++AKKQ/IVca7O/ZyyB8hR58=" + if Signature(appsecret, method, params, RequestURL) != signature { + t.Error("Signature error") + } +} From 8a2b6976251fba8a319cf421d54e456f1469ac5e Mon Sep 17 00:00:00 2001 From: awengo Date: Fri, 10 Feb 2017 17:45:47 +0900 Subject: [PATCH 32/40] Add http methods --- router.go | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/router.go b/router.go index 74cf02a1..9f573f26 100644 --- a/router.go +++ b/router.go @@ -51,15 +51,22 @@ const ( var ( // HTTPMETHOD list the supported http methods. HTTPMETHOD = map[string]string{ - "GET": "GET", - "POST": "POST", - "PUT": "PUT", - "DELETE": "DELETE", - "PATCH": "PATCH", - "OPTIONS": "OPTIONS", - "HEAD": "HEAD", - "TRACE": "TRACE", - "CONNECT": "CONNECT", + "GET": "GET", + "POST": "POST", + "PUT": "PUT", + "DELETE": "DELETE", + "PATCH": "PATCH", + "OPTIONS": "OPTIONS", + "HEAD": "HEAD", + "TRACE": "TRACE", + "CONNECT": "CONNECT", + "MKCOL": "MKCOL", + "COPY": "COPY", + "MOVE": "MOVE", + "PROPFIND": "PROPFIND", + "PROPPATCH": "PROPPATCH", + "LOCK": "LOCK", + "UNLOCK": "UNLOCK", } // these beego.Controller's methods shouldn't reflect to AutoRouter exceptMethod = []string{"Init", "Prepare", "Finish", "Render", "RenderString", From fc2c0f4fbac53d3df06c706c7d79c77fc61e0ee1 Mon Sep 17 00:00:00 2001 From: Eyal Post Date: Sat, 11 Feb 2017 22:00:30 +0200 Subject: [PATCH 33/40] Don't allow AddViewPath after beego run --- hooks.go | 1 + template.go | 16 +++++++++++++++- 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/hooks.go b/hooks.go index 528b58cc..091ecbc7 100644 --- a/hooks.go +++ b/hooks.go @@ -72,6 +72,7 @@ func registerSession() error { } func registerTemplate() error { + defer lockViewPaths() if err := AddViewPath(BConfig.WebConfig.ViewsPath); err != nil { if BConfig.RunMode == DEV { logs.Warn(err) diff --git a/template.go b/template.go index d4d0ed12..2dba9d1f 100644 --- a/template.go +++ b/template.go @@ -32,7 +32,8 @@ import ( var ( beegoTplFuncMap = make(template.FuncMap) - // beeViewPathTemplates caching map and supported template file extensions. + beeViewPathTemplateLocked = false + // beeViewPathTemplates caching map and supported template file extensions per view beeViewPathTemplates = make(map[string]map[string]*template.Template) templatesLock sync.RWMutex // beeTemplateExt stores the template extension which will build @@ -48,6 +49,9 @@ func ExecuteTemplate(wr io.Writer, name string, data interface{}) error { return ExecuteViewPathTemplate(wr,name, BConfig.WebConfig.ViewsPath, data) } +// ExecuteViewPathTemplate applies the template with name and from specific viewPath to the specified data object, +// writing the output to wr. +// A template will be executed safely in parallel. func ExecuteViewPathTemplate(wr io.Writer, name string, viewPath string, data interface{}) error { if BConfig.RunMode == DEV { templatesLock.RLock() @@ -156,11 +160,21 @@ func AddTemplateExt(ext string) { beeTemplateExt = append(beeTemplateExt, ext) } +// AddViewPath adds a new path to the supported view paths. +//Can later be used by setting a controller ViewPath to this folder +//will panic if called after beego.Run() func AddViewPath(viewPath string) error { + if beeViewPathTemplateLocked { + panic("Can not add new view paths after beego.Run()") + } beeViewPathTemplates[viewPath] = make(map[string]*template.Template) return BuildTemplate(viewPath) } +func lockViewPaths() { + beeViewPathTemplateLocked = true +} + // BuildTemplate will build all template files in a directory. // it makes beego can render any template file in view directory. func BuildTemplate(dir string, files ...string) error { From 393e4c4969f62c71ce4d65a68fca96ea60e84b0e Mon Sep 17 00:00:00 2001 From: jiayukun Date: Wed, 22 Feb 2017 17:38:26 +0800 Subject: [PATCH 34/40] Improve json coding performance --- context/output.go | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/context/output.go b/context/output.go index 4b513dd8..564ef96d 100644 --- a/context/output.go +++ b/context/output.go @@ -331,16 +331,17 @@ func (output *BeegoOutput) IsServerError() bool { func stringsToJSON(str string) string { rs := []rune(str) - jsons := "" + var jsons bytes.Buffer for _, r := range rs { rint := int(r) if rint < 128 { - jsons += string(r) + jsons.WriteRune(r) } else { - jsons += "\\u" + strconv.FormatInt(int64(rint), 16) // json + jsons.WriteString("\\u") + jsons.WriteString(strconv.FormatInt(int64(rint), 16)) } } - return jsons + return jsons.String() } // Session sets session item value with given key. From 28011a5835229ad64d90a9d8f0805571cbf07d40 Mon Sep 17 00:00:00 2001 From: astaxie Date: Sun, 26 Feb 2017 22:20:11 +0800 Subject: [PATCH 35/40] fix #2462 --- config.go | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/config.go b/config.go index 52e647db..42b7c70c 100644 --- a/config.go +++ b/config.go @@ -254,6 +254,9 @@ func parseConfig(appConfigPath string) (err error) { } func assignConfig(ac config.Configer) error { + for _, i := range []interface{}{&BConfig.Listen, &BConfig.WebConfig, &BConfig.Log, &BConfig.WebConfig.Session} { + assignSingleConfig(i, ac) + } // set the run mode first if envRunMode := os.Getenv("BEEGO_RUNMODE"); envRunMode != "" { BConfig.RunMode = envRunMode @@ -261,10 +264,6 @@ func assignConfig(ac config.Configer) error { BConfig.RunMode = runMode } - for _, i := range []interface{}{BConfig, &BConfig.Listen, &BConfig.WebConfig, &BConfig.Log, &BConfig.WebConfig.Session} { - assignSingleConfig(i, ac) - } - if sd := ac.String("StaticDir"); sd != "" { BConfig.WebConfig.StaticDir = map[string]string{} sds := strings.Fields(sd) From 50e294be32da818b9fea72c42a668033f396674d Mon Sep 17 00:00:00 2001 From: astaxie Date: Mon, 27 Feb 2017 09:41:15 +0800 Subject: [PATCH 36/40] fix the broken test --- config.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config.go b/config.go index 51ee08a1..3c202e53 100644 --- a/config.go +++ b/config.go @@ -256,7 +256,7 @@ func parseConfig(appConfigPath string) (err error) { } func assignConfig(ac config.Configer) error { - for _, i := range []interface{}{&BConfig.Listen, &BConfig.WebConfig, &BConfig.Log, &BConfig.WebConfig.Session} { + for _, i := range []interface{}{BConfig, &BConfig.Listen, &BConfig.WebConfig, &BConfig.Log, &BConfig.WebConfig.Session} { assignSingleConfig(i, ac) } // set the run mode first From 74045090cc834b4b13587a210ed92d7a188a2c1c Mon Sep 17 00:00:00 2001 From: Anton Khalikov Date: Mon, 27 Feb 2017 14:14:16 +0500 Subject: [PATCH 37/40] This fixes issue #2467 with ssdb cache IsExist --- cache/ssdb/ssdb.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cache/ssdb/ssdb.go b/cache/ssdb/ssdb.go index bfee69ce..bbc43606 100644 --- a/cache/ssdb/ssdb.go +++ b/cache/ssdb/ssdb.go @@ -152,7 +152,7 @@ func (rc *Cache) IsExist(key string) bool { if err != nil { return false } - if resp[1] == "1" { + if len(resp) == 2 && resp[1] == "1" { return true } return false From e0250e287137fd061217c107b061077c98b25dd5 Mon Sep 17 00:00:00 2001 From: Mariano Fevola Date: Fri, 3 Mar 2017 16:24:02 +0000 Subject: [PATCH 38/40] Fix typo --- config/ini.go | 2 +- config/yaml/yaml.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/config/ini.go b/config/ini.go index 6b78f02a..416d48f0 100644 --- a/config/ini.go +++ b/config/ini.go @@ -300,7 +300,7 @@ func (c *IniConfigContainer) GetSection(section string) (map[string]string, erro if v, ok := c.data[section]; ok { return v, nil } - return nil, errors.New("not exist setction") + return nil, errors.New("not exist section") } // SaveConfigFile save the config into file. diff --git a/config/yaml/yaml.go b/config/yaml/yaml.go index bcef4a20..51fe44d3 100644 --- a/config/yaml/yaml.go +++ b/config/yaml/yaml.go @@ -252,7 +252,7 @@ func (c *ConfigContainer) GetSection(section string) (map[string]string, error) if v, ok := c.data[section]; ok { return v.(map[string]string), nil } - return nil, errors.New("not exist setction") + return nil, errors.New("not exist section") } // SaveConfigFile save the config into file From 6d997366ed4d300bebfe851dde62598d592b5d7e Mon Sep 17 00:00:00 2001 From: ysqi Date: Sat, 4 Mar 2017 20:23:55 +0800 Subject: [PATCH 39/40] Fixed 2456 and strengthen bind --- context/input.go | 11 +++- context/input_test.go | 138 +++++++++++++++++++++++------------------- 2 files changed, 87 insertions(+), 62 deletions(-) diff --git a/context/input.go b/context/input.go index 1e6eaf71..d9015ce3 100644 --- a/context/input.go +++ b/context/input.go @@ -413,7 +413,13 @@ func (input *BeegoInput) Bind(dest interface{}, key string) error { if !value.CanSet() { return errors.New("beego: non-settable variable passed to Bind: " + key) } - rv := input.bind(key, value.Type()) + typ := value.Type() + // Get real type if dest define with interface{}. + // e.g var dest interface{} dest=1.0 + if value.Kind() == reflect.Interface { + typ = value.Elem().Type() + } + rv := input.bind(key, typ) if !rv.IsValid() { return errors.New("beego: reflect value is empty") } @@ -422,6 +428,9 @@ func (input *BeegoInput) Bind(dest interface{}, key string) error { } func (input *BeegoInput) bind(key string, typ reflect.Type) reflect.Value { + if input.Context.Request.Form == nil { + input.Context.Request.ParseForm() + } rv := reflect.Zero(typ) switch typ.Kind() { case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: diff --git a/context/input_test.go b/context/input_test.go index e64addba..9853e398 100644 --- a/context/input_test.go +++ b/context/input_test.go @@ -15,81 +15,97 @@ package context import ( - "fmt" "net/http" "net/http/httptest" "reflect" "testing" ) -func TestParse(t *testing.T) { - r, _ := http.NewRequest("GET", "/?id=123&isok=true&ft=1.2&ol[0]=1&ol[1]=2&ul[]=str&ul[]=array&user.Name=astaxie", nil) - beegoInput := NewInput() - beegoInput.Context = NewContext() - beegoInput.Context.Reset(httptest.NewRecorder(), r) - beegoInput.ParseFormOrMulitForm(1 << 20) +func TestBind(t *testing.T) { + type testItem struct { + field string + empty interface{} + want interface{} + } + type Human struct { + ID int + Nick string + Pwd string + Ms bool + } - var id int - err := beegoInput.Bind(&id, "id") - if id != 123 || err != nil { - t.Fatal("id should has int value") - } - fmt.Println(id) + cases := []struct { + request string + valueGp []testItem + }{ + {"/?p=str", []testItem{{"p", interface{}(""), interface{}("str")}}}, - var isok bool - err = beegoInput.Bind(&isok, "isok") - if !isok || err != nil { - t.Fatal("isok should be true") - } - fmt.Println(isok) + {"/?p=", []testItem{{"p", "", ""}}}, + {"/?p=str", []testItem{{"p", "", "str"}}}, - var float float64 - err = beegoInput.Bind(&float, "ft") - if float != 1.2 || err != nil { - t.Fatal("float should be equal to 1.2") - } - fmt.Println(float) + {"/?p=123", []testItem{{"p", 0, 123}}}, + {"/?p=123", []testItem{{"p", uint(0), uint(123)}}}, - ol := make([]int, 0, 2) - err = beegoInput.Bind(&ol, "ol") - if len(ol) != 2 || err != nil || ol[0] != 1 || ol[1] != 2 { - t.Fatal("ol should has two elements") - } - fmt.Println(ol) + {"/?p=1.0", []testItem{{"p", 0.0, 1.0}}}, + {"/?p=1", []testItem{{"p", false, true}}}, - ul := make([]string, 0, 2) - err = beegoInput.Bind(&ul, "ul") - if len(ul) != 2 || err != nil || ul[0] != "str" || ul[1] != "array" { - t.Fatal("ul should has two elements") - } - fmt.Println(ul) + {"/?p=true", []testItem{{"p", false, true}}}, + {"/?p=ON", []testItem{{"p", false, true}}}, + {"/?p=on", []testItem{{"p", false, true}}}, + {"/?p=1", []testItem{{"p", false, true}}}, + {"/?p=2", []testItem{{"p", false, false}}}, + {"/?p=false", []testItem{{"p", false, false}}}, - type User struct { - Name string - } - user := User{} - err = beegoInput.Bind(&user, "user") - if err != nil || user.Name != "astaxie" { - t.Fatal("user should has name") - } - fmt.Println(user) -} + {"/?p[a]=1&p[b]=2&p[c]=3", []testItem{{"p", map[string]int{}, map[string]int{"a": 1, "b": 2, "c": 3}}}}, + {"/?p[a]=v1&p[b]=v2&p[c]=v3", []testItem{{"p", map[string]string{}, map[string]string{"a": "v1", "b": "v2", "c": "v3"}}}}, -func TestParse2(t *testing.T) { - r, _ := http.NewRequest("GET", "/?user[0][Username]=Raph&user[1].Username=Leo&user[0].Password=123456&user[1][Password]=654321", nil) - beegoInput := NewInput() - beegoInput.Context = NewContext() - beegoInput.Context.Reset(httptest.NewRecorder(), r) - beegoInput.ParseFormOrMulitForm(1 << 20) - type User struct { - Username string - Password string + {"/?p[]=8&p[]=9&p[]=10", []testItem{{"p", []int{}, []int{8, 9, 10}}}}, + {"/?p[0]=8&p[1]=9&p[2]=10", []testItem{{"p", []int{}, []int{8, 9, 10}}}}, + {"/?p[0]=8&p[1]=9&p[2]=10&p[5]=14", []testItem{{"p", []int{}, []int{8, 9, 10, 0, 0, 14}}}}, + {"/?p[0]=8.0&p[1]=9.0&p[2]=10.0", []testItem{{"p", []float64{}, []float64{8.0, 9.0, 10.0}}}}, + + {"/?p[]=10&p[]=9&p[]=8", []testItem{{"p", []string{}, []string{"10", "9", "8"}}}}, + {"/?p[0]=8&p[1]=9&p[2]=10", []testItem{{"p", []string{}, []string{"8", "9", "10"}}}}, + + {"/?p[0]=true&p[1]=false&p[2]=true&p[5]=1&p[6]=ON&p[7]=other", []testItem{{"p", []bool{}, []bool{true, false, true, false, false, true, true, false}}}}, + + {"/?human.Nick=astaxie", []testItem{{"human", Human{}, Human{Nick: "astaxie"}}}}, + {"/?human.ID=888&human.Nick=astaxie&human.Ms=true&human[Pwd]=pass", []testItem{{"human", Human{}, Human{ID: 888, Nick: "astaxie", Ms: true, Pwd: "pass"}}}}, + {"/?human[0].ID=888&human[0].Nick=astaxie&human[0].Ms=true&human[0][Pwd]=pass01&human[1].ID=999&human[1].Nick=ysqi&human[1].Ms=On&human[1].Pwd=pass02", + []testItem{{"human", []Human{}, []Human{ + Human{ID: 888, Nick: "astaxie", Ms: true, Pwd: "pass01"}, + Human{ID: 999, Nick: "ysqi", Ms: true, Pwd: "pass02"}, + }}}}, + + { + "/?id=123&isok=true&ft=1.2&ol[0]=1&ol[1]=2&ul[]=str&ul[]=array&human.Nick=astaxie", + []testItem{ + {"id", 0, 123}, + {"isok", false, true}, + {"ft", 0.0, 1.2}, + {"ol", []int{}, []int{1, 2}}, + {"ul", []string{}, []string{"str", "array"}}, + {"human", Human{}, Human{Nick: "astaxie"}}, + }, + }, } - var users []User - err := beegoInput.Bind(&users, "user") - fmt.Println(users) - if err != nil || users[0].Username != "Raph" || users[0].Password != "123456" || users[1].Username != "Leo" || users[1].Password != "654321" { - t.Fatal("users info wrong") + for _, c := range cases { + r, _ := http.NewRequest("GET", c.request, nil) + beegoInput := NewInput() + beegoInput.Context = NewContext() + beegoInput.Context.Reset(httptest.NewRecorder(), r) + + for _, item := range c.valueGp { + got := item.empty + err := beegoInput.Bind(&got, item.field) + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(got, item.want) { + t.Fatalf("Bind %q error,should be:\n%#v \ngot:\n%#v", item.field, item.want, got) + } + } + } } From c5f838e7856f07c96173717e4e04126ff0dc7978 Mon Sep 17 00:00:00 2001 From: astaxie Date: Mon, 6 Mar 2017 21:29:41 +0800 Subject: [PATCH 40/40] beego 1.8.0 --- beego.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beego.go b/beego.go index 1bc8bb85..c06b499c 100644 --- a/beego.go +++ b/beego.go @@ -23,7 +23,7 @@ import ( const ( // VERSION represent beego web framework version. - VERSION = "1.7.2" + VERSION = "1.8.0" // DEV is for develop DEV = "dev"