lbsadmin/Beego
1
0
Fork 0
mirror of https://github.com/astaxie/beego.git synced 2024-11-21 23:30:55 +00:00
Code Issues Releases Activity

Merge pull request #2485 from astaxie/develop

Browse Source 
beego 1.8.0
This commit is contained in:
astaxie 2017-03-06 21:59:04 +08:00 committed by GitHub
commit 323a1c4214
50 changed files with 3336 additions and 300 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@ -1,4 +1,5 @@
.idea
.vscode
.DS_Store
*.swp
*.swo

2
.travis.yml
View File

@ -31,6 +31,8 @@ install:
- go get github.com/siddontang/ledisdb/config
- go get github.com/siddontang/ledisdb/ledis
- go get github.com/ssdb/gossdb/ssdb
- go get github.com/cloudflare/golz4
- go get github.com/gogo/protobuf/proto
before_script:
- psql --version
- sh -c "if [ '$ORM_DRIVER' = 'postgres' ]; then psql -c 'create database orm_test;' -U postgres; fi"

2
beego.go
View File

@ -23,7 +23,7 @@ import (
const (
// VERSION represent beego web framework version.
VERSION = "1.7.2"
VERSION = "1.8.0"
// DEV is for develop
DEV = "dev"

25
cache/file.go vendored
View File

@ -22,6 +22,7 @@ import (
"encoding/json"
"fmt"
"io"
"io/ioutil"
"os"
"path/filepath"
"reflect"
@ -222,33 +223,13 @@ func exists(path string) (bool, error) {
// FileGetContents Get bytes to file.
// if non-exist, create this file.
func FileGetContents(filename string) (data []byte, e error) {
f, e := os.OpenFile(filename, os.O_RDWR|os.O_CREATE, os.ModePerm)
if e != nil {
return
}
defer f.Close()
stat, e := f.Stat()
if e != nil {
return
}
data = make([]byte, stat.Size())
result, e := f.Read(data)
if e != nil || int64(result) != stat.Size() {
return nil, e
}
return
return ioutil.ReadFile(filename)
}
// FilePutContents Put bytes to file.
// if non-exist, create this file.
func FilePutContents(filename string, content []byte) error {
fp, err := os.OpenFile(filename, os.O_RDWR|os.O_CREATE, os.ModePerm)
if err != nil {
return err
}
defer fp.Close()
_, err = fp.Write(content)
return err
return ioutil.WriteFile(filename, content, os.ModePerm)
}
// GobEncode Gob encodes file cache item.

2
cache/ssdb/ssdb.go vendored
View File

@ -152,7 +152,7 @@ func (rc *Cache) IsExist(key string) bool {
if err != nil {
return false
}
if resp[1] == "1" {
if len(resp) == 2 && resp[1] == "1" {
return true
}
return false

14
config.go
View File

@ -41,6 +41,7 @@ type Config struct {
EnableGzip bool
MaxMemory int64
EnableErrorsShow bool
EnableErrorsRender bool
Listen Listen
WebConfig WebConfig
Log LogConfig
@ -144,9 +145,6 @@ func init() {
if err = parseConfig(appConfigPath); err != nil {
panic(err)
}
if err = os.Chdir(AppPath); err != nil {
panic(err)
}
}
func recoverPanic(ctx *context.Context) {
@ -174,7 +172,7 @@ func recoverPanic(ctx *context.Context) {
logs.Critical(fmt.Sprintf("%s:%d", file, line))
stack = stack + fmt.Sprintln(fmt.Sprintf("%s:%d", file, line))
}
if BConfig.RunMode == DEV {
if BConfig.RunMode == DEV && BConfig.EnableErrorsRender {
showErr(err, ctx, stack)
}
}
@ -192,6 +190,7 @@ func newBConfig() *Config {
EnableGzip: false,
MaxMemory: 1 << 26, //64MB
EnableErrorsShow: true,
EnableErrorsRender: true,
Listen: Listen{
Graceful: false,
ServerTimeOut: 0,
@ -257,6 +256,9 @@ func parseConfig(appConfigPath string) (err error) {
}
func assignConfig(ac config.Configer) error {
for _, i := range []interface{}{BConfig, &BConfig.Listen, &BConfig.WebConfig, &BConfig.Log, &BConfig.WebConfig.Session} {
assignSingleConfig(i, ac)
}
// set the run mode first
if envRunMode := os.Getenv("BEEGO_RUNMODE"); envRunMode != "" {
BConfig.RunMode = envRunMode
@ -264,10 +266,6 @@ func assignConfig(ac config.Configer) error {
BConfig.RunMode = runMode
}
for _, i := range []interface{}{BConfig, &BConfig.Listen, &BConfig.WebConfig, &BConfig.Log, &BConfig.WebConfig.Session} {
assignSingleConfig(i, ac)
}
if sd := ac.String("StaticDir"); sd != "" {
BConfig.WebConfig.StaticDir = map[string]string{}
sds := strings.Fields(sd)

85
config/env/env.go vendored Normal file
View File

@ -0,0 +1,85 @@
// Copyright 2014 beego Author. All Rights Reserved.
// Copyright 2017 Faissal Elamraoui. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package env
import (
"fmt"
"os"
"strings"
"github.com/astaxie/beego/utils"
)
var env *utils.BeeMap
func init() {
env = utils.NewBeeMap()
for _, e := range os.Environ() {
splits := strings.Split(e, "=")
env.Set(splits[0], os.Getenv(splits[0]))
}
}
// Get returns a value by key.
// If the key does not exist, the default value will be returned.
func Get(key string, defVal string) string {
if val := env.Get(key); val != nil {
return val.(string)
}
return defVal
}
// MustGet returns a value by key.
// If the key does not exist, it will return an error.
func MustGet(key string) (string, error) {
if val := env.Get(key); val != nil {
return val.(string), nil
}
return "", fmt.Errorf("no env variable with %s", key)
}
// Set sets a value in the ENV copy.
// This does not affect the child process environment.
func Set(key string, value string) {
env.Set(key, value)
}
// MustSet sets a value in the ENV copy and the child process environment.
// It returns an error in case the set operation failed.
func MustSet(key string, value string) error {
err := os.Setenv(key, value)
if err != nil {
return err
}
env.Set(key, value)
return nil
}
// GetAll returns all keys/values in the current child process environment.
func GetAll() map[string]string {
items := env.Items()
envs := make(map[string]string, env.Count())
for key, val := range items {
switch key := key.(type) {
case string:
switch val := val.(type) {
case string:
envs[key] = val
}
}
}
return envs
}

75
config/env/env_test.go vendored Normal file
View File

@ -0,0 +1,75 @@
// Copyright 2014 beego Author. All Rights Reserved.
// Copyright 2017 Faissal Elamraoui. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package env
import (
"os"
"testing"
)
func TestEnvGet(t *testing.T) {
gopath := Get("GOPATH", "")
if gopath != os.Getenv("GOPATH") {
t.Error("expected GOPATH not empty.")
}
noExistVar := Get("NOEXISTVAR", "foo")
if noExistVar != "foo" {
t.Errorf("expected NOEXISTVAR to equal foo, got %s.", noExistVar)
}
}
func TestEnvMustGet(t *testing.T) {
gopath, err := MustGet("GOPATH")
if err != nil {
t.Error(err)
}
if gopath != os.Getenv("GOPATH") {
t.Errorf("expected GOPATH to be the same, got %s.", gopath)
}
_, err = MustGet("NOEXISTVAR")
if err == nil {
t.Error("expected error to be non-nil")
}
}
func TestEnvSet(t *testing.T) {
Set("MYVAR", "foo")
myVar := Get("MYVAR", "bar")
if myVar != "foo" {
t.Errorf("expected MYVAR to equal foo, got %s.", myVar)
}
}
func TestEnvMustSet(t *testing.T) {
err := MustSet("FOO", "bar")
if err != nil {
t.Error(err)
}
fooVar := os.Getenv("FOO")
if fooVar != "bar" {
t.Errorf("expected FOO variable to equal bar, got %s.", fooVar)
}
}
func TestEnvGetAll(t *testing.T) {
envMap := GetAll()
if len(envMap) == 0 {
t.Error("expected environment not empty.")
}
}

46
config/ini.go
View File

@ -18,16 +18,13 @@ import (
"bufio"
"bytes"
"errors"
"fmt"
"io"
"io/ioutil"
"os"
"path"
"path/filepath"
"strconv"
"strings"
"sync"
"time"
)
var (
@ -52,24 +49,26 @@ func (ini *IniConfig) Parse(name string) (Configer, error) {
}
func (ini *IniConfig) parseFile(name string) (*IniConfigContainer, error) {
file, err := os.Open(name)
data, err := ioutil.ReadFile(name)
if err != nil {
return nil, err
}
return ini.parseData(filepath.Dir(name), data)
}
func (ini *IniConfig) parseData(dir string, data []byte) (*IniConfigContainer, error) {
cfg := &IniConfigContainer{
file.Name(),
make(map[string]map[string]string),
make(map[string]string),
make(map[string]string),
sync.RWMutex{},
data: make(map[string]map[string]string),
sectionComment: make(map[string]string),
keyComment: make(map[string]string),
RWMutex: sync.RWMutex{},
}
cfg.Lock()
defer cfg.Unlock()
defer file.Close()
var comment bytes.Buffer
buf := bufio.NewReader(file)
buf := bufio.NewReader(bytes.NewBuffer(data))
// check the BOM
head, err := buf.Peek(3)
if err == nil && head[0] == 239 && head[1] == 187 && head[2] == 191 {
@ -130,16 +129,20 @@ func (ini *IniConfig) parseFile(name string) (*IniConfigContainer, error) {
// handle include "other.conf"
if len(keyValue) == 1 && strings.HasPrefix(key, "include") {
includefiles := strings.Fields(key)
if includefiles[0] == "include" && len(includefiles) == 2 {
otherfile := strings.Trim(includefiles[1], "\"")
if !filepath.IsAbs(otherfile) {
otherfile = filepath.Join(filepath.Dir(name), otherfile)
otherfile = filepath.Join(dir, otherfile)
}
i, err := ini.parseFile(otherfile)
if err != nil {
return nil, err
}
for sec, dt := range i.data {
if _, ok := cfg.data[sec]; !ok {
cfg.data[sec] = make(map[string]string)
@ -148,12 +151,15 @@ func (ini *IniConfig) parseFile(name string) (*IniConfigContainer, error) {
cfg.data[sec][k] = v
}
}
for sec, comm := range i.sectionComment {
cfg.sectionComment[sec] = comm
}
for k, comm := range i.keyComment {
cfg.keyComment[k] = comm
}
continue
}
}
@ -177,20 +183,18 @@ func (ini *IniConfig) parseFile(name string) (*IniConfigContainer, error) {
}
// ParseData parse ini the data
// When include other.conf,other.conf is either absolute directory
// or under beego in default temporary directory(/tmp/beego).
func (ini *IniConfig) ParseData(data []byte) (Configer, error) {
// Save memory data to temporary file
tmpName := path.Join(os.TempDir(), "beego", fmt.Sprintf("%d", time.Now().Nanosecond()))
os.MkdirAll(path.Dir(tmpName), os.ModePerm)
if err := ioutil.WriteFile(tmpName, data, 0655); err != nil {
return nil, err
}
return ini.Parse(tmpName)
dir := filepath.Join(os.TempDir(), "beego")
os.MkdirAll(dir, os.ModePerm)
return ini.parseData(dir, data)
}
// IniConfigContainer A Config represents the ini configuration.
// When set and get value, support key as section:name type.
type IniConfigContainer struct {
filename string
data map[string]map[string]string // section=> key:val
sectionComment map[string]string // section : comment
keyComment map[string]string // id: []{comment, key...}; id 1 is for main comment.
@ -297,7 +301,7 @@ func (c *IniConfigContainer) GetSection(section string) (map[string]string, erro
if v, ok := c.data[section]; ok {
return v, nil
}
return nil, errors.New("not exist setction")
return nil, errors.New("not exist section")
}
// SaveConfigFile save the config into file.

28
config/xml/xml.go
View File

@ -35,11 +35,9 @@ import (
"fmt"
"io/ioutil"
"os"
"path"
"strconv"
"strings"
"sync"
"time"
"github.com/astaxie/beego/config"
"github.com/beego/x2j"
@ -52,36 +50,26 @@ type Config struct{}
// Parse returns a ConfigContainer with parsed xml config map.
func (xc *Config) Parse(filename string) (config.Configer, error) {
file, err := os.Open(filename)
context, err := ioutil.ReadFile(filename)
if err != nil {
return nil, err
}
defer file.Close()
return xc.ParseData(context)
}
// ParseData xml data
func (xc *Config) ParseData(data []byte) (config.Configer, error) {
x := &ConfigContainer{data: make(map[string]interface{})}
content, err := ioutil.ReadAll(file)
if err != nil {
return nil, err
}
d, err := x2j.DocToMap(string(content))
d, err := x2j.DocToMap(string(data))
if err != nil {
return nil, err
}
x.data = config.ExpandValueEnvForMap(d["config"].(map[string]interface{}))
return x, nil
}
// ParseData xml data
func (xc *Config) ParseData(data []byte) (config.Configer, error) {
// Save memory data to temporary file
tmpName := path.Join(os.TempDir(), "beego", fmt.Sprintf("%d", time.Now().Nanosecond()))
os.MkdirAll(path.Dir(tmpName), os.ModePerm)
if err := ioutil.WriteFile(tmpName, data, 0655); err != nil {
return nil, err
}
return xc.Parse(tmpName)
return x, nil
}
// ConfigContainer A Config represents the xml configuration.

26
config/yaml/yaml.go
View File

@ -37,10 +37,8 @@ import (
"io/ioutil"
"log"
"os"
"path"
"strings"
"sync"
"time"
"github.com/astaxie/beego/config"
"github.com/beego/goyaml2"
@ -63,26 +61,30 @@ func (yaml *Config) Parse(filename string) (y config.Configer, err error) {
// ParseData parse yaml data
func (yaml *Config) ParseData(data []byte) (config.Configer, error) {
// Save memory data to temporary file
tmpName := path.Join(os.TempDir(), "beego", fmt.Sprintf("%d", time.Now().Nanosecond()))
os.MkdirAll(path.Dir(tmpName), os.ModePerm)
if err := ioutil.WriteFile(tmpName, data, 0655); err != nil {
cnf, err := parseYML(data)
if err != nil {
return nil, err
}
return yaml.Parse(tmpName)
return &ConfigContainer{
data: cnf,
}, nil
}
// ReadYmlReader Read yaml file to map.
// if json like, use json package, unless goyaml2 package.
func ReadYmlReader(path string) (cnf map[string]interface{}, err error) {
f, err := os.Open(path)
buf, err := ioutil.ReadFile(path)
if err != nil {
return
}
defer f.Close()
buf, err := ioutil.ReadAll(f)
if err != nil || len(buf) < 3 {
return parseYML(buf)
}
// parseYML parse yaml formatted []byte to map.
func parseYML(buf []byte) (cnf map[string]interface{}, err error) {
if len(buf) < 3 {
return
}
@ -250,7 +252,7 @@ func (c *ConfigContainer) GetSection(section string) (map[string]string, error)
if v, ok := c.data[section]; ok {
return v.(map[string]string), nil
}
return nil, errors.New("not exist setction")
return nil, errors.New("not exist section")
}
// SaveConfigFile save the config into file

11
context/input.go
View File

@ -413,7 +413,13 @@ func (input *BeegoInput) Bind(dest interface{}, key string) error {
if !value.CanSet() {
return errors.New("beego: non-settable variable passed to Bind: " + key)
}
rv := input.bind(key, value.Type())
typ := value.Type()
// Get real type if dest define with interface{}.
// e.g var dest interface{} dest=1.0
if value.Kind() == reflect.Interface {
typ = value.Elem().Type()
}
rv := input.bind(key, typ)
if !rv.IsValid() {
return errors.New("beego: reflect value is empty")
}
@ -422,6 +428,9 @@ func (input *BeegoInput) Bind(dest interface{}, key string) error {
}
func (input *BeegoInput) bind(key string, typ reflect.Type) reflect.Value {
if input.Context.Request.Form == nil {
input.Context.Request.ParseForm()
}
rv := reflect.Zero(typ)
switch typ.Kind() {
case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:

138
context/input_test.go
View File

@ -15,81 +15,97 @@
package context
import (
"fmt"
"net/http"
"net/http/httptest"
"reflect"
"testing"
)
func TestParse(t *testing.T) {
r, _ := http.NewRequest("GET", "/?id=123&isok=true&ft=1.2&ol[0]=1&ol[1]=2&ul[]=str&ul[]=array&user.Name=astaxie", nil)
beegoInput := NewInput()
beegoInput.Context = NewContext()
beegoInput.Context.Reset(httptest.NewRecorder(), r)
beegoInput.ParseFormOrMulitForm(1 << 20)
func TestBind(t *testing.T) {
type testItem struct {
field string
empty interface{}
want interface{}
}
type Human struct {
ID int
Nick string
Pwd string
Ms bool
}
var id int
err := beegoInput.Bind(&id, "id")
if id != 123 || err != nil {
t.Fatal("id should has int value")
}
fmt.Println(id)
cases := []struct {
request string
valueGp []testItem
}{
{"/?p=str", []testItem{{"p", interface{}(""), interface{}("str")}}},
var isok bool
err = beegoInput.Bind(&isok, "isok")
if !isok || err != nil {
t.Fatal("isok should be true")
}
fmt.Println(isok)
{"/?p=", []testItem{{"p", "", ""}}},
{"/?p=str", []testItem{{"p", "", "str"}}},
var float float64
err = beegoInput.Bind(&float, "ft")
if float != 1.2 || err != nil {
t.Fatal("float should be equal to 1.2")
}
fmt.Println(float)
{"/?p=123", []testItem{{"p", 0, 123}}},
{"/?p=123", []testItem{{"p", uint(0), uint(123)}}},
ol := make([]int, 0, 2)
err = beegoInput.Bind(&ol, "ol")
if len(ol) != 2 || err != nil || ol[0] != 1 || ol[1] != 2 {
t.Fatal("ol should has two elements")
}
fmt.Println(ol)
{"/?p=1.0", []testItem{{"p", 0.0, 1.0}}},
{"/?p=1", []testItem{{"p", false, true}}},
ul := make([]string, 0, 2)
err = beegoInput.Bind(&ul, "ul")
if len(ul) != 2 || err != nil || ul[0] != "str" || ul[1] != "array" {
t.Fatal("ul should has two elements")
}
fmt.Println(ul)
{"/?p=true", []testItem{{"p", false, true}}},
{"/?p=ON", []testItem{{"p", false, true}}},
{"/?p=on", []testItem{{"p", false, true}}},
{"/?p=1", []testItem{{"p", false, true}}},
{"/?p=2", []testItem{{"p", false, false}}},
{"/?p=false", []testItem{{"p", false, false}}},
type User struct {
Name string
}
user := User{}
err = beegoInput.Bind(&user, "user")
if err != nil || user.Name != "astaxie" {
t.Fatal("user should has name")
}
fmt.Println(user)
}
{"/?p[a]=1&p[b]=2&p[c]=3", []testItem{{"p", map[string]int{}, map[string]int{"a": 1, "b": 2, "c": 3}}}},
{"/?p[a]=v1&p[b]=v2&p[c]=v3", []testItem{{"p", map[string]string{}, map[string]string{"a": "v1", "b": "v2", "c": "v3"}}}},
func TestParse2(t *testing.T) {
r, _ := http.NewRequest("GET", "/?user[0][Username]=Raph&user[1].Username=Leo&user[0].Password=123456&user[1][Password]=654321", nil)
beegoInput := NewInput()
beegoInput.Context = NewContext()
beegoInput.Context.Reset(httptest.NewRecorder(), r)
beegoInput.ParseFormOrMulitForm(1 << 20)
type User struct {
Username string
Password string
{"/?p[]=8&p[]=9&p[]=10", []testItem{{"p", []int{}, []int{8, 9, 10}}}},
{"/?p[0]=8&p[1]=9&p[2]=10", []testItem{{"p", []int{}, []int{8, 9, 10}}}},
{"/?p[0]=8&p[1]=9&p[2]=10&p[5]=14", []testItem{{"p", []int{}, []int{8, 9, 10, 0, 0, 14}}}},
{"/?p[0]=8.0&p[1]=9.0&p[2]=10.0", []testItem{{"p", []float64{}, []float64{8.0, 9.0, 10.0}}}},
{"/?p[]=10&p[]=9&p[]=8", []testItem{{"p", []string{}, []string{"10", "9", "8"}}}},
{"/?p[0]=8&p[1]=9&p[2]=10", []testItem{{"p", []string{}, []string{"8", "9", "10"}}}},
{"/?p[0]=true&p[1]=false&p[2]=true&p[5]=1&p[6]=ON&p[7]=other", []testItem{{"p", []bool{}, []bool{true, false, true, false, false, true, true, false}}}},
{"/?human.Nick=astaxie", []testItem{{"human", Human{}, Human{Nick: "astaxie"}}}},
{"/?human.ID=888&human.Nick=astaxie&human.Ms=true&human[Pwd]=pass", []testItem{{"human", Human{}, Human{ID: 888, Nick: "astaxie", Ms: true, Pwd: "pass"}}}},
{"/?human[0].ID=888&human[0].Nick=astaxie&human[0].Ms=true&human[0][Pwd]=pass01&human[1].ID=999&human[1].Nick=ysqi&human[1].Ms=On&human[1].Pwd=pass02",
[]testItem{{"human", []Human{}, []Human{
Human{ID: 888, Nick: "astaxie", Ms: true, Pwd: "pass01"},
Human{ID: 999, Nick: "ysqi", Ms: true, Pwd: "pass02"},
}}}},
{
"/?id=123&isok=true&ft=1.2&ol[0]=1&ol[1]=2&ul[]=str&ul[]=array&human.Nick=astaxie",
[]testItem{
{"id", 0, 123},
{"isok", false, true},
{"ft", 0.0, 1.2},
{"ol", []int{}, []int{1, 2}},
{"ul", []string{}, []string{"str", "array"}},
{"human", Human{}, Human{Nick: "astaxie"}},
},
},
}
var users []User
err := beegoInput.Bind(&users, "user")
fmt.Println(users)
if err != nil || users[0].Username != "Raph" || users[0].Password != "123456" || users[1].Username != "Leo" || users[1].Password != "654321" {
t.Fatal("users info wrong")
for _, c := range cases {
r, _ := http.NewRequest("GET", c.request, nil)
beegoInput := NewInput()
beegoInput.Context = NewContext()
beegoInput.Context.Reset(httptest.NewRecorder(), r)
for _, item := range c.valueGp {
got := item.empty
err := beegoInput.Bind(&got, item.field)
if err != nil {
t.Fatal(err)
}
if !reflect.DeepEqual(got, item.want) {
t.Fatalf("Bind %q error,should be:\n%#v \ngot:\n%#v", item.field, item.want, got)
}
}
}
}

10
context/output.go
View File

@ -67,6 +67,7 @@ func (output *BeegoOutput) Body(content []byte) error {
}
if b, n, _ := WriteBody(encoding, buf, content); b {
output.Header("Content-Encoding", n)
output.Header("Content-Length", strconv.Itoa(buf.Len()))
} else {
output.Header("Content-Length", strconv.Itoa(len(content)))
}
@ -330,16 +331,17 @@ func (output *BeegoOutput) IsServerError() bool {
func stringsToJSON(str string) string {
rs := []rune(str)
jsons := ""
var jsons bytes.Buffer
for _, r := range rs {
rint := int(r)
if rint < 128 {
jsons += string(r)
jsons.WriteRune(r)
} else {
jsons += "\\u" + strconv.FormatInt(int64(rint), 16) // json
jsons.WriteString("\\u")
jsons.WriteString(strconv.FormatInt(int64(rint), 16))
}
}
return jsons
return jsons.String()
}
// Session sets session item value with given key.

22
controller.go
View File

@ -69,6 +69,7 @@ type Controller struct {
// template data
TplName string
ViewPath string
Layout string
LayoutSections map[string]string // the key is the section name and the value is the template name
TplPrefix string
@ -185,7 +186,11 @@ func (c *Controller) Render() error {
if err != nil {
return err
}
c.Ctx.Output.Header("Content-Type", "text/html; charset=utf-8")
if c.Ctx.ResponseWriter.Header().Get("Content-Type") == "" {
c.Ctx.Output.Header("Content-Type", "text/html; charset=utf-8")
}
return c.Ctx.Output.Body(rb)
}
@ -209,7 +214,7 @@ func (c *Controller) RenderBytes() ([]byte, error) {
continue
}
buf.Reset()
err = ExecuteTemplate(&buf, sectionTpl, c.Data)
err = ExecuteViewPathTemplate(&buf, sectionTpl, c.viewPath(), c.Data)
if err != nil {
return nil, err
}
@ -218,7 +223,7 @@ func (c *Controller) RenderBytes() ([]byte, error) {
}
buf.Reset()
ExecuteTemplate(&buf, c.Layout, c.Data)
ExecuteViewPathTemplate(&buf, c.Layout, c.viewPath() ,c.Data)
}
return buf.Bytes(), err
}
@ -244,9 +249,16 @@ func (c *Controller) renderTemplate() (bytes.Buffer, error) {
}
}
}
BuildTemplate(BConfig.WebConfig.ViewsPath, buildFiles...)
BuildTemplate(c.viewPath() , buildFiles...)
}
return buf, ExecuteTemplate(&buf, c.TplName, c.Data)
return buf, ExecuteViewPathTemplate(&buf, c.TplName, c.viewPath(), c.Data)
}
func (c *Controller) viewPath() string {
if c.ViewPath == "" {
return BConfig.WebConfig.ViewsPath
}
return c.ViewPath
}
// Redirect sends the redirection response to url with status code.

58
controller_test.go
View File

@ -20,6 +20,8 @@ import (
"testing"
"github.com/astaxie/beego/context"
"os"
"path/filepath"
)
func TestGetInt(t *testing.T) {
@ -121,3 +123,59 @@ func TestGetUint64(t *testing.T) {
t.Errorf("TestGetUint64 expect %v,get %T,%v", uint64(math.MaxUint64), val, val)
}
}
func TestAdditionalViewPaths(t *testing.T) {
dir1 := "_beeTmp"
dir2 := "_beeTmp2"
defer os.RemoveAll(dir1)
defer os.RemoveAll(dir2)
dir1file := "file1.tpl"
dir2file := "file2.tpl"
genFile := func(dir string, name string, content string) {
os.MkdirAll(filepath.Dir(filepath.Join(dir, name)), 0777)
if f, err := os.Create(filepath.Join(dir, name)); err != nil {
t.Fatal(err)
} else {
defer f.Close()
f.WriteString(content)
f.Close()
}
}
genFile(dir1, dir1file, `<div>{{.Content}}</div>`)
genFile(dir2, dir2file, `<html>{{.Content}}</html>`)
AddViewPath(dir1)
AddViewPath(dir2)
ctrl := Controller{
TplName: "file1.tpl",
ViewPath: dir1,
}
ctrl.Data = map[interface{}]interface{}{
"Content": "value2",
}
if result, err := ctrl.RenderString(); err != nil {
t.Fatal(err)
} else {
if result != "<div>value2</div>" {
t.Fatalf("TestAdditionalViewPaths expect %s got %s", "<div>value2</div>", result)
}
}
func() {
ctrl.TplName = "file2.tpl"
defer func() {
if r := recover(); r == nil {
t.Fatal("TestAdditionalViewPaths expected error")
}
}()
ctrl.RenderString();
}()
ctrl.TplName = "file2.tpl"
ctrl.ViewPath = dir2
ctrl.RenderString();
}

16
grace/grace.go
View File

@ -85,23 +85,31 @@ var (
isChild bool
socketOrder string
once sync.Once
hookableSignals []os.Signal
)
func onceInit() {
regLock = &sync.Mutex{}
func init() {
flag.BoolVar(&isChild, "graceful", false, "listen on open fd (after forking)")
flag.StringVar(&socketOrder, "socketorder", "", "previous initialization order - used when more than one listener was started")
regLock = &sync.Mutex{}
runningServers = make(map[string]*Server)
runningServersOrder = []string{}
socketPtrOffsetMap = make(map[string]uint)
hookableSignals = []os.Signal{
syscall.SIGHUP,
syscall.SIGINT,
syscall.SIGTERM,
}
}
// NewServer returns a new graceServer.
func NewServer(addr string, handler http.Handler) (srv *Server) {
once.Do(onceInit)
regLock.Lock()
defer regLock.Unlock()
if !flag.Parsed() {
flag.Parse()
}

20
grace/server.go
View File

@ -162,9 +162,7 @@ func (srv *Server) handleSignals() {
signal.Notify(
srv.sigChan,
syscall.SIGHUP,
syscall.SIGINT,
syscall.SIGTERM,
hookableSignals...,
)
pid := syscall.Getpid()
@ -290,3 +288,19 @@ func (srv *Server) fork() (err error) {
return
}
// RegisterSignalHook registers a function to be run PreSignal or PostSignal for a given signal.
func (srv *Server) RegisterSignalHook(ppFlag int, sig os.Signal, f func()) (err error) {
if ppFlag != PreSignal && ppFlag != PostSignal {
err = fmt.Errorf("Invalid ppFlag argument. Must be either grace.PreSignal or grace.PostSignal.")
return
}
for _, s := range hookableSignals {
if s == sig {
srv.SignalHooks[ppFlag][sig] = append(srv.SignalHooks[ppFlag][sig], f)
return
}
}
err = fmt.Errorf("Signal '%v' is not supported.", sig)
return
}

3
hooks.go
View File

@ -72,7 +72,8 @@ func registerSession() error {
}
func registerTemplate() error {
if err := BuildTemplate(BConfig.WebConfig.ViewsPath); err != nil {
defer lockViewPaths()
if err := AddViewPath(BConfig.WebConfig.ViewsPath); err != nil {
if BConfig.RunMode == DEV {
logs.Warn(err)
}

23
httplib/httplib.go
View File

@ -140,6 +140,7 @@ type BeegoHTTPSettings struct {
EnableCookie bool
Gzip bool
DumpBody bool
Retries int // if set to -1 means will retry forever
}
// BeegoHTTPRequest provides more useful methods for requesting one url than http.Request.
@ -189,6 +190,15 @@ func (b *BeegoHTTPRequest) Debug(isdebug bool) *BeegoHTTPRequest {
return b
}
// Retries sets Retries times.
// default is 0 means no retried.
// -1 means retried forever.
// others means retried times.
func (b *BeegoHTTPRequest) Retries(times int) *BeegoHTTPRequest {
b.setting.Retries = times
return b
}
// DumpBody setting whether need to Dump the Body.
func (b *BeegoHTTPRequest) DumpBody(isdump bool) *BeegoHTTPRequest {
b.setting.DumpBody = isdump
@ -390,7 +400,7 @@ func (b *BeegoHTTPRequest) getResponse() (*http.Response, error) {
}
// DoRequest will do the client.Do
func (b *BeegoHTTPRequest) DoRequest() (*http.Response, error) {
func (b *BeegoHTTPRequest) DoRequest() (resp *http.Response, err error) {
var paramBody string
if len(b.params) > 0 {
var buf bytes.Buffer
@ -467,7 +477,16 @@ func (b *BeegoHTTPRequest) DoRequest() (*http.Response, error) {
}
b.dump = dump
}
return client.Do(b.req)
// retries default value is 0, it will run once.
// retries equal to -1, it will run forever until success
// retries is setted, it will retries fixed times.
for i := 0; b.setting.Retries == -1 || i <= b.setting.Retries; i++ {
resp, err = client.Do(b.req)
if err == nil {
break
}
}
return resp, err
}
// String returns the body string in response.

192
logs/alils/alils.go Normal file
View File

@ -0,0 +1,192 @@
package alils
import (
"encoding/json"
"github.com/astaxie/beego/logs"
"github.com/gogo/protobuf/proto"
"strings"
"sync"
"time"
)
const (
CacheSize int = 64
Delimiter string = "##"
)
type AliLSConfig struct {
Project string `json:"project"`
Endpoint string `json:"endpoint"`
KeyID string `json:"key_id"`
KeySecret string `json:"key_secret"`
LogStore string `json:"log_store"`
Topics []string `json:"topics"`
Source string `json:"source"`
Level int `json:"level"`
FlushWhen int `json:"flush_when"`
}
// aliLSWriter implements LoggerInterface.
// it writes messages in keep-live tcp connection.
type aliLSWriter struct {
store *LogStore
group []*LogGroup
withMap bool
groupMap map[string]*LogGroup
lock *sync.Mutex
AliLSConfig
}
// 创建提供Logger接口的日志服务
func NewAliLS() logs.Logger {
alils := new(aliLSWriter)
alils.Level = logs.LevelTrace
return alils
}
// 读取配置
// 初始化必要的数据结构
func (c *aliLSWriter) Init(jsonConfig string) (err error) {
json.Unmarshal([]byte(jsonConfig), c)
if c.FlushWhen > CacheSize {
c.FlushWhen = CacheSize
}
// 初始化Project
prj := &LogProject{
Name: c.Project,
Endpoint: c.Endpoint,
AccessKeyId: c.KeyID,
AccessKeySecret: c.KeySecret,
}
// 获取logstore
c.store, err = prj.GetLogStore(c.LogStore)
if err != nil {
return err
}
// 创建默认Log Group
c.group = append(c.group, &LogGroup{
Topic: proto.String(""),
Source: proto.String(c.Source),
Logs: make([]*Log, 0, c.FlushWhen),
})
// 创建其它Log Group
c.groupMap = make(map[string]*LogGroup)
for _, topic := range c.Topics {
lg := &LogGroup{
Topic: proto.String(topic),
Source: proto.String(c.Source),
Logs: make([]*Log, 0, c.FlushWhen),
}
c.group = append(c.group, lg)
c.groupMap[topic] = lg
}
if len(c.group) == 1 {
c.withMap = false
} else {
c.withMap = true
}
c.lock = &sync.Mutex{}
return nil
}
// WriteMsg write message in connection.
// if connection is down, try to re-connect.
func (c *aliLSWriter) WriteMsg(when time.Time, msg string, level int) (err error) {
if level > c.Level {
return nil
}
var topic string
var content string
var lg *LogGroup
if c.withMap {
// 解析出Topic，并匹配LogGroup
strs := strings.SplitN(msg, Delimiter, 2)
if len(strs) == 2 {
pos := strings.LastIndex(strs[0], " ")
topic = strs[0][pos+1 : len(strs[0])]
content = strs[0][0:pos] + strs[1]
lg = c.groupMap[topic]
}
// 默认发到空Topic
if lg == nil {
topic = ""
content = msg
lg = c.group[0]
}
} else {
topic = ""
content = msg
lg = c.group[0]
}
// 生成日志
c1 := &Log_Content{
Key: proto.String("msg"),
Value: proto.String(content),
}
l := &Log{
Time: proto.Uint32(uint32(when.Unix())), // 填写日志时间
Contents: []*Log_Content{
c1,
},
}
c.lock.Lock()
lg.Logs = append(lg.Logs, l)
c.lock.Unlock()
// 满足条件则Flush
if len(lg.Logs) >= c.FlushWhen {
c.flush(lg)
}
return nil
}
// Flush implementing method. empty.
func (c *aliLSWriter) Flush() {
// flush所有group
for _, lg := range c.group {
c.flush(lg)
}
}
// Destroy destroy connection writer and close tcp listener.
func (c *aliLSWriter) Destroy() {
}
func (c *aliLSWriter) flush(lg *LogGroup) {
c.lock.Lock()
defer c.lock.Unlock()
// 把以上的LogGroup推送到SLS服务器，
// SLS服务器会根据该logstore的shard个数自动进行负载均衡。
err := c.store.PutLogs(lg)
if err != nil {
return
}
lg.Logs = make([]*Log, 0, c.FlushWhen)
}
func init() {
logs.Register(logs.AdapterAliLS, NewAliLS)
}

13
logs/alils/config.go Executable file
View File

@ -0,0 +1,13 @@
package alils
const (
version = "0.5.0" // SDK version
signatureMethod = "hmac-sha1" // Signature method
// OffsetNewest stands for the log head offset, i.e. the offset that will be
// assigned to the next message that will be produced to the shard.
OffsetNewest = "end"
// OffsetOldest stands for the oldest offset available on the logstore for a
// shard.
OffsetOldest = "begin"
)

984
logs/alils/log.pb.go Executable file
View File

@ -0,0 +1,984 @@
package alils
import "github.com/gogo/protobuf/proto"
import "fmt"
import "math"
// discarding unused import gogoproto "."
import github_com_gogo_protobuf_proto "github.com/gogo/protobuf/proto"
import "io"
// Reference imports to suppress errors if they are not otherwise used.
var _ = proto.Marshal
var _ = fmt.Errorf
var _ = math.Inf
type Log struct {
Time *uint32 `protobuf:"varint,1,req,name=Time" json:"Time,omitempty"`
Contents []*Log_Content `protobuf:"bytes,2,rep,name=Contents" json:"Contents,omitempty"`
XXX_unrecognized []byte `json:"-"`
}
func (m *Log) Reset() { *m = Log{} }
func (m *Log) String() string { return proto.CompactTextString(m) }
func (*Log) ProtoMessage() {}
func (m *Log) GetTime() uint32 {
if m != nil && m.Time != nil {
return *m.Time
}
return 0
}
func (m *Log) GetContents() []*Log_Content {
if m != nil {
return m.Contents
}
return nil
}
type Log_Content struct {
Key *string `protobuf:"bytes,1,req,name=Key" json:"Key,omitempty"`
Value *string `protobuf:"bytes,2,req,name=Value" json:"Value,omitempty"`
XXX_unrecognized []byte `json:"-"`
}
func (m *Log_Content) Reset() { *m = Log_Content{} }
func (m *Log_Content) String() string { return proto.CompactTextString(m) }
func (*Log_Content) ProtoMessage() {}
func (m *Log_Content) GetKey() string {
if m != nil && m.Key != nil {
return *m.Key
}
return ""
}
func (m *Log_Content) GetValue() string {
if m != nil && m.Value != nil {
return *m.Value
}
return ""
}
type LogGroup struct {
Logs []*Log `protobuf:"bytes,1,rep,name=Logs" json:"Logs,omitempty"`
Reserved *string `protobuf:"bytes,2,opt,name=Reserved" json:"Reserved,omitempty"`
Topic *string `protobuf:"bytes,3,opt,name=Topic" json:"Topic,omitempty"`
Source *string `protobuf:"bytes,4,opt,name=Source" json:"Source,omitempty"`
XXX_unrecognized []byte `json:"-"`
}
func (m *LogGroup) Reset() { *m = LogGroup{} }
func (m *LogGroup) String() string { return proto.CompactTextString(m) }
func (*LogGroup) ProtoMessage() {}
func (m *LogGroup) GetLogs() []*Log {
if m != nil {
return m.Logs
}
return nil
}
func (m *LogGroup) GetReserved() string {
if m != nil && m.Reserved != nil {
return *m.Reserved
}
return ""
}
func (m *LogGroup) GetTopic() string {
if m != nil && m.Topic != nil {
return *m.Topic
}
return ""
}
func (m *LogGroup) GetSource() string {
if m != nil && m.Source != nil {
return *m.Source
}
return ""
}
type LogGroupList struct {
LogGroups []*LogGroup `protobuf:"bytes,1,rep,name=logGroups" json:"logGroups,omitempty"`
XXX_unrecognized []byte `json:"-"`
}
func (m *LogGroupList) Reset() { *m = LogGroupList{} }
func (m *LogGroupList) String() string { return proto.CompactTextString(m) }
func (*LogGroupList) ProtoMessage() {}
func (m *LogGroupList) GetLogGroups() []*LogGroup {
if m != nil {
return m.LogGroups
}
return nil
}
func (m *Log) Marshal() (data []byte, err error) {
size := m.Size()
data = make([]byte, size)
n, err := m.MarshalTo(data)
if err != nil {
return nil, err
}
return data[:n], nil
}
func (m *Log) MarshalTo(data []byte) (int, error) {
var i int
_ = i
var l int
_ = l
if m.Time == nil {
return 0, github_com_gogo_protobuf_proto.NewRequiredNotSetError("Time")
} else {
data[i] = 0x8
i++
i = encodeVarintLog(data, i, uint64(*m.Time))
}
if len(m.Contents) > 0 {
for _, msg := range m.Contents {
data[i] = 0x12
i++
i = encodeVarintLog(data, i, uint64(msg.Size()))
n, err := msg.MarshalTo(data[i:])
if err != nil {
return 0, err
}
i += n
}
}
if m.XXX_unrecognized != nil {
i += copy(data[i:], m.XXX_unrecognized)
}
return i, nil
}
func (m *Log_Content) Marshal() (data []byte, err error) {
size := m.Size()
data = make([]byte, size)
n, err := m.MarshalTo(data)
if err != nil {
return nil, err
}
return data[:n], nil
}
func (m *Log_Content) MarshalTo(data []byte) (int, error) {
var i int
_ = i
var l int
_ = l
if m.Key == nil {
return 0, github_com_gogo_protobuf_proto.NewRequiredNotSetError("Key")
} else {
data[i] = 0xa
i++
i = encodeVarintLog(data, i, uint64(len(*m.Key)))
i += copy(data[i:], *m.Key)
}
if m.Value == nil {
return 0, github_com_gogo_protobuf_proto.NewRequiredNotSetError("Value")
} else {
data[i] = 0x12
i++
i = encodeVarintLog(data, i, uint64(len(*m.Value)))
i += copy(data[i:], *m.Value)
}
if m.XXX_unrecognized != nil {
i += copy(data[i:], m.XXX_unrecognized)
}
return i, nil
}
func (m *LogGroup) Marshal() (data []byte, err error) {
size := m.Size()
data = make([]byte, size)
n, err := m.MarshalTo(data)
if err != nil {
return nil, err
}
return data[:n], nil
}
func (m *LogGroup) MarshalTo(data []byte) (int, error) {
var i int
_ = i
var l int
_ = l
if len(m.Logs) > 0 {
for _, msg := range m.Logs {
data[i] = 0xa
i++
i = encodeVarintLog(data, i, uint64(msg.Size()))
n, err := msg.MarshalTo(data[i:])
if err != nil {
return 0, err
}
i += n
}
}
if m.Reserved != nil {
data[i] = 0x12
i++
i = encodeVarintLog(data, i, uint64(len(*m.Reserved)))
i += copy(data[i:], *m.Reserved)
}
if m.Topic != nil {
data[i] = 0x1a
i++
i = encodeVarintLog(data, i, uint64(len(*m.Topic)))
i += copy(data[i:], *m.Topic)
}
if m.Source != nil {
data[i] = 0x22
i++
i = encodeVarintLog(data, i, uint64(len(*m.Source)))
i += copy(data[i:], *m.Source)
}
if m.XXX_unrecognized != nil {
i += copy(data[i:], m.XXX_unrecognized)
}
return i, nil
}
func (m *LogGroupList) Marshal() (data []byte, err error) {
size := m.Size()
data = make([]byte, size)
n, err := m.MarshalTo(data)
if err != nil {
return nil, err
}
return data[:n], nil
}
func (m *LogGroupList) MarshalTo(data []byte) (int, error) {
var i int
_ = i
var l int
_ = l
if len(m.LogGroups) > 0 {
for _, msg := range m.LogGroups {
data[i] = 0xa
i++
i = encodeVarintLog(data, i, uint64(msg.Size()))
n, err := msg.MarshalTo(data[i:])
if err != nil {
return 0, err
}
i += n
}
}
if m.XXX_unrecognized != nil {
i += copy(data[i:], m.XXX_unrecognized)
}
return i, nil
}
func encodeFixed64Log(data []byte, offset int, v uint64) int {
data[offset] = uint8(v)
data[offset+1] = uint8(v >> 8)
data[offset+2] = uint8(v >> 16)
data[offset+3] = uint8(v >> 24)
data[offset+4] = uint8(v >> 32)
data[offset+5] = uint8(v >> 40)
data[offset+6] = uint8(v >> 48)
data[offset+7] = uint8(v >> 56)
return offset + 8
}
func encodeFixed32Log(data []byte, offset int, v uint32) int {
data[offset] = uint8(v)
data[offset+1] = uint8(v >> 8)
data[offset+2] = uint8(v >> 16)
data[offset+3] = uint8(v >> 24)
return offset + 4
}
func encodeVarintLog(data []byte, offset int, v uint64) int {
for v >= 1<<7 {
data[offset] = uint8(v&0x7f | 0x80)
v >>= 7
offset++
}
data[offset] = uint8(v)
return offset + 1
}
func (m *Log) Size() (n int) {
var l int
_ = l
if m.Time != nil {
n += 1 + sovLog(uint64(*m.Time))
}
if len(m.Contents) > 0 {
for _, e := range m.Contents {
l = e.Size()
n += 1 + l + sovLog(uint64(l))
}
}
if m.XXX_unrecognized != nil {
n += len(m.XXX_unrecognized)
}
return n
}
func (m *Log_Content) Size() (n int) {
var l int
_ = l
if m.Key != nil {
l = len(*m.Key)
n += 1 + l + sovLog(uint64(l))
}
if m.Value != nil {
l = len(*m.Value)
n += 1 + l + sovLog(uint64(l))
}
if m.XXX_unrecognized != nil {
n += len(m.XXX_unrecognized)
}
return n
}
func (m *LogGroup) Size() (n int) {
var l int
_ = l
if len(m.Logs) > 0 {
for _, e := range m.Logs {
l = e.Size()
n += 1 + l + sovLog(uint64(l))
}
}
if m.Reserved != nil {
l = len(*m.Reserved)
n += 1 + l + sovLog(uint64(l))
}
if m.Topic != nil {
l = len(*m.Topic)
n += 1 + l + sovLog(uint64(l))
}
if m.Source != nil {
l = len(*m.Source)
n += 1 + l + sovLog(uint64(l))
}
if m.XXX_unrecognized != nil {
n += len(m.XXX_unrecognized)
}
return n
}
func (m *LogGroupList) Size() (n int) {
var l int
_ = l
if len(m.LogGroups) > 0 {
for _, e := range m.LogGroups {
l = e.Size()
n += 1 + l + sovLog(uint64(l))
}
}
if m.XXX_unrecognized != nil {
n += len(m.XXX_unrecognized)
}
return n
}
func sovLog(x uint64) (n int) {
for {
n++
x >>= 7
if x == 0 {
break
}
}
return n
}
func sozLog(x uint64) (n int) {
return sovLog(uint64((x << 1) ^ uint64((int64(x) >> 63))))
}
func (m *Log) Unmarshal(data []byte) error {
var hasFields [1]uint64
l := len(data)
iNdEx := 0
for iNdEx < l {
preIndex := iNdEx
var wire uint64
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowLog
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := data[iNdEx]
iNdEx++
wire |= (uint64(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
fieldNum := int32(wire >> 3)
wireType := int(wire & 0x7)
if wireType == 4 {
return fmt.Errorf("proto: Log: wiretype end group for non-group")
}
if fieldNum <= 0 {
return fmt.Errorf("proto: Log: illegal tag %d (wire type %d)", fieldNum, wire)
}
switch fieldNum {
case 1:
if wireType != 0 {
return fmt.Errorf("proto: wrong wireType = %d for field Time", wireType)
}
var v uint32
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowLog
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := data[iNdEx]
iNdEx++
v |= (uint32(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
m.Time = &v
hasFields[0] |= uint64(0x00000001)
case 2:
if wireType != 2 {
return fmt.Errorf("proto: wrong wireType = %d for field Contents", wireType)
}
var msglen int
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowLog
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := data[iNdEx]
iNdEx++
msglen |= (int(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
if msglen < 0 {
return ErrInvalidLengthLog
}
postIndex := iNdEx + msglen
if postIndex > l {
return io.ErrUnexpectedEOF
}
m.Contents = append(m.Contents, &Log_Content{})
if err := m.Contents[len(m.Contents)-1].Unmarshal(data[iNdEx:postIndex]); err != nil {
return err
}
iNdEx = postIndex
default:
iNdEx = preIndex
skippy, err := skipLog(data[iNdEx:])
if err != nil {
return err
}
if skippy < 0 {
return ErrInvalidLengthLog
}
if (iNdEx + skippy) > l {
return io.ErrUnexpectedEOF
}
m.XXX_unrecognized = append(m.XXX_unrecognized, data[iNdEx:iNdEx+skippy]...)
iNdEx += skippy
}
}
if hasFields[0]&uint64(0x00000001) == 0 {
return github_com_gogo_protobuf_proto.NewRequiredNotSetError("Time")
}
if iNdEx > l {
return io.ErrUnexpectedEOF
}
return nil
}
func (m *Log_Content) Unmarshal(data []byte) error {
var hasFields [1]uint64
l := len(data)
iNdEx := 0
for iNdEx < l {
preIndex := iNdEx
var wire uint64
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowLog
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := data[iNdEx]
iNdEx++
wire |= (uint64(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
fieldNum := int32(wire >> 3)
wireType := int(wire & 0x7)
if wireType == 4 {
return fmt.Errorf("proto: Content: wiretype end group for non-group")
}
if fieldNum <= 0 {
return fmt.Errorf("proto: Content: illegal tag %d (wire type %d)", fieldNum, wire)
}
switch fieldNum {
case 1:
if wireType != 2 {
return fmt.Errorf("proto: wrong wireType = %d for field Key", wireType)
}
var stringLen uint64
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowLog
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := data[iNdEx]
iNdEx++
stringLen |= (uint64(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
intStringLen := int(stringLen)
if intStringLen < 0 {
return ErrInvalidLengthLog
}
postIndex := iNdEx + intStringLen
if postIndex > l {
return io.ErrUnexpectedEOF
}
s := string(data[iNdEx:postIndex])
m.Key = &s
iNdEx = postIndex
hasFields[0] |= uint64(0x00000001)
case 2:
if wireType != 2 {
return fmt.Errorf("proto: wrong wireType = %d for field Value", wireType)
}
var stringLen uint64
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowLog
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := data[iNdEx]
iNdEx++
stringLen |= (uint64(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
intStringLen := int(stringLen)
if intStringLen < 0 {
return ErrInvalidLengthLog
}
postIndex := iNdEx + intStringLen
if postIndex > l {
return io.ErrUnexpectedEOF
}
s := string(data[iNdEx:postIndex])
m.Value = &s
iNdEx = postIndex
hasFields[0] |= uint64(0x00000002)
default:
iNdEx = preIndex
skippy, err := skipLog(data[iNdEx:])
if err != nil {
return err
}
if skippy < 0 {
return ErrInvalidLengthLog
}
if (iNdEx + skippy) > l {
return io.ErrUnexpectedEOF
}
m.XXX_unrecognized = append(m.XXX_unrecognized, data[iNdEx:iNdEx+skippy]...)
iNdEx += skippy
}
}
if hasFields[0]&uint64(0x00000001) == 0 {
return github_com_gogo_protobuf_proto.NewRequiredNotSetError("Key")
}
if hasFields[0]&uint64(0x00000002) == 0 {
return github_com_gogo_protobuf_proto.NewRequiredNotSetError("Value")
}
if iNdEx > l {
return io.ErrUnexpectedEOF
}
return nil
}
func (m *LogGroup) Unmarshal(data []byte) error {
l := len(data)
iNdEx := 0
for iNdEx < l {
preIndex := iNdEx
var wire uint64
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowLog
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := data[iNdEx]
iNdEx++
wire |= (uint64(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
fieldNum := int32(wire >> 3)
wireType := int(wire & 0x7)
if wireType == 4 {
return fmt.Errorf("proto: LogGroup: wiretype end group for non-group")
}
if fieldNum <= 0 {
return fmt.Errorf("proto: LogGroup: illegal tag %d (wire type %d)", fieldNum, wire)
}
switch fieldNum {
case 1:
if wireType != 2 {
return fmt.Errorf("proto: wrong wireType = %d for field Logs", wireType)
}
var msglen int
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowLog
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := data[iNdEx]
iNdEx++
msglen |= (int(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
if msglen < 0 {
return ErrInvalidLengthLog
}
postIndex := iNdEx + msglen
if postIndex > l {
return io.ErrUnexpectedEOF
}
m.Logs = append(m.Logs, &Log{})
if err := m.Logs[len(m.Logs)-1].Unmarshal(data[iNdEx:postIndex]); err != nil {
return err
}
iNdEx = postIndex
case 2:
if wireType != 2 {
return fmt.Errorf("proto: wrong wireType = %d for field Reserved", wireType)
}
var stringLen uint64
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowLog
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := data[iNdEx]
iNdEx++
stringLen |= (uint64(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
intStringLen := int(stringLen)
if intStringLen < 0 {
return ErrInvalidLengthLog
}
postIndex := iNdEx + intStringLen
if postIndex > l {
return io.ErrUnexpectedEOF
}
s := string(data[iNdEx:postIndex])
m.Reserved = &s
iNdEx = postIndex
case 3:
if wireType != 2 {
return fmt.Errorf("proto: wrong wireType = %d for field Topic", wireType)
}
var stringLen uint64
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowLog
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := data[iNdEx]
iNdEx++
stringLen |= (uint64(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
intStringLen := int(stringLen)
if intStringLen < 0 {
return ErrInvalidLengthLog
}
postIndex := iNdEx + intStringLen
if postIndex > l {
return io.ErrUnexpectedEOF
}
s := string(data[iNdEx:postIndex])
m.Topic = &s
iNdEx = postIndex
case 4:
if wireType != 2 {
return fmt.Errorf("proto: wrong wireType = %d for field Source", wireType)
}
var stringLen uint64
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowLog
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := data[iNdEx]
iNdEx++
stringLen |= (uint64(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
intStringLen := int(stringLen)
if intStringLen < 0 {
return ErrInvalidLengthLog
}
postIndex := iNdEx + intStringLen
if postIndex > l {
return io.ErrUnexpectedEOF
}
s := string(data[iNdEx:postIndex])
m.Source = &s
iNdEx = postIndex
default:
iNdEx = preIndex
skippy, err := skipLog(data[iNdEx:])
if err != nil {
return err
}
if skippy < 0 {
return ErrInvalidLengthLog
}
if (iNdEx + skippy) > l {
return io.ErrUnexpectedEOF
}
m.XXX_unrecognized = append(m.XXX_unrecognized, data[iNdEx:iNdEx+skippy]...)
iNdEx += skippy
}
}
if iNdEx > l {
return io.ErrUnexpectedEOF
}
return nil
}
func (m *LogGroupList) Unmarshal(data []byte) error {
l := len(data)
iNdEx := 0
for iNdEx < l {
preIndex := iNdEx
var wire uint64
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowLog
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := data[iNdEx]
iNdEx++
wire |= (uint64(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
fieldNum := int32(wire >> 3)
wireType := int(wire & 0x7)
if wireType == 4 {
return fmt.Errorf("proto: LogGroupList: wiretype end group for non-group")
}
if fieldNum <= 0 {
return fmt.Errorf("proto: LogGroupList: illegal tag %d (wire type %d)", fieldNum, wire)
}
switch fieldNum {
case 1:
if wireType != 2 {
return fmt.Errorf("proto: wrong wireType = %d for field LogGroups", wireType)
}
var msglen int
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowLog
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := data[iNdEx]
iNdEx++
msglen |= (int(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
if msglen < 0 {
return ErrInvalidLengthLog
}
postIndex := iNdEx + msglen
if postIndex > l {
return io.ErrUnexpectedEOF
}
m.LogGroups = append(m.LogGroups, &LogGroup{})
if err := m.LogGroups[len(m.LogGroups)-1].Unmarshal(data[iNdEx:postIndex]); err != nil {
return err
}
iNdEx = postIndex
default:
iNdEx = preIndex
skippy, err := skipLog(data[iNdEx:])
if err != nil {
return err
}
if skippy < 0 {
return ErrInvalidLengthLog
}
if (iNdEx + skippy) > l {
return io.ErrUnexpectedEOF
}
m.XXX_unrecognized = append(m.XXX_unrecognized, data[iNdEx:iNdEx+skippy]...)
iNdEx += skippy
}
}
if iNdEx > l {
return io.ErrUnexpectedEOF
}
return nil
}
func skipLog(data []byte) (n int, err error) {
l := len(data)
iNdEx := 0
for iNdEx < l {
var wire uint64
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return 0, ErrIntOverflowLog
}
if iNdEx >= l {
return 0, io.ErrUnexpectedEOF
}
b := data[iNdEx]
iNdEx++
wire |= (uint64(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
wireType := int(wire & 0x7)
switch wireType {
case 0:
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return 0, ErrIntOverflowLog
}
if iNdEx >= l {
return 0, io.ErrUnexpectedEOF
}
iNdEx++
if data[iNdEx-1] < 0x80 {
break
}
}
return iNdEx, nil
case 1:
iNdEx += 8
return iNdEx, nil
case 2:
var length int
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return 0, ErrIntOverflowLog
}
if iNdEx >= l {
return 0, io.ErrUnexpectedEOF
}
b := data[iNdEx]
iNdEx++
length |= (int(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
iNdEx += length
if length < 0 {
return 0, ErrInvalidLengthLog
}
return iNdEx, nil
case 3:
for {
var innerWire uint64
var start int = iNdEx
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return 0, ErrIntOverflowLog
}
if iNdEx >= l {
return 0, io.ErrUnexpectedEOF
}
b := data[iNdEx]
iNdEx++
innerWire |= (uint64(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
innerWireType := int(innerWire & 0x7)
if innerWireType == 4 {
break
}
next, err := skipLog(data[start:])
if err != nil {
return 0, err
}
iNdEx = start + next
}
return iNdEx, nil
case 4:
return iNdEx, nil
case 5:
iNdEx += 4
return iNdEx, nil
default:
return 0, fmt.Errorf("proto: illegal wireType %d", wireType)
}
}
panic("unreachable")
}
var (
ErrInvalidLengthLog = fmt.Errorf("proto: negative length found during unmarshaling")
ErrIntOverflowLog = fmt.Errorf("proto: integer overflow")
)

39
logs/alils/log_config.go Executable file
View File

@ -0,0 +1,39 @@
package alils
type InputDetail struct {
LogType string `json:"logType"`
LogPath string `json:"logPath"`
FilePattern string `json:"filePattern"`
LocalStorage bool `json:"localStorage"`
TimeFormat string `json:"timeFormat"`
LogBeginRegex string `json:"logBeginRegex"`
Regex string `json:"regex"`
Keys []string `json:"key"`
FilterKeys []string `json:"filterKey"`
FilterRegex []string `json:"filterRegex"`
TopicFormat string `json:"topicFormat"`
}
type OutputDetail struct {
Endpoint string `json:"endpoint"`
LogStoreName string `json:"logstoreName"`
}
type LogConfig struct {
Name string `json:"configName"`
InputType string `json:"inputType"`
InputDetail InputDetail `json:"inputDetail"`
OutputType string `json:"outputType"`
OutputDetail OutputDetail `json:"outputDetail"`
CreateTime uint32
LastModifyTime uint32
project *LogProject
}
// GetAppliedMachineGroup returns applied machine group of this config.
func (c *LogConfig) GetAppliedMachineGroup(confName string) (groupNames []string, err error) {
groupNames, err = c.project.GetAppliedMachineGroups(c.Name)
return
}

818
logs/alils/log_project.go Executable file
View File

@ -0,0 +1,818 @@
/*
Package sls implements the SDK(v0.5.0) of Simple Log Service(abbr. SLS).
For more description about SLS, please read this article:
http://gitlab.alibaba-inc.com/sls/doc.
*/
package alils
import (
"encoding/json"
"fmt"
"io/ioutil"
"net/http"
"net/http/httputil"
)
// Error message in SLS HTTP response.
type errorMessage struct {
Code string `json:"errorCode"`
Message string `json:"errorMessage"`
}
type LogProject struct {
Name string // Project name
Endpoint string // IP or hostname of SLS endpoint
AccessKeyId string
AccessKeySecret string
}
// NewLogProject creates a new SLS project.
func NewLogProject(name, endpoint, accessKeyId, accessKeySecret string) (p *LogProject, err error) {
p = &LogProject{
Name: name,
Endpoint: endpoint,
AccessKeyId: accessKeyId,
AccessKeySecret: accessKeySecret,
}
return p, nil
}
// ListLogStore returns all logstore names of project p.
func (p *LogProject) ListLogStore() (storeNames []string, err error) {
h := map[string]string{
"x-sls-bodyrawsize": "0",
}
uri := fmt.Sprintf("/logstores")
r, err := request(p, "GET", uri, h, nil)
if err != nil {
return
}
buf, err := ioutil.ReadAll(r.Body)
if err != nil {
return
}
if r.StatusCode != http.StatusOK {
errMsg := &errorMessage{}
err = json.Unmarshal(buf, errMsg)
if err != nil {
err = fmt.Errorf("failed to list logstore")
dump, _ := httputil.DumpResponse(r, true)
fmt.Printf("%s\n", dump)
return
}
err = fmt.Errorf("%v:%v", errMsg.Code, errMsg.Message)
return
}
type Body struct {
Count int
LogStores []string
}
body := &Body{}
err = json.Unmarshal(buf, body)
if err != nil {
return
}
storeNames = body.LogStores
return
}
// GetLogStore returns logstore according by logstore name.
func (p *LogProject) GetLogStore(name string) (s *LogStore, err error) {
h := map[string]string{
"x-sls-bodyrawsize": "0",
}
r, err := request(p, "GET", "/logstores/"+name, h, nil)
if err != nil {
return
}
buf, err := ioutil.ReadAll(r.Body)
if err != nil {
return
}
if r.StatusCode != http.StatusOK {
errMsg := &errorMessage{}
err = json.Unmarshal(buf, errMsg)
if err != nil {
err = fmt.Errorf("failed to get logstore")
dump, _ := httputil.DumpResponse(r, true)
fmt.Printf("%s\n", dump)
return
}
err = fmt.Errorf("%v:%v", errMsg.Code, errMsg.Message)
return
}
s = &LogStore{}
err = json.Unmarshal(buf, s)
if err != nil {
return
}
s.project = p
return
}
// CreateLogStore creates a new logstore in SLS,
// where name is logstore name,
// and ttl is time-to-live(in day) of logs,
// and shardCnt is the number of shards.
func (p *LogProject) CreateLogStore(name string, ttl, shardCnt int) (err error) {
type Body struct {
Name string `json:"logstoreName"`
TTL int `json:"ttl"`
ShardCount int `json:"shardCount"`
}
store := &Body{
Name: name,
TTL: ttl,
ShardCount: shardCnt,
}
body, err := json.Marshal(store)
if err != nil {
return
}
h := map[string]string{
"x-sls-bodyrawsize": fmt.Sprintf("%v", len(body)),
"Content-Type": "application/json",
"Accept-Encoding": "deflate", // TODO: support lz4
}
r, err := request(p, "POST", "/logstores", h, body)
if err != nil {
return
}
body, err = ioutil.ReadAll(r.Body)
if err != nil {
return
}
if r.StatusCode != http.StatusOK {
errMsg := &errorMessage{}
err = json.Unmarshal(body, errMsg)
if err != nil {
err = fmt.Errorf("failed to create logstore")
dump, _ := httputil.DumpResponse(r, true)
fmt.Printf("%s\n", dump)
return
}
err = fmt.Errorf("%v:%v", errMsg.Code, errMsg.Message)
return
}
return
}
// DeleteLogStore deletes a logstore according by logstore name.
func (p *LogProject) DeleteLogStore(name string) (err error) {
h := map[string]string{
"x-sls-bodyrawsize": "0",
}
r, err := request(p, "DELETE", "/logstores/"+name, h, nil)
if err != nil {
return
}
body, err := ioutil.ReadAll(r.Body)
if err != nil {
return
}
if r.StatusCode != http.StatusOK {
errMsg := &errorMessage{}
err = json.Unmarshal(body, errMsg)
if err != nil {
err = fmt.Errorf("failed to delete logstore")
dump, _ := httputil.DumpResponse(r, true)
fmt.Printf("%s\n", dump)
return
}
err = fmt.Errorf("%v:%v", errMsg.Code, errMsg.Message)
return
}
return
}
// UpdateLogStore updates a logstore according by logstore name,
// obviously we can't modify the logstore name itself.
func (p *LogProject) UpdateLogStore(name string, ttl, shardCnt int) (err error) {
type Body struct {
Name string `json:"logstoreName"`
TTL int `json:"ttl"`
ShardCount int `json:"shardCount"`
}
store := &Body{
Name: name,
TTL: ttl,
ShardCount: shardCnt,
}
body, err := json.Marshal(store)
if err != nil {
return
}
h := map[string]string{
"x-sls-bodyrawsize": fmt.Sprintf("%v", len(body)),
"Content-Type": "application/json",
"Accept-Encoding": "deflate", // TODO: support lz4
}
r, err := request(p, "PUT", "/logstores", h, body)
if err != nil {
return
}
body, err = ioutil.ReadAll(r.Body)
if err != nil {
return
}
if r.StatusCode != http.StatusOK {
errMsg := &errorMessage{}
err = json.Unmarshal(body, errMsg)
if err != nil {
err = fmt.Errorf("failed to update logstore")
dump, _ := httputil.DumpResponse(r, true)
fmt.Printf("%s\n", dump)
return
}
err = fmt.Errorf("%v:%v", errMsg.Code, errMsg.Message)
return
}
return
}
// ListMachineGroup returns machine group name list and the total number of machine groups.
// The offset starts from 0 and the size is the max number of machine groups could be returned.
func (p *LogProject) ListMachineGroup(offset, size int) (m []string, total int, err error) {
h := map[string]string{
"x-sls-bodyrawsize": "0",
}
if size <= 0 {
size = 500
}
uri := fmt.Sprintf("/machinegroups?offset=%v&size=%v", offset, size)
r, err := request(p, "GET", uri, h, nil)
if err != nil {
return
}
buf, err := ioutil.ReadAll(r.Body)
if err != nil {
return
}
if r.StatusCode != http.StatusOK {
errMsg := &errorMessage{}
err = json.Unmarshal(buf, errMsg)
if err != nil {
err = fmt.Errorf("failed to list machine group")
dump, _ := httputil.DumpResponse(r, true)
fmt.Printf("%s\n", dump)
return
}
err = fmt.Errorf("%v:%v", errMsg.Code, errMsg.Message)
return
}
type Body struct {
MachineGroups []string
Count int
Total int
}
body := &Body{}
err = json.Unmarshal(buf, body)
if err != nil {
return
}
m = body.MachineGroups
total = body.Total
return
}
// GetMachineGroup retruns machine group according by machine group name.
func (p *LogProject) GetMachineGroup(name string) (m *MachineGroup, err error) {
h := map[string]string{
"x-sls-bodyrawsize": "0",
}
r, err := request(p, "GET", "/machinegroups/"+name, h, nil)
if err != nil {
return
}
buf, err := ioutil.ReadAll(r.Body)
if err != nil {
return
}
if r.StatusCode != http.StatusOK {
errMsg := &errorMessage{}
err = json.Unmarshal(buf, errMsg)
if err != nil {
err = fmt.Errorf("failed to get machine group:%v", name)
dump, _ := httputil.DumpResponse(r, true)
fmt.Printf("%s\n", dump)
return
}
err = fmt.Errorf("%v:%v", errMsg.Code, errMsg.Message)
return
}
m = &MachineGroup{}
err = json.Unmarshal(buf, m)
if err != nil {
return
}
m.project = p
return
}
// CreateMachineGroup creates a new machine group in SLS.
func (p *LogProject) CreateMachineGroup(m *MachineGroup) (err error) {
body, err := json.Marshal(m)
if err != nil {
return
}
h := map[string]string{
"x-sls-bodyrawsize": fmt.Sprintf("%v", len(body)),
"Content-Type": "application/json",
"Accept-Encoding": "deflate", // TODO: support lz4
}
r, err := request(p, "POST", "/machinegroups", h, body)
if err != nil {
return
}
body, err = ioutil.ReadAll(r.Body)
if err != nil {
return
}
if r.StatusCode != http.StatusOK {
errMsg := &errorMessage{}
err = json.Unmarshal(body, errMsg)
if err != nil {
err = fmt.Errorf("failed to create machine group")
dump, _ := httputil.DumpResponse(r, true)
fmt.Printf("%s\n", dump)
return
}
err = fmt.Errorf("%v:%v", errMsg.Code, errMsg.Message)
return
}
return
}
// UpdateMachineGroup updates a machine group.
func (p *LogProject) UpdateMachineGroup(m *MachineGroup) (err error) {
body, err := json.Marshal(m)
if err != nil {
return
}
h := map[string]string{
"x-sls-bodyrawsize": fmt.Sprintf("%v", len(body)),
"Content-Type": "application/json",
"Accept-Encoding": "deflate", // TODO: support lz4
}
r, err := request(p, "PUT", "/machinegroups/"+m.Name, h, body)
if err != nil {
return
}
body, err = ioutil.ReadAll(r.Body)
if err != nil {
return
}
if r.StatusCode != http.StatusOK {
errMsg := &errorMessage{}
err = json.Unmarshal(body, errMsg)
if err != nil {
err = fmt.Errorf("failed to update machine group")
dump, _ := httputil.DumpResponse(r, true)
fmt.Printf("%s\n", dump)
return
}
err = fmt.Errorf("%v:%v", errMsg.Code, errMsg.Message)
return
}
return
}
// DeleteMachineGroup deletes machine group according machine group name.
func (p *LogProject) DeleteMachineGroup(name string) (err error) {
h := map[string]string{
"x-sls-bodyrawsize": "0",
}
r, err := request(p, "DELETE", "/machinegroups/"+name, h, nil)
if err != nil {
return
}
body, err := ioutil.ReadAll(r.Body)
if err != nil {
return
}
if r.StatusCode != http.StatusOK {
errMsg := &errorMessage{}
err = json.Unmarshal(body, errMsg)
if err != nil {
err = fmt.Errorf("failed to delete machine group")
dump, _ := httputil.DumpResponse(r, true)
fmt.Printf("%s\n", dump)
return
}
err = fmt.Errorf("%v:%v", errMsg.Code, errMsg.Message)
return
}
return
}
// ListConfig returns config names list and the total number of configs.
// The offset starts from 0 and the size is the max number of configs could be returned.
func (p *LogProject) ListConfig(offset, size int) (cfgNames []string, total int, err error) {
h := map[string]string{
"x-sls-bodyrawsize": "0",
}
if size <= 0 {
size = 100
}
uri := fmt.Sprintf("/configs?offset=%v&size=%v", offset, size)
r, err := request(p, "GET", uri, h, nil)
if err != nil {
return
}
buf, err := ioutil.ReadAll(r.Body)
if err != nil {
return
}
if r.StatusCode != http.StatusOK {
errMsg := &errorMessage{}
err = json.Unmarshal(buf, errMsg)
if err != nil {
err = fmt.Errorf("failed to delete machine group")
dump, _ := httputil.DumpResponse(r, true)
fmt.Printf("%s\n", dump)
return
}
err = fmt.Errorf("%v:%v", errMsg.Code, errMsg.Message)
return
}
type Body struct {
Total int
Configs []string
}
body := &Body{}
err = json.Unmarshal(buf, body)
if err != nil {
return
}
cfgNames = body.Configs
total = body.Total
return
}
// GetConfig returns config according by config name.
func (p *LogProject) GetConfig(name string) (c *LogConfig, err error) {
h := map[string]string{
"x-sls-bodyrawsize": "0",
}
r, err := request(p, "GET", "/configs/"+name, h, nil)
if err != nil {
return
}
buf, err := ioutil.ReadAll(r.Body)
if err != nil {
return
}
if r.StatusCode != http.StatusOK {
errMsg := &errorMessage{}
err = json.Unmarshal(buf, errMsg)
if err != nil {
err = fmt.Errorf("failed to delete config")
dump, _ := httputil.DumpResponse(r, true)
fmt.Printf("%s\n", dump)
return
}
err = fmt.Errorf("%v:%v", errMsg.Code, errMsg.Message)
return
}
c = &LogConfig{}
err = json.Unmarshal(buf, c)
if err != nil {
return
}
c.project = p
return
}
// UpdateConfig updates a config.
func (p *LogProject) UpdateConfig(c *LogConfig) (err error) {
body, err := json.Marshal(c)
if err != nil {
return
}
h := map[string]string{
"x-sls-bodyrawsize": fmt.Sprintf("%v", len(body)),
"Content-Type": "application/json",
"Accept-Encoding": "deflate", // TODO: support lz4
}
r, err := request(p, "PUT", "/configs/"+c.Name, h, body)
if err != nil {
return
}
body, err = ioutil.ReadAll(r.Body)
if err != nil {
return
}
if r.StatusCode != http.StatusOK {
errMsg := &errorMessage{}
err = json.Unmarshal(body, errMsg)
if err != nil {
err = fmt.Errorf("failed to update config")
dump, _ := httputil.DumpResponse(r, true)
fmt.Printf("%s\n", dump)
return
}
err = fmt.Errorf("%v:%v", errMsg.Code, errMsg.Message)
return
}
return
}
// CreateConfig creates a new config in SLS.
func (p *LogProject) CreateConfig(c *LogConfig) (err error) {
body, err := json.Marshal(c)
if err != nil {
return
}
h := map[string]string{
"x-sls-bodyrawsize": fmt.Sprintf("%v", len(body)),
"Content-Type": "application/json",
"Accept-Encoding": "deflate", // TODO: support lz4
}
r, err := request(p, "POST", "/configs", h, body)
if err != nil {
return
}
body, err = ioutil.ReadAll(r.Body)
if err != nil {
return
}
if r.StatusCode != http.StatusOK {
errMsg := &errorMessage{}
err = json.Unmarshal(body, errMsg)
if err != nil {
err = fmt.Errorf("failed to update config")
dump, _ := httputil.DumpResponse(r, true)
fmt.Printf("%s\n", dump)
return
}
err = fmt.Errorf("%v:%v", errMsg.Code, errMsg.Message)
return
}
return
}
// DeleteConfig deletes a config according by config name.
func (p *LogProject) DeleteConfig(name string) (err error) {
h := map[string]string{
"x-sls-bodyrawsize": "0",
}
r, err := request(p, "DELETE", "/configs/"+name, h, nil)
if err != nil {
return
}
body, err := ioutil.ReadAll(r.Body)
if err != nil {
return
}
if r.StatusCode != http.StatusOK {
errMsg := &errorMessage{}
err = json.Unmarshal(body, errMsg)
if err != nil {
err = fmt.Errorf("failed to delete config")
dump, _ := httputil.DumpResponse(r, true)
fmt.Printf("%s\n", dump)
return
}
err = fmt.Errorf("%v:%v", errMsg.Code, errMsg.Message)
return
}
return
}
// GetAppliedMachineGroups returns applied machine group names list according config name.
func (p *LogProject) GetAppliedMachineGroups(confName string) (groupNames []string, err error) {
h := map[string]string{
"x-sls-bodyrawsize": "0",
}
uri := fmt.Sprintf("/configs/%v/machinegroups", confName)
r, err := request(p, "GET", uri, h, nil)
if err != nil {
return
}
buf, err := ioutil.ReadAll(r.Body)
if err != nil {
return
}
if r.StatusCode != http.StatusOK {
errMsg := &errorMessage{}
err = json.Unmarshal(buf, errMsg)
if err != nil {
err = fmt.Errorf("failed to get applied machine groups")
dump, _ := httputil.DumpResponse(r, true)
fmt.Printf("%s\n", dump)
return
}
err = fmt.Errorf("%v:%v", errMsg.Code, errMsg.Message)
return
}
type Body struct {
Count int
Machinegroups []string
}
body := &Body{}
err = json.Unmarshal(buf, body)
if err != nil {
return
}
groupNames = body.Machinegroups
return
}
// GetAppliedConfigs returns applied config names list according machine group name groupName.
func (p *LogProject) GetAppliedConfigs(groupName string) (confNames []string, err error) {
h := map[string]string{
"x-sls-bodyrawsize": "0",
}
uri := fmt.Sprintf("/machinegroups/%v/configs", groupName)
r, err := request(p, "GET", uri, h, nil)
if err != nil {
return
}
buf, err := ioutil.ReadAll(r.Body)
if err != nil {
return
}
if r.StatusCode != http.StatusOK {
errMsg := &errorMessage{}
err = json.Unmarshal(buf, errMsg)
if err != nil {
err = fmt.Errorf("failed to applied configs")
dump, _ := httputil.DumpResponse(r, true)
fmt.Printf("%s\n", dump)
return
}
err = fmt.Errorf("%v:%v", errMsg.Code, errMsg.Message)
return
}
type Cfg struct {
Count int `json:"count"`
Configs []string `json:"configs"`
}
body := &Cfg{}
err = json.Unmarshal(buf, body)
if err != nil {
return
}
confNames = body.Configs
return
}
// ApplyConfigToMachineGroup applies config to machine group.
func (p *LogProject) ApplyConfigToMachineGroup(confName, groupName string) (err error) {
h := map[string]string{
"x-sls-bodyrawsize": "0",
}
uri := fmt.Sprintf("/machinegroups/%v/configs/%v", groupName, confName)
r, err := request(p, "PUT", uri, h, nil)
if err != nil {
return
}
buf, err := ioutil.ReadAll(r.Body)
if err != nil {
return
}
if r.StatusCode != http.StatusOK {
errMsg := &errorMessage{}
err = json.Unmarshal(buf, errMsg)
if err != nil {
err = fmt.Errorf("failed to apply config to machine group")
dump, _ := httputil.DumpResponse(r, true)
fmt.Printf("%s\n", dump)
return
}
err = fmt.Errorf("%v:%v", errMsg.Code, errMsg.Message)
return
}
return
}
// RemoveConfigFromMachineGroup removes config from machine group.
func (p *LogProject) RemoveConfigFromMachineGroup(confName, groupName string) (err error) {
h := map[string]string{
"x-sls-bodyrawsize": "0",
}
uri := fmt.Sprintf("/machinegroups/%v/configs/%v", groupName, confName)
r, err := request(p, "DELETE", uri, h, nil)
if err != nil {
return
}
buf, err := ioutil.ReadAll(r.Body)
if err != nil {
return
}
if r.StatusCode != http.StatusOK {
errMsg := &errorMessage{}
err = json.Unmarshal(buf, errMsg)
if err != nil {
err = fmt.Errorf("failed to remove config from machine group")
dump, _ := httputil.DumpResponse(r, true)
fmt.Printf("%s\n", dump)
return
}
err = fmt.Errorf("%v:%v", errMsg.Code, errMsg.Message)
return
}
return
}

269
logs/alils/log_store.go Executable file
View File

@ -0,0 +1,269 @@
package alils
import (
"encoding/json"
"fmt"
"io/ioutil"
"net/http"
"net/http/httputil"
"strconv"
lz4 "github.com/cloudflare/golz4"
"github.com/gogo/protobuf/proto"
)
type LogStore struct {
Name string `json:"logstoreName"`
TTL int
ShardCount int
CreateTime uint32
LastModifyTime uint32
project *LogProject
}
type Shard struct {
ShardID int `json:"shardID"`
}
// ListShards returns shard id list of this logstore.
func (s *LogStore) ListShards() (shardIDs []int, err error) {
h := map[string]string{
"x-sls-bodyrawsize": "0",
}
uri := fmt.Sprintf("/logstores/%v/shards", s.Name)
r, err := request(s.project, "GET", uri, h, nil)
if err != nil {
return
}
buf, err := ioutil.ReadAll(r.Body)
if err != nil {
return
}
if r.StatusCode != http.StatusOK {
errMsg := &errorMessage{}
err = json.Unmarshal(buf, errMsg)
if err != nil {
err = fmt.Errorf("failed to list logstore")
dump, _ := httputil.DumpResponse(r, true)
fmt.Println(dump)
return
}
err = fmt.Errorf("%v:%v", errMsg.Code, errMsg.Message)
return
}
var shards []*Shard
err = json.Unmarshal(buf, &shards)
if err != nil {
return
}
for _, v := range shards {
shardIDs = append(shardIDs, v.ShardID)
}
return
}
// PutLogs put logs into logstore.
// The callers should transform user logs into LogGroup.
func (s *LogStore) PutLogs(lg *LogGroup) (err error) {
body, err := proto.Marshal(lg)
if err != nil {
return
}
// Compresse body with lz4
out := make([]byte, lz4.CompressBound(body))
n, err := lz4.Compress(body, out)
if err != nil {
return
}
h := map[string]string{
"x-sls-compresstype": "lz4",
"x-sls-bodyrawsize": fmt.Sprintf("%v", len(body)),
"Content-Type": "application/x-protobuf",
}
uri := fmt.Sprintf("/logstores/%v", s.Name)
r, err := request(s.project, "POST", uri, h, out[:n])
if err != nil {
return
}
buf, err := ioutil.ReadAll(r.Body)
if err != nil {
return
}
if r.StatusCode != http.StatusOK {
errMsg := &errorMessage{}
err = json.Unmarshal(buf, errMsg)
if err != nil {
err = fmt.Errorf("failed to put logs")
dump, _ := httputil.DumpResponse(r, true)
fmt.Println(dump)
return
}
err = fmt.Errorf("%v:%v", errMsg.Code, errMsg.Message)
return
}
return
}
// GetCursor gets log cursor of one shard specified by shardId.
// The from can be in three form: a) unix timestamp in seccond, b) "begin", c) "end".
// For more detail please read: http://gitlab.alibaba-inc.com/sls/doc/blob/master/api/shard.md#logstore
func (s *LogStore) GetCursor(shardId int, from string) (cursor string, err error) {
h := map[string]string{
"x-sls-bodyrawsize": "0",
}
uri := fmt.Sprintf("/logstores/%v/shards/%v?type=cursor&from=%v",
s.Name, shardId, from)
r, err := request(s.project, "GET", uri, h, nil)
if err != nil {
return
}
buf, err := ioutil.ReadAll(r.Body)
if err != nil {
return
}
if r.StatusCode != http.StatusOK {
errMsg := &errorMessage{}
err = json.Unmarshal(buf, errMsg)
if err != nil {
err = fmt.Errorf("failed to get cursor")
dump, _ := httputil.DumpResponse(r, true)
fmt.Println(dump)
return
}
err = fmt.Errorf("%v:%v", errMsg.Code, errMsg.Message)
return
}
type Body struct {
Cursor string
}
body := &Body{}
err = json.Unmarshal(buf, body)
if err != nil {
return
}
cursor = body.Cursor
return
}
// GetLogsBytes gets logs binary data from shard specified by shardId according cursor.
// The logGroupMaxCount is the max number of logGroup could be returned.
// The nextCursor is the next curosr can be used to read logs at next time.
func (s *LogStore) GetLogsBytes(shardId int, cursor string,
logGroupMaxCount int) (out []byte, nextCursor string, err error) {
h := map[string]string{
"x-sls-bodyrawsize": "0",
"Accept": "application/x-protobuf",
"Accept-Encoding": "lz4",
}
uri := fmt.Sprintf("/logstores/%v/shards/%v?type=logs&cursor=%v&count=%v",
s.Name, shardId, cursor, logGroupMaxCount)
r, err := request(s.project, "GET", uri, h, nil)
if err != nil {
return
}
buf, err := ioutil.ReadAll(r.Body)
if err != nil {
return
}
if r.StatusCode != http.StatusOK {
errMsg := &errorMessage{}
err = json.Unmarshal(buf, errMsg)
if err != nil {
err = fmt.Errorf("failed to get cursor")
dump, _ := httputil.DumpResponse(r, true)
fmt.Println(dump)
return
}
err = fmt.Errorf("%v:%v", errMsg.Code, errMsg.Message)
return
}
v, ok := r.Header["X-Sls-Compresstype"]
if !ok || len(v) == 0 {
err = fmt.Errorf("can't find 'x-sls-compresstype' header")
return
}
if v[0] != "lz4" {
err = fmt.Errorf("unexpected compress type:%v", v[0])
return
}
v, ok = r.Header["X-Sls-Cursor"]
if !ok || len(v) == 0 {
err = fmt.Errorf("can't find 'x-sls-cursor' header")
return
}
nextCursor = v[0]
v, ok = r.Header["X-Sls-Bodyrawsize"]
if !ok || len(v) == 0 {
err = fmt.Errorf("can't find 'x-sls-bodyrawsize' header")
return
}
bodyRawSize, err := strconv.Atoi(v[0])
if err != nil {
return
}
out = make([]byte, bodyRawSize)
err = lz4.Uncompress(buf, out)
if err != nil {
return
}
return
}
// LogsBytesDecode decodes logs binary data retruned by GetLogsBytes API
func LogsBytesDecode(data []byte) (gl *LogGroupList, err error) {
gl = &LogGroupList{}
err = proto.Unmarshal(data, gl)
if err != nil {
return
}
return
}
// GetLogs gets logs from shard specified by shardId according cursor.
// The logGroupMaxCount is the max number of logGroup could be returned.
// The nextCursor is the next curosr can be used to read logs at next time.
func (s *LogStore) GetLogs(shardId int, cursor string,
logGroupMaxCount int) (gl *LogGroupList, nextCursor string, err error) {
out, nextCursor, err := s.GetLogsBytes(shardId, cursor, logGroupMaxCount)
if err != nil {
return
}
gl, err = LogsBytesDecode(out)
if err != nil {
return
}
return
}

87
logs/alils/machine_group.go Executable file
View File

@ -0,0 +1,87 @@
package alils
import (
"encoding/json"
"fmt"
"io/ioutil"
"net/http"
"net/http/httputil"
)
type MachinGroupAttribute struct {
ExternalName string `json:"externalName"`
TopicName string `json:"groupTopic"`
}
type MachineGroup struct {
Name string `json:"groupName"`
Type string `json:"groupType"`
MachineIdType string `json:"machineIdentifyType"`
MachineIdList []string `json:"machineList"`
Attribute MachinGroupAttribute `json:"groupAttribute"`
CreateTime uint32
LastModifyTime uint32
project *LogProject
}
type Machine struct {
IP string
UniqueId string `json:"machine-uniqueid"`
UserdefinedId string `json:"userdefined-id"`
}
type MachineList struct {
Total int
Machines []*Machine
}
// ListMachines returns machine list of this machine group.
func (m *MachineGroup) ListMachines() (ms []*Machine, total int, err error) {
h := map[string]string{
"x-sls-bodyrawsize": "0",
}
uri := fmt.Sprintf("/machinegroups/%v/machines", m.Name)
r, err := request(m.project, "GET", uri, h, nil)
if err != nil {
return
}
buf, err := ioutil.ReadAll(r.Body)
if err != nil {
return
}
if r.StatusCode != http.StatusOK {
errMsg := &errorMessage{}
err = json.Unmarshal(buf, errMsg)
if err != nil {
err = fmt.Errorf("failed to remove config from machine group")
dump, _ := httputil.DumpResponse(r, true)
fmt.Println(dump)
return
}
err = fmt.Errorf("%v:%v", errMsg.Code, errMsg.Message)
return
}
body := &MachineList{}
err = json.Unmarshal(buf, body)
if err != nil {
return
}
ms = body.Machines
total = body.Total
return
}
// GetAppliedConfigs returns applied configs of this machine group.
func (m *MachineGroup) GetAppliedConfigs() (confNames []string, err error) {
confNames, err = m.project.GetAppliedConfigs(m.Name)
return
}

62
logs/alils/request.go Executable file
View File

@ -0,0 +1,62 @@
package alils
import (
"bytes"
"crypto/md5"
"fmt"
"net/http"
)
// request sends a request to SLS.
func request(project *LogProject, method, uri string, headers map[string]string,
body []byte) (resp *http.Response, err error) {
// The caller should provide 'x-sls-bodyrawsize' header
if _, ok := headers["x-sls-bodyrawsize"]; !ok {
err = fmt.Errorf("Can't find 'x-sls-bodyrawsize' header")
return
}
// SLS public request headers
headers["Host"] = project.Name + "." + project.Endpoint
headers["Date"] = nowRFC1123()
headers["x-sls-apiversion"] = version
headers["x-sls-signaturemethod"] = signatureMethod
if body != nil {
bodyMD5 := fmt.Sprintf("%X", md5.Sum(body))
headers["Content-MD5"] = bodyMD5
if _, ok := headers["Content-Type"]; !ok {
err = fmt.Errorf("Can't find 'Content-Type' header")
return
}
}
// Calc Authorization
// Authorization = "SLS <AccessKeyId>:<Signature>"
digest, err := signature(project, method, uri, headers)
if err != nil {
return
}
auth := fmt.Sprintf("SLS %v:%v", project.AccessKeyId, digest)
headers["Authorization"] = auth
// Initialize http request
reader := bytes.NewReader(body)
urlStr := fmt.Sprintf("http://%v.%v%v", project.Name, project.Endpoint, uri)
req, err := http.NewRequest(method, urlStr, reader)
if err != nil {
return
}
for k, v := range headers {
req.Header.Add(k, v)
}
// Get ready to do request
resp, err = http.DefaultClient.Do(req)
if err != nil {
return
}
return
}

112
logs/alils/signature.go Executable file
View File

@ -0,0 +1,112 @@
package alils
import (
"crypto/hmac"
"crypto/sha1"
"encoding/base64"
"fmt"
"net/url"
"sort"
"strings"
"time"
)
// GMT location
var gmtLoc = time.FixedZone("GMT", 0)
// NowRFC1123 returns now time in RFC1123 format with GMT timezone,
// eg. "Mon, 02 Jan 2006 15:04:05 GMT".
func nowRFC1123() string {
return time.Now().In(gmtLoc).Format(time.RFC1123)
}
// signature calculates a request's signature digest.
func signature(project *LogProject, method, uri string,
headers map[string]string) (digest string, err error) {
var contentMD5, contentType, date, canoHeaders, canoResource string
var slsHeaderKeys sort.StringSlice
// SignString = VERB + "\n"
// + CONTENT-MD5 + "\n"
// + CONTENT-TYPE + "\n"
// + DATE + "\n"
// + CanonicalizedSLSHeaders + "\n"
// + CanonicalizedResource
if val, ok := headers["Content-MD5"]; ok {
contentMD5 = val
}
if val, ok := headers["Content-Type"]; ok {
contentType = val
}
date, ok := headers["Date"]
if !ok {
err = fmt.Errorf("Can't find 'Date' header")
return
}
// Calc CanonicalizedSLSHeaders
slsHeaders := make(map[string]string, len(headers))
for k, v := range headers {
l := strings.TrimSpace(strings.ToLower(k))
if strings.HasPrefix(l, "x-sls-") {
slsHeaders[l] = strings.TrimSpace(v)
slsHeaderKeys = append(slsHeaderKeys, l)
}
}
sort.Sort(slsHeaderKeys)
for i, k := range slsHeaderKeys {
canoHeaders += k + ":" + slsHeaders[k]
if i+1 < len(slsHeaderKeys) {
canoHeaders += "\n"
}
}
// Calc CanonicalizedResource
u, err := url.Parse(uri)
if err != nil {
return
}
canoResource += url.QueryEscape(u.Path)
if u.RawQuery != "" {
var keys sort.StringSlice
vals := u.Query()
for k, _ := range vals {
keys = append(keys, k)
}
sort.Sort(keys)
canoResource += "?"
for i, k := range keys {
if i > 0 {
canoResource += "&"
}
for _, v := range vals[k] {
canoResource += k + "=" + v
}
}
}
signStr := method + "\n" +
contentMD5 + "\n" +
contentType + "\n" +
date + "\n" +
canoHeaders + "\n" +
canoResource
// Signature = base64(hmac-sha1(UTF8-Encoding-Of(SignString)，AccessKeySecret))
mac := hmac.New(sha1.New, []byte(project.AccessKeySecret))
_, err = mac.Write([]byte(signStr))
if err != nil {
return
}
digest = base64.StdEncoding.EncodeToString(mac.Sum(nil))
return
}

1
logs/file.go
View File

@ -270,6 +270,7 @@ func (w *fileLogWriter) doRotate(logTime time.Time) error {
// Rename the file to its new found name
// even if occurs error,we MUST guarantee to restart new logger
err = os.Rename(w.Filename, fName)
err = os.Chmod(fName, os.FileMode(440))
// re-start logger
RESTART_LOGGER:

1
logs/log.go
View File

@ -71,6 +71,7 @@ const (
AdapterEs = "es"
AdapterJianLiao = "jianliao"
AdapterSlack = "slack"
AdapterAliLS = "alils"
)
// Legacy log level constants to ensure backwards compatibility.

2
orm/db_utils.go
View File

@ -41,6 +41,8 @@ func getExistPk(mi *modelInfo, ind reflect.Value) (column string, value interfac
vu := v.Int()
exist = true
value = vu
} else if fi.fieldType&IsRelField > 0 {
_, value, exist = getExistPk(fi.relModelInfo, reflect.Indirect(v))
} else {
vu := v.String()
exist = vu != ""

2
orm/models_boot.go
View File

@ -117,7 +117,7 @@ func bootStrap() {
name := getFullName(elm)
mii, ok := modelCache.getByFullName(name)
if !ok || mii.pkg != elm.PkgPath() {
err = fmt.Errorf("can not found rel in field `%s`, `%s` may be miss register", fi.fullName, elm.String())
err = fmt.Errorf("can not find rel in field `%s`, `%s` may be miss register", fi.fullName, elm.String())
goto end
}
fi.relModelInfo = mii

5
orm/models_test.go
View File

@ -406,6 +406,11 @@ type UintPk struct {
Name string
}
type PtrPk struct {
ID *IntegerPk `orm:"pk;rel(one)"`
Positive bool
}
var DBARGS = struct {
Driver string
Source string

2
orm/orm.go
View File

@ -153,6 +153,8 @@ func (o *orm) ReadOrCreate(md interface{}, col1 string, cols ...string) (bool, i
id, vid := int64(0), ind.FieldByIndex(mi.fields.pk.fieldIndex)
if mi.fields.pk.fieldType&IsPositiveIntegerField > 0 {
id = int64(vid.Uint())
} else if mi.fields.pk.rel {
return o.ReadOrCreate(vid.Interface(), mi.fields.pk.relModelInfo.fields.pk.name)
} else {
id = vid.Int()
}

5
orm/orm_queryset.go
View File

@ -153,6 +153,11 @@ func (o querySet) SetCond(cond *Condition) QuerySeter {
return &o
}
// get condition from QuerySeter
func (o querySet) GetCond() *Condition {
return o.cond
}
// return QuerySeter execution result number
func (o *querySet) Count() (int64, error) {
return o.orm.alias.DbBaser.Count(o.orm.db, o, o.mi, o.cond, o.orm.alias.TZ)

44
orm/orm_test.go
View File

@ -193,6 +193,7 @@ func TestSyncDb(t *testing.T) {
RegisterModel(new(InLineOneToOne))
RegisterModel(new(IntegerPk))
RegisterModel(new(UintPk))
RegisterModel(new(PtrPk))
err := RunSyncdb("default", true, Debug)
throwFail(t, err)
@ -216,6 +217,7 @@ func TestRegisterModels(t *testing.T) {
RegisterModel(new(InLineOneToOne))
RegisterModel(new(IntegerPk))
RegisterModel(new(UintPk))
RegisterModel(new(PtrPk))
BootStrap()
@ -2144,6 +2146,48 @@ func TestUintPk(t *testing.T) {
dORM.Delete(u)
}
func TestPtrPk(t *testing.T) {
parent := &IntegerPk{ID: 10, Value: "10"}
id, _ := dORM.Insert(parent)
if !IsMysql {
// MySql does not support last_insert_id in this case: see #2382
throwFail(t, AssertIs(id, 10))
}
ptr := PtrPk{ID: parent, Positive: true}
num, err := dORM.InsertMulti(2, []PtrPk{ptr})
throwFail(t, err)
throwFail(t, AssertIs(num, 1))
throwFail(t, AssertIs(ptr.ID, parent))
nptr := &PtrPk{ID: parent}
created, pk, err := dORM.ReadOrCreate(nptr, "ID")
throwFail(t, err)
throwFail(t, AssertIs(created, false))
throwFail(t, AssertIs(pk, 10))
throwFail(t, AssertIs(nptr.ID, parent))
throwFail(t, AssertIs(nptr.Positive, true))
nptr = &PtrPk{Positive: true}
created, pk, err = dORM.ReadOrCreate(nptr, "Positive")
throwFail(t, err)
throwFail(t, AssertIs(created, false))
throwFail(t, AssertIs(pk, 10))
throwFail(t, AssertIs(nptr.ID, parent))
nptr.Positive = false
num, err = dORM.Update(nptr)
throwFail(t, err)
throwFail(t, AssertIs(num, 1))
throwFail(t, AssertIs(nptr.ID, parent))
throwFail(t, AssertIs(nptr.Positive, false))
num, err = dORM.Delete(nptr)
throwFail(t, err)
throwFail(t, AssertIs(num, 1))
}
func TestSnake(t *testing.T) {
cases := map[string]string{
"i": "i",

10
orm/types.go
View File

@ -145,6 +145,16 @@ type QuerySeter interface {
// //sql-> WHERE T0.`profile_id` IS NOT NULL AND NOT T0.`Status` IN (?) OR T1.`age` > 2000
// num, err := qs.SetCond(cond1).Count()
SetCond(*Condition) QuerySeter
// get condition from QuerySeter.
// sql's where condition
// cond := orm.NewCondition()
// cond = cond.And("profile__isnull", false).AndNot("status__in", 1)
// qs = qs.SetCond(cond)
// cond = qs.GetCond()
// cond := cond.Or("profile__age__gt", 2000)
// //sql-> WHERE T0.`profile_id` IS NOT NULL AND NOT T0.`Status` IN (?) OR T1.`age` > 2000
// num, err := qs.SetCond(cond).Count()
GetCond() *Condition
// add LIMIT value.
// args[0] means offset, e.g. LIMIT num,offset.
// if Limit <= 0 then Limit will be set to default limit ,eg 1000

21
orm/utils.go
View File

@ -219,22 +219,17 @@ func snakeString(s string) string {
// camel string, xx_yy to XxYy
func camelString(s string) string {
data := make([]byte, 0, len(s))
j := false
k := false
num := len(s) - 1
flag, num := true, len(s)-1
for i := 0; i <= num; i++ {
d := s[i]
if k == false && d >= 'A' && d <= 'Z' {
k = true
}
if d >= 'a' && d <= 'z' && (j || k == false) {
d = d - 32
j = false
k = true
}
if k && d == '_' && num > i && s[i+1] >= 'a' && s[i+1] <= 'z' {
j = true
if d == '_' {
flag = true
continue
} else if flag == true {
if d >= 'a' && d <= 'z' {
d = d - 32
}
flag = false
}
data = append(data, d)
}

36
orm/utils_test.go Normal file
View File

@ -0,0 +1,36 @@
// Copyright 2014 beego Author. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package orm
import (
"testing"
)
func TestCamelString(t *testing.T) {
snake := []string{"pic_url", "hello_world_", "hello__World", "_HelLO_Word", "pic_url_1", "pic_url__1"}
camel := []string{"PicUrl", "HelloWorld", "HelloWorld", "HelLOWord", "PicUrl1", "PicUrl1"}
answer := make(map[string]string)
for i, v := range snake {
answer[v] = camel[i]
}
for _, v := range snake {
res := camelString(v)
if res != answer[v] {
t.Error("Unit Test Fail:", v, res, answer[v])
}
}
}

52
plugins/apiauth/apiauth.go
View File

@ -56,6 +56,7 @@
package apiauth
import (
"bytes"
"crypto/hmac"
"crypto/sha256"
"encoding/base64"
@ -128,53 +129,32 @@ func APISecretAuth(f AppIDToAppSecret, timeout int) beego.FilterFunc {
// Signature used to generate signature with the appsecret/method/params/RequestURI
func Signature(appsecret, method string, params url.Values, RequestURL string) (result string) {
var query string
var b bytes.Buffer
keys := make([]string, len(params))
pa := make(map[string]string)
for k, v := range params {
pa[k] = v[0]
keys = append(keys, k)
}
vs := mapSorter(pa)
vs.Sort()
for i := 0; i < vs.Len(); i++ {
if vs.Keys[i] == "signature" {
sort.Strings(keys)
for _, key := range keys {
if key == "signature" {
continue
}
if vs.Keys[i] != "" && vs.Vals[i] != "" {
query = fmt.Sprintf("%v%v%v", query, vs.Keys[i], vs.Vals[i])
val := pa[key]
if key != "" && val != "" {
b.WriteString(key)
b.WriteString(val)
}
}
stringToSign := fmt.Sprintf("%v\n%v\n%v\n", method, query, RequestURL)
stringToSign := fmt.Sprintf("%v\n%v\n%v\n", method, b.String(), RequestURL)
sha256 := sha256.New
hash := hmac.New(sha256, []byte(appsecret))
hash.Write([]byte(stringToSign))
return base64.StdEncoding.EncodeToString(hash.Sum(nil))
}
type valSorter struct {
Keys []string
Vals []string
}
func mapSorter(m map[string]string) *valSorter {
vs := &valSorter{
Keys: make([]string, 0, len(m)),
Vals: make([]string, 0, len(m)),
}
for k, v := range m {
vs.Keys = append(vs.Keys, k)
vs.Vals = append(vs.Vals, v)
}
return vs
}
func (vs *valSorter) Sort() {
sort.Sort(vs)
}
func (vs *valSorter) Len() int { return len(vs.Keys) }
func (vs *valSorter) Less(i, j int) bool { return vs.Keys[i] < vs.Keys[j] }
func (vs *valSorter) Swap(i, j int) {
vs.Vals[i], vs.Vals[j] = vs.Vals[j], vs.Vals[i]
vs.Keys[i], vs.Keys[j] = vs.Keys[j], vs.Keys[i]
}

20
plugins/apiauth/apiauth_test.go Normal file
View File

@ -0,0 +1,20 @@
package apiauth
import (
"net/url"
"testing"
)
func TestSignature(t *testing.T) {
appsecret := "beego secret"
method := "GET"
RequestURL := "http://localhost/test/url"
params := make(url.Values)
params.Add("arg1", "hello")
params.Add("arg2", "beego")
signature := "mFdpvLh48ca4mDVEItE9++AKKQ/IVca7O/ZyyB8hR58="
if Signature(appsecret, method, params, RequestURL) != signature {
t.Error("Signature error")
}
}

25
router.go
View File

@ -51,15 +51,22 @@ const (
var (
// HTTPMETHOD list the supported http methods.
HTTPMETHOD = map[string]string{
"GET": "GET",
"POST": "POST",
"PUT": "PUT",
"DELETE": "DELETE",
"PATCH": "PATCH",
"OPTIONS": "OPTIONS",
"HEAD": "HEAD",
"TRACE": "TRACE",
"CONNECT": "CONNECT",
"GET": "GET",
"POST": "POST",
"PUT": "PUT",
"DELETE": "DELETE",
"PATCH": "PATCH",
"OPTIONS": "OPTIONS",
"HEAD": "HEAD",
"TRACE": "TRACE",
"CONNECT": "CONNECT",
"MKCOL": "MKCOL",
"COPY": "COPY",
"MOVE": "MOVE",
"PROPFIND": "PROPFIND",
"PROPPATCH": "PROPPATCH",
"LOCK": "LOCK",
"UNLOCK": "UNLOCK",
}
// these beego.Controller's methods shouldn't reflect to AutoRouter
exceptMethod = []string{"Init", "Prepare", "Finish", "Render", "RenderString",

2
session/mysql/sess_mysql.go
View File

@ -143,6 +143,7 @@ func (mp *Provider) SessionInit(maxlifetime int64, savePath string) error {
// SessionRead get mysql session by sid
func (mp *Provider) SessionRead(sid string) (session.Store, error) {
c := mp.connectInit()
defer c.Close()
row := c.QueryRow("select session_data from "+TableName+" where session_key=?", sid)
var sessiondata []byte
err := row.Scan(&sessiondata)
@ -179,6 +180,7 @@ func (mp *Provider) SessionExist(sid string) bool {
// SessionRegenerate generate new sid for mysql session
func (mp *Provider) SessionRegenerate(oldsid, sid string) (session.Store, error) {
c := mp.connectInit()
defer c.Close()
row := c.QueryRow("select session_data from "+TableName+" where session_key=?", oldsid)
var sessiondata []byte
err := row.Scan(&sessiondata)

85
session/sess_file.go
View File

@ -15,8 +15,7 @@
package session
import (
"errors"
"io"
"fmt"
"io/ioutil"
"net/http"
"os"
@ -135,6 +134,9 @@ func (fp *FileProvider) SessionRead(sid string) (Store, error) {
} else {
return nil, err
}
defer f.Close()
os.Chtimes(path.Join(fp.savePath, string(sid[0]), string(sid[1]), sid), time.Now(), time.Now())
var kv map[interface{}]interface{}
b, err := ioutil.ReadAll(f)
@ -149,7 +151,7 @@ func (fp *FileProvider) SessionRead(sid string) (Store, error) {
return nil, err
}
}
f.Close()
ss := &FileSessionStore{sid: sid, values: kv}
return ss, nil
}
@ -204,49 +206,58 @@ func (fp *FileProvider) SessionRegenerate(oldsid, sid string) (Store, error) {
filepder.lock.Lock()
defer filepder.lock.Unlock()
err := os.MkdirAll(path.Join(fp.savePath, string(oldsid[0]), string(oldsid[1])), 0777)
if err != nil {
SLogger.Println(err.Error())
}
err = os.MkdirAll(path.Join(fp.savePath, string(sid[0]), string(sid[1])), 0777)
if err != nil {
SLogger.Println(err.Error())
}
_, err = os.Stat(path.Join(fp.savePath, string(sid[0]), string(sid[1]), sid))
var newf *os.File
oldPath := path.Join(fp.savePath, string(oldsid[0]), string(oldsid[1]))
oldSidFile := path.Join(oldPath, oldsid)
newPath := path.Join(fp.savePath, string(sid[0]), string(sid[1]))
newSidFile := path.Join(newPath, sid)
// new sid file is exist
_, err := os.Stat(newSidFile)
if err == nil {
return nil, errors.New("newsid exist")
} else if os.IsNotExist(err) {
newf, err = os.Create(path.Join(fp.savePath, string(sid[0]), string(sid[1]), sid))
return nil, fmt.Errorf("newsid %s exist", newSidFile)
}
_, err = os.Stat(path.Join(fp.savePath, string(oldsid[0]), string(oldsid[1]), oldsid))
var f *os.File
if err == nil {
f, err = os.OpenFile(path.Join(fp.savePath, string(oldsid[0]), string(oldsid[1]), oldsid), os.O_RDWR, 0777)
io.Copy(newf, f)
} else if os.IsNotExist(err) {
newf, err = os.Create(path.Join(fp.savePath, string(sid[0]), string(sid[1]), sid))
} else {
return nil, err
}
f.Close()
os.Remove(path.Join(fp.savePath, string(oldsid[0]), string(oldsid[1])))
os.Chtimes(path.Join(fp.savePath, string(sid[0]), string(sid[1]), sid), time.Now(), time.Now())
var kv map[interface{}]interface{}
b, err := ioutil.ReadAll(newf)
err = os.MkdirAll(newPath, 0777)
if err != nil {
return nil, err
SLogger.Println(err.Error())
}
if len(b) == 0 {
kv = make(map[interface{}]interface{})
} else {
kv, err = DecodeGob(b)
// if old sid file exist
// 1.read and parse file content
// 2.write content to new sid file
// 3.remove old sid file, change new sid file atime and ctime
// 4.return FileSessionStore
_, err = os.Stat(oldSidFile)
if err == nil {
b, err := ioutil.ReadFile(oldSidFile)
if err != nil {
return nil, err
}
var kv map[interface{}]interface{}
if len(b) == 0 {
kv = make(map[interface{}]interface{})
} else {
kv, err = DecodeGob(b)
if err != nil {
return nil, err
}
}
ioutil.WriteFile(newSidFile, b, 0777)
os.Remove(oldSidFile)
os.Chtimes(newSidFile, time.Now(), time.Now())
ss := &FileSessionStore{sid: sid, values: kv}
return ss, nil
}
ss := &FileSessionStore{sid: sid, values: kv}
// if old sid file not exist, just create new sid file and return
newf, err := os.Create(newSidFile)
if err != nil {
return nil, err
}
newf.Close()
ss := &FileSessionStore{sid: sid, values: make(map[interface{}]interface{})}
return ss, nil
}

58
template.go
View File

@ -32,8 +32,9 @@ import (
var (
beegoTplFuncMap = make(template.FuncMap)
// beeTemplates caching map and supported template file extensions.
beeTemplates = make(map[string]*template.Template)
beeViewPathTemplateLocked = false
// beeViewPathTemplates caching map and supported template file extensions per view
beeViewPathTemplates = make(map[string]map[string]*template.Template)
templatesLock sync.RWMutex
// beeTemplateExt stores the template extension which will build
beeTemplateExt = []string{"tpl", "html"}
@ -45,23 +46,33 @@ var (
// writing the output to wr.
// A template will be executed safely in parallel.
func ExecuteTemplate(wr io.Writer, name string, data interface{}) error {
return ExecuteViewPathTemplate(wr,name, BConfig.WebConfig.ViewsPath, data)
}
// ExecuteViewPathTemplate applies the template with name and from specific viewPath to the specified data object,
// writing the output to wr.
// A template will be executed safely in parallel.
func ExecuteViewPathTemplate(wr io.Writer, name string, viewPath string, data interface{}) error {
if BConfig.RunMode == DEV {
templatesLock.RLock()
defer templatesLock.RUnlock()
}
if t, ok := beeTemplates[name]; ok {
var err error
if t.Lookup(name) != nil {
err = t.ExecuteTemplate(wr, name, data)
} else {
err = t.Execute(wr, data)
if beeTemplates,ok := beeViewPathTemplates[viewPath]; ok {
if t, ok := beeTemplates[name]; ok {
var err error
if t.Lookup(name) != nil {
err = t.ExecuteTemplate(wr, name, data)
} else {
err = t.Execute(wr, data)
}
if err != nil {
logs.Trace("template Execute err:", err)
}
return err
}
if err != nil {
logs.Trace("template Execute err:", err)
}
return err
panic("can't find templatefile in the path:" + viewPath + "/" + name)
}
panic("can't find templatefile in the path:" + name)
panic("Uknown view path:" + viewPath)
}
func init() {
@ -149,6 +160,21 @@ func AddTemplateExt(ext string) {
beeTemplateExt = append(beeTemplateExt, ext)
}
// AddViewPath adds a new path to the supported view paths.
//Can later be used by setting a controller ViewPath to this folder
//will panic if called after beego.Run()
func AddViewPath(viewPath string) error {
if beeViewPathTemplateLocked {
panic("Can not add new view paths after beego.Run()")
}
beeViewPathTemplates[viewPath] = make(map[string]*template.Template)
return BuildTemplate(viewPath)
}
func lockViewPaths() {
beeViewPathTemplateLocked = true
}
// BuildTemplate will build all template files in a directory.
// it makes beego can render any template file in view directory.
func BuildTemplate(dir string, files ...string) error {
@ -158,6 +184,10 @@ func BuildTemplate(dir string, files ...string) error {
}
return errors.New("dir open err")
}
beeTemplates,ok := beeViewPathTemplates[dir];
if !ok {
panic("Unknown view path: " + dir)
}
self := &templateFile{
root: dir,
files: make(map[string][]string),
@ -224,7 +254,7 @@ func getTplDeep(root, file, parent string, t *template.Template) (*template.Temp
if !HasTemplateExt(m[1]) {
continue
}
t, _, err = getTplDeep(root, m[1], file, t)
_, _, err = getTplDeep(root, m[1], file, t)
if err != nil {
return nil, [][]string{}, err
}

10
template_test.go
View File

@ -67,9 +67,10 @@ func TestTemplate(t *testing.T) {
f.Close()
}
}
if err := BuildTemplate(dir); err != nil {
if err := AddViewPath(dir); err != nil {
t.Fatal(err)
}
beeTemplates := beeViewPathTemplates[dir]
if len(beeTemplates) != 3 {
t.Fatalf("should be 3 but got %v", len(beeTemplates))
}
@ -103,6 +104,12 @@ var user = `<!DOCTYPE html>
func TestRelativeTemplate(t *testing.T) {
dir := "_beeTmp"
//Just add dir to known viewPaths
if err := AddViewPath(dir); err != nil {
t.Fatal(err)
}
files := []string{
"easyui/public/menu.tpl",
"easyui/rbac/user.tpl",
@ -126,6 +133,7 @@ func TestRelativeTemplate(t *testing.T) {
if err := BuildTemplate(dir, files[1]); err != nil {
t.Fatal(err)
}
beeTemplates := beeViewPathTemplates[dir]
if err := beeTemplates["easyui/rbac/user.tpl"].ExecuteTemplate(os.Stdout, "easyui/rbac/user.tpl", nil); err != nil {
t.Fatal(err)
}

4
toolbox/statistics.go
View File

@ -117,7 +117,9 @@ func (m *URLMap) GetMap() map[string]interface{} {
// GetMapData return all mapdata
func (m *URLMap) GetMapData() []map[string]interface{} {
m.lock.Lock()
defer m.lock.Unlock()
var resultLists []map[string]interface{}
for k, v := range m.urlmap {

13
utils/safemap.go
View File

@ -61,10 +61,8 @@ func (m *BeeMap) Set(k interface{}, v interface{}) bool {
func (m *BeeMap) Check(k interface{}) bool {
m.lock.RLock()
defer m.lock.RUnlock()
if _, ok := m.bm[k]; !ok {
return false
}
return true
_, ok := m.bm[k]
return ok
}
// Delete the given key and value.
@ -84,3 +82,10 @@ func (m *BeeMap) Items() map[interface{}]interface{} {
}
return r
}
// Count returns the number of items within the map.
func (m *BeeMap) Count() int {
m.lock.RLock()
defer m.lock.RUnlock()
return len(m.bm)
}

55
utils/safemap_test.go
View File

@ -14,25 +14,44 @@
package utils
import (
"testing"
)
import "testing"
func Test_beemap(t *testing.T) {
bm := NewBeeMap()
if !bm.Set("astaxie", 1) {
t.Error("set Error")
}
if !bm.Check("astaxie") {
t.Error("check err")
}
var safeMap *BeeMap
if v := bm.Get("astaxie"); v.(int) != 1 {
t.Error("get err")
}
bm.Delete("astaxie")
if bm.Check("astaxie") {
t.Error("delete err")
func TestNewBeeMap(t *testing.T) {
safeMap = NewBeeMap()
if safeMap == nil {
t.Fatal("expected to return non-nil BeeMap", "got", safeMap)
}
}
func TestSet(t *testing.T) {
if ok := safeMap.Set("astaxie", 1); !ok {
t.Error("expected", true, "got", false)
}
}
func TestCheck(t *testing.T) {
if exists := safeMap.Check("astaxie"); !exists {
t.Error("expected", true, "got", false)
}
}
func TestGet(t *testing.T) {
if val := safeMap.Get("astaxie"); val.(int) != 1 {
t.Error("expected value", 1, "got", val)
}
}
func TestDelete(t *testing.T) {
safeMap.Delete("astaxie")
if exists := safeMap.Check("astaxie"); exists {
t.Error("expected element to be deleted")
}
}
func TestCount(t *testing.T) {
if count := safeMap.Count(); count != 0 {
t.Error("expected count to be", 0, "got", count)
}
}