Merge pull request #4262 from flycash/ftr/toml

Support toml config

core/config/config.go

@@ -72,6 +72,8 @@ type Configer interface {
 	DefaultInt64(ctx context.Context, key string, defaultVal int64) int64
 	DefaultBool(ctx context.Context, key string, defaultVal bool) bool
 	DefaultFloat(ctx context.Context, key string, defaultVal float64) float64
+
+	// DIY return the original value
 	DIY(ctx context.Context, key string) (interface{}, error)
 
 	GetSection(ctx context.Context, section string) (map[string]string, error)

core/config/error.go

@@ -0,0 +1,25 @@
+// Copyright 2020
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package config
+
+import (
+	"github.com/pkg/errors"
+)
+
+// now not all implementation return those error codes
+var (
+	KeyNotFoundError      = errors.New("the key is not found")
+	InvalidValueTypeError = errors.New("the value is not expected type")
+)

core/config/toml/toml.go

@@ -0,0 +1,358 @@
+// Copyright 2020
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package toml
+
+import (
+	"context"
+	"io/ioutil"
+	"os"
+	"strings"
+
+	"github.com/pelletier/go-toml"
+
+	"github.com/astaxie/beego/core/config"
+)
+
+const keySeparator = "."
+
+type Config struct {
+	tree *toml.Tree
+}
+
+// Parse accepts filename as the parameter
+func (c *Config) Parse(filename string) (config.Configer, error) {
+	ctx, err := ioutil.ReadFile(filename)
+	if err != nil {
+		return nil, err
+	}
+	return c.ParseData(ctx)
+}
+
+func (c *Config) ParseData(data []byte) (config.Configer, error) {
+	t, err := toml.LoadBytes(data)
+	if err != nil {
+		return nil, err
+	}
+	return &configContainer{
+		t: t,
+	}, nil
+
+}
+
+// configContainer support key looks like "a.b.c"
+type configContainer struct {
+	t *toml.Tree
+}
+
+// Set put key, val
+func (c *configContainer) Set(ctx context.Context, key, val string) error {
+	path := strings.Split(key, keySeparator)
+	sub, err := subTree(c.t, path[0:len(path)-1])
+	if err != nil {
+		return err
+	}
+	sub.Set(path[len(path)-1], val)
+	return nil
+}
+
+// String return the value.
+// return error if key not found or value is invalid type
+func (c *configContainer) String(ctx context.Context, key string) (string, error) {
+	res, err := c.get(key)
+
+	if err != nil {
+		return "", err
+	}
+
+	if res == nil {
+		return "", config.KeyNotFoundError
+	}
+
+	if str, ok := res.(string); ok {
+		return str, nil
+	} else {
+		return "", config.InvalidValueTypeError
+	}
+}
+
+// Strings return []string
+// return error if key not found or value is invalid type
+func (c *configContainer) Strings(ctx context.Context, key string) ([]string, error) {
+	val, err := c.get(key)
+
+	if err != nil {
+		return []string{}, err
+	}
+	if val == nil {
+		return []string{}, config.KeyNotFoundError
+	}
+	if arr, ok := val.([]interface{}); ok {
+		res := make([]string, 0, len(arr))
+		for _, ele := range arr {
+			if str, ok := ele.(string); ok {
+				res = append(res, str)
+			} else {
+				return []string{}, config.InvalidValueTypeError
+			}
+		}
+		return res, nil
+	} else {
+		return []string{}, config.InvalidValueTypeError
+	}
+}
+
+// Int return int value
+// return error if key not found or value is invalid type
+func (c *configContainer) Int(ctx context.Context, key string) (int, error) {
+	val, err := c.Int64(ctx, key)
+	return int(val), err
+}
+
+// Int64 return int64 value
+// return error if key not found or value is invalid type
+func (c *configContainer) Int64(ctx context.Context, key string) (int64, error) {
+	res, err := c.get(key)
+	if err != nil {
+		return 0, err
+	}
+	if res == nil {
+		return 0, config.KeyNotFoundError
+	}
+	if i, ok := res.(int); ok {
+		return int64(i), nil
+	} else if i64, ok := res.(int64); ok {
+		return i64, nil
+	} else {
+		return 0, config.InvalidValueTypeError
+	}
+}
+
+// bool return bool value
+// return error if key not found or value is invalid type
+func (c *configContainer) Bool(ctx context.Context, key string) (bool, error) {
+
+	res, err := c.get(key)
+
+	if err != nil {
+		return false, err
+	}
+
+	if res == nil {
+		return false, config.KeyNotFoundError
+	}
+	if b, ok := res.(bool); ok {
+		return b, nil
+	} else {
+		return false, config.InvalidValueTypeError
+	}
+}
+
+// Float return float value
+// return error if key not found or value is invalid type
+func (c *configContainer) Float(ctx context.Context, key string) (float64, error) {
+	res, err := c.get(key)
+	if err != nil {
+		return 0, err
+	}
+
+	if res == nil {
+		return 0, config.KeyNotFoundError
+	}
+
+	if f, ok := res.(float64); ok {
+		return f, nil
+	} else {
+		return 0, config.InvalidValueTypeError
+	}
+}
+
+// DefaultString return string value
+// return default value if key not found or value is invalid type
+func (c *configContainer) DefaultString(ctx context.Context, key string, defaultVal string) string {
+	res, err := c.get(key)
+	if err != nil {
+		return defaultVal
+	}
+	if str, ok := res.(string); ok {
+		return str
+	} else {
+		return defaultVal
+	}
+}
+
+// DefaultStrings return []string
+// return default value if key not found or value is invalid type
+func (c *configContainer) DefaultStrings(ctx context.Context, key string, defaultVal []string) []string {
+	val, err := c.get(key)
+	if err != nil {
+		return defaultVal
+	}
+	if arr, ok := val.([]interface{}); ok {
+		res := make([]string, 0, len(arr))
+		for _, ele := range arr {
+			if str, ok := ele.(string); ok {
+				res = append(res, str)
+			} else {
+				return defaultVal
+			}
+		}
+		return res
+	} else {
+		return defaultVal
+	}
+}
+
+// DefaultInt return int value
+// return default value if key not found or value is invalid type
+func (c *configContainer) DefaultInt(ctx context.Context, key string, defaultVal int) int {
+	return int(c.DefaultInt64(ctx, key, int64(defaultVal)))
+}
+
+// DefaultInt64 return int64 value
+// return default value if key not found or value is invalid type
+func (c *configContainer) DefaultInt64(ctx context.Context, key string, defaultVal int64) int64 {
+	res, err := c.get(key)
+	if err != nil {
+		return defaultVal
+	}
+	if i, ok := res.(int); ok {
+		return int64(i)
+	} else if i64, ok := res.(int64); ok {
+		return i64
+	} else {
+		return defaultVal
+	}
+}
+
+// DefaultBool return bool value
+// return default value if key not found or value is invalid type
+func (c *configContainer) DefaultBool(ctx context.Context, key string, defaultVal bool) bool {
+	res, err := c.get(key)
+	if err != nil {
+		return defaultVal
+	}
+	if b, ok := res.(bool); ok {
+		return b
+	} else {
+		return defaultVal
+	}
+}
+
+// DefaultFloat return float value
+// return default value if key not found or value is invalid type
+func (c *configContainer) DefaultFloat(ctx context.Context, key string, defaultVal float64) float64 {
+	res, err := c.get(key)
+	if err != nil {
+		return defaultVal
+	}
+	if f, ok := res.(float64); ok {
+		return f
+	} else {
+		return defaultVal
+	}
+}
+
+// DIY returns the original value
+func (c *configContainer) DIY(ctx context.Context, key string) (interface{}, error) {
+	return c.get(key)
+}
+
+// GetSection return error if the value is not valid toml doc
+func (c *configContainer) GetSection(ctx context.Context, section string) (map[string]string, error) {
+	val, err := subTree(c.t, strings.Split(section, keySeparator))
+	if err != nil {
+		return map[string]string{}, err
+	}
+	m := val.ToMap()
+	res := make(map[string]string, len(m))
+	for k, v := range m {
+		res[k] = config.ToString(v)
+	}
+	return res, nil
+}
+
+func (c *configContainer) Unmarshaler(ctx context.Context, prefix string, obj interface{}, opt ...config.DecodeOption) error {
+	if len(prefix) > 0 {
+		t, err := subTree(c.t, strings.Split(prefix, keySeparator))
+		if err != nil {
+			return err
+		}
+		return t.Unmarshal(obj)
+	}
+	return c.t.Unmarshal(obj)
+}
+
+// Sub return sub configer
+// return error if key not found or the value is not a sub doc
+func (c *configContainer) Sub(ctx context.Context, key string) (config.Configer, error) {
+	val, err := subTree(c.t, strings.Split(key, keySeparator))
+	if err != nil {
+		return nil, err
+	}
+	return &configContainer{
+		t: val,
+	}, nil
+}
+
+// OnChange do nothing
+func (c *configContainer) OnChange(ctx context.Context, key string, fn func(value string)) {
+	// do nothing
+}
+
+// SaveConfigFile create or override the file
+func (c *configContainer) SaveConfigFile(ctx context.Context, filename string) error {
+	// Write configuration file by filename.
+	f, err := os.Create(filename)
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+
+	_, err = c.t.WriteTo(f)
+	return err
+}
+
+func (c *configContainer) get(key string) (interface{}, error) {
+	if len(key) == 0 {
+		return nil, config.KeyNotFoundError
+	}
+
+	segs := strings.Split(key, keySeparator)
+	t, err := subTree(c.t, segs[0:len(segs)-1])
+
+	if err != nil {
+		return nil, err
+	}
+	return t.Get(segs[len(segs)-1]), nil
+}
+
+func subTree(t *toml.Tree, path []string) (*toml.Tree, error) {
+	res := t
+	for i := 0; i < len(path); i++ {
+		if subTree, ok := res.Get(path[i]).(*toml.Tree); ok {
+			res = subTree
+		} else {
+			return nil, config.InvalidValueTypeError
+		}
+	}
+	if res == nil {
+		return nil, config.KeyNotFoundError
+	}
+	return res, nil
+}
+
+func init() {
+	config.Register("toml", &Config{})
+}

core/config/toml/toml_test.go

@@ -0,0 +1,380 @@
+// Copyright 2020
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package toml
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+
+	"github.com/astaxie/beego/core/config"
+)
+
+func TestConfig_Parse(t *testing.T) {
+	// file not found
+	cfg := &Config{}
+	_, err := cfg.Parse("invalid_file_name.txt")
+	assert.NotNil(t, err)
+}
+
+func TestConfig_ParseData(t *testing.T) {
+	data := `
+name="Tom"
+`
+	cfg := &Config{}
+	c, err := cfg.ParseData([]byte(data))
+	assert.Nil(t, err)
+	assert.NotNil(t, c)
+}
+
+func TestConfigContainer_Bool(t *testing.T) {
+	data := `
+Man=true
+Woman="true"
+`
+	cfg := &Config{}
+	c, err := cfg.ParseData([]byte(data))
+	assert.Nil(t, err)
+	assert.NotNil(t, c)
+
+	val, err := c.Bool(context.Background(), "Man")
+	assert.Nil(t, err)
+	assert.True(t, val)
+
+	_, err = c.Bool(context.Background(), "Woman")
+	assert.NotNil(t, err)
+	assert.Equal(t, config.InvalidValueTypeError, err)
+}
+
+func TestConfigContainer_DefaultBool(t *testing.T) {
+	data := `
+Man=true
+Woman="false"
+`
+	cfg := &Config{}
+	c, err := cfg.ParseData([]byte(data))
+	assert.Nil(t, err)
+	assert.NotNil(t, c)
+
+	val := c.DefaultBool(context.Background(), "Man11", true)
+	assert.True(t, val)
+
+	val = c.DefaultBool(context.Background(), "Man", false)
+	assert.True(t, val)
+
+	val = c.DefaultBool(context.Background(), "Woman", true)
+	assert.True(t, val)
+}
+
+func TestConfigContainer_DefaultFloat(t *testing.T) {
+	data := `
+Price=12.3
+PriceInvalid="12.3"
+`
+	cfg := &Config{}
+	c, err := cfg.ParseData([]byte(data))
+	assert.Nil(t, err)
+	assert.NotNil(t, c)
+
+	val := c.DefaultFloat(context.Background(), "Price", 11.2)
+	assert.Equal(t, 12.3, val)
+
+	val = c.DefaultFloat(context.Background(), "Price11", 11.2)
+	assert.Equal(t, 11.2, val)
+
+	val = c.DefaultFloat(context.Background(), "PriceInvalid", 11.2)
+	assert.Equal(t, 11.2, val)
+}
+
+func TestConfigContainer_DefaultInt(t *testing.T) {
+	data := `
+Age=12
+AgeInvalid="13"
+`
+	cfg := &Config{}
+	c, err := cfg.ParseData([]byte(data))
+	assert.Nil(t, err)
+	assert.NotNil(t, c)
+
+	val := c.DefaultInt(context.Background(), "Age", 11)
+	assert.Equal(t, 12, val)
+
+	val = c.DefaultInt(context.Background(), "Price11", 11)
+	assert.Equal(t, 11, val)
+
+	val = c.DefaultInt(context.Background(), "PriceInvalid", 11)
+	assert.Equal(t, 11, val)
+}
+
+func TestConfigContainer_DefaultString(t *testing.T) {
+	data := `
+Name="Tom"
+NameInvalid=13
+`
+	cfg := &Config{}
+	c, err := cfg.ParseData([]byte(data))
+	assert.Nil(t, err)
+	assert.NotNil(t, c)
+
+	val := c.DefaultString(context.Background(), "Name", "Jerry")
+	assert.Equal(t, "Tom", val)
+
+	val = c.DefaultString(context.Background(), "Name11", "Jerry")
+	assert.Equal(t, "Jerry", val)
+
+	val = c.DefaultString(context.Background(), "NameInvalid", "Jerry")
+	assert.Equal(t, "Jerry", val)
+}
+
+func TestConfigContainer_DefaultStrings(t *testing.T) {
+	data := `
+Name=["Tom", "Jerry"]
+NameInvalid="Tom"
+`
+	cfg := &Config{}
+	c, err := cfg.ParseData([]byte(data))
+	assert.Nil(t, err)
+	assert.NotNil(t, c)
+
+	val := c.DefaultStrings(context.Background(), "Name", []string{"Jerry"})
+	assert.Equal(t, []string{"Tom", "Jerry"}, val)
+
+	val = c.DefaultStrings(context.Background(), "Name11", []string{"Jerry"})
+	assert.Equal(t, []string{"Jerry"}, val)
+
+	val = c.DefaultStrings(context.Background(), "NameInvalid", []string{"Jerry"})
+	assert.Equal(t, []string{"Jerry"}, val)
+}
+
+func TestConfigContainer_DIY(t *testing.T) {
+	data := `
+Name=["Tom", "Jerry"]
+`
+	cfg := &Config{}
+	c, err := cfg.ParseData([]byte(data))
+	assert.Nil(t, err)
+	assert.NotNil(t, c)
+
+	_, err = c.DIY(context.Background(), "Name")
+	assert.Nil(t, err)
+}
+
+func TestConfigContainer_Float(t *testing.T) {
+	data := `
+Price=12.3
+PriceInvalid="12.3"
+`
+	cfg := &Config{}
+	c, err := cfg.ParseData([]byte(data))
+	assert.Nil(t, err)
+	assert.NotNil(t, c)
+
+	val, err := c.Float(context.Background(), "Price")
+	assert.Nil(t, err)
+	assert.Equal(t, 12.3, val)
+
+	_, err = c.Float(context.Background(), "Price11")
+	assert.Equal(t, config.KeyNotFoundError, err)
+
+	_, err = c.Float(context.Background(), "PriceInvalid")
+	assert.Equal(t, config.InvalidValueTypeError, err)
+}
+
+func TestConfigContainer_Int(t *testing.T) {
+	data := `
+Age=12
+AgeInvalid="13"
+`
+	cfg := &Config{}
+	c, err := cfg.ParseData([]byte(data))
+	assert.Nil(t, err)
+	assert.NotNil(t, c)
+
+	val, err := c.Int(context.Background(), "Age")
+	assert.Nil(t, err)
+	assert.Equal(t, 12, val)
+
+	_, err = c.Int(context.Background(), "Age11")
+	assert.Equal(t, config.KeyNotFoundError, err)
+
+	_, err = c.Int(context.Background(), "AgeInvalid")
+	assert.Equal(t, config.InvalidValueTypeError, err)
+}
+
+func TestConfigContainer_GetSection(t *testing.T) {
+	data := `
+[servers]
+
+  # You can indent as you please. Tabs or spaces. TOML don't care.
+  [servers.alpha]
+  ip = "10.0.0.1"
+  dc = "eqdc10"
+
+  [servers.beta]
+  ip = "10.0.0.2"
+  dc = "eqdc10"
+`
+	cfg := &Config{}
+	c, err := cfg.ParseData([]byte(data))
+	assert.Nil(t, err)
+	assert.NotNil(t, c)
+
+	m, err := c.GetSection(context.Background(), "servers")
+	assert.Nil(t, err)
+	assert.NotNil(t, m)
+	assert.Equal(t, 2, len(m))
+}
+
+func TestConfigContainer_String(t *testing.T) {
+	data := `
+Name="Tom"
+NameInvalid=13
+[Person]
+Name="Jerry"
+`
+	cfg := &Config{}
+	c, err := cfg.ParseData([]byte(data))
+	assert.Nil(t, err)
+	assert.NotNil(t, c)
+
+	val, err := c.String(context.Background(), "Name")
+	assert.Nil(t, err)
+	assert.Equal(t, "Tom", val)
+
+	_, err = c.String(context.Background(), "Name11")
+	assert.Equal(t, config.KeyNotFoundError, err)
+
+	_, err = c.String(context.Background(), "NameInvalid")
+	assert.Equal(t, config.InvalidValueTypeError, err)
+
+	val, err = c.String(context.Background(), "Person.Name")
+	assert.Nil(t, err)
+	assert.Equal(t, "Jerry", val)
+}
+
+func TestConfigContainer_Strings(t *testing.T) {
+	data := `
+Name=["Tom", "Jerry"]
+NameInvalid="Tom"
+`
+	cfg := &Config{}
+	c, err := cfg.ParseData([]byte(data))
+	assert.Nil(t, err)
+	assert.NotNil(t, c)
+
+	val, err := c.Strings(context.Background(), "Name")
+	assert.Nil(t, err)
+	assert.Equal(t, []string{"Tom", "Jerry"}, val)
+
+	_, err = c.Strings(context.Background(), "Name11")
+	assert.Equal(t, config.KeyNotFoundError, err)
+
+	_, err = c.Strings(context.Background(), "NameInvalid")
+	assert.Equal(t, config.InvalidValueTypeError, err)
+}
+
+func TestConfigContainer_Set(t *testing.T) {
+	data := `
+Name=["Tom", "Jerry"]
+NameInvalid="Tom"
+`
+	cfg := &Config{}
+	c, err := cfg.ParseData([]byte(data))
+	assert.Nil(t, err)
+	assert.NotNil(t, c)
+
+	err = c.Set(context.Background(), "Age", "11")
+	assert.Nil(t, err)
+	age, err := c.String(context.Background(), "Age")
+	assert.Nil(t, err)
+	assert.Equal(t, "11", age)
+}
+
+func TestConfigContainer_SubAndMushall(t *testing.T) {
+	data := `
+[servers]
+
+  # You can indent as you please. Tabs or spaces. TOML don't care.
+  [servers.alpha]
+  ip = "10.0.0.1"
+  dc = "eqdc10"
+
+  [servers.beta]
+  ip = "10.0.0.2"
+  dc = "eqdc10"
+`
+	cfg := &Config{}
+	c, err := cfg.ParseData([]byte(data))
+	assert.Nil(t, err)
+	assert.NotNil(t, c)
+
+	sub, err := c.Sub(context.Background(), "servers")
+	assert.Nil(t, err)
+	assert.NotNil(t, sub)
+
+	sub, err = sub.Sub(context.Background(), "alpha")
+	assert.Nil(t, err)
+	assert.NotNil(t, sub)
+	ip, err := sub.String(context.Background(), "ip")
+	assert.Nil(t, err)
+	assert.Equal(t, "10.0.0.1", ip)
+
+	svr := &Server{}
+	err = sub.Unmarshaler(context.Background(), "", svr)
+	assert.Nil(t, err)
+	assert.Equal(t, "10.0.0.1", svr.Ip)
+
+	svr = &Server{}
+	err = c.Unmarshaler(context.Background(), "servers.alpha", svr)
+	assert.Nil(t, err)
+	assert.Equal(t, "10.0.0.1", svr.Ip)
+}
+
+func TestConfigContainer_SaveConfigFile(t *testing.T) {
+	filename := "test_config.toml"
+	path := os.TempDir() + string(os.PathSeparator) + filename
+	data := `
+[servers]
+
+  # You can indent as you please. Tabs or spaces. TOML don't care.
+  [servers.alpha]
+  ip = "10.0.0.1"
+  dc = "eqdc10"
+
+  [servers.beta]
+  ip = "10.0.0.2"
+  dc = "eqdc10"
+`
+	cfg := &Config{}
+	c, err := cfg.ParseData([]byte(data))
+
+	fmt.Println(path)
+
+	assert.Nil(t, err)
+	assert.NotNil(t, c)
+
+	sub, err := c.Sub(context.Background(), "servers")
+	assert.Nil(t, err)
+
+	err = sub.SaveConfigFile(context.Background(), path)
+	assert.Nil(t, err)
+}
+
+type Server struct {
+	Ip string `toml:"ip"`
+}

go.mod

@@ -32,7 +32,7 @@ require (
 	github.com/mattn/go-sqlite3 v2.0.3+incompatible
 	github.com/mitchellh/mapstructure v1.3.3
 	github.com/opentracing/opentracing-go v1.2.0
-	github.com/pelletier/go-toml v1.2.0 // indirect
+	github.com/pelletier/go-toml v1.2.0
 	github.com/pkg/errors v0.9.1
 	github.com/prometheus/client_golang v1.7.0
 	github.com/shiena/ansicolor v0.0.0-20151119151921-a422bbe96644