From 34d6a733e9be883779fd0a5d70784a3bce2eafe9 Mon Sep 17 00:00:00 2001 From: Ming Deng Date: Sun, 11 Oct 2020 23:22:19 +0800 Subject: [PATCH] Support toml config --- core/config/config.go | 2 + core/config/error.go | 25 +++ core/config/toml/toml.go | 358 ++++++++++++++++++++++++++++++++ core/config/toml/toml_test.go | 380 ++++++++++++++++++++++++++++++++++ go.mod | 2 +- 5 files changed, 766 insertions(+), 1 deletion(-) create mode 100644 core/config/error.go create mode 100644 core/config/toml/toml.go create mode 100644 core/config/toml/toml_test.go diff --git a/core/config/config.go b/core/config/config.go index deac8e3e..cfbe5724 100644 --- a/core/config/config.go +++ b/core/config/config.go @@ -72,6 +72,8 @@ type Configer interface { DefaultInt64(ctx context.Context, key string, defaultVal int64) int64 DefaultBool(ctx context.Context, key string, defaultVal bool) bool DefaultFloat(ctx context.Context, key string, defaultVal float64) float64 + + // DIY return the original value DIY(ctx context.Context, key string) (interface{}, error) GetSection(ctx context.Context, section string) (map[string]string, error) diff --git a/core/config/error.go b/core/config/error.go new file mode 100644 index 00000000..e4636c45 --- /dev/null +++ b/core/config/error.go @@ -0,0 +1,25 @@ +// Copyright 2020 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package config + +import ( + "github.com/pkg/errors" +) + +// now not all implementation return those error codes +var ( + KeyNotFoundError = errors.New("the key is not found") + InvalidValueTypeError = errors.New("the value is not expected type") +) diff --git a/core/config/toml/toml.go b/core/config/toml/toml.go new file mode 100644 index 00000000..47ea6a25 --- /dev/null +++ b/core/config/toml/toml.go @@ -0,0 +1,358 @@ +// Copyright 2020 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package toml + +import ( + "context" + "io/ioutil" + "os" + "strings" + + "github.com/pelletier/go-toml" + + "github.com/astaxie/beego/core/config" +) + +const keySeparator = "." + +type Config struct { + tree *toml.Tree +} + +// Parse accepts filename as the parameter +func (c *Config) Parse(filename string) (config.Configer, error) { + ctx, err := ioutil.ReadFile(filename) + if err != nil { + return nil, err + } + return c.ParseData(ctx) +} + +func (c *Config) ParseData(data []byte) (config.Configer, error) { + t, err := toml.LoadBytes(data) + if err != nil { + return nil, err + } + return &configContainer{ + t: t, + }, nil + +} + +// configContainer support key looks like "a.b.c" +type configContainer struct { + t *toml.Tree +} + +// Set put key, val +func (c *configContainer) Set(ctx context.Context, key, val string) error { + path := strings.Split(key, keySeparator) + sub, err := subTree(c.t, path[0:len(path)-1]) + if err != nil { + return err + } + sub.Set(path[len(path)-1], val) + return nil +} + +// String return the value. +// return error if key not found or value is invalid type +func (c *configContainer) String(ctx context.Context, key string) (string, error) { + res, err := c.get(key) + + if err != nil { + return "", err + } + + if res == nil { + return "", config.KeyNotFoundError + } + + if str, ok := res.(string); ok { + return str, nil + } else { + return "", config.InvalidValueTypeError + } +} + +// Strings return []string +// return error if key not found or value is invalid type +func (c *configContainer) Strings(ctx context.Context, key string) ([]string, error) { + val, err := c.get(key) + + if err != nil { + return []string{}, err + } + if val == nil { + return []string{}, config.KeyNotFoundError + } + if arr, ok := val.([]interface{}); ok { + res := make([]string, 0, len(arr)) + for _, ele := range arr { + if str, ok := ele.(string); ok { + res = append(res, str) + } else { + return []string{}, config.InvalidValueTypeError + } + } + return res, nil + } else { + return []string{}, config.InvalidValueTypeError + } +} + +// Int return int value +// return error if key not found or value is invalid type +func (c *configContainer) Int(ctx context.Context, key string) (int, error) { + val, err := c.Int64(ctx, key) + return int(val), err +} + +// Int64 return int64 value +// return error if key not found or value is invalid type +func (c *configContainer) Int64(ctx context.Context, key string) (int64, error) { + res, err := c.get(key) + if err != nil { + return 0, err + } + if res == nil { + return 0, config.KeyNotFoundError + } + if i, ok := res.(int); ok { + return int64(i), nil + } else if i64, ok := res.(int64); ok { + return i64, nil + } else { + return 0, config.InvalidValueTypeError + } +} + +// bool return bool value +// return error if key not found or value is invalid type +func (c *configContainer) Bool(ctx context.Context, key string) (bool, error) { + + res, err := c.get(key) + + if err != nil { + return false, err + } + + if res == nil { + return false, config.KeyNotFoundError + } + if b, ok := res.(bool); ok { + return b, nil + } else { + return false, config.InvalidValueTypeError + } +} + +// Float return float value +// return error if key not found or value is invalid type +func (c *configContainer) Float(ctx context.Context, key string) (float64, error) { + res, err := c.get(key) + if err != nil { + return 0, err + } + + if res == nil { + return 0, config.KeyNotFoundError + } + + if f, ok := res.(float64); ok { + return f, nil + } else { + return 0, config.InvalidValueTypeError + } +} + +// DefaultString return string value +// return default value if key not found or value is invalid type +func (c *configContainer) DefaultString(ctx context.Context, key string, defaultVal string) string { + res, err := c.get(key) + if err != nil { + return defaultVal + } + if str, ok := res.(string); ok { + return str + } else { + return defaultVal + } +} + +// DefaultStrings return []string +// return default value if key not found or value is invalid type +func (c *configContainer) DefaultStrings(ctx context.Context, key string, defaultVal []string) []string { + val, err := c.get(key) + if err != nil { + return defaultVal + } + if arr, ok := val.([]interface{}); ok { + res := make([]string, 0, len(arr)) + for _, ele := range arr { + if str, ok := ele.(string); ok { + res = append(res, str) + } else { + return defaultVal + } + } + return res + } else { + return defaultVal + } +} + +// DefaultInt return int value +// return default value if key not found or value is invalid type +func (c *configContainer) DefaultInt(ctx context.Context, key string, defaultVal int) int { + return int(c.DefaultInt64(ctx, key, int64(defaultVal))) +} + +// DefaultInt64 return int64 value +// return default value if key not found or value is invalid type +func (c *configContainer) DefaultInt64(ctx context.Context, key string, defaultVal int64) int64 { + res, err := c.get(key) + if err != nil { + return defaultVal + } + if i, ok := res.(int); ok { + return int64(i) + } else if i64, ok := res.(int64); ok { + return i64 + } else { + return defaultVal + } +} + +// DefaultBool return bool value +// return default value if key not found or value is invalid type +func (c *configContainer) DefaultBool(ctx context.Context, key string, defaultVal bool) bool { + res, err := c.get(key) + if err != nil { + return defaultVal + } + if b, ok := res.(bool); ok { + return b + } else { + return defaultVal + } +} + +// DefaultFloat return float value +// return default value if key not found or value is invalid type +func (c *configContainer) DefaultFloat(ctx context.Context, key string, defaultVal float64) float64 { + res, err := c.get(key) + if err != nil { + return defaultVal + } + if f, ok := res.(float64); ok { + return f + } else { + return defaultVal + } +} + +// DIY returns the original value +func (c *configContainer) DIY(ctx context.Context, key string) (interface{}, error) { + return c.get(key) +} + +// GetSection return error if the value is not valid toml doc +func (c *configContainer) GetSection(ctx context.Context, section string) (map[string]string, error) { + val, err := subTree(c.t, strings.Split(section, keySeparator)) + if err != nil { + return map[string]string{}, err + } + m := val.ToMap() + res := make(map[string]string, len(m)) + for k, v := range m { + res[k] = config.ToString(v) + } + return res, nil +} + +func (c *configContainer) Unmarshaler(ctx context.Context, prefix string, obj interface{}, opt ...config.DecodeOption) error { + if len(prefix) > 0 { + t, err := subTree(c.t, strings.Split(prefix, keySeparator)) + if err != nil { + return err + } + return t.Unmarshal(obj) + } + return c.t.Unmarshal(obj) +} + +// Sub return sub configer +// return error if key not found or the value is not a sub doc +func (c *configContainer) Sub(ctx context.Context, key string) (config.Configer, error) { + val, err := subTree(c.t, strings.Split(key, keySeparator)) + if err != nil { + return nil, err + } + return &configContainer{ + t: val, + }, nil +} + +// OnChange do nothing +func (c *configContainer) OnChange(ctx context.Context, key string, fn func(value string)) { + // do nothing +} + +// SaveConfigFile create or override the file +func (c *configContainer) SaveConfigFile(ctx context.Context, filename string) error { + // Write configuration file by filename. + f, err := os.Create(filename) + if err != nil { + return err + } + defer f.Close() + + _, err = c.t.WriteTo(f) + return err +} + +func (c *configContainer) get(key string) (interface{}, error) { + if len(key) == 0 { + return nil, config.KeyNotFoundError + } + + segs := strings.Split(key, keySeparator) + t, err := subTree(c.t, segs[0:len(segs)-1]) + + if err != nil { + return nil, err + } + return t.Get(segs[len(segs)-1]), nil +} + +func subTree(t *toml.Tree, path []string) (*toml.Tree, error) { + res := t + for i := 0; i < len(path); i++ { + if subTree, ok := res.Get(path[i]).(*toml.Tree); ok { + res = subTree + } else { + return nil, config.InvalidValueTypeError + } + } + if res == nil { + return nil, config.KeyNotFoundError + } + return res, nil +} + +func init() { + config.Register("toml", &Config{}) +} diff --git a/core/config/toml/toml_test.go b/core/config/toml/toml_test.go new file mode 100644 index 00000000..2af15596 --- /dev/null +++ b/core/config/toml/toml_test.go @@ -0,0 +1,380 @@ +// Copyright 2020 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package toml + +import ( + "context" + "fmt" + "os" + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/astaxie/beego/core/config" +) + +func TestConfig_Parse(t *testing.T) { + // file not found + cfg := &Config{} + _, err := cfg.Parse("invalid_file_name.txt") + assert.NotNil(t, err) +} + +func TestConfig_ParseData(t *testing.T) { + data := ` +name="Tom" +` + cfg := &Config{} + c, err := cfg.ParseData([]byte(data)) + assert.Nil(t, err) + assert.NotNil(t, c) +} + +func TestConfigContainer_Bool(t *testing.T) { + data := ` +Man=true +Woman="true" +` + cfg := &Config{} + c, err := cfg.ParseData([]byte(data)) + assert.Nil(t, err) + assert.NotNil(t, c) + + val, err := c.Bool(context.Background(), "Man") + assert.Nil(t, err) + assert.True(t, val) + + _, err = c.Bool(context.Background(), "Woman") + assert.NotNil(t, err) + assert.Equal(t, config.InvalidValueTypeError, err) +} + +func TestConfigContainer_DefaultBool(t *testing.T) { + data := ` +Man=true +Woman="false" +` + cfg := &Config{} + c, err := cfg.ParseData([]byte(data)) + assert.Nil(t, err) + assert.NotNil(t, c) + + val := c.DefaultBool(context.Background(), "Man11", true) + assert.True(t, val) + + val = c.DefaultBool(context.Background(), "Man", false) + assert.True(t, val) + + val = c.DefaultBool(context.Background(), "Woman", true) + assert.True(t, val) +} + +func TestConfigContainer_DefaultFloat(t *testing.T) { + data := ` +Price=12.3 +PriceInvalid="12.3" +` + cfg := &Config{} + c, err := cfg.ParseData([]byte(data)) + assert.Nil(t, err) + assert.NotNil(t, c) + + val := c.DefaultFloat(context.Background(), "Price", 11.2) + assert.Equal(t, 12.3, val) + + val = c.DefaultFloat(context.Background(), "Price11", 11.2) + assert.Equal(t, 11.2, val) + + val = c.DefaultFloat(context.Background(), "PriceInvalid", 11.2) + assert.Equal(t, 11.2, val) +} + +func TestConfigContainer_DefaultInt(t *testing.T) { + data := ` +Age=12 +AgeInvalid="13" +` + cfg := &Config{} + c, err := cfg.ParseData([]byte(data)) + assert.Nil(t, err) + assert.NotNil(t, c) + + val := c.DefaultInt(context.Background(), "Age", 11) + assert.Equal(t, 12, val) + + val = c.DefaultInt(context.Background(), "Price11", 11) + assert.Equal(t, 11, val) + + val = c.DefaultInt(context.Background(), "PriceInvalid", 11) + assert.Equal(t, 11, val) +} + +func TestConfigContainer_DefaultString(t *testing.T) { + data := ` +Name="Tom" +NameInvalid=13 +` + cfg := &Config{} + c, err := cfg.ParseData([]byte(data)) + assert.Nil(t, err) + assert.NotNil(t, c) + + val := c.DefaultString(context.Background(), "Name", "Jerry") + assert.Equal(t, "Tom", val) + + val = c.DefaultString(context.Background(), "Name11", "Jerry") + assert.Equal(t, "Jerry", val) + + val = c.DefaultString(context.Background(), "NameInvalid", "Jerry") + assert.Equal(t, "Jerry", val) +} + +func TestConfigContainer_DefaultStrings(t *testing.T) { + data := ` +Name=["Tom", "Jerry"] +NameInvalid="Tom" +` + cfg := &Config{} + c, err := cfg.ParseData([]byte(data)) + assert.Nil(t, err) + assert.NotNil(t, c) + + val := c.DefaultStrings(context.Background(), "Name", []string{"Jerry"}) + assert.Equal(t, []string{"Tom", "Jerry"}, val) + + val = c.DefaultStrings(context.Background(), "Name11", []string{"Jerry"}) + assert.Equal(t, []string{"Jerry"}, val) + + val = c.DefaultStrings(context.Background(), "NameInvalid", []string{"Jerry"}) + assert.Equal(t, []string{"Jerry"}, val) +} + +func TestConfigContainer_DIY(t *testing.T) { + data := ` +Name=["Tom", "Jerry"] +` + cfg := &Config{} + c, err := cfg.ParseData([]byte(data)) + assert.Nil(t, err) + assert.NotNil(t, c) + + _, err = c.DIY(context.Background(), "Name") + assert.Nil(t, err) +} + +func TestConfigContainer_Float(t *testing.T) { + data := ` +Price=12.3 +PriceInvalid="12.3" +` + cfg := &Config{} + c, err := cfg.ParseData([]byte(data)) + assert.Nil(t, err) + assert.NotNil(t, c) + + val, err := c.Float(context.Background(), "Price") + assert.Nil(t, err) + assert.Equal(t, 12.3, val) + + _, err = c.Float(context.Background(), "Price11") + assert.Equal(t, config.KeyNotFoundError, err) + + _, err = c.Float(context.Background(), "PriceInvalid") + assert.Equal(t, config.InvalidValueTypeError, err) +} + +func TestConfigContainer_Int(t *testing.T) { + data := ` +Age=12 +AgeInvalid="13" +` + cfg := &Config{} + c, err := cfg.ParseData([]byte(data)) + assert.Nil(t, err) + assert.NotNil(t, c) + + val, err := c.Int(context.Background(), "Age") + assert.Nil(t, err) + assert.Equal(t, 12, val) + + _, err = c.Int(context.Background(), "Age11") + assert.Equal(t, config.KeyNotFoundError, err) + + _, err = c.Int(context.Background(), "AgeInvalid") + assert.Equal(t, config.InvalidValueTypeError, err) +} + +func TestConfigContainer_GetSection(t *testing.T) { + data := ` +[servers] + + # You can indent as you please. Tabs or spaces. TOML don't care. + [servers.alpha] + ip = "10.0.0.1" + dc = "eqdc10" + + [servers.beta] + ip = "10.0.0.2" + dc = "eqdc10" +` + cfg := &Config{} + c, err := cfg.ParseData([]byte(data)) + assert.Nil(t, err) + assert.NotNil(t, c) + + m, err := c.GetSection(context.Background(), "servers") + assert.Nil(t, err) + assert.NotNil(t, m) + assert.Equal(t, 2, len(m)) +} + +func TestConfigContainer_String(t *testing.T) { + data := ` +Name="Tom" +NameInvalid=13 +[Person] +Name="Jerry" +` + cfg := &Config{} + c, err := cfg.ParseData([]byte(data)) + assert.Nil(t, err) + assert.NotNil(t, c) + + val, err := c.String(context.Background(), "Name") + assert.Nil(t, err) + assert.Equal(t, "Tom", val) + + _, err = c.String(context.Background(), "Name11") + assert.Equal(t, config.KeyNotFoundError, err) + + _, err = c.String(context.Background(), "NameInvalid") + assert.Equal(t, config.InvalidValueTypeError, err) + + val, err = c.String(context.Background(), "Person.Name") + assert.Nil(t, err) + assert.Equal(t, "Jerry", val) +} + +func TestConfigContainer_Strings(t *testing.T) { + data := ` +Name=["Tom", "Jerry"] +NameInvalid="Tom" +` + cfg := &Config{} + c, err := cfg.ParseData([]byte(data)) + assert.Nil(t, err) + assert.NotNil(t, c) + + val, err := c.Strings(context.Background(), "Name") + assert.Nil(t, err) + assert.Equal(t, []string{"Tom", "Jerry"}, val) + + _, err = c.Strings(context.Background(), "Name11") + assert.Equal(t, config.KeyNotFoundError, err) + + _, err = c.Strings(context.Background(), "NameInvalid") + assert.Equal(t, config.InvalidValueTypeError, err) +} + +func TestConfigContainer_Set(t *testing.T) { + data := ` +Name=["Tom", "Jerry"] +NameInvalid="Tom" +` + cfg := &Config{} + c, err := cfg.ParseData([]byte(data)) + assert.Nil(t, err) + assert.NotNil(t, c) + + err = c.Set(context.Background(), "Age", "11") + assert.Nil(t, err) + age, err := c.String(context.Background(), "Age") + assert.Nil(t, err) + assert.Equal(t, "11", age) +} + +func TestConfigContainer_SubAndMushall(t *testing.T) { + data := ` +[servers] + + # You can indent as you please. Tabs or spaces. TOML don't care. + [servers.alpha] + ip = "10.0.0.1" + dc = "eqdc10" + + [servers.beta] + ip = "10.0.0.2" + dc = "eqdc10" +` + cfg := &Config{} + c, err := cfg.ParseData([]byte(data)) + assert.Nil(t, err) + assert.NotNil(t, c) + + sub, err := c.Sub(context.Background(), "servers") + assert.Nil(t, err) + assert.NotNil(t, sub) + + sub, err = sub.Sub(context.Background(), "alpha") + assert.Nil(t, err) + assert.NotNil(t, sub) + ip, err := sub.String(context.Background(), "ip") + assert.Nil(t, err) + assert.Equal(t, "10.0.0.1", ip) + + svr := &Server{} + err = sub.Unmarshaler(context.Background(), "", svr) + assert.Nil(t, err) + assert.Equal(t, "10.0.0.1", svr.Ip) + + svr = &Server{} + err = c.Unmarshaler(context.Background(), "servers.alpha", svr) + assert.Nil(t, err) + assert.Equal(t, "10.0.0.1", svr.Ip) +} + +func TestConfigContainer_SaveConfigFile(t *testing.T) { + filename := "test_config.toml" + path := os.TempDir() + string(os.PathSeparator) + filename + data := ` +[servers] + + # You can indent as you please. Tabs or spaces. TOML don't care. + [servers.alpha] + ip = "10.0.0.1" + dc = "eqdc10" + + [servers.beta] + ip = "10.0.0.2" + dc = "eqdc10" +` + cfg := &Config{} + c, err := cfg.ParseData([]byte(data)) + + fmt.Println(path) + + assert.Nil(t, err) + assert.NotNil(t, c) + + sub, err := c.Sub(context.Background(), "servers") + assert.Nil(t, err) + + err = sub.SaveConfigFile(context.Background(), path) + assert.Nil(t, err) +} + +type Server struct { + Ip string `toml:"ip"` +} diff --git a/go.mod b/go.mod index ab7f5e39..697c9951 100644 --- a/go.mod +++ b/go.mod @@ -32,7 +32,7 @@ require ( github.com/mattn/go-sqlite3 v2.0.3+incompatible github.com/mitchellh/mapstructure v1.3.3 github.com/opentracing/opentracing-go v1.2.0 - github.com/pelletier/go-toml v1.2.0 // indirect + github.com/pelletier/go-toml v1.2.0 github.com/pkg/errors v0.9.1 github.com/prometheus/client_golang v1.7.0 github.com/shiena/ansicolor v0.0.0-20151119151921-a422bbe96644