From 63cd8e4e15de50618bf0da81e59c0789864e4975 Mon Sep 17 00:00:00 2001 From: Ming Deng Date: Fri, 11 Sep 2020 21:10:12 +0800 Subject: [PATCH] refactor log module --- pkg/infrastructure/logs/alils/alils.go | 56 ++--- pkg/infrastructure/logs/conn.go | 52 ++--- pkg/infrastructure/logs/console.go | 63 +++-- pkg/infrastructure/logs/es/es.go | 65 +++--- pkg/infrastructure/logs/file.go | 65 +++--- pkg/infrastructure/logs/file_test.go | 5 + pkg/infrastructure/logs/formatter.go | 34 +++ pkg/infrastructure/logs/jianliao.go | 58 ++--- pkg/infrastructure/logs/log.go | 215 ++++++------------ pkg/infrastructure/logs/log_formatter_test.go | 35 --- pkg/infrastructure/logs/multifile.go | 43 ++-- pkg/infrastructure/logs/slack.go | 43 ++-- pkg/infrastructure/logs/smtp.go | 34 +-- 13 files changed, 346 insertions(+), 422 deletions(-) create mode 100644 pkg/infrastructure/logs/formatter.go delete mode 100644 pkg/infrastructure/logs/log_formatter_test.go diff --git a/pkg/infrastructure/logs/alils/alils.go b/pkg/infrastructure/logs/alils/alils.go index 03e97045..0689aae0 100644 --- a/pkg/infrastructure/logs/alils/alils.go +++ b/pkg/infrastructure/logs/alils/alils.go @@ -2,12 +2,14 @@ package alils import ( "encoding/json" + "fmt" "strings" "sync" - "github.com/astaxie/beego/pkg/infrastructure/logs" - "github.com/astaxie/beego/pkg/infrastructure/utils" "github.com/gogo/protobuf/proto" + "github.com/pkg/errors" + + "github.com/astaxie/beego/pkg/infrastructure/logs" ) const ( @@ -28,40 +30,35 @@ type Config struct { Source string `json:"source"` Level int `json:"level"` FlushWhen int `json:"flush_when"` + Formatter string `json:"formatter"` } // aliLSWriter implements LoggerInterface. // Writes messages in keep-live tcp connection. type aliLSWriter struct { - store *LogStore - group []*LogGroup - withMap bool - groupMap map[string]*LogGroup - lock *sync.Mutex - customFormatter func(*logs.LogMsg) string + store *LogStore + group []*LogGroup + withMap bool + groupMap map[string]*LogGroup + lock *sync.Mutex Config + formatter logs.LogFormatter } // NewAliLS creates a new Logger func NewAliLS() logs.Logger { alils := new(aliLSWriter) alils.Level = logs.LevelTrace + alils.formatter = alils return alils } // Init parses config and initializes struct -func (c *aliLSWriter) Init(jsonConfig string, opts ...utils.KV) error { - - for _, elem := range opts { - if elem.GetKey() == "formatter" { - formatter, err := logs.GetFormatter(elem) - if err != nil { - return err - } - c.customFormatter = formatter - } +func (c *aliLSWriter) Init(config string) error { + err := json.Unmarshal([]byte(config), c) + if err != nil { + return err } - json.Unmarshal([]byte(jsonConfig), c) if c.FlushWhen > CacheSize { c.FlushWhen = CacheSize @@ -110,11 +107,23 @@ func (c *aliLSWriter) Init(jsonConfig string, opts ...utils.KV) error { c.lock = &sync.Mutex{} + if len(c.Formatter) > 0 { + fmtr, ok := logs.GetFormatter(c.Formatter) + if !ok { + return errors.New(fmt.Sprintf("the formatter with name: %s not found", c.Formatter)) + } + c.formatter = fmtr + } + return nil } func (c *aliLSWriter) Format(lm *logs.LogMsg) string { - return lm.Msg + return lm.OldStyleFormat() +} + +func (c *aliLSWriter) SetFormatter(f logs.LogFormatter) { + c.formatter = f } // WriteMsg writes a message in connection. @@ -145,11 +154,7 @@ func (c *aliLSWriter) WriteMsg(lm *logs.LogMsg) error { lg = c.group[0] } - if c.customFormatter != nil { - content = c.customFormatter(lm) - } else { - content = c.Format(lm) - } + content = c.formatter.Format(lm) c1 := &LogContent{ Key: proto.String("msg"), @@ -170,7 +175,6 @@ func (c *aliLSWriter) WriteMsg(lm *logs.LogMsg) error { if len(lg.Logs) >= c.FlushWhen { c.flush(lg) } - return nil } diff --git a/pkg/infrastructure/logs/conn.go b/pkg/infrastructure/logs/conn.go index f7d44d7f..1fd71be7 100644 --- a/pkg/infrastructure/logs/conn.go +++ b/pkg/infrastructure/logs/conn.go @@ -16,51 +16,55 @@ package logs import ( "encoding/json" + "fmt" "io" "net" - "github.com/astaxie/beego/pkg/infrastructure/utils" + "github.com/pkg/errors" ) // connWriter implements LoggerInterface. // Writes messages in keep-live tcp connection. type connWriter struct { - lg *logWriter - innerWriter io.WriteCloser - customFormatter func(*LogMsg) string - ReconnectOnMsg bool `json:"reconnectOnMsg"` - Reconnect bool `json:"reconnect"` - Net string `json:"net"` - Addr string `json:"addr"` - Level int `json:"level"` + lg *logWriter + innerWriter io.WriteCloser + formatter LogFormatter + Formatter string `json:"formatter"` + ReconnectOnMsg bool `json:"reconnectOnMsg"` + Reconnect bool `json:"reconnect"` + Net string `json:"net"` + Addr string `json:"addr"` + Level int `json:"level"` } // NewConn creates new ConnWrite returning as LoggerInterface. func NewConn() Logger { conn := new(connWriter) conn.Level = LevelTrace + conn.formatter = conn return conn } func (c *connWriter) Format(lm *LogMsg) string { - return lm.Msg + return lm.OldStyleFormat() } // Init initializes a connection writer with json config. // json config only needs they "level" key -func (c *connWriter) Init(jsonConfig string, opts ...utils.KV) error { - - for _, elem := range opts { - if elem.GetKey() == "formatter" { - formatter, err := GetFormatter(elem) - if err != nil { - return err - } - c.customFormatter = formatter +func (c *connWriter) Init(config string) error { + res := json.Unmarshal([]byte(config), c) + if res == nil && len(c.Formatter) > 0 { + fmtr, ok := GetFormatter(c.Formatter) + if !ok { + return errors.New(fmt.Sprintf("the formatter with name: %s not found", c.Formatter)) } + c.formatter = fmtr } + return res +} - return json.Unmarshal([]byte(jsonConfig), c) +func (c *connWriter) SetFormatter(f LogFormatter) { + c.formatter = f } // WriteMsg writes message in connection. @@ -80,13 +84,7 @@ func (c *connWriter) WriteMsg(lm *LogMsg) error { defer c.innerWriter.Close() } - msg := "" - if c.customFormatter != nil { - msg = c.customFormatter(lm) - } else { - msg = c.Format(lm) - - } + msg := c.formatter.Format(lm) _, err := c.lg.writeln(msg) if err != nil { diff --git a/pkg/infrastructure/logs/console.go b/pkg/infrastructure/logs/console.go index 802d79f5..f99ef11b 100644 --- a/pkg/infrastructure/logs/console.go +++ b/pkg/infrastructure/logs/console.go @@ -16,11 +16,11 @@ package logs import ( "encoding/json" + "fmt" "os" "strings" - "github.com/astaxie/beego/pkg/infrastructure/utils" - + "github.com/pkg/errors" "github.com/shiena/ansicolor" ) @@ -49,20 +49,25 @@ var colors = []brush{ // consoleWriter implements LoggerInterface and writes messages to terminal. type consoleWriter struct { - lg *logWriter - customFormatter func(*LogMsg) string - Level int `json:"level"` - Colorful bool `json:"color"` //this filed is useful only when system's terminal supports color + lg *logWriter + formatter LogFormatter + Formatter string `json:"formatter"` + Level int `json:"level"` + Colorful bool `json:"color"` // this filed is useful only when system's terminal supports color } func (c *consoleWriter) Format(lm *LogMsg) string { - msg := lm.Msg - + msg := lm.OldStyleFormat() + if c.Colorful { + msg = strings.Replace(lm.Msg, levelPrefix[lm.Level], colors[lm.Level](levelPrefix[lm.Level]), 1) + } h, _, _ := formatTimeHeader(lm.When) bytes := append(append(h, msg...), '\n') - return string(bytes) +} +func (c *consoleWriter) SetFormatter(f LogFormatter) { + c.formatter = f } // NewConsole creates ConsoleWriter returning as LoggerInterface. @@ -72,28 +77,27 @@ func NewConsole() Logger { Level: LevelDebug, Colorful: true, } + cw.formatter = cw return cw } // Init initianlizes the console logger. // jsonConfig must be in the format '{"level":LevelTrace}' -func (c *consoleWriter) Init(jsonConfig string, opts ...utils.KV) error { +func (c *consoleWriter) Init(config string) error { - for _, elem := range opts { - if elem.GetKey() == "formatter" { - formatter, err := GetFormatter(elem) - if err != nil { - return err - } - c.customFormatter = formatter - } - } - - if len(jsonConfig) == 0 { + if len(config) == 0 { return nil } - return json.Unmarshal([]byte(jsonConfig), c) + res := json.Unmarshal([]byte(config), c) + if res == nil && len(c.Formatter) > 0 { + fmtr, ok := GetFormatter(c.Formatter) + if !ok { + return errors.New(fmt.Sprintf("the formatter with name: %s not found", c.Formatter)) + } + c.formatter = fmtr + } + return res } // WriteMsg writes message in console. @@ -101,20 +105,7 @@ func (c *consoleWriter) WriteMsg(lm *LogMsg) error { if lm.Level > c.Level { return nil } - - msg := "" - - if c.Colorful { - lm.Msg = strings.Replace(lm.Msg, levelPrefix[lm.Level], colors[lm.Level](levelPrefix[lm.Level]), 1) - } - - if c.customFormatter != nil { - msg = c.customFormatter(lm) - } else { - msg = c.Format(lm) - - } - + msg := c.formatter.Format(lm) c.lg.writeln(msg) return nil } diff --git a/pkg/infrastructure/logs/es/es.go b/pkg/infrastructure/logs/es/es.go index 857a1a34..438a6da6 100644 --- a/pkg/infrastructure/logs/es/es.go +++ b/pkg/infrastructure/logs/es/es.go @@ -13,7 +13,6 @@ import ( "github.com/elastic/go-elasticsearch/v6/esapi" "github.com/astaxie/beego/pkg/infrastructure/logs" - "github.com/astaxie/beego/pkg/infrastructure/utils" ) // NewES returns a LoggerInterface @@ -32,29 +31,34 @@ func NewES() logs.Logger { // import _ "github.com/astaxie/beego/logs/es" type esLogger struct { *elasticsearch.Client - DSN string `json:"dsn"` - Level int `json:"level"` - customFormatter func(*logs.LogMsg) string + DSN string `json:"dsn"` + Level int `json:"level"` + formatter logs.LogFormatter + Formatter string `json:"formatter"` } func (el *esLogger) Format(lm *logs.LogMsg) string { - return lm.Msg + + msg := lm.OldStyleFormat() + idx := LogDocument{ + Timestamp: lm.When.Format(time.RFC3339), + Msg: msg, + } + body, err := json.Marshal(idx) + if err != nil { + return msg + } + return string(body) +} + +func (el *esLogger) SetFormatter(f logs.LogFormatter) { + el.formatter = f } // {"dsn":"http://localhost:9200/","level":1} -func (el *esLogger) Init(jsonConfig string, opts ...utils.KV) error { +func (el *esLogger) Init(config string) error { - for _, elem := range opts { - if elem.GetKey() == "formatter" { - formatter, err := logs.GetFormatter(elem) - if err != nil { - return err - } - el.customFormatter = formatter - } - } - - err := json.Unmarshal([]byte(jsonConfig), el) + err := json.Unmarshal([]byte(config), el) if err != nil { return err } @@ -73,6 +77,13 @@ func (el *esLogger) Init(jsonConfig string, opts ...utils.KV) error { } el.Client = conn } + if len(el.Formatter) > 0 { + fmtr, ok := logs.GetFormatter(el.Formatter) + if !ok { + return errors.New(fmt.Sprintf("the formatter with name: %s not found", el.Formatter)) + } + el.formatter = fmtr + } return nil } @@ -82,28 +93,14 @@ func (el *esLogger) WriteMsg(lm *logs.LogMsg) error { return nil } - msg := "" - if el.customFormatter != nil { - msg = el.customFormatter(lm) - } else { - msg = el.Format(lm) - } + msg := el.formatter.Format(lm) - idx := LogDocument{ - Timestamp: lm.When.Format(time.RFC3339), - Msg: msg, - } - - body, err := json.Marshal(idx) - if err != nil { - return err - } req := esapi.IndexRequest{ Index: fmt.Sprintf("%04d.%02d.%02d", lm.When.Year(), lm.When.Month(), lm.When.Day()), DocumentType: "logs", - Body: strings.NewReader(string(body)), + Body: strings.NewReader(msg), } - _, err = req.Do(context.Background(), el.Client) + _, err := req.Do(context.Background(), el.Client) return err } diff --git a/pkg/infrastructure/logs/file.go b/pkg/infrastructure/logs/file.go index 0c96918c..b01be357 100644 --- a/pkg/infrastructure/logs/file.go +++ b/pkg/infrastructure/logs/file.go @@ -27,8 +27,6 @@ import ( "strings" "sync" "time" - - "github.com/astaxie/beego/pkg/infrastructure/utils" ) // fileLogWriter implements LoggerInterface. @@ -62,8 +60,6 @@ type fileLogWriter struct { hourlyOpenDate int hourlyOpenTime time.Time - customFormatter func(*LogMsg) string - Rotate bool `json:"rotate"` Level int `json:"level"` @@ -73,6 +69,9 @@ type fileLogWriter struct { RotatePerm string `json:"rotateperm"` fileNameOnly, suffix string // like "project.log", project is fileNameOnly and .log is suffix + + formatter LogFormatter + Formatter string `json:"formatter"` } // newFileWriter creates a FileLogWriter returning as LoggerInterface. @@ -90,11 +89,19 @@ func newFileWriter() Logger { MaxFiles: 999, MaxSize: 1 << 28, } + w.formatter = w return w } func (w *fileLogWriter) Format(lm *LogMsg) string { - return lm.Msg + msg := lm.OldStyleFormat() + hd, _, _ := formatTimeHeader(lm.When) + msg = fmt.Sprintf("%s %s\n", string(hd), msg) + return msg +} + +func (w *fileLogWriter) SetFormatter(f LogFormatter) { + w.formatter = f } // Init file logger with json config. @@ -108,19 +115,9 @@ func (w *fileLogWriter) Format(lm *LogMsg) string { // "rotate":true, // "perm":"0600" // } -func (w *fileLogWriter) Init(jsonConfig string, opts ...utils.KV) error { +func (w *fileLogWriter) Init(config string) error { - for _, elem := range opts { - if elem.GetKey() == "formatter" { - formatter, err := GetFormatter(elem) - if err != nil { - return err - } - w.customFormatter = formatter - } - } - - err := json.Unmarshal([]byte(jsonConfig), w) + err := json.Unmarshal([]byte(config), w) if err != nil { return err } @@ -132,6 +129,14 @@ func (w *fileLogWriter) Init(jsonConfig string, opts ...utils.KV) error { if w.suffix == "" { w.suffix = ".log" } + + if len(w.Formatter) > 0 { + fmtr, ok := GetFormatter(w.Formatter) + if !ok { + return errors.New(fmt.Sprintf("the formatter with name: %s not found", w.Formatter)) + } + w.formatter = fmtr + } err = w.startLogger() return err } @@ -149,13 +154,13 @@ func (w *fileLogWriter) startLogger() error { return w.initFd() } -func (w *fileLogWriter) needRotateDaily(size int, day int) bool { +func (w *fileLogWriter) needRotateDaily(day int) bool { return (w.MaxLines > 0 && w.maxLinesCurLines >= w.MaxLines) || (w.MaxSize > 0 && w.maxSizeCurSize >= w.MaxSize) || (w.Daily && day != w.dailyOpenDate) } -func (w *fileLogWriter) needRotateHourly(size int, hour int) bool { +func (w *fileLogWriter) needRotateHourly(hour int) bool { return (w.MaxLines > 0 && w.maxLinesCurLines >= w.MaxLines) || (w.MaxSize > 0 && w.maxSizeCurSize >= w.MaxSize) || (w.Hourly && hour != w.hourlyOpenDate) @@ -167,31 +172,25 @@ func (w *fileLogWriter) WriteMsg(lm *LogMsg) error { if lm.Level > w.Level { return nil } - hd, d, h := formatTimeHeader(lm.When) - msg := "" - if w.customFormatter != nil { - msg = w.customFormatter(lm) - } else { - msg = w.Format(lm) - } + _, d, h := formatTimeHeader(lm.When) - msg = fmt.Sprintf("%s %s\n", string(hd), msg) + msg := w.formatter.Format(lm) if w.Rotate { w.RLock() - if w.needRotateHourly(len(lm.Msg), h) { + if w.needRotateHourly(h) { w.RUnlock() w.Lock() - if w.needRotateHourly(len(lm.Msg), h) { + if w.needRotateHourly(h) { if err := w.doRotate(lm.When); err != nil { fmt.Fprintf(os.Stderr, "FileLogWriter(%q): %s\n", w.Filename, err) } } w.Unlock() - } else if w.needRotateDaily(len(lm.Msg), d) { + } else if w.needRotateDaily(d) { w.RUnlock() w.Lock() - if w.needRotateDaily(len(lm.Msg), d) { + if w.needRotateDaily(d) { if err := w.doRotate(lm.When); err != nil { fmt.Fprintf(os.Stderr, "FileLogWriter(%q): %s\n", w.Filename, err) } @@ -263,7 +262,7 @@ func (w *fileLogWriter) dailyRotate(openTime time.Time) { tm := time.NewTimer(time.Duration(nextDay.UnixNano() - openTime.UnixNano() + 100)) <-tm.C w.Lock() - if w.needRotateDaily(0, time.Now().Day()) { + if w.needRotateDaily(time.Now().Day()) { if err := w.doRotate(time.Now()); err != nil { fmt.Fprintf(os.Stderr, "FileLogWriter(%q): %s\n", w.Filename, err) } @@ -278,7 +277,7 @@ func (w *fileLogWriter) hourlyRotate(openTime time.Time) { tm := time.NewTimer(time.Duration(nextHour.UnixNano() - openTime.UnixNano() + 100)) <-tm.C w.Lock() - if w.needRotateHourly(0, time.Now().Hour()) { + if w.needRotateHourly(time.Now().Hour()) { if err := w.doRotate(time.Now()); err != nil { fmt.Fprintf(os.Stderr, "FileLogWriter(%q): %s\n", w.Filename, err) } diff --git a/pkg/infrastructure/logs/file_test.go b/pkg/infrastructure/logs/file_test.go index 7f2a3590..494d0a9e 100644 --- a/pkg/infrastructure/logs/file_test.go +++ b/pkg/infrastructure/logs/file_test.go @@ -268,6 +268,7 @@ func testFileRotate(t *testing.T, fn1, fn2 string, daily, hourly bool) { Perm: "0660", RotatePerm: "0440", } + fw.formatter = fw if daily { fw.Init(fmt.Sprintf(`{"filename":"%v","maxdays":1}`, fn1)) @@ -308,6 +309,8 @@ func testFileDailyRotate(t *testing.T, fn1, fn2 string) { Perm: "0660", RotatePerm: "0440", } + fw.formatter = fw + fw.Init(fmt.Sprintf(`{"filename":"%v","maxdays":1}`, fn1)) fw.dailyOpenTime = time.Now().Add(-24 * time.Hour) fw.dailyOpenDate = fw.dailyOpenTime.Day() @@ -340,6 +343,8 @@ func testFileHourlyRotate(t *testing.T, fn1, fn2 string) { Perm: "0660", RotatePerm: "0440", } + + fw.formatter = fw fw.Init(fmt.Sprintf(`{"filename":"%v","maxhours":1}`, fn1)) fw.hourlyOpenTime = time.Now().Add(-1 * time.Hour) fw.hourlyOpenDate = fw.hourlyOpenTime.Hour() diff --git a/pkg/infrastructure/logs/formatter.go b/pkg/infrastructure/logs/formatter.go new file mode 100644 index 00000000..b2599f2d --- /dev/null +++ b/pkg/infrastructure/logs/formatter.go @@ -0,0 +1,34 @@ +// Copyright 2020 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package logs + +var formatterMap = make(map[string]LogFormatter, 4) + +type LogFormatter interface { + Format(lm *LogMsg) string +} + +// RegisterFormatter register an formatter. Usually you should use this to extend your custom formatter +// for example: +// RegisterFormatter("my-fmt", &MyFormatter{}) +// logs.SetFormatter(Console, `{"formatter": "my-fmt"}`) +func RegisterFormatter(name string, fmtr LogFormatter) { + formatterMap[name] = fmtr +} + +func GetFormatter(name string) (LogFormatter, bool) { + res, ok := formatterMap[name] + return res, ok +} diff --git a/pkg/infrastructure/logs/jianliao.go b/pkg/infrastructure/logs/jianliao.go index 88750125..9757a7d5 100644 --- a/pkg/infrastructure/logs/jianliao.go +++ b/pkg/infrastructure/logs/jianliao.go @@ -6,42 +6,49 @@ import ( "net/http" "net/url" - "github.com/astaxie/beego/pkg/infrastructure/utils" + "github.com/pkg/errors" ) // JLWriter implements beego LoggerInterface and is used to send jiaoliao webhook type JLWriter struct { - AuthorName string `json:"authorname"` - Title string `json:"title"` - WebhookURL string `json:"webhookurl"` - RedirectURL string `json:"redirecturl,omitempty"` - ImageURL string `json:"imageurl,omitempty"` - Level int `json:"level"` - customFormatter func(*LogMsg) string + AuthorName string `json:"authorname"` + Title string `json:"title"` + WebhookURL string `json:"webhookurl"` + RedirectURL string `json:"redirecturl,omitempty"` + ImageURL string `json:"imageurl,omitempty"` + Level int `json:"level"` + + formatter LogFormatter + Formatter string `json:"formatter"` } // newJLWriter creates jiaoliao writer. func newJLWriter() Logger { - return &JLWriter{Level: LevelTrace} + res := &JLWriter{Level: LevelTrace} + res.formatter = res + return res } // Init JLWriter with json config string -func (s *JLWriter) Init(jsonConfig string, opts ...utils.KV) error { - for _, elem := range opts { - if elem.GetKey() == "formatter" { - formatter, err := GetFormatter(elem) - if err != nil { - return err - } - s.customFormatter = formatter - } - } +func (s *JLWriter) Init(config string) error { - return json.Unmarshal([]byte(jsonConfig), s) + res := json.Unmarshal([]byte(config), s) + if res == nil && len(s.Formatter) > 0 { + fmtr, ok := GetFormatter(s.Formatter) + if !ok { + return errors.New(fmt.Sprintf("the formatter with name: %s not found", s.Formatter)) + } + s.formatter = fmtr + } + return res } func (s *JLWriter) Format(lm *LogMsg) string { - return lm.Msg + return lm.OldStyleFormat() +} + +func (s *JLWriter) SetFormatter(f LogFormatter) { + s.formatter = f } // WriteMsg writes message in smtp writer. @@ -51,14 +58,7 @@ func (s *JLWriter) WriteMsg(lm *LogMsg) error { return nil } - text := "" - - if s.customFormatter != nil { - text = fmt.Sprintf("%s %s", lm.When.Format("2006-01-02 15:04:05"), s.customFormatter(lm)) - } else { - text = fmt.Sprintf("%s %s", lm.When.Format("2006-01-02 15:04:05"), s.Format(lm)) - - } + text := s.formatter.Format(lm) form := url.Values{} form.Add("authorName", s.AuthorName) diff --git a/pkg/infrastructure/logs/log.go b/pkg/infrastructure/logs/log.go index 2d400eba..480cecab 100644 --- a/pkg/infrastructure/logs/log.go +++ b/pkg/infrastructure/logs/log.go @@ -38,13 +38,12 @@ import ( "log" "os" "path" - "reflect" "runtime" "strings" "sync" "time" - "github.com/astaxie/beego/pkg/infrastructure/utils" + "github.com/pkg/errors" ) // RFC5424 log message levels. @@ -87,11 +86,11 @@ type newLoggerFunc func() Logger // Logger defines the behavior of a log provider. type Logger interface { - Init(config string, opts ...utils.KV) error + Init(config string) error WriteMsg(lm *LogMsg) error - Format(lm *LogMsg) string Destroy() Flush() + SetFormatter(f LogFormatter) } var adapters = make(map[string]newLoggerFunc) @@ -118,7 +117,6 @@ type BeeLogger struct { init bool enableFuncCallDepth bool loggerFuncCallDepth int - globalFormatter func(*LogMsg) string enableFullFilePath bool asynchronous bool prefix string @@ -127,6 +125,7 @@ type BeeLogger struct { signalChan chan string wg sync.WaitGroup outputs []*nameLogger + globalFormatter string } const defaultAsyncMsgLen = 1e3 @@ -137,15 +136,15 @@ type nameLogger struct { } type LogMsg struct { - Level int - Msg string - When time.Time - FilePath string - LineNumber int -} - -type LogFormatter interface { - Format(lm *LogMsg) string + Level int + Msg string + When time.Time + FilePath string + LineNumber int + Args []interface{} + Prefix string + enableFullFilePath bool + enableFuncCallDepth bool } var logMsgPool *sync.Pool @@ -188,8 +187,25 @@ func (bl *BeeLogger) Async(msgLen ...int64) *BeeLogger { return bl } -func Format(lm *LogMsg) string { - return lm.Msg +// OldStyleFormat you should never invoke this +func (lm *LogMsg) OldStyleFormat() string { + msg := lm.Msg + + if len(lm.Args) > 0 { + lm.Msg = fmt.Sprintf(lm.Msg, lm.Args...) + } + + msg = lm.Prefix + " " + msg + + if lm.enableFuncCallDepth { + if !lm.enableFullFilePath { + _, lm.FilePath = path.Split(lm.FilePath) + } + msg = fmt.Sprintf("[%s:%d] %s", lm.FilePath, lm.LineNumber, msg) + } + + msg = levelPrefix[lm.Level] + " " + msg + return msg } // SetLogger provides a given logger adapter into BeeLogger with config string. @@ -208,16 +224,18 @@ func (bl *BeeLogger) setLogger(adapterName string, configs ...string) error { } lg := logAdapter() - var err error // Global formatter overrides the default set formatter - // but not adapter specific formatters set with logs.SetLoggerWithOpts() - if bl.globalFormatter != nil { - err = lg.Init(config, &utils.SimpleKV{Key: "formatter", Value: bl.globalFormatter}) - } else { - err = lg.Init(config) + if len(bl.globalFormatter) > 0 { + fmtr, ok := GetFormatter(bl.globalFormatter) + if !ok { + return errors.New(fmt.Sprintf("the formatter with name: %s not found", bl.globalFormatter)) + } + lg.SetFormatter(fmtr) } + err := lg.Init(config) + if err != nil { fmt.Fprintln(os.Stderr, "logs.BeeLogger.SetLogger: "+err.Error()) return err @@ -287,46 +305,34 @@ func (bl *BeeLogger) Write(p []byte) (n int, err error) { return 0, err } -func (bl *BeeLogger) writeMsg(lm *LogMsg, v ...interface{}) error { +func (bl *BeeLogger) writeMsg(lm *LogMsg) error { if !bl.init { bl.lock.Lock() bl.setLogger(AdapterConsole) bl.lock.Unlock() } - if len(v) > 0 { - lm.Msg = fmt.Sprintf(lm.Msg, v...) - } - - lm.Msg = bl.prefix + " " + lm.Msg - var ( file string line int ok bool ) - if bl.enableFuncCallDepth { - _, file, line, ok = runtime.Caller(bl.loggerFuncCallDepth) - if !ok { - file = "???" - line = 0 - } - - if !bl.enableFullFilePath { - _, file = path.Split(file) - } - lm.FilePath = file - lm.LineNumber = line - lm.Msg = fmt.Sprintf("[%s:%d] %s", lm.FilePath, lm.LineNumber, lm.Msg) + _, file, line, ok = runtime.Caller(bl.loggerFuncCallDepth) + if !ok { + file = "???" + line = 0 } + lm.FilePath = file + lm.LineNumber = line + + lm.enableFullFilePath = bl.enableFullFilePath + lm.enableFuncCallDepth = bl.enableFuncCallDepth // set level info in front of filename info if lm.Level == levelLoggerImpl { // set to emergency to ensure all log will be print out correctly lm.Level = LevelEmergency - } else { - lm.Msg = levelPrefix[lm.Level] + " " + lm.Msg } if bl.asynchronous { @@ -334,6 +340,10 @@ func (bl *BeeLogger) writeMsg(lm *LogMsg, v ...interface{}) error { logM.Level = lm.Level logM.Msg = lm.Msg logM.When = lm.When + logM.Args = lm.Args + logM.FilePath = lm.FilePath + logM.LineNumber = lm.LineNumber + logM.Prefix = lm.Prefix if bl.outputs != nil { bl.msgChan <- lm } else { @@ -404,84 +414,14 @@ func (bl *BeeLogger) startLogger() { } } -// Get the formatter from the opts common.SimpleKV structure -// Looks for a key: "formatter" with value: func(*LogMsg) string -func GetFormatter(opts utils.KV) (func(*LogMsg) string, error) { - if strings.ToLower(opts.GetKey().(string)) == "formatter" { - formatterInterface := reflect.ValueOf(opts.GetValue()).Interface() - formatterFunc := formatterInterface.(func(*LogMsg) string) - return formatterFunc, nil - } - - return nil, fmt.Errorf("no \"formatter\" key given in simpleKV") -} - -// SetLoggerWithOpts sets a log adapter with a user defined logging format. Config must be valid JSON -// such as: {"interval":360} -func (bl *BeeLogger) setLoggerWithOpts(adapterName string, opts utils.KV, configs ...string) error { - config := append(configs, "{}")[0] - for _, l := range bl.outputs { - if l.name == adapterName { - return fmt.Errorf("logs: duplicate adaptername %q (you have set this logger before)", adapterName) - } - } - - logAdapter, ok := adapters[adapterName] - if !ok { - return fmt.Errorf("logs: unknown adaptername %q (forgotten Register?)", adapterName) - } - - if opts.GetKey() == nil { - return fmt.Errorf("No SimpleKV struct set for %s log adapter", adapterName) - } - - lg := logAdapter() - err := lg.Init(config, opts) - if err != nil { - fmt.Fprintln(os.Stderr, "logs.BeeLogger.SetLogger: "+err.Error()) - return err - } - - bl.outputs = append(bl.outputs, &nameLogger{ - name: adapterName, - Logger: lg, - }) - - return nil -} - -// SetLogger provides a given logger adapter into BeeLogger with config string. -func (bl *BeeLogger) SetLoggerWithOpts(adapterName string, opts utils.KV, configs ...string) error { - bl.lock.Lock() - defer bl.lock.Unlock() - if !bl.init { - bl.outputs = []*nameLogger{} - bl.init = true - } - return bl.setLoggerWithOpts(adapterName, opts, configs...) -} - -// SetLoggerWIthOpts sets a given log adapter with a custom log adapter. -// Log Adapter must be given in the form common.SimpleKV{Key: "formatter": Value: struct.FormatFunc} -// where FormatFunc has the signature func(*LogMsg) string -// func SetLoggerWithOpts(adapter string, config []string, formatterFunc func(*LogMsg) string) error { -func SetLoggerWithOpts(adapter string, config []string, opts utils.KV) error { - err := beeLogger.SetLoggerWithOpts(adapter, opts, config...) - if err != nil { - log.Fatal(err) - } - return nil - -} - -func (bl *BeeLogger) setGlobalFormatter(fmtter func(*LogMsg) string) error { +func (bl *BeeLogger) setGlobalFormatter(fmtter string) error { bl.globalFormatter = fmtter return nil } // SetGlobalFormatter sets the global formatter for all log adapters -// This overrides and other individually set adapter -func SetGlobalFormatter(fmtter func(*LogMsg) string) error { +// don't forget to register the formatter by invoking RegisterFormatter +func SetGlobalFormatter(fmtter string) error { return beeLogger.setGlobalFormatter(fmtter) } @@ -513,11 +453,8 @@ func (bl *BeeLogger) Alert(format string, v ...interface{}) { Level: LevelAlert, Msg: format, When: time.Now(), + Args: v, } - if len(v) > 0 { - lm.Msg = fmt.Sprintf(lm.Msg, v...) - } - bl.writeMsg(lm) } @@ -530,9 +467,7 @@ func (bl *BeeLogger) Critical(format string, v ...interface{}) { Level: LevelCritical, Msg: format, When: time.Now(), - } - if len(v) > 0 { - lm.Msg = fmt.Sprintf(lm.Msg, v...) + Args: v, } bl.writeMsg(lm) @@ -547,9 +482,7 @@ func (bl *BeeLogger) Error(format string, v ...interface{}) { Level: LevelError, Msg: format, When: time.Now(), - } - if len(v) > 0 { - lm.Msg = fmt.Sprintf(lm.Msg, v...) + Args: v, } bl.writeMsg(lm) @@ -564,9 +497,7 @@ func (bl *BeeLogger) Warning(format string, v ...interface{}) { Level: LevelWarn, Msg: format, When: time.Now(), - } - if len(v) > 0 { - lm.Msg = fmt.Sprintf(lm.Msg, v...) + Args: v, } bl.writeMsg(lm) @@ -581,9 +512,7 @@ func (bl *BeeLogger) Notice(format string, v ...interface{}) { Level: LevelNotice, Msg: format, When: time.Now(), - } - if len(v) > 0 { - lm.Msg = fmt.Sprintf(lm.Msg, v...) + Args: v, } bl.writeMsg(lm) @@ -598,9 +527,7 @@ func (bl *BeeLogger) Informational(format string, v ...interface{}) { Level: LevelInfo, Msg: format, When: time.Now(), - } - if len(v) > 0 { - lm.Msg = fmt.Sprintf(lm.Msg, v...) + Args: v, } bl.writeMsg(lm) @@ -615,9 +542,7 @@ func (bl *BeeLogger) Debug(format string, v ...interface{}) { Level: LevelDebug, Msg: format, When: time.Now(), - } - if len(v) > 0 { - lm.Msg = fmt.Sprintf(lm.Msg, v...) + Args: v, } bl.writeMsg(lm) @@ -633,9 +558,7 @@ func (bl *BeeLogger) Warn(format string, v ...interface{}) { Level: LevelWarn, Msg: format, When: time.Now(), - } - if len(v) > 0 { - lm.Msg = fmt.Sprintf(lm.Msg, v...) + Args: v, } bl.writeMsg(lm) @@ -651,9 +574,7 @@ func (bl *BeeLogger) Info(format string, v ...interface{}) { Level: LevelInfo, Msg: format, When: time.Now(), - } - if len(v) > 0 { - lm.Msg = fmt.Sprintf(lm.Msg, v...) + Args: v, } bl.writeMsg(lm) @@ -669,9 +590,7 @@ func (bl *BeeLogger) Trace(format string, v ...interface{}) { Level: LevelDebug, Msg: format, When: time.Now(), - } - if len(v) > 0 { - lm.Msg = fmt.Sprintf(lm.Msg, v...) + Args: v, } bl.writeMsg(lm) diff --git a/pkg/infrastructure/logs/log_formatter_test.go b/pkg/infrastructure/logs/log_formatter_test.go deleted file mode 100644 index 73281cf6..00000000 --- a/pkg/infrastructure/logs/log_formatter_test.go +++ /dev/null @@ -1,35 +0,0 @@ -package logs - -import ( - "fmt" - "testing" - - "github.com/astaxie/beego/pkg/infrastructure/utils" -) - -func customFormatter(lm *LogMsg) string { - return fmt.Sprintf("[CUSTOM CONSOLE LOGGING] %s", lm.Msg) -} - -func globalFormatter(lm *LogMsg) string { - return fmt.Sprintf("[GLOBAL] %s", lm.Msg) -} - -func TestCustomLoggingFormatter(t *testing.T) { - // beego.BConfig.Log.AccessLogs = true - - SetLoggerWithOpts("console", []string{`{"color":true}`}, &utils.SimpleKV{Key: "formatter", Value: customFormatter}) - - // Message will be formatted by the customFormatter with colorful text set to true - Informational("Test message") -} - -func TestGlobalLoggingFormatter(t *testing.T) { - SetGlobalFormatter(globalFormatter) - - SetLogger("console", `{"color":true}`) - - // Message will be formatted by globalFormatter - Informational("Test message") - -} diff --git a/pkg/infrastructure/logs/multifile.go b/pkg/infrastructure/logs/multifile.go index bf589b91..79178211 100644 --- a/pkg/infrastructure/logs/multifile.go +++ b/pkg/infrastructure/logs/multifile.go @@ -16,8 +16,6 @@ package logs import ( "encoding/json" - - "github.com/astaxie/beego/pkg/infrastructure/utils" ) // A filesLogWriter manages several fileLogWriter @@ -26,10 +24,9 @@ import ( // and write the error-level logs to project.error.log and write the debug-level logs to project.debug.log // the rotate attribute also acts like fileLogWriter type multiFileLogWriter struct { - writers [LevelDebug + 1 + 1]*fileLogWriter // the last one for fullLogWriter - fullLogWriter *fileLogWriter - Separate []string `json:"separate"` - customFormatter func(*LogMsg) string + writers [LevelDebug + 1 + 1]*fileLogWriter // the last one for fullLogWriter + fullLogWriter *fileLogWriter + Separate []string `json:"separate"` } var levelNames = [...]string{"emergency", "alert", "critical", "error", "warning", "notice", "info", "debug"} @@ -47,30 +44,27 @@ var levelNames = [...]string{"emergency", "alert", "critical", "error", "warning // "separate":["emergency", "alert", "critical", "error", "warning", "notice", "info", "debug"], // } -func (f *multiFileLogWriter) Init(jsonConfig string, opts ...utils.KV) error { - for _, elem := range opts { - if elem.GetKey() == "formatter" { - formatter, err := GetFormatter(elem) - if err != nil { - return err - } - f.customFormatter = formatter - } - } +func (f *multiFileLogWriter) Init(config string) error { writer := newFileWriter().(*fileLogWriter) - err := writer.Init(jsonConfig) + err := writer.Init(config) if err != nil { return err } f.fullLogWriter = writer f.writers[LevelDebug+1] = writer - //unmarshal "separate" field to f.Separate - json.Unmarshal([]byte(jsonConfig), f) + // unmarshal "separate" field to f.Separate + err = json.Unmarshal([]byte(config), f) + if err != nil { + return err + } jsonMap := map[string]interface{}{} - json.Unmarshal([]byte(jsonConfig), &jsonMap) + err = json.Unmarshal([]byte(config), &jsonMap) + if err != nil { + return err + } for i := LevelEmergency; i < LevelDebug+1; i++ { for _, v := range f.Separate { @@ -91,7 +85,11 @@ func (f *multiFileLogWriter) Init(jsonConfig string, opts ...utils.KV) error { } func (f *multiFileLogWriter) Format(lm *LogMsg) string { - return lm.Msg + return lm.OldStyleFormat() +} + +func (f *multiFileLogWriter) SetFormatter(fmt LogFormatter) { + f.fullLogWriter.SetFormatter(f) } func (f *multiFileLogWriter) Destroy() { @@ -126,7 +124,8 @@ func (f *multiFileLogWriter) Flush() { // newFilesWriter create a FileLogWriter returning as LoggerInterface. func newFilesWriter() Logger { - return &multiFileLogWriter{} + res := &multiFileLogWriter{} + return res } func init() { diff --git a/pkg/infrastructure/logs/slack.go b/pkg/infrastructure/logs/slack.go index d56b9acd..b6e2f170 100644 --- a/pkg/infrastructure/logs/slack.go +++ b/pkg/infrastructure/logs/slack.go @@ -6,35 +6,46 @@ import ( "net/http" "net/url" - "github.com/astaxie/beego/pkg/infrastructure/utils" + "github.com/pkg/errors" ) // SLACKWriter implements beego LoggerInterface and is used to send jiaoliao webhook type SLACKWriter struct { - WebhookURL string `json:"webhookurl"` - Level int `json:"level"` - UseCustomFormatter bool - CustomFormatter func(*LogMsg) string + WebhookURL string `json:"webhookurl"` + Level int `json:"level"` + formatter LogFormatter + Formatter string `json:"formatter"` } // newSLACKWriter creates jiaoliao writer. func newSLACKWriter() Logger { - return &SLACKWriter{Level: LevelTrace} + res := &SLACKWriter{Level: LevelTrace} + res.formatter = res + return res } func (s *SLACKWriter) Format(lm *LogMsg) string { - return lm.Msg + text := fmt.Sprintf("{\"text\": \"%s %s\"}", lm.When.Format("2006-01-02 15:04:05"), lm.OldStyleFormat()) + return text +} + +func (s *SLACKWriter) SetFormatter(f LogFormatter) { + s.formatter = f } // Init SLACKWriter with json config string -func (s *SLACKWriter) Init(jsonConfig string, opts ...utils.KV) error { - // if elem != nil { - // s.UseCustomFormatter = true - // s.CustomFormatter = elem - // } - // } +func (s *SLACKWriter) Init(config string) error { + res := json.Unmarshal([]byte(config), s) - return json.Unmarshal([]byte(jsonConfig), s) + if res == nil && len(s.Formatter) > 0 { + fmtr, ok := GetFormatter(s.Formatter) + if !ok { + return errors.New(fmt.Sprintf("the formatter with name: %s not found", s.Formatter)) + } + s.formatter = fmtr + } + + return res } // WriteMsg write message in smtp writer. @@ -44,10 +55,8 @@ func (s *SLACKWriter) WriteMsg(lm *LogMsg) error { return nil } msg := s.Format(lm) - text := fmt.Sprintf("{\"text\": \"%s %s\"}", lm.When.Format("2006-01-02 15:04:05"), msg) - form := url.Values{} - form.Add("payload", text) + form.Add("payload", msg) resp, err := http.PostForm(s.WebhookURL, form) if err != nil { diff --git a/pkg/infrastructure/logs/smtp.go b/pkg/infrastructure/logs/smtp.go index 904a89df..40891a7c 100644 --- a/pkg/infrastructure/logs/smtp.go +++ b/pkg/infrastructure/logs/smtp.go @@ -22,7 +22,7 @@ import ( "net/smtp" "strings" - "github.com/astaxie/beego/pkg/infrastructure/utils" + "github.com/pkg/errors" ) // SMTPWriter implements LoggerInterface and is used to send emails via given SMTP-server. @@ -34,12 +34,15 @@ type SMTPWriter struct { FromAddress string `json:"fromAddress"` RecipientAddresses []string `json:"sendTos"` Level int `json:"level"` - customFormatter func(*LogMsg) string + formatter LogFormatter + Formatter string `json:"formatter"` } // NewSMTPWriter creates the smtp writer. func newSMTPWriter() Logger { - return &SMTPWriter{Level: LevelTrace} + res := &SMTPWriter{Level: LevelTrace} + res.formatter = res + return res } // Init smtp writer with json config. @@ -53,19 +56,16 @@ func newSMTPWriter() Logger { // "sendTos":["email1","email2"], // "level":LevelError // } -func (s *SMTPWriter) Init(jsonConfig string, opts ...utils.KV) error { - - for _, elem := range opts { - if elem.GetKey() == "formatter" { - formatter, err := GetFormatter(elem) - if err != nil { - return err - } - s.customFormatter = formatter +func (s *SMTPWriter) Init(config string) error { + res := json.Unmarshal([]byte(config), s) + if res == nil && len(s.Formatter) > 0 { + fmtr, ok := GetFormatter(s.Formatter) + if !ok { + return errors.New(fmt.Sprintf("the formatter with name: %s not found", s.Formatter)) } + s.formatter = fmtr } - - return json.Unmarshal([]byte(jsonConfig), s) + return res } func (s *SMTPWriter) getSMTPAuth(host string) smtp.Auth { @@ -80,6 +80,10 @@ func (s *SMTPWriter) getSMTPAuth(host string) smtp.Auth { ) } +func (s *SMTPWriter) SetFormatter(f LogFormatter) { + s.formatter = f +} + func (s *SMTPWriter) sendMail(hostAddressWithPort string, auth smtp.Auth, fromAddress string, recipients []string, msgContent []byte) error { client, err := smtp.Dial(hostAddressWithPort) if err != nil { @@ -129,7 +133,7 @@ func (s *SMTPWriter) sendMail(hostAddressWithPort string, auth smtp.Auth, fromAd } func (s *SMTPWriter) Format(lm *LogMsg) string { - return lm.Msg + return lm.OldStyleFormat() } // WriteMsg writes message in smtp writer.