Merge pull request #4188 from IamCathal/custom-log-formatter-third-pr

Custom logging format PR#3

pkg/logs/alils/alils.go

@@ -5,6 +5,7 @@ import (
 	"strings"
 	"sync"
 
+	"github.com/astaxie/beego/pkg/common"
 	"github.com/astaxie/beego/pkg/logs"
 	"github.com/gogo/protobuf/proto"
 )
@@ -37,6 +38,7 @@ type aliLSWriter struct {
 	withMap         bool
 	groupMap        map[string]*LogGroup
 	lock            *sync.Mutex
+	customFormatter func(*logs.LogMsg) string
 	Config
 }
 
@@ -48,8 +50,17 @@ func NewAliLS() logs.Logger {
 }
 
 // Init parses config and initializes struct
-func (c *aliLSWriter) Init(jsonConfig string) (err error) {
+func (c *aliLSWriter) Init(jsonConfig string, opts ...common.SimpleKV) error {
 
+	for _, elem := range opts {
+		if elem.Key == "formatter" {
+			formatter, err := logs.GetFormatter(elem)
+			if err != nil {
+				return err
+			}
+			c.customFormatter = formatter
+		}
+	}
 	json.Unmarshal([]byte(jsonConfig), c)
 
 	if c.FlushWhen > CacheSize {
@@ -63,11 +74,13 @@ func (c *aliLSWriter) Init(jsonConfig string) (err error) {
 		AccessKeySecret: c.KeySecret,
 	}
 
-	c.store, err = prj.GetLogStore(c.LogStore)
+	store, err := prj.GetLogStore(c.LogStore)
 	if err != nil {
 		return err
 	}
 
+	c.store = store
+
 	// Create default Log Group
 	c.group = append(c.group, &LogGroup{
 		Topic:  proto.String(""),
@@ -121,20 +134,23 @@ func (c *aliLSWriter) WriteMsg(lm *logs.LogMsg) error {
 		if len(strs) == 2 {
 			pos := strings.LastIndex(strs[0], " ")
 			topic = strs[0][pos+1 : len(strs[0])]
-			content = strs[0][0:pos] + strs[1]
 			lg = c.groupMap[topic]
 		}
 
 		// send to empty Topic
 		if lg == nil {
-			content = lm.Msg
 			lg = c.group[0]
 		}
 	} else {
-		content = lm.Msg
 		lg = c.group[0]
 	}
 
+	if c.customFormatter != nil {
+		content = c.customFormatter(lm)
+	} else {
+		content = c.Format(lm)
+	}
+
 	c1 := &LogContent{
 		Key:   proto.String("msg"),
 		Value: proto.String(content),

pkg/logs/conn.go

@@ -18,6 +18,8 @@ import (
 	"encoding/json"
 	"io"
 	"net"
+
+	"github.com/astaxie/beego/pkg/common"
 )
 
 // connWriter implements LoggerInterface.
@@ -25,6 +27,7 @@ import (
 type connWriter struct {
 	lg              *logWriter
 	innerWriter     io.WriteCloser
+	customFormatter func(*LogMsg) string
 	ReconnectOnMsg  bool   `json:"reconnectOnMsg"`
 	Reconnect       bool   `json:"reconnect"`
 	Net             string `json:"net"`
@@ -45,7 +48,18 @@ func (c *connWriter) Format(lm *LogMsg) string {
 
 // Init initializes a connection writer with json config.
 // json config only needs they "level" key
-func (c *connWriter) Init(jsonConfig string) error {
+func (c *connWriter) Init(jsonConfig string, opts ...common.SimpleKV) error {
+
+	for _, elem := range opts {
+		if elem.Key == "formatter" {
+			formatter, err := GetFormatter(elem)
+			if err != nil {
+				return err
+			}
+			c.customFormatter = formatter
+		}
+	}
+
 	return json.Unmarshal([]byte(jsonConfig), c)
 }
 
@@ -66,7 +80,14 @@ func (c *connWriter) WriteMsg(lm *LogMsg) error {
 		defer c.innerWriter.Close()
 	}
 
-	msg := c.Format(lm)
+	msg := ""
+	if c.customFormatter != nil {
+		msg = c.customFormatter(lm)
+	} else {
+		msg = c.Format(lm)
+
+	}
+
 	_, err := c.lg.writeln(msg)
 	if err != nil {
 		return err

pkg/logs/console.go

@@ -19,6 +19,8 @@ import (
 	"os"
 	"strings"
 
+	"github.com/astaxie/beego/pkg/common"
+
 	"github.com/shiena/ansicolor"
 )
 
@@ -48,6 +50,7 @@ var colors = []brush{
 // consoleWriter implements LoggerInterface and writes messages to terminal.
 type consoleWriter struct {
 	lg              *logWriter
+	customFormatter func(*LogMsg) string
 	Level           int  `json:"level"`
 	Colorful        bool `json:"color"` //this filed is useful only when system's terminal supports color
 }
@@ -55,14 +58,10 @@ type consoleWriter struct {
 func (c *consoleWriter) Format(lm *LogMsg) string {
 	msg := lm.Msg
 
-	if c.Colorful {
-		msg = strings.Replace(lm.Msg, levelPrefix[lm.Level], colors[lm.Level](levelPrefix[lm.Level]), 1)
-	}
-
 	h, _, _ := formatTimeHeader(lm.When)
 	bytes := append(append(h, msg...), '\n')
 
-	return "eee" + string(bytes)
+	return string(bytes)
 
 }
 
@@ -78,10 +77,22 @@ func NewConsole() Logger {
 
 // Init initianlizes the console logger.
 // jsonConfig must be in the format '{"level":LevelTrace}'
-func (c *consoleWriter) Init(jsonConfig string) error {
+func (c *consoleWriter) Init(jsonConfig string, opts ...common.SimpleKV) error {
+
+	for _, elem := range opts {
+		if elem.Key == "formatter" {
+			formatter, err := GetFormatter(elem)
+			if err != nil {
+				return err
+			}
+			c.customFormatter = formatter
+		}
+	}
+
 	if len(jsonConfig) == 0 {
 		return nil
 	}
+
 	return json.Unmarshal([]byte(jsonConfig), c)
 }
 
@@ -90,11 +101,20 @@ func (c *consoleWriter) WriteMsg(lm *LogMsg) error {
 	if lm.Level > c.Level {
 		return nil
 	}
-	// fmt.Printf("Formatted: %s\n\n", c.fmtter.Format(lm))
+
+	msg := ""
+
 	if c.Colorful {
 		lm.Msg = strings.Replace(lm.Msg, levelPrefix[lm.Level], colors[lm.Level](levelPrefix[lm.Level]), 1)
 	}
-	msg := c.Format(lm)
+
+	if c.customFormatter != nil {
+		msg = c.customFormatter(lm)
+	} else {
+		msg = c.Format(lm)
+
+	}
+
 	c.lg.writeln(msg)
 	return nil
 }

pkg/logs/es/es.go

@@ -12,6 +12,7 @@ import (
 	"github.com/elastic/go-elasticsearch/v6"
 	"github.com/elastic/go-elasticsearch/v6/esapi"
 
+	"github.com/astaxie/beego/pkg/common"
 	"github.com/astaxie/beego/pkg/logs"
 )
 
@@ -33,6 +34,7 @@ type esLogger struct {
 	*elasticsearch.Client
 	DSN             string `json:"dsn"`
 	Level           int    `json:"level"`
+	customFormatter func(*logs.LogMsg) string
 }
 
 func (el *esLogger) Format(lm *logs.LogMsg) string {
@@ -40,8 +42,19 @@ func (el *esLogger) Format(lm *logs.LogMsg) string {
 }
 
 // {"dsn":"http://localhost:9200/","level":1}
-func (el *esLogger) Init(jsonconfig string) error {
+func (el *esLogger) Init(jsonConfig string, opts ...common.SimpleKV) error {
-	err := json.Unmarshal([]byte(jsonconfig), el)
+
+	for _, elem := range opts {
+		if elem.Key == "formatter" {
+			formatter, err := logs.GetFormatter(elem)
+			if err != nil {
+				return err
+			}
+			el.customFormatter = formatter
+		}
+	}
+
+	err := json.Unmarshal([]byte(jsonConfig), el)
 	if err != nil {
 		return err
 	}
@@ -69,9 +82,16 @@ func (el *esLogger) WriteMsg(lm *logs.LogMsg) error {
 		return nil
 	}
 
+	msg := ""
+	if el.customFormatter != nil {
+		msg = el.customFormatter(lm)
+	} else {
+		msg = el.Format(lm)
+	}
+
 	idx := LogDocument{
 		Timestamp: lm.When.Format(time.RFC3339),
-		Msg:       el.Format(lm),
+		Msg:       msg,
 	}
 
 	body, err := json.Marshal(idx)

pkg/logs/file.go

@@ -27,6 +27,8 @@ import (
 	"strings"
 	"sync"
 	"time"
+
+	"github.com/astaxie/beego/pkg/common"
 )
 
 // fileLogWriter implements LoggerInterface.
@@ -60,6 +62,8 @@ type fileLogWriter struct {
 	hourlyOpenDate int
 	hourlyOpenTime time.Time
 
+	customFormatter func(*LogMsg) string
+
 	Rotate bool `json:"rotate"`
 
 	Level int `json:"level"`
@@ -104,7 +108,18 @@ func (w *fileLogWriter) Format(lm *LogMsg) string {
 //  "rotate":true,
 //      "perm":"0600"
 //  }
-func (w *fileLogWriter) Init(jsonConfig string) error {
+func (w *fileLogWriter) Init(jsonConfig string, opts ...common.SimpleKV) error {
+
+	for _, elem := range opts {
+		if elem.Key == "formatter" {
+			formatter, err := GetFormatter(elem)
+			if err != nil {
+				return err
+			}
+			w.customFormatter = formatter
+		}
+	}
+
 	err := json.Unmarshal([]byte(jsonConfig), w)
 	if err != nil {
 		return err
@@ -153,7 +168,14 @@ func (w *fileLogWriter) WriteMsg(lm *LogMsg) error {
 		return nil
 	}
 	hd, d, h := formatTimeHeader(lm.When)
-	msg := w.Format(lm)
+	msg := ""
+
+	if w.customFormatter != nil {
+		msg = w.customFormatter(lm)
+	} else {
+		msg = w.Format(lm)
+	}
+
 	msg = fmt.Sprintf("%s %s\n", string(hd), msg)
 	if w.Rotate {
 		w.RLock()

pkg/logs/jianliao.go

@@ -5,6 +5,8 @@ import (
 	"fmt"
 	"net/http"
 	"net/url"
+
+	"github.com/astaxie/beego/pkg/common"
 )
 
 // JLWriter implements beego LoggerInterface and is used to send jiaoliao webhook
@@ -15,6 +17,7 @@ type JLWriter struct {
 	RedirectURL     string `json:"redirecturl,omitempty"`
 	ImageURL        string `json:"imageurl,omitempty"`
 	Level           int    `json:"level"`
+	customFormatter func(*LogMsg) string
 }
 
 // newJLWriter creates jiaoliao writer.
@@ -23,8 +26,18 @@ func newJLWriter() Logger {
 }
 
 // Init JLWriter with json config string
-func (s *JLWriter) Init(jsonconfig string) error {
+func (s *JLWriter) Init(jsonConfig string, opts ...common.SimpleKV) error {
-	return json.Unmarshal([]byte(jsonconfig), s)
+	for _, elem := range opts {
+		if elem.Key == "formatter" {
+			formatter, err := GetFormatter(elem)
+			if err != nil {
+				return err
+			}
+			s.customFormatter = formatter
+		}
+	}
+
+	return json.Unmarshal([]byte(jsonConfig), s)
 }
 
 func (s *JLWriter) Format(lm *LogMsg) string {
@@ -38,7 +51,15 @@ func (s *JLWriter) WriteMsg(lm *LogMsg) error {
 		return nil
 	}
 
-	text := fmt.Sprintf("%s %s", lm.When.Format("2006-01-02 15:04:05"), s.Format(lm))
+	text := ""
+
+	if s.customFormatter != nil {
+		text = fmt.Sprintf("%s %s", lm.When.Format("2006-01-02 15:04:05"), s.customFormatter(lm))
+	} else {
+		text = fmt.Sprintf("%s %s", lm.When.Format("2006-01-02 15:04:05"), s.Format(lm))
+
+	}
+
 	form := url.Values{}
 	form.Add("authorName", s.AuthorName)
 	form.Add("title", s.Title)

pkg/logs/log.go

@@ -38,10 +38,13 @@ import (
 	"log"
 	"os"
 	"path"
+	"reflect"
 	"runtime"
 	"strings"
 	"sync"
 	"time"
+
+	"github.com/astaxie/beego/pkg/common"
 )
 
 // RFC5424 log message levels.
@@ -84,7 +87,7 @@ type newLoggerFunc func() Logger
 
 // Logger defines the behavior of a log provider.
 type Logger interface {
-	Init(config string) error
+	Init(config string, opts ...common.SimpleKV) error
 	WriteMsg(lm *LogMsg) error
 	Format(lm *LogMsg) string
 	Destroy()
@@ -115,6 +118,7 @@ type BeeLogger struct {
 	init                bool
 	enableFuncCallDepth bool
 	loggerFuncCallDepth int
+	globalFormatter     func(*LogMsg) string
 	enableFullFilePath  bool
 	asynchronous        bool
 	prefix              string
@@ -129,8 +133,6 @@ const defaultAsyncMsgLen = 1e3
 
 type nameLogger struct {
 	Logger
-	// Formatter func(*LogMsg) string
-	LogFormatter
 	name string
 }
 
@@ -206,7 +208,16 @@ func (bl *BeeLogger) setLogger(adapterName string, configs ...string) error {
 	}
 
 	lg := logAdapter()
-	err := lg.Init(config)
+	var err error
+
+	// Global formatter overrides the default set formatter
+	// but not adapter specific formatters set with logs.SetLoggerWithOpts()
+	if bl.globalFormatter != nil {
+		err = lg.Init(config, common.SimpleKV{Key: "formatter", Value: bl.globalFormatter})
+	} else {
+		err = lg.Init(config)
+	}
+
 	if err != nil {
 		fmt.Fprintln(os.Stderr, "logs.BeeLogger.SetLogger: "+err.Error())
 		return err
@@ -248,7 +259,6 @@ func (bl *BeeLogger) DelLogger(adapterName string) error {
 
 func (bl *BeeLogger) writeToLoggers(lm *LogMsg) {
 	for _, l := range bl.outputs {
-		// fmt.Println("Formatted: ", l.Format(lm))
 		err := l.WriteMsg(lm)
 		if err != nil {
 			fmt.Fprintf(os.Stderr, "unable to WriteMsg to adapter:%v,error:%v\n", l.name, err)
@@ -394,6 +404,87 @@ func (bl *BeeLogger) startLogger() {
 	}
 }
 
+// Get the formatter from the opts common.SimpleKV structure
+// Looks for a key: "formatter" with value: func(*LogMsg) string
+func GetFormatter(opts common.SimpleKV) (func(*LogMsg) string, error) {
+	if strings.ToLower(opts.Key.(string)) == "formatter" {
+		formatterInterface := reflect.ValueOf(opts.Value).Interface()
+		formatterFunc := formatterInterface.(func(*LogMsg) string)
+		return formatterFunc, nil
+	}
+
+	return nil, fmt.Errorf("no \"formatter\" key given in simpleKV")
+}
+
+// SetLoggerWithOpts sets a log adapter with a user defined logging format. Config must be valid JSON
+// such as: {"interval":360}
+func (bl *BeeLogger) setLoggerWithOpts(adapterName string, opts common.SimpleKV, configs ...string) error {
+	config := append(configs, "{}")[0]
+	for _, l := range bl.outputs {
+		if l.name == adapterName {
+			return fmt.Errorf("logs: duplicate adaptername %q (you have set this logger before)", adapterName)
+		}
+	}
+
+	logAdapter, ok := adapters[adapterName]
+	if !ok {
+		return fmt.Errorf("logs: unknown adaptername %q (forgotten Register?)", adapterName)
+	}
+
+	if opts.Key == nil {
+		return fmt.Errorf("No SimpleKV struct set for %s log adapter", adapterName)
+	}
+
+	lg := logAdapter()
+	err := lg.Init(config, opts)
+	if err != nil {
+		fmt.Fprintln(os.Stderr, "logs.BeeLogger.SetLogger: "+err.Error())
+		return err
+	}
+
+	bl.outputs = append(bl.outputs, &nameLogger{
+		name:   adapterName,
+		Logger: lg,
+	})
+
+	return nil
+}
+
+// SetLogger provides a given logger adapter into BeeLogger with config string.
+func (bl *BeeLogger) SetLoggerWithOpts(adapterName string, opts common.SimpleKV, configs ...string) error {
+	bl.lock.Lock()
+	defer bl.lock.Unlock()
+	if !bl.init {
+		bl.outputs = []*nameLogger{}
+		bl.init = true
+	}
+	return bl.setLoggerWithOpts(adapterName, opts, configs...)
+}
+
+// SetLoggerWIthOpts sets a given log adapter with a custom log adapter.
+// Log Adapter must be given in the form common.SimpleKV{Key: "formatter": Value: struct.FormatFunc}
+// where FormatFunc has the signature func(*LogMsg) string
+// func SetLoggerWithOpts(adapter string, config []string, formatterFunc func(*LogMsg) string) error {
+func SetLoggerWithOpts(adapter string, config []string, opts common.SimpleKV) error {
+	err := beeLogger.SetLoggerWithOpts(adapter, opts, config...)
+	if err != nil {
+		log.Fatal(err)
+	}
+	return nil
+
+}
+
+func (bl *BeeLogger) setGlobalFormatter(fmtter func(*LogMsg) string) error {
+	bl.globalFormatter = fmtter
+	return nil
+}
+
+// SetGlobalFormatter sets the global formatter for all log adapters
+// This overrides and other individually set adapter
+func SetGlobalFormatter(fmtter func(*LogMsg) string) error {
+	return beeLogger.setGlobalFormatter(fmtter)
+}
+
 // Emergency Log EMERGENCY level message.
 func (bl *BeeLogger) Emergency(format string, v ...interface{}) {
 	if LevelEmergency > bl.level {

pkg/logs/logformattertest/log_formatter_test.go

@@ -0,0 +1,36 @@
+package logformattertest
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/astaxie/beego/pkg/common"
+	"github.com/astaxie/beego/pkg/logs"
+)
+
+func customFormatter(lm *logs.LogMsg) string {
+	return fmt.Sprintf("[CUSTOM CONSOLE LOGGING] %s", lm.Msg)
+}
+
+func globalFormatter(lm *logs.LogMsg) string {
+	return fmt.Sprintf("[GLOBAL] %s", lm.Msg)
+}
+
+func TestCustomLoggingFormatter(t *testing.T) {
+	// beego.BConfig.Log.AccessLogs = true
+
+	logs.SetLoggerWithOpts("console", []string{`{"color":true}`}, common.SimpleKV{Key: "formatter", Value: customFormatter})
+
+	// Message will be formatted by the customFormatter with colorful text set to true
+	logs.Informational("Test message")
+}
+
+func TestGlobalLoggingFormatter(t *testing.T) {
+	logs.SetGlobalFormatter(globalFormatter)
+
+	logs.SetLogger("console", `{"color":true}`)
+
+	// Message will be formatted by globalFormatter
+	logs.Informational("Test message")
+
+}

pkg/logs/multifile.go

@@ -16,6 +16,8 @@ package logs
 
 import (
 	"encoding/json"
+
+	"github.com/astaxie/beego/pkg/common"
 )
 
 // A filesLogWriter manages several fileLogWriter
@@ -27,6 +29,7 @@ type multiFileLogWriter struct {
 	writers         [LevelDebug + 1 + 1]*fileLogWriter // the last one for fullLogWriter
 	fullLogWriter   *fileLogWriter
 	Separate        []string `json:"separate"`
+	customFormatter func(*LogMsg) string
 }
 
 var levelNames = [...]string{"emergency", "alert", "critical", "error", "warning", "notice", "info", "debug"}
@@ -44,9 +47,19 @@ var levelNames = [...]string{"emergency", "alert", "critical", "error", "warning
 //	"separate":["emergency", "alert", "critical", "error", "warning", "notice", "info", "debug"],
 //	}
 
-func (f *multiFileLogWriter) Init(config string) error {
+func (f *multiFileLogWriter) Init(jsonConfig string, opts ...common.SimpleKV) error {
+	for _, elem := range opts {
+		if elem.Key == "formatter" {
+			formatter, err := GetFormatter(elem)
+			if err != nil {
+				return err
+			}
+			f.customFormatter = formatter
+		}
+	}
+
 	writer := newFileWriter().(*fileLogWriter)
-	err := writer.Init(config)
+	err := writer.Init(jsonConfig)
 	if err != nil {
 		return err
 	}
@@ -54,10 +67,10 @@ func (f *multiFileLogWriter) Init(config string) error {
 	f.writers[LevelDebug+1] = writer
 
 	//unmarshal "separate" field to f.Separate
-	json.Unmarshal([]byte(config), f)
+	json.Unmarshal([]byte(jsonConfig), f)
 
 	jsonMap := map[string]interface{}{}
-	json.Unmarshal([]byte(config), &jsonMap)
+	json.Unmarshal([]byte(jsonConfig), &jsonMap)
 
 	for i := LevelEmergency; i < LevelDebug+1; i++ {
 		for _, v := range f.Separate {
@@ -74,7 +87,6 @@ func (f *multiFileLogWriter) Init(config string) error {
 			}
 		}
 	}
-
 	return nil
 }
 

pkg/logs/slack.go

@@ -5,12 +5,16 @@ import (
 	"fmt"
 	"net/http"
 	"net/url"
+
+	"github.com/astaxie/beego/pkg/common"
 )
 
 // SLACKWriter implements beego LoggerInterface and is used to send jiaoliao webhook
 type SLACKWriter struct {
 	WebhookURL         string `json:"webhookurl"`
 	Level              int    `json:"level"`
+	UseCustomFormatter bool
+	CustomFormatter    func(*LogMsg) string
 }
 
 // newSLACKWriter creates jiaoliao writer.
@@ -23,8 +27,14 @@ func (s *SLACKWriter) Format(lm *LogMsg) string {
 }
 
 // Init SLACKWriter with json config string
-func (s *SLACKWriter) Init(jsonconfig string) error {
+func (s *SLACKWriter) Init(jsonConfig string, opts ...common.SimpleKV) error {
-	return json.Unmarshal([]byte(jsonconfig), s)
+	// 	if elem != nil {
+	// 		s.UseCustomFormatter = true
+	// 		s.CustomFormatter = elem
+	// 	}
+	// }
+
+	return json.Unmarshal([]byte(jsonConfig), s)
 }
 
 // WriteMsg write message in smtp writer.

pkg/logs/smtp.go

@@ -21,6 +21,8 @@ import (
 	"net"
 	"net/smtp"
 	"strings"
+
+	"github.com/astaxie/beego/pkg/common"
 )
 
 // SMTPWriter implements LoggerInterface and is used to send emails via given SMTP-server.
@@ -32,6 +34,7 @@ type SMTPWriter struct {
 	FromAddress        string   `json:"fromAddress"`
 	RecipientAddresses []string `json:"sendTos"`
 	Level              int      `json:"level"`
+	customFormatter    func(*LogMsg) string
 }
 
 // NewSMTPWriter creates the smtp writer.
@@ -50,8 +53,19 @@ func newSMTPWriter() Logger {
 //		"sendTos":["email1","email2"],
 //		"level":LevelError
 //	}
-func (s *SMTPWriter) Init(jsonconfig string) error {
+func (s *SMTPWriter) Init(jsonConfig string, opts ...common.SimpleKV) error {
-	return json.Unmarshal([]byte(jsonconfig), s)
+
+	for _, elem := range opts {
+		if elem.Key == "formatter" {
+			formatter, err := GetFormatter(elem)
+			if err != nil {
+				return err
+			}
+			s.customFormatter = formatter
+		}
+	}
+
+	return json.Unmarshal([]byte(jsonConfig), s)
 }
 
 func (s *SMTPWriter) getSMTPAuth(host string) smtp.Auth {