From f92973794e8cdd12f0d651e8621c5bd42273d0ae Mon Sep 17 00:00:00 2001 From: astaxie Date: Tue, 27 Aug 2013 23:48:58 +0800 Subject: [PATCH] finish logs module --- logs/conn.go | 111 ++++++++++++++++++++ logs/conn_test.go | 11 ++ logs/console.go | 47 +++++++++ logs/console_test.go | 21 ++++ logs/file.go | 242 +++++++++++++++++++++++++++++++++++++++++++ logs/file_test.go | 101 ++++++++++++++++++ logs/log.go | 147 ++++++++++++++++++++++++++ logs/smtp.go | 97 +++++++++++++++++ logs/smtp_test.go | 11 ++ 9 files changed, 788 insertions(+) create mode 100644 logs/conn.go create mode 100644 logs/conn_test.go create mode 100644 logs/console.go create mode 100644 logs/console_test.go create mode 100644 logs/file.go create mode 100644 logs/file_test.go create mode 100644 logs/log.go create mode 100644 logs/smtp.go create mode 100644 logs/smtp_test.go diff --git a/logs/conn.go b/logs/conn.go new file mode 100644 index 00000000..c3adaf35 --- /dev/null +++ b/logs/conn.go @@ -0,0 +1,111 @@ +package logs + +import ( + "encoding/json" + "io" + "log" + "net" +) + +type ConnWriter struct { + lg *log.Logger + innerWriter io.WriteCloser + reconnectOnMsg bool + reconnect bool + net string + addr string + level int +} + +func NewConn() LoggerInterface { + conn := new(ConnWriter) + conn.level = LevelTrace + return conn +} + +func (c *ConnWriter) Init(jsonconfig string) error { + var m map[string]interface{} + err := json.Unmarshal([]byte(jsonconfig), &m) + if err != nil { + return err + } + if rom, ok := m["reconnectOnMsg"]; ok { + c.reconnectOnMsg = rom.(bool) + } + if rc, ok := m["reconnect"]; ok { + c.reconnect = rc.(bool) + } + if nt, ok := m["net"]; ok { + c.net = nt.(string) + } + if addr, ok := m["addr"]; ok { + c.addr = addr.(string) + } + if lv, ok := m["level"]; ok { + c.level = int(lv.(float64)) + } + return nil +} + +func (c *ConnWriter) WriteMsg(msg string, level int) error { + if level < c.level { + return nil + } + if c.neddedConnectOnMsg() { + err := c.connect() + if err != nil { + return err + } + } + + if c.reconnectOnMsg { + defer c.innerWriter.Close() + } + c.lg.Println(msg) + return nil +} + +func (c *ConnWriter) Destroy() { + if c.innerWriter == nil { + return + } + c.innerWriter.Close() +} + +func (c *ConnWriter) connect() error { + if c.innerWriter != nil { + c.innerWriter.Close() + c.innerWriter = nil + } + + conn, err := net.Dial(c.net, c.addr) + if err != nil { + return err + } + + tcpConn, ok := conn.(*net.TCPConn) + if ok { + tcpConn.SetKeepAlive(true) + } + + c.innerWriter = conn + c.lg = log.New(conn, "", log.Ldate|log.Ltime) + return nil +} + +func (c *ConnWriter) neddedConnectOnMsg() bool { + if c.reconnect { + c.reconnect = false + return true + } + + if c.innerWriter == nil { + return true + } + + return c.reconnectOnMsg +} + +func init() { + Register("conn", NewConn) +} diff --git a/logs/conn_test.go b/logs/conn_test.go new file mode 100644 index 00000000..0d55ddd8 --- /dev/null +++ b/logs/conn_test.go @@ -0,0 +1,11 @@ +package logs + +import ( + "testing" +) + +func TestConn(t *testing.T) { + log := NewLogger(1000) + log.SetLogger("conn", `{"net":"tcp","addr":":7020"}`) + log.Info("info") +} diff --git a/logs/console.go b/logs/console.go new file mode 100644 index 00000000..a14afc4a --- /dev/null +++ b/logs/console.go @@ -0,0 +1,47 @@ +package logs + +import ( + "encoding/json" + "log" + "os" +) + +type ConsoleWriter struct { + lg *log.Logger + level int +} + +func NewConsole() LoggerInterface { + cw := new(ConsoleWriter) + cw.lg = log.New(os.Stdout, "", log.Ldate|log.Ltime) + cw.level = LevelTrace + return cw +} + +func (c *ConsoleWriter) Init(jsonconfig string) error { + var m map[string]interface{} + err := json.Unmarshal([]byte(jsonconfig), &m) + if err != nil { + return err + } + if lv, ok := m["level"]; ok { + c.level = int(lv.(float64)) + } + return nil +} + +func (c *ConsoleWriter) WriteMsg(msg string, level int) error { + if level < c.level { + return nil + } + c.lg.Println(msg) + return nil +} + +func (c *ConsoleWriter) Destroy() { + +} + +func init() { + Register("console", NewConsole) +} diff --git a/logs/console_test.go b/logs/console_test.go new file mode 100644 index 00000000..5ce0b1e1 --- /dev/null +++ b/logs/console_test.go @@ -0,0 +1,21 @@ +package logs + +import ( + "testing" +) + +func TestConsole(t *testing.T) { + log := NewLogger(10000) + log.Trace("trace") + log.Info("info") + log.Warn("warning") + log.Debug("debug") + log.Critical("critical") + log2 := NewLogger(100) + log2.SetLogger("console", `{"level":1}`) + log.Trace("trace") + log.Info("info") + log.Warn("warning") + log.Debug("debug") + log.Critical("critical") +} diff --git a/logs/file.go b/logs/file.go new file mode 100644 index 00000000..b63cc8d8 --- /dev/null +++ b/logs/file.go @@ -0,0 +1,242 @@ +package logs + +import ( + "encoding/json" + "errors" + "fmt" + "io/ioutil" + "log" + "os" + "path" + "path/filepath" + "strings" + "sync" + "time" +) + +type FileLogWriter struct { + *log.Logger + mw *MuxWriter + // The opened file + filename string + + maxlines int + maxlines_curlines int + + // Rotate at size + maxsize int + maxsize_cursize int + + // Rotate daily + daily bool + maxdays int64 + daily_opendate int + + rotate bool + + startLock sync.Mutex // Only one log can write to the file + + level int +} + +type MuxWriter struct { + sync.Mutex + fd *os.File +} + +func (l *MuxWriter) Write(b []byte) (int, error) { + l.Lock() + defer l.Unlock() + return l.fd.Write(b) +} + +func (l *MuxWriter) SetFd(fd *os.File) { + if l.fd != nil { + l.fd.Close() + } + l.fd = fd +} + +func NewFileWriter() LoggerInterface { + w := &FileLogWriter{ + filename: "", + maxlines: 1000000, + maxsize: 1 << 28, //256 MB + daily: true, + maxdays: 7, + rotate: true, + level: LevelTrace, + } + // use MuxWriter instead direct use os.File for lock write when rotate + w.mw = new(MuxWriter) + // set MuxWriter as Logger's io.Writer + w.Logger = log.New(w.mw, "", log.Ldate|log.Ltime) + return w +} + +// jsonconfig like this +//{ +// "filename":"logs/beego.log", +// "maxlines":10000, +// "maxsize":1<<30, +// "daily":true, +// "maxdays":15, +// "rotate":true +//} +func (w *FileLogWriter) Init(jsonconfig string) error { + var m map[string]interface{} + err := json.Unmarshal([]byte(jsonconfig), &m) + if err != nil { + return err + } + if fn, ok := m["filename"]; !ok { + return errors.New("jsonconfig must have filename") + } else { + w.filename = fn.(string) + } + if ml, ok := m["maxlines"]; ok { + w.maxlines = int(ml.(float64)) + } + if ms, ok := m["maxsize"]; ok { + w.maxsize = int(ms.(float64)) + } + if dl, ok := m["daily"]; ok { + w.daily = dl.(bool) + } + if md, ok := m["maxdays"]; ok { + w.maxdays = int64(md.(float64)) + } + if rt, ok := m["rotate"]; ok { + w.rotate = rt.(bool) + } + if lv, ok := m["level"]; ok { + w.level = int(lv.(float64)) + } + err = w.StartLogger() + return err +} + +func (w *FileLogWriter) StartLogger() error { + fd, err := w.createLogFile() + if err != nil { + return err + } + w.mw.SetFd(fd) + err = w.initFd() + if err != nil { + return err + } + return nil +} + +func (w *FileLogWriter) docheck(size int) { + w.startLock.Lock() + defer w.startLock.Unlock() + if (w.maxlines > 0 && w.maxlines_curlines >= w.maxlines) || + (w.maxsize > 0 && w.maxsize_cursize >= w.maxsize) || + (w.daily && time.Now().Day() != w.daily_opendate) { + if err := w.DoRotate(); err != nil { + fmt.Fprintf(os.Stderr, "FileLogWriter(%q): %s\n", w.filename, err) + return + } + } + w.maxlines_curlines++ + w.maxsize_cursize += size +} + +func (w *FileLogWriter) WriteMsg(msg string, level int) error { + if level < w.level { + return nil + } + n := 24 + len(msg) // 24 stand for the length "2013/06/23 21:00:22 [T] " + w.docheck(n) + w.Logger.Println(msg) + return nil +} + +func (w *FileLogWriter) createLogFile() (*os.File, error) { + // Open the log file + fd, err := os.OpenFile(w.filename, os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0660) + return fd, err +} + +func (w *FileLogWriter) initFd() error { + fd := w.mw.fd + finfo, err := fd.Stat() + if err != nil { + return fmt.Errorf("get stat err: %s\n", err) + } + w.maxsize_cursize = int(finfo.Size()) + w.daily_opendate = time.Now().Day() + if finfo.Size() > 0 { + content, err := ioutil.ReadFile(w.filename) + if err != nil { + return err + } + w.maxlines_curlines = len(strings.Split(string(content), "\n")) + } else { + w.maxlines_curlines = 0 + } + return nil +} + +func (w *FileLogWriter) DoRotate() error { + _, err := os.Lstat(w.filename) + if err == nil { // file exists + // Find the next available number + num := 1 + fname := "" + for ; err == nil && num <= 999; num++ { + fname = w.filename + fmt.Sprintf(".%s.%03d", time.Now().Format("2006-01-02"), num) + _, err = os.Lstat(fname) + } + // return error if the last file checked still existed + if err == nil { + return fmt.Errorf("Rotate: Cannot find free log number to rename %s\n", w.filename) + } + + // block Logger's io.Writer + w.mw.Lock() + defer w.mw.Unlock() + + fd := w.mw.fd + fd.Close() + + // close fd before rename + // Rename the file to its newfound home + err = os.Rename(w.filename, fname) + if err != nil { + return fmt.Errorf("Rotate: %s\n", err) + } + + // re-start logger + err = w.StartLogger() + if err != nil { + return fmt.Errorf("Rotate StartLogger: %s\n", err) + } + + go w.deleteOldLog() + } + + return nil +} + +func (w *FileLogWriter) deleteOldLog() { + dir := path.Dir(w.filename) + filepath.Walk(dir, func(path string, info os.FileInfo, err error) error { + if !info.IsDir() && info.ModTime().Unix() < (time.Now().Unix()-60*60*24*w.maxdays) { + if strings.HasPrefix(filepath.Base(path), filepath.Base(w.filename)) { + os.Remove(path) + } + } + return nil + }) +} + +func (w *FileLogWriter) Destroy() { + w.mw.fd.Close() +} + +func init() { + Register("file", NewFileWriter) +} diff --git a/logs/file_test.go b/logs/file_test.go new file mode 100644 index 00000000..c9b99eef --- /dev/null +++ b/logs/file_test.go @@ -0,0 +1,101 @@ +package logs + +import ( + "bufio" + "fmt" + "os" + "testing" + "time" +) + +func TestFile(t *testing.T) { + log := NewLogger(10000) + log.SetLogger("file", `{"filename":"test.log"}`) + log.Trace("test") + log.Info("info") + log.Debug("debug") + log.Warn("warning") + log.Error("error") + log.Critical("critical") + time.Sleep(time.Second * 4) + f, err := os.Open("test.log") + if err != nil { + t.Fatal(err) + } + b := bufio.NewReader(f) + linenum := 0 + for { + line, _, err := b.ReadLine() + if err != nil { + break + } + if len(line) > 0 { + linenum++ + } + } + if linenum != 6 { + t.Fatal(linenum, "not line 6") + } + os.Remove("test.log") +} + +func TestFile2(t *testing.T) { + log := NewLogger(10000) + log.SetLogger("file", `{"filename":"test2.log","level":2}`) + log.Trace("test") + log.Info("info") + log.Debug("debug") + log.Warn("warning") + log.Error("error") + log.Critical("critical") + time.Sleep(time.Second * 4) + f, err := os.Open("test2.log") + if err != nil { + t.Fatal(err) + } + b := bufio.NewReader(f) + linenum := 0 + for { + line, _, err := b.ReadLine() + if err != nil { + break + } + if len(line) > 0 { + linenum++ + } + } + if linenum != 4 { + t.Fatal(linenum, "not line 4") + } + os.Remove("test2.log") +} + +func TestFileRotate(t *testing.T) { + log := NewLogger(10000) + log.SetLogger("file", `{"filename":"test3.log","maxlines":4}`) + log.Trace("test") + log.Info("info") + log.Debug("debug") + log.Warn("warning") + log.Error("error") + log.Critical("critical") + time.Sleep(time.Second * 4) + rotatename := "test3.log" + fmt.Sprintf(".%s.%03d", time.Now().Format("2006-01-02"), 1) + b, err := exists(rotatename) + if !b || err != nil { + t.Fatal("rotate not gen") + } + os.Remove(rotatename) + os.Remove("test3.log") +} + +func exists(path string) (bool, error) { + _, err := os.Stat(path) + if err == nil { + return true, nil + } + if os.IsNotExist(err) { + return false, nil + } + return false, err +} diff --git a/logs/log.go b/logs/log.go new file mode 100644 index 00000000..8a9ac3c0 --- /dev/null +++ b/logs/log.go @@ -0,0 +1,147 @@ +package logs + +import ( + "fmt" + "sync" +) + +const ( + LevelTrace = iota + LevelDebug + LevelInfo + LevelWarn + LevelError + LevelCritical +) + +type loggerType func() LoggerInterface + +type LoggerInterface interface { + Init(config string) error + WriteMsg(msg string, level int) error + Destroy() +} + +var adapters = make(map[string]loggerType) + +// Register makes a log provide available by the provided name. +// If Register is called twice with the same name or if driver is nil, +// it panics. +func Register(name string, log loggerType) { + if log == nil { + panic("logs: Register provide is nil") + } + if _, dup := adapters[name]; dup { + panic("logs: Register called twice for provider " + name) + } + adapters[name] = log +} + +type BeeLogger struct { + lock sync.Mutex + level int + msg chan *logMsg + outputs map[string]LoggerInterface +} + +type logMsg struct { + level int + msg string +} + +// config need to be correct JSON as string: {"interval":360} +func NewLogger(channellen int64) *BeeLogger { + bl := new(BeeLogger) + bl.msg = make(chan *logMsg, channellen) + bl.outputs = make(map[string]LoggerInterface) + bl.SetLogger("console", "") // default output to console + go bl.StartLogger() + return bl +} + +func (bl *BeeLogger) SetLogger(adaptername string, config string) error { + bl.lock.Lock() + defer bl.lock.Unlock() + if log, ok := adapters[adaptername]; ok { + lg := log() + lg.Init(config) + bl.outputs[adaptername] = lg + return nil + } else { + return fmt.Errorf("logs: unknown adaptername %q (forgotten Register?)", adaptername) + } +} + +func (bl *BeeLogger) DelLogger(adaptername string) error { + bl.lock.Lock() + defer bl.lock.Unlock() + if lg, ok := bl.outputs[adaptername]; ok { + lg.Destroy() + delete(bl.outputs, adaptername) + return nil + } else { + return fmt.Errorf("logs: unknown adaptername %q (forgotten Register?)", adaptername) + } +} + +func (bl *BeeLogger) writerMsg(loglevel int, msg string) error { + if bl.level > loglevel { + return nil + } + lm := new(logMsg) + lm.level = loglevel + lm.msg = msg + bl.msg <- lm + return nil +} + +func (bl *BeeLogger) SetLevel(l int) { + bl.level = l +} + +func (bl *BeeLogger) StartLogger() { + for { + select { + case bm := <-bl.msg: + for _, l := range bl.outputs { + l.WriteMsg(bm.msg, bm.level) + } + } + } +} + +func (bl *BeeLogger) Trace(format string, v ...interface{}) { + msg := fmt.Sprintf("[T] "+format, v...) + bl.writerMsg(LevelTrace, msg) +} + +func (bl *BeeLogger) Debug(format string, v ...interface{}) { + msg := fmt.Sprintf("[D] "+format, v...) + bl.writerMsg(LevelDebug, msg) +} + +func (bl *BeeLogger) Info(format string, v ...interface{}) { + msg := fmt.Sprintf("[I] "+format, v...) + bl.writerMsg(LevelInfo, msg) +} + +func (bl *BeeLogger) Warn(format string, v ...interface{}) { + msg := fmt.Sprintf("[W] "+format, v...) + bl.writerMsg(LevelWarn, msg) +} + +func (bl *BeeLogger) Error(format string, v ...interface{}) { + msg := fmt.Sprintf("[E] "+format, v...) + bl.writerMsg(LevelError, msg) +} + +func (bl *BeeLogger) Critical(format string, v ...interface{}) { + msg := fmt.Sprintf("[C] "+format, v...) + bl.writerMsg(LevelCritical, msg) +} + +func (bl *BeeLogger) Close() { + for _, l := range bl.outputs { + l.Destroy() + } +} diff --git a/logs/smtp.go b/logs/smtp.go new file mode 100644 index 00000000..19a03337 --- /dev/null +++ b/logs/smtp.go @@ -0,0 +1,97 @@ +package logs + +import ( + "encoding/json" + "errors" + "net/smtp" + "strings" +) + +const ( + subjectPhrase = "Diagnostic message from server" +) + +// smtpWriter is used to send emails via given SMTP-server. +type SmtpWriter struct { + username string + password string + host string + subject string + recipientAddresses []string + level int +} + +func NewSmtpWriter() LoggerInterface { + return &SmtpWriter{level: LevelTrace} +} + +func (s *SmtpWriter) Init(jsonconfig string) error { + var m map[string]interface{} + err := json.Unmarshal([]byte(jsonconfig), &m) + if err != nil { + return err + } + if username, ok := m["username"]; !ok { + return errors.New("smtp config must have auth username") + } else if password, ok := m["password"]; !ok { + return errors.New("smtp config must have auth password") + } else if hostname, ok := m["host"]; !ok { + return errors.New("smtp config must have host like 'mail.example.com:25'") + } else if sendTos, ok := m["sendTos"]; !ok { + return errors.New("smtp config must have sendTos") + } else { + s.username = username.(string) + s.password = password.(string) + s.host = hostname.(string) + for _, v := range sendTos.([]interface{}) { + s.recipientAddresses = append(s.recipientAddresses, v.(string)) + } + } + + if subject, ok := m["subject"]; ok { + s.subject = subject.(string) + } else { + s.subject = subjectPhrase + } + if lv, ok := m["level"]; ok { + s.level = int(lv.(float64)) + } + return nil +} + +func (s *SmtpWriter) WriteMsg(msg string, level int) error { + if level < s.level { + return nil + } + + hp := strings.Split(s.host, ":") + + // Set up authentication information. + auth := smtp.PlainAuth( + "", + s.username, + s.password, + hp[0], + ) + // Connect to the server, authenticate, set the sender and recipient, + // and send the email all in one step. + content_type := "Content-Type: text/plain" + "; charset=UTF-8" + mailmsg := []byte("To: " + strings.Join(s.recipientAddresses, ";") + "\r\nFrom: " + s.username + "<" + s.username + + ">\r\nSubject: " + s.subject + "\r\n" + content_type + "\r\n\r\n" + msg) + err := smtp.SendMail( + s.host, + auth, + s.username, + s.recipientAddresses, + mailmsg, + ) + return err +} + +func (s *SmtpWriter) Destroy() { + return +} + +func init() { + Register("smtp", NewSmtpWriter) +} diff --git a/logs/smtp_test.go b/logs/smtp_test.go new file mode 100644 index 00000000..c391c6ab --- /dev/null +++ b/logs/smtp_test.go @@ -0,0 +1,11 @@ +package logs + +import ( + "testing" +) + +func TestSmtp(t *testing.T) { + log := NewLogger(10000) + log.SetLogger("smtp", `{"username":"xxxxxx@gmail.com","password":"xxxxxxx","host":"smtp.gmail.com:587","sendTos":["xiemengjun@gmail.com"]}`) + log.Critical("sendmail critical") +}