From 6c13bdde25ca7df87ceed58fd42faed00ea37db5 Mon Sep 17 00:00:00 2001 From: astaxie Date: Wed, 13 Nov 2013 21:11:03 +0800 Subject: [PATCH] support profile & statistics in another port --- admin.go | 42 +++++++++++ admin/profile.go | 157 +++++++++++++++++++++++++++++++++++++++ admin/statistics.go | 79 ++++++++++++++++++++ admin/statistics_test.go | 17 +++++ admin/task.go | 1 + beego.go | 4 + config.go | 24 ++++++ context/output.go | 4 + router.go | 63 +++++++++------- 9 files changed, 364 insertions(+), 27 deletions(-) create mode 100644 admin.go create mode 100644 admin/profile.go create mode 100644 admin/statistics.go create mode 100644 admin/statistics_test.go create mode 100644 admin/task.go diff --git a/admin.go b/admin.go new file mode 100644 index 00000000..b570a3fa --- /dev/null +++ b/admin.go @@ -0,0 +1,42 @@ +package beego + +import ( + "fmt" + "net/http" +) + +var BeeAdminApp *AdminApp + +func init() { + BeeAdminApp = &AdminApp{ + routers: make(map[string]http.HandlerFunc), + } + BeeAdminApp.Route("/", AdminIndex) +} + +func AdminIndex(rw http.ResponseWriter, r *http.Request) { + rw.Write([]byte("Welcome to Admin Dashboard")) +} + +type AdminApp struct { + routers map[string]http.HandlerFunc +} + +func (admin *AdminApp) Route(pattern string, f http.HandlerFunc) { + admin.routers[pattern] = f +} + +func (admin *AdminApp) Run() { + addr := AdminHttpAddr + + if AdminHttpPort != 0 { + addr = fmt.Sprintf("%s:%d", AdminHttpAddr, AdminHttpPort) + } + for p, f := range admin.routers { + http.Handle(p, f) + } + err := http.ListenAndServe(addr, nil) + if err != nil { + BeeLogger.Critical("Admin ListenAndServe: ", err) + } +} diff --git a/admin/profile.go b/admin/profile.go new file mode 100644 index 00000000..ec90e8e0 --- /dev/null +++ b/admin/profile.go @@ -0,0 +1,157 @@ +package admin + +import ( + "fmt" + "log" + "os" + "runtime" + "runtime/debug" + "runtime/pprof" + "strconv" + "time" +) + +var heapProfileCounter int32 +var startTime = time.Now() +var pid int + +func init() { + pid = os.Getpid() +} + +func ProcessInput(input string) { + switch input { + case "lookup goroutine": + p := pprof.Lookup("goroutine") + p.WriteTo(os.Stdout, 2) + case "lookup heap": + p := pprof.Lookup("heap") + p.WriteTo(os.Stdout, 2) + case "lookup threadcreate": + p := pprof.Lookup("threadcreate") + p.WriteTo(os.Stdout, 2) + case "lookup block": + p := pprof.Lookup("block") + p.WriteTo(os.Stdout, 2) + case "start cpuprof": + StartCPUProfile() + case "stop cpuprof": + StopCPUProfile() + case "get memprof": + MemProf() + case "gc summary": + PrintGCSummary() + } +} + +func MemProf() { + if f, err := os.Create("mem-" + strconv.Itoa(pid) + ".memprof"); err != nil { + log.Fatal("record memory profile failed: %v", err) + } else { + runtime.GC() + pprof.WriteHeapProfile(f) + f.Close() + } +} + +func StartCPUProfile() { + f, err := os.Create("cpu-" + strconv.Itoa(pid) + ".pprof") + if err != nil { + log.Fatal(err) + } + pprof.StartCPUProfile(f) +} + +func StopCPUProfile() { + pprof.StopCPUProfile() +} + +func PrintGCSummary() { + memStats := &runtime.MemStats{} + runtime.ReadMemStats(memStats) + gcstats := &debug.GCStats{PauseQuantiles: make([]time.Duration, 100)} + debug.ReadGCStats(gcstats) + + printGC(memStats, gcstats) +} + +func printGC(memStats *runtime.MemStats, gcstats *debug.GCStats) { + + if gcstats.NumGC > 0 { + lastPause := gcstats.Pause[0] + elapsed := time.Now().Sub(startTime) + overhead := float64(gcstats.PauseTotal) / float64(elapsed) * 100 + allocatedRate := float64(memStats.TotalAlloc) / elapsed.Seconds() + + fmt.Printf("NumGC:%d Pause:%s Pause(Avg):%s Overhead:%3.2f%% Alloc:%s Sys:%s Alloc(Rate):%s/s Histogram:%s %s %s \n", + gcstats.NumGC, + toS(lastPause), + toS(avg(gcstats.Pause)), + overhead, + toH(memStats.Alloc), + toH(memStats.Sys), + toH(uint64(allocatedRate)), + toS(gcstats.PauseQuantiles[94]), + toS(gcstats.PauseQuantiles[98]), + toS(gcstats.PauseQuantiles[99])) + } else { + // while GC has disabled + elapsed := time.Now().Sub(startTime) + allocatedRate := float64(memStats.TotalAlloc) / elapsed.Seconds() + + fmt.Printf("Alloc:%s Sys:%s Alloc(Rate):%s/s\n", + toH(memStats.Alloc), + toH(memStats.Sys), + toH(uint64(allocatedRate))) + } +} + +func avg(items []time.Duration) time.Duration { + var sum time.Duration + for _, item := range items { + sum += item + } + return time.Duration(int64(sum) / int64(len(items))) +} + +// human readable format +func toH(bytes uint64) string { + switch { + case bytes < 1024: + return fmt.Sprintf("%dB", bytes) + case bytes < 1024*1024: + return fmt.Sprintf("%.2fK", float64(bytes)/1024) + case bytes < 1024*1024*1024: + return fmt.Sprintf("%.2fM", float64(bytes)/1024/1024) + default: + return fmt.Sprintf("%.2fG", float64(bytes)/1024/1024/1024) + } +} + +// short string format +func toS(d time.Duration) string { + + u := uint64(d) + if u < uint64(time.Second) { + switch { + case u == 0: + return "0" + case u < uint64(time.Microsecond): + return fmt.Sprintf("%.2fns", float64(u)) + case u < uint64(time.Millisecond): + return fmt.Sprintf("%.2fus", float64(u)/1000) + default: + return fmt.Sprintf("%.2fms", float64(u)/1000/1000) + } + } else { + switch { + case u < uint64(time.Minute): + return fmt.Sprintf("%.2fs", float64(u)/1000/1000/1000) + case u < uint64(time.Hour): + return fmt.Sprintf("%.2fm", float64(u)/1000/1000/1000/60) + default: + return fmt.Sprintf("%.2fh", float64(u)/1000/1000/1000/60/60) + } + } + +} diff --git a/admin/statistics.go b/admin/statistics.go new file mode 100644 index 00000000..55ebb79a --- /dev/null +++ b/admin/statistics.go @@ -0,0 +1,79 @@ +package admin + +import ( + "encoding/json" + "sync" + "time" +) + +type Statistics struct { + RequestUrl string + RequestController string + RequestNum int64 + MinTime time.Duration + MaxTime time.Duration + TotalTime time.Duration +} + +type UrlMap struct { + lock sync.RWMutex + urlmap map[string]map[string]*Statistics +} + +func (m *UrlMap) AddStatistics(requestMethod, requestUrl, requestController string, requesttime time.Duration) { + m.lock.Lock() + defer m.lock.Unlock() + if method, ok := m.urlmap[requestUrl]; ok { + if s, ok := method[requestMethod]; ok { + s.RequestNum += 1 + if s.MaxTime < requesttime { + s.MaxTime = requesttime + } + if s.MinTime > requesttime { + s.MinTime = requesttime + } + s.TotalTime += requesttime + } else { + nb := &Statistics{ + RequestUrl: requestUrl, + RequestController: requestController, + RequestNum: 1, + MinTime: requesttime, + MaxTime: requesttime, + TotalTime: requesttime, + } + m.urlmap[requestUrl][requestMethod] = nb + } + + } else { + methodmap := make(map[string]*Statistics) + nb := &Statistics{ + RequestUrl: requestUrl, + RequestController: requestController, + RequestNum: 1, + MinTime: requesttime, + MaxTime: requesttime, + TotalTime: requesttime, + } + methodmap[requestMethod] = nb + m.urlmap[requestUrl] = methodmap + } +} + +func (m *UrlMap) GetMap() []byte { + m.lock.RLock() + defer m.lock.RUnlock() + r, err := json.Marshal(m.urlmap) + if err != nil { + return []byte("") + } + return r +} + +var StatisticsMap *UrlMap + +func init() { + StatisticsMap = &UrlMap{ + urlmap: make(map[string]map[string]*Statistics), + } +} diff --git a/admin/statistics_test.go b/admin/statistics_test.go new file mode 100644 index 00000000..4ef0bfcb --- /dev/null +++ b/admin/statistics_test.go @@ -0,0 +1,17 @@ +package admin + +import ( + "testing" + "time" +) + +func TestStatics(t *testing.T) { + StatisticsMap.AddStatistics("POST", "/api/user", "&admin.user", time.Duration(1000000)) + StatisticsMap.AddStatistics("POST", "/api/user", "&admin.user", time.Duration(1200000)) + StatisticsMap.AddStatistics("GET", "/api/user", "&admin.user", time.Duration(1300000)) + StatisticsMap.AddStatistics("POST", "/api/admin", "&admin.user", time.Duration(1400000)) + StatisticsMap.AddStatistics("POST", "/api/user/astaxie", "&admin.user", time.Duration(1200000)) + StatisticsMap.AddStatistics("POST", "/api/user/xiemengjun", "&admin.user", time.Duration(1300000)) + StatisticsMap.AddStatistics("DELETE", "/api/user", "&admin.user", time.Duration(1400000)) + s := StatisticsMap.GetMap() +} diff --git a/admin/task.go b/admin/task.go new file mode 100644 index 00000000..bf48fd8c --- /dev/null +++ b/admin/task.go @@ -0,0 +1 @@ +package admin \ No newline at end of file diff --git a/beego.go b/beego.go index e9fc0dab..aa0dc85b 100644 --- a/beego.go +++ b/beego.go @@ -93,5 +93,9 @@ func Run() { middleware.AppName = AppName middleware.RegisterErrorHander() + if EnableAdmin { + go BeeAdminApp.Run() + } + BeeApp.Run() } diff --git a/config.go b/config.go index 14792175..08d0ce94 100644 --- a/config.go +++ b/config.go @@ -53,6 +53,9 @@ var ( TemplateLeft string TemplateRight string BeegoServerName string + EnableAdmin bool //enable admin module to log api time + AdminHttpAddr string //admin module http addr + AdminHttpPort int ) func init() { @@ -89,6 +92,9 @@ func init() { TemplateLeft = "{{" TemplateRight = "}}" BeegoServerName = "beegoServer" + EnableAdmin = true + AdminHttpAddr = "localhost" + AdminHttpPort = 8088 ParseConfig() runtime.GOMAXPROCS(runtime.NumCPU()) } @@ -311,6 +317,24 @@ func ParseConfig() (err error) { } } } + if enableadmin, err := AppConfig.Bool("enableadmin"); err == nil { + EnableAdmin = enableadmin + } + if enableadmin, err := AppConfig.Bool("EnableAdmin"); err == nil { + EnableAdmin = enableadmin + } + if adminhttpaddr := AppConfig.String("admintttpaddr"); adminhttpaddr != "" { + AdminHttpAddr = adminhttpaddr + } + if adminhttpaddr := AppConfig.String("AdminHttpAddr"); adminhttpaddr != "" { + AdminHttpAddr = adminhttpaddr + } + if adminhttpport, err := AppConfig.Int("adminhttpport"); err == nil { + AdminHttpPort = adminhttpport + } + if adminhttpport, err := AppConfig.Int("AdminHttpPort"); err == nil { + AdminHttpPort = adminhttpport + } } return nil } diff --git a/context/output.go b/context/output.go index 0fa6a581..fa38f4e4 100644 --- a/context/output.go +++ b/context/output.go @@ -258,3 +258,7 @@ func stringsToJson(str string) string { } return jsons } + +func (output *BeegoOutput) Session(name interface{}, value interface{}) { + output.Context.Input.CruSession.Set(name, value) +} diff --git a/router.go b/router.go index 3a8cb4e6..8620d71f 100644 --- a/router.go +++ b/router.go @@ -2,6 +2,7 @@ package beego import ( "fmt" + "github.com/astaxie/beego/admin" beecontext "github.com/astaxie/beego/context" "github.com/astaxie/beego/middleware" "net/http" @@ -12,6 +13,7 @@ import ( "runtime" "strconv" "strings" + "time" ) var HTTPMETHOD = []string{"get", "post", "put", "delete", "patch", "options", "head"} @@ -406,6 +408,12 @@ func (p *ControllerRegistor) ServeHTTP(rw http.ResponseWriter, r *http.Request) } }() + starttime := time.Now() + requestPath := r.URL.Path + var runrouter *controllerInfo + var findrouter bool + params := make(map[string]string) + w := &responseWriter{writer: rw} w.Header().Set("Server", BeegoServerName) context := &beecontext.Context{ @@ -422,27 +430,18 @@ func (p *ControllerRegistor) ServeHTTP(rw http.ResponseWriter, r *http.Request) context.Output = beecontext.NewOutput(rw) } - if SessionOn { - context.Input.CruSession = GlobalSessions.SessionStart(w, r) - } - if !inSlice(strings.ToLower(r.Method), HTTPMETHOD) { http.Error(w, "Method Not Allowed", 405) - return + goto Admin } - var runrouter *controllerInfo - var findrouter bool - - params := make(map[string]string) - if p.enableFilter { if l, ok := p.filters["BeforRouter"]; ok { for _, filterR := range l { if filterR.ValidRouter(r.URL.Path) { filterR.filterFunc(context) if w.started { - return + goto Admin } } } @@ -455,7 +454,7 @@ func (p *ControllerRegistor) ServeHTTP(rw http.ResponseWriter, r *http.Request) file := staticDir + r.URL.Path http.ServeFile(w, r, file) w.started = true - return + goto Admin } if strings.HasPrefix(r.URL.Path, prefix) { file := staticDir + r.URL.Path[len(prefix):] @@ -465,32 +464,36 @@ func (p *ControllerRegistor) ServeHTTP(rw http.ResponseWriter, r *http.Request) Warn(err) } http.NotFound(w, r) - return + goto Admin } //if the request is dir and DirectoryIndex is false then if finfo.IsDir() && !DirectoryIndex { middleware.Exception("403", rw, r, "403 Forbidden") - return + goto Admin } http.ServeFile(w, r, file) w.started = true - return + goto Admin } } + // session init after static file + if SessionOn { + context.Input.CruSession = GlobalSessions.SessionStart(w, r) + } + if p.enableFilter { if l, ok := p.filters["AfterStatic"]; ok { for _, filterR := range l { if filterR.ValidRouter(r.URL.Path) { filterR.filterFunc(context) if w.started { - return + goto Admin } } } } } - requestPath := r.URL.Path if CopyRequestBody { context.Input.Body() @@ -509,7 +512,7 @@ func (p *ControllerRegistor) ServeHTTP(rw http.ResponseWriter, r *http.Request) if requestPath[n-1] != '/' && len(route.pattern) == n+1 && route.pattern[n] == '/' && route.pattern[:n] == requestPath { http.Redirect(w, r, requestPath+"/", 301) - return + goto Admin } } @@ -546,6 +549,7 @@ func (p *ControllerRegistor) ServeHTTP(rw http.ResponseWriter, r *http.Request) break } } + context.Input.Param = params if runrouter != nil { @@ -559,7 +563,7 @@ func (p *ControllerRegistor) ServeHTTP(rw http.ResponseWriter, r *http.Request) if filterR.ValidRouter(r.URL.Path) { filterR.filterFunc(context) if w.started { - return + goto Admin } } } @@ -714,7 +718,7 @@ func (p *ControllerRegistor) ServeHTTP(rw http.ResponseWriter, r *http.Request) if filterR.ValidRouter(r.URL.Path) { filterR.filterFunc(context) if w.started { - return + goto Admin } } } @@ -740,7 +744,7 @@ func (p *ControllerRegistor) ServeHTTP(rw http.ResponseWriter, r *http.Request) if strings.ToLower(requestPath) == "/"+cName { http.Redirect(w, r, requestPath+"/", 301) - return + goto Admin } if strings.ToLower(requestPath) == "/"+cName+"/" { @@ -754,6 +758,8 @@ func (p *ControllerRegistor) ServeHTTP(rw http.ResponseWriter, r *http.Request) if r.Method == "POST" { r.ParseMultipartForm(MaxMemory) } + // set find + findrouter = true //execute middleware filters if p.enableFilter { if l, ok := p.filters["BeforExec"]; ok { @@ -761,7 +767,7 @@ func (p *ControllerRegistor) ServeHTTP(rw http.ResponseWriter, r *http.Request) if filterR.ValidRouter(r.URL.Path) { filterR.filterFunc(context) if w.started { - return + goto Admin } } } @@ -816,7 +822,7 @@ func (p *ControllerRegistor) ServeHTTP(rw http.ResponseWriter, r *http.Request) if filterR.ValidRouter(r.URL.Path) { filterR.filterFunc(context) if w.started { - return + goto Admin } } } @@ -824,9 +830,7 @@ func (p *ControllerRegistor) ServeHTTP(rw http.ResponseWriter, r *http.Request) } method = vc.MethodByName("Destructor") method.Call(in) - // set find - findrouter = true - goto Last + goto Admin } } } @@ -834,11 +838,16 @@ func (p *ControllerRegistor) ServeHTTP(rw http.ResponseWriter, r *http.Request) } } -Last: //if no matches to url, throw a not found exception if !findrouter { middleware.Exception("404", rw, r, "") } + +Admin: + //admin module record QPS + if EnableAdmin { + go admin.StatisticsMap.AddStatistics(r.Method, requestPath, runrouter.controllerType.Name(), time.Since(starttime)) + } } //responseWriter is a wrapper for the http.ResponseWriter