From 8b8191d8562ae6d790164d3a18ac94f6a6624fe7 Mon Sep 17 00:00:00 2001 From: Samu Tamminen Date: Mon, 21 Mar 2022 15:17:23 +0800 Subject: [PATCH] Turn profiling on/off runtime (#543) * profile api status command * start profiling endpoints * stop path * egctl profile client * Apply suggestions from code review Co-authored-by: Yun Long Co-authored-by: Bomin Zhang * validate profiling paths and do not modify static options * remove filepath validation and move state to profile/profile.go * error handling * use lock to synchronize concurrent modifications * change cluster level lock to profile level lock Co-authored-by: Yun Long Co-authored-by: Bomin Zhang --- cmd/client/command/common.go | 4 + cmd/client/command/profile.go | 114 +++++++++++++++++++++++++++++ cmd/client/main.go | 1 + cmd/server/main.go | 4 +- pkg/api/api.go | 1 + pkg/api/profile.go | 134 ++++++++++++++++++++++++++++++++++ pkg/api/server.go | 5 +- pkg/profile/profile.go | 86 ++++++++++++++++++---- 8 files changed, 330 insertions(+), 19 deletions(-) create mode 100644 cmd/client/command/profile.go create mode 100644 pkg/api/profile.go diff --git a/cmd/client/command/common.go b/cmd/client/command/common.go index 261cb1a8be..955c2092dc 100644 --- a/cmd/client/command/common.go +++ b/cmd/client/command/common.go @@ -68,6 +68,10 @@ const ( customDataURL = apiURL + "/customdata/%s" customDataItemURL = apiURL + "/customdata/%s/%s" + profileURL = apiURL + "/profile" + profileStartURL = apiURL + "/profile/start/%s" + profileStopURL = apiURL + "/profile/stop" + // MeshTenantsURL is the mesh tenant prefix. MeshTenantsURL = apiURL + "/mesh/tenants" diff --git a/cmd/client/command/profile.go b/cmd/client/command/profile.go new file mode 100644 index 0000000000..0ddd3511e1 --- /dev/null +++ b/cmd/client/command/profile.go @@ -0,0 +1,114 @@ +/* + * Copyright (c) 2017, MegaEase + * All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package command + +import ( + "errors" + "net/http" + + "github.com/spf13/cobra" +) + +// ProfileCmd defines member command. +func ProfileCmd() *cobra.Command { + cmd := &cobra.Command{ + Use: "profile", + Short: "Start and stop CPU and memory profilers", + } + + cmd.AddCommand(infoProfileCmd()) + cmd.AddCommand(startProfilingCmd()) + cmd.AddCommand(stopProfilingCmd()) + return cmd +} + +func infoProfileCmd() *cobra.Command { + cmd := &cobra.Command{ + Use: "info", + Short: "Show memory and CPU profile file paths", + Run: func(cmd *cobra.Command, args []string) { + handleRequest(http.MethodGet, makeURL(profileURL), nil, cmd) + }, + } + + return cmd +} + +func startProfilingCmd() *cobra.Command { + cmd := &cobra.Command{ + Use: "start", + Short: "Prepare measuring CPU or memory", + } + cmd.AddCommand(startCPUCmd()) + cmd.AddCommand(startMemoryCmd()) + + return cmd +} + +func startCPUCmd() *cobra.Command { + cmd := &cobra.Command{ + Use: "cpu", + Short: "Prepare measuring CPU", + Example: "egctl profile start cpu ", + Args: func(cmd *cobra.Command, args []string) error { + if len(args) != 1 { + return errors.New("requires one file path") + } + return nil + }, + Run: func(cmd *cobra.Command, args []string) { + body := []byte("path: " + args[0]) + handleRequest(http.MethodPost, makeURL(profileStartURL, "cpu"), body, cmd) + }, + } + + return cmd +} + +func startMemoryCmd() *cobra.Command { + cmd := &cobra.Command{ + Use: "memory", + Short: "Prepare measuring memory", + Example: "egctl profile start memory ", + Args: func(cmd *cobra.Command, args []string) error { + if len(args) != 1 { + return errors.New("requires one file path") + } + return nil + }, + Run: func(cmd *cobra.Command, args []string) { + body := []byte("path: " + args[0]) + handleRequest(http.MethodPost, makeURL(profileStartURL, "memory"), body, cmd) + }, + } + + return cmd +} + +func stopProfilingCmd() *cobra.Command { + cmd := &cobra.Command{ + Use: "stop", + Short: "Stop profiling.", + Example: "egctl profile stop", + Run: func(cmd *cobra.Command, args []string) { + handleRequest(http.MethodPost, makeURL(profileStopURL), nil, cmd) + }, + } + + return cmd +} diff --git a/cmd/client/main.go b/cmd/client/main.go index 1bdd1e5bd9..99e6fd857d 100644 --- a/cmd/client/main.go +++ b/cmd/client/main.go @@ -112,6 +112,7 @@ func main() { command.WasmCmd(), command.CustomDataKindCmd(), command.CustomDataCmd(), + command.ProfileCmd(), completionCmd, ) diff --git a/cmd/server/main.go b/cmd/server/main.go index caffe6d5fc..9c5f8a2ec0 100644 --- a/cmd/server/main.go +++ b/cmd/server/main.go @@ -98,7 +98,7 @@ func main() { super := supervisor.MustNew(opt, cls) - apiServer := api.MustNewServer(opt, cls, super) + apiServer := api.MustNewServer(opt, cls, super, profile) if graceupdate.CallOriProcessTerm(super.FirstHandleDone()) { pidfile.Write(opt) @@ -113,7 +113,7 @@ func main() { } restartCls := func() { cls.StartServer() - apiServer = api.MustNewServer(opt, cls, super) + apiServer = api.MustNewServer(opt, cls, super, profile) } if err := graceupdate.NotifySigUsr2(closeCls, restartCls); err != nil { log.Printf("failed to notify signal: %v", err) diff --git a/pkg/api/api.go b/pkg/api/api.go index b26ee2cdde..cd04587b31 100644 --- a/pkg/api/api.go +++ b/pkg/api/api.go @@ -102,6 +102,7 @@ func (s *Server) registerAPIs() { group.Entries = append(group.Entries, s.healthAPIEntries()...) group.Entries = append(group.Entries, s.aboutAPIEntries()...) group.Entries = append(group.Entries, s.customDataAPIEntries()...) + group.Entries = append(group.Entries, s.profileAPIEntries()...) for _, fn := range appendAddonAPIs { fn(s, group) diff --git a/pkg/api/profile.go b/pkg/api/profile.go new file mode 100644 index 0000000000..601aa9cbee --- /dev/null +++ b/pkg/api/profile.go @@ -0,0 +1,134 @@ +/* + * Copyright (c) 2017, MegaEase + * All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package api + +import ( + "fmt" + "gopkg.in/yaml.v2" + "net/http" +) + +const ( + // ProfilePrefix is the URL prefix of profile APIs + ProfilePrefix = "/profile" + // StartAction is the URL for starting profiling + StartAction = "start" + // StopAction is the URL for stopping profiling + StopAction = "stop" +) + +type ( + // ProfileStatusResponse contains cpu and memory profile file paths + ProfileStatusResponse struct { + CPUPath string `yaml:"cpuPath"` + MemoryPath string `yaml:"memoryPath"` + } + + // StartProfilingRequest contains file path to profile file + StartProfilingRequest struct { + Path string `yaml:"path"` + } +) + +func (s *Server) profileAPIEntries() []*Entry { + return []*Entry{ + { + Path: ProfilePrefix, + Method: http.MethodGet, + Handler: s.getProfileStatus, + }, + { + Path: fmt.Sprintf("%s/%s/cpu", ProfilePrefix, StartAction), + Method: http.MethodPost, + Handler: s.startCPUProfile, + }, + { + Path: fmt.Sprintf("%s/%s/memory", ProfilePrefix, StartAction), + Method: http.MethodPost, + Handler: s.startMemoryProfile, + }, + { + Path: fmt.Sprintf("%s/%s", ProfilePrefix, StopAction), + Method: http.MethodPost, + Handler: s.stopProfile, + }, + } +} + +func (s *Server) getProfileStatus(w http.ResponseWriter, r *http.Request) { + cpuFile := s.profile.CPUFileName() + memFile := s.profile.MemoryFileName() + + result := &ProfileStatusResponse{CPUPath: cpuFile, MemoryPath: memFile} + w.Header().Set("Content-Type", "text/vnd.yaml") + err := yaml.NewEncoder(w).Encode(result) + if err != nil { + panic(err) + } +} + +func (s *Server) startCPUProfile(w http.ResponseWriter, r *http.Request) { + spr := StartProfilingRequest{} + err := yaml.NewDecoder(r.Body).Decode(&spr) + if err != nil { + HandleAPIError(w, r, http.StatusBadRequest, fmt.Errorf("bad request")) + return + } + + if spr.Path == "" { + HandleAPIError(w, r, http.StatusBadRequest, fmt.Errorf("missing path")) + return + } + + s.profile.Lock() + defer s.profile.Unlock() + + err = s.profile.UpdateCPUProfile(spr.Path) + if err != nil { + HandleAPIError(w, r, http.StatusBadRequest, fmt.Errorf(err.Error())) + return + } +} + +func (s *Server) startMemoryProfile(w http.ResponseWriter, r *http.Request) { + spr := StartProfilingRequest{} + err := yaml.NewDecoder(r.Body).Decode(&spr) + if err != nil { + HandleAPIError(w, r, http.StatusBadRequest, fmt.Errorf("bad request")) + return + } + + if spr.Path == "" { + HandleAPIError(w, r, http.StatusBadRequest, fmt.Errorf("missing path")) + return + } + + s.profile.Lock() + defer s.profile.Unlock() + + // Memory profile is flushed only at stop/exit + s.profile.UpdateMemoryProfile(spr.Path) +} + +func (s *Server) stopProfile(w http.ResponseWriter, r *http.Request) { + s.profile.Lock() + defer s.profile.Unlock() + + s.profile.StopCPUProfile() + s.profile.StopMemoryProfile(s.profile.MemoryFileName()) +} diff --git a/pkg/api/server.go b/pkg/api/server.go index acb7f57f74..637a9d1159 100644 --- a/pkg/api/server.go +++ b/pkg/api/server.go @@ -27,6 +27,7 @@ import ( "github.com/megaease/easegress/pkg/cluster/customdata" "github.com/megaease/easegress/pkg/logger" "github.com/megaease/easegress/pkg/option" + pprof "github.com/megaease/easegress/pkg/profile" "github.com/megaease/easegress/pkg/supervisor" ) @@ -39,6 +40,7 @@ type ( cluster cluster.Cluster super *supervisor.Supervisor cds *customdata.Store + profile pprof.Profile mutex cluster.Mutex mutexMutex sync.Mutex @@ -59,11 +61,12 @@ type ( ) // MustNewServer creates an api server. -func MustNewServer(opt *option.Options, cls cluster.Cluster, super *supervisor.Supervisor) *Server { +func MustNewServer(opt *option.Options, cls cluster.Cluster, super *supervisor.Supervisor, profile pprof.Profile) *Server { s := &Server{ opt: opt, cluster: cls, super: super, + profile: profile, } s.router = newDynamicMux(s) s.server = http.Server{Addr: opt.APIAddr, Handler: s.router} diff --git a/pkg/profile/profile.go b/pkg/profile/profile.go index 10f738f900..88274502a8 100644 --- a/pkg/profile/profile.go +++ b/pkg/profile/profile.go @@ -31,12 +31,28 @@ import ( // Profile is the Profile interface. type Profile interface { + StartCPUProfile(fp string) error + + UpdateCPUProfile(fp string) error + UpdateMemoryProfile(fp string) + + StopCPUProfile() + StopMemoryProfile(fp string) + + CPUFileName() string + MemoryFileName() string + Close(wg *sync.WaitGroup) + Lock() + Unlock() } type profile struct { - cpuFile *os.File - opt *option.Options + cpuFile *os.File + opt *option.Options + memFileName string + + mutex sync.Mutex } // New creates a profile. @@ -45,7 +61,7 @@ func New(opt *option.Options) (Profile, error) { opt: opt, } - err := p.startCPUProfile() + err := p.StartCPUProfile("") if err != nil { return nil, err } @@ -53,12 +69,30 @@ func New(opt *option.Options) (Profile, error) { return p, nil } -func (p *profile) startCPUProfile() error { - if p.opt.CPUProfileFile == "" { +func (p *profile) CPUFileName() string { + if p.cpuFile == nil { + return "" + } + return p.cpuFile.Name() +} + +func (p *profile) MemoryFileName() string { + return p.memFileName +} + +func (p *profile) UpdateMemoryProfile(fp string) { + p.memFileName = fp +} + +func (p *profile) StartCPUProfile(filepath string) error { + if p.opt.CPUProfileFile == "" && filepath == "" { return nil } + if filepath == "" { + filepath = p.opt.CPUProfileFile + } - f, err := os.Create(p.opt.CPUProfileFile) + f, err := os.Create(filepath) if err != nil { return fmt.Errorf("create cpu profile failed: %v", err) } @@ -69,21 +103,24 @@ func (p *profile) startCPUProfile() error { p.cpuFile = f - logger.Infof("cpu profile: %s", p.opt.CPUProfileFile) + logger.Infof("cpu profile: %s", filepath) return nil } -func (p *profile) memoryProfile() { - if p.opt.MemoryProfileFile == "" { +func (p *profile) StopMemoryProfile(filepath string) { + if p.opt.MemoryProfileFile == "" && filepath == "" { return } + if filepath == "" { + filepath = p.opt.MemoryProfileFile + } // to include every allocated block in the profile runtime.MemProfileRate = 1 - logger.Infof("memory profile: %s", p.opt.MemoryProfileFile) - f, err := os.Create(p.opt.MemoryProfileFile) + logger.Infof("memory profile: %s", filepath) + f, err := os.Create(filepath) if err != nil { logger.Errorf("create memory profile failed: %v", err) return @@ -102,16 +139,33 @@ func (p *profile) memoryProfile() { } } -func (p *profile) Close(wg *sync.WaitGroup) { - defer wg.Done() - +func (p *profile) StopCPUProfile() { if p.cpuFile != nil { + fname := p.cpuFile.Name() pprof.StopCPUProfile() err := p.cpuFile.Close() if err != nil { - logger.Errorf("close %s failed: %v", p.opt.CPUProfileFile, err) + logger.Errorf("close %s failed: %v", fname, err) } + p.cpuFile = nil } +} + +func (p *profile) UpdateCPUProfile(filepath string) error { + p.StopCPUProfile() + return p.StartCPUProfile(filepath) +} + +func (p *profile) Close(wg *sync.WaitGroup) { + defer wg.Done() + p.StopCPUProfile() + p.StopMemoryProfile("") +} + +func (p *profile) Lock() { + p.mutex.Lock() +} - p.memoryProfile() +func (p *profile) Unlock() { + p.mutex.Unlock() }