Turn profiling on/off runtime (#543)

* profile api status command

* start profiling endpoints

* stop path

* egctl profile client

* Apply suggestions from code review

Co-authored-by: Yun Long <[email protected]>
Co-authored-by: Bomin Zhang <[email protected]>

* validate profiling paths and do not modify static options

* remove filepath validation and move state to profile/profile.go

* error handling

* use lock to synchronize concurrent modifications

* change cluster level lock to profile level lock

Co-authored-by: Yun Long <[email protected]>
Co-authored-by: Bomin Zhang <[email protected]>

cmd/client/command/common.go

@@ -68,6 +68,10 @@ const (
  customDataURL = apiURL + "/customdata/%s"
  customDataItemURL = apiURL + "/customdata/%s/%s"
 
+ profileURL = apiURL + "/profile"
+ profileStartURL = apiURL + "/profile/start/%s"
+ profileStopURL = apiURL + "/profile/stop"
+
  // MeshTenantsURL is the mesh tenant prefix.
  MeshTenantsURL = apiURL + "/mesh/tenants"
 

cmd/client/command/profile.go

@@ -0,0 +1,114 @@
+/*
+ * Copyright (c) 2017, MegaEase
+ * All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http:https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package command
+
+import (
+ "errors"
+ "net/http"
+
+ "github.com/spf13/cobra"
+)
+
+// ProfileCmd defines member command.
+func ProfileCmd() *cobra.Command {
+ cmd := &cobra.Command{
+ Use: "profile",
+ Short: "Start and stop CPU and memory profilers",
+ }
+
+ cmd.AddCommand(infoProfileCmd())
+ cmd.AddCommand(startProfilingCmd())
+ cmd.AddCommand(stopProfilingCmd())
+ return cmd
+}
+
+func infoProfileCmd() *cobra.Command {
+ cmd := &cobra.Command{
+ Use: "info",
+ Short: "Show memory and CPU profile file paths",
+ Run: func(cmd *cobra.Command, args []string) {
+ handleRequest(http.MethodGet, makeURL(profileURL), nil, cmd)
+ },
+ }
+
+ return cmd
+}
+
+func startProfilingCmd() *cobra.Command {
+ cmd := &cobra.Command{
+ Use: "start",
+ Short: "Prepare measuring CPU or memory",
+ }
+ cmd.AddCommand(startCPUCmd())
+ cmd.AddCommand(startMemoryCmd())
+
+ return cmd
+}
+
+func startCPUCmd() *cobra.Command {
+ cmd := &cobra.Command{
+ Use: "cpu",
+ Short: "Prepare measuring CPU",
+ Example: "egctl profile start cpu <path/to/cpu-prof-file>",
+ Args: func(cmd *cobra.Command, args []string) error {
+ if len(args) != 1 {
+ return errors.New("requires one file path")
+ }
+ return nil
+ },
+ Run: func(cmd *cobra.Command, args []string) {
+ body := []byte("path: " + args[0])
+ handleRequest(http.MethodPost, makeURL(profileStartURL, "cpu"), body, cmd)
+ },
+ }
+
+ return cmd
+}
+
+func startMemoryCmd() *cobra.Command {
+ cmd := &cobra.Command{
+ Use: "memory",
+ Short: "Prepare measuring memory",
+ Example: "egctl profile start memory <path/to/memory-prof-file>",
+ Args: func(cmd *cobra.Command, args []string) error {
+ if len(args) != 1 {
+ return errors.New("requires one file path")
+ }
+ return nil
+ },
+ Run: func(cmd *cobra.Command, args []string) {
+ body := []byte("path: " + args[0])
+ handleRequest(http.MethodPost, makeURL(profileStartURL, "memory"), body, cmd)
+ },
+ }
+
+ return cmd
+}
+
+func stopProfilingCmd() *cobra.Command {
+ cmd := &cobra.Command{
+ Use: "stop",
+ Short: "Stop profiling.",
+ Example: "egctl profile stop",
+ Run: func(cmd *cobra.Command, args []string) {
+ handleRequest(http.MethodPost, makeURL(profileStopURL), nil, cmd)
+ },
+ }
+
+ return cmd
+}

cmd/client/main.go

@@ -112,6 +112,7 @@ func main() {
  command.WasmCmd(),
  command.CustomDataKindCmd(),
  command.CustomDataCmd(),
+ command.ProfileCmd(),
  completionCmd,
  )
 

cmd/server/main.go

@@ -98,7 +98,7 @@ func main() {
 
  super := supervisor.MustNew(opt, cls)
 
- apiServer := api.MustNewServer(opt, cls, super)
+ apiServer := api.MustNewServer(opt, cls, super, profile)
 
  if graceupdate.CallOriProcessTerm(super.FirstHandleDone()) {
  pidfile.Write(opt)
@@ -113,7 +113,7 @@ func main() {
  }
  restartCls := func() {
  cls.StartServer()
- apiServer = api.MustNewServer(opt, cls, super)
+ apiServer = api.MustNewServer(opt, cls, super, profile)
  }
  if err := graceupdate.NotifySigUsr2(closeCls, restartCls); err != nil {
  log.Printf("failed to notify signal: %v", err)

pkg/api/api.go

@@ -102,6 +102,7 @@ func (s *Server) registerAPIs() {
  group.Entries = append(group.Entries, s.healthAPIEntries()...)
  group.Entries = append(group.Entries, s.aboutAPIEntries()...)
  group.Entries = append(group.Entries, s.customDataAPIEntries()...)
+ group.Entries = append(group.Entries, s.profileAPIEntries()...)
 
  for _, fn := range appendAddonAPIs {
  fn(s, group)

pkg/api/profile.go

@@ -0,0 +1,134 @@
+/*
+ * Copyright (c) 2017, MegaEase
+ * All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http:https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package api
+
+import (
+ "fmt"
+ "gopkg.in/yaml.v2"
+ "net/http"
+)
+
+const (
+ // ProfilePrefix is the URL prefix of profile APIs
+ ProfilePrefix = "/profile"
+ // StartAction is the URL for starting profiling
+ StartAction = "start"
+ // StopAction is the URL for stopping profiling
+ StopAction = "stop"
+)
+
+type (
+ // ProfileStatusResponse contains cpu and memory profile file paths
+ ProfileStatusResponse struct {
+ CPUPath string `yaml:"cpuPath"`
+ MemoryPath string `yaml:"memoryPath"`
+ }
+
+ // StartProfilingRequest contains file path to profile file
+ StartProfilingRequest struct {
+ Path string `yaml:"path"`
+ }
+)
+
+func (s *Server) profileAPIEntries() []*Entry {
+ return []*Entry{
+ {
+ Path: ProfilePrefix,
+ Method: http.MethodGet,
+ Handler: s.getProfileStatus,
+ },
+ {
+ Path: fmt.Sprintf("%s/%s/cpu", ProfilePrefix, StartAction),
+ Method: http.MethodPost,
+ Handler: s.startCPUProfile,
+ },
+ {
+ Path: fmt.Sprintf("%s/%s/memory", ProfilePrefix, StartAction),
+ Method: http.MethodPost,
+ Handler: s.startMemoryProfile,
+ },
+ {
+ Path: fmt.Sprintf("%s/%s", ProfilePrefix, StopAction),
+ Method: http.MethodPost,
+ Handler: s.stopProfile,
+ },
+ }
+}
+
+func (s *Server) getProfileStatus(w http.ResponseWriter, r *http.Request) {
+ cpuFile := s.profile.CPUFileName()
+ memFile := s.profile.MemoryFileName()
+
+ result := &ProfileStatusResponse{CPUPath: cpuFile, MemoryPath: memFile}
+ w.Header().Set("Content-Type", "text/vnd.yaml")
+ err := yaml.NewEncoder(w).Encode(result)
+ if err != nil {
+ panic(err)
+ }
+}
+
+func (s *Server) startCPUProfile(w http.ResponseWriter, r *http.Request) {
+ spr := StartProfilingRequest{}
+ err := yaml.NewDecoder(r.Body).Decode(&spr)
+ if err != nil {
+ HandleAPIError(w, r, http.StatusBadRequest, fmt.Errorf("bad request"))
+ return
+ }
+
+ if spr.Path == "" {
+ HandleAPIError(w, r, http.StatusBadRequest, fmt.Errorf("missing path"))
+ return
+ }
+
+ s.profile.Lock()
+ defer s.profile.Unlock()
+
+ err = s.profile.UpdateCPUProfile(spr.Path)
+ if err != nil {
+ HandleAPIError(w, r, http.StatusBadRequest, fmt.Errorf(err.Error()))
+ return
+ }
+}
+
+func (s *Server) startMemoryProfile(w http.ResponseWriter, r *http.Request) {
+ spr := StartProfilingRequest{}
+ err := yaml.NewDecoder(r.Body).Decode(&spr)
+ if err != nil {
+ HandleAPIError(w, r, http.StatusBadRequest, fmt.Errorf("bad request"))
+ return
+ }
+
+ if spr.Path == "" {
+ HandleAPIError(w, r, http.StatusBadRequest, fmt.Errorf("missing path"))
+ return
+ }
+
+ s.profile.Lock()
+ defer s.profile.Unlock()
+
+ // Memory profile is flushed only at stop/exit
+ s.profile.UpdateMemoryProfile(spr.Path)
+}
+
+func (s *Server) stopProfile(w http.ResponseWriter, r *http.Request) {
+ s.profile.Lock()
+ defer s.profile.Unlock()
+
+ s.profile.StopCPUProfile()
+ s.profile.StopMemoryProfile(s.profile.MemoryFileName())
+}

pkg/api/server.go

@@ -27,6 +27,7 @@ import (
  "github.com/megaease/easegress/pkg/cluster/customdata"
  "github.com/megaease/easegress/pkg/logger"
  "github.com/megaease/easegress/pkg/option"
+ pprof "github.com/megaease/easegress/pkg/profile"
  "github.com/megaease/easegress/pkg/supervisor"
 )
 
@@ -39,6 +40,7 @@ type (
  cluster cluster.Cluster
  super *supervisor.Supervisor
  cds *customdata.Store
+ profile pprof.Profile
 
  mutex cluster.Mutex
  mutexMutex sync.Mutex
@@ -59,11 +61,12 @@ type (
 )
 
 // MustNewServer creates an api server.
-func MustNewServer(opt *option.Options, cls cluster.Cluster, super *supervisor.Supervisor) *Server {
+func MustNewServer(opt *option.Options, cls cluster.Cluster, super *supervisor.Supervisor, profile pprof.Profile) *Server {
  s := &Server{
  opt: opt,
  cluster: cls,
  super: super,
+ profile: profile,
  }
  s.router = newDynamicMux(s)
  s.server = http.Server{Addr: opt.APIAddr, Handler: s.router}