fix(monitor): send alert to correct webhook addr (#794)

tkestack · Oct 9, 2020 · 9e212bd · 9e212bd
1 parent b62fc41
commit 9e212bd
Show file tree

Hide file tree

Showing 2 changed files with 41 additions and 9 deletions.
diff --git a/pkg/gateway/proxy/proxy.go b/pkg/gateway/proxy/proxy.go
@@ -77,10 +77,6 @@ func componentPrefix() map[moduleName][]modulePath {
  },
  },
  moduleNameNotify: {
- modulePath{
- prefix: "/webhook/",
- protected: false,
- },
  modulePath{
  prefix: fmt.Sprintf("%s/%s/", apiPrefix, notify.GroupName),
  protected: true,
@@ -210,6 +206,15 @@ func RegisterRoute(m *mux.PathRecorderMux, cfg *gatewayconfig.GatewayConfigurati
  m.Handle(path.prefix, handler)
  }
  }
+ // proxy /webhook to tke-notify-api for alert
+ if cfg.Components.Notify.Passthrough != nil {
+ handler, err := passthrough.NewHandler(cfg.Components.Notify.Address, cfg.Components.Notify.Passthrough, false)
+ if err != nil {
+ return err
+ }
+ log.Info("Registered reverse proxy of passthrough mode for backend component", log.String("path", "/webhook"), log.Bool("protected", false), log.String("address", cfg.Components.Notify.Address))
+ m.Handle("/webhook", handler)
+ }
  return nil
 }
 

diff --git a/pkg/monitor/controller/prometheus/controller.go b/pkg/monitor/controller/prometheus/controller.go
@@ -62,6 +62,7 @@ import (
  platformutil "tkestack.io/tke/pkg/platform/util"
  "tkestack.io/tke/pkg/util/apiclient"
  containerregistryutil "tkestack.io/tke/pkg/util/containerregistry"
+ utilhttp "tkestack.io/tke/pkg/util/http"
  "tkestack.io/tke/pkg/util/log"
  "tkestack.io/tke/pkg/util/metrics"
 )
@@ -689,11 +690,9 @@ func (c *Controller) installPrometheus(ctx context.Context, prometheus *v1.Prome
  extensionsAPIGroup := apiclient.ClusterVersionIsBefore19(kubeClient)
 
  // get notify webhook address
- var webhookAddr string
- if prometheus.Spec.NotifyWebhook != "" {
- webhookAddr = prometheus.Spec.NotifyWebhook
- } else {
- webhookAddr = c.notifyAPIAddress + "/webhook"
+ webhookAddr, err := c.getWebhookAddr(ctx, prometheus)
+ if err != nil {
+ return fmt.Errorf("get webhook address failed: %v", err)
  }
 
  log.Infof("Start to create alertmanager")
@@ -828,6 +827,34 @@ func (c *Controller) installPrometheus(ctx context.Context, prometheus *v1.Prome
  return nil
 }
 
+func (c *Controller) getWebhookAddr(ctx context.Context, prometheus *v1.Prometheus) (webhookAddr string, err error) {
+ if prometheus.Spec.NotifyWebhook != "" {
+ webhookAddr = prometheus.Spec.NotifyWebhook
+ } else {
+ // use notify api address directly in global cluster
+ webhookAddr = c.notifyAPIAddress + "/webhook"
+ // use tke-gateway as proxy in non-global cluster
+ if prometheus.Spec.ClusterName != "global" {
+ globalCluster, err := c.platformClient.Clusters().Get(ctx, "global", metav1.GetOptions{})
+ if err != nil {
+ return "", fmt.Errorf("get global cluster failed: %v", err)
+ }
+ gatewayAddr := globalCluster.Spec.Machines[0].IP
+ if globalCluster.Spec.Features.HA != nil {
+ if globalCluster.Spec.Features.HA.TKEHA != nil {
+ gatewayAddr = globalCluster.Spec.Features.HA.TKEHA.VIP
+ }
+ if globalCluster.Spec.Features.HA.ThirdPartyHA != nil {
+ gatewayAddr = globalCluster.Spec.Features.HA.ThirdPartyHA.VIP
+ }
+ }
+ webhookAddr = utilhttp.MakeEndpoint("https", gatewayAddr,
+ 443, "/webhook")
+ }
+ }
+ return webhookAddr, nil
+}
+
 var selectorForPrometheusOperator = metav1.LabelSelector{
  MatchLabels: map[string]string{specialLabelName: specialLabelValue, "k8s-app": "prometheus-operator"},
 }