diff --git a/CHANGELOG.md b/CHANGELOG.md index b4a1fa4..ee06989 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,7 +1,24 @@ # Changelog +## [v1.6.0] - 2024-03-18 +Add: + - Slog logging + - PagerDuty push + - Report state change + +Fix: + - Add missing buddynsconfig option to config file + +Change: + - Upgrade Go + dependencies + - config element nsconfig is now called buddynsconfig + +Deprecated: + - weblog config element + ## [v1.5.4] - 2024-03-06 Fix: - Logic error in relations to DisableMetrics + - Wrong state of timeoutprobe Add: - Basic Prometheus docs diff --git a/README.md b/README.md index 37ddcee..f39e126 100644 --- a/README.md +++ b/README.md @@ -153,6 +153,36 @@ Either by using a enviroment variable, Or adding a line to the `dashgoat.yaml` `prometheusurl: http://localhost:9090` +To show the timeline in dashGoat select the change time and the timeline will appear, this feature needs some rework in the UI. + +## PagerDuty + +You can forward events to PagerDutys diffrent tecnical services depending on hosts + services, tags and severity, there is also a catch all. +
Minimum config, catch all: +```yaml +pagerdutyconfig: + pagerdutyservicemaps: + - hostservice: + tag: + eapikey: 12345acc39464b01d0105f1234567890 +``` +All options looks like this, first matching the host `host-1` and service `cache` to one technical service key, second matching all services with tag `customer23` to another tecnical service key. Both only being forwarded to PageDuty if severity is `error` or higher. +```yaml +pagerdutyconfig: + url: https://events.pagerduty.com/v2/enqueue + timeout: 10s + triggerlevel: error + pagerdutymode: push + pagerdutyservicemaps: + - hostservice: host-1cache + tag: + eapikey: 12345acc39464b01d0105f1234567890 + - hostservice: + tag: customer23 + eapikey: ffff5acc39464b01d0105f123456ffff +``` + + ## Docker Hello world ```docker run -e UPDATEKEY=my-precious! -p 2000:2000 --rm --name=dashgoat analogbear/dashgoat``` @@ -228,6 +258,8 @@ To include the config file: * ~~Better TTL handling~~ * ~~add dependecy to service hosts+tags~~ * ~~Dynamic favicon with most critical colour~~ + * Delete event in dashboard + * MS teams support * API tests (in progress) * Configuration tests * Save state @@ -238,5 +270,6 @@ To include the config file: * Users +gravatar? * Auth on delete * Automatic event cleanup - * Better logging + * ~~Better logging~~ + * ~~PagerDuty support~~ * dashGoat client diff --git a/cmd/dashgoat/buddy.go b/cmd/dashgoat/buddy.go index 0756138..bf7abe4 100644 --- a/cmd/dashgoat/buddy.go +++ b/cmd/dashgoat/buddy.go @@ -92,7 +92,7 @@ func talkToBuddyApi(event dg.ServiceState, host Buddy, delete string) { jsonMapAsStringFormat, err := json.Marshal(event) if err != nil { - fmt.Println(err) + logger.Error("talkToBuddyApi json marshall", err) return } @@ -107,7 +107,7 @@ func talkToBuddyApi(event dg.ServiceState, host Buddy, delete string) { res, err := http.DefaultClient.Do(req) if err != nil { - fmt.Println(err) + logger.Error("talkToBuddyApi problems talking to "+host.Url, err) tellBuddyState(host.Name, false, event.Host) return } @@ -123,7 +123,7 @@ func findBuddy(buddyConfig []Buddy) { if buddyAmount < 1 { setDashGoatReady(true) - fmt.Println("Buddy not found") + logger.Info("Buddy not found") } firstRound := true @@ -134,7 +134,7 @@ func findBuddy(buddyConfig []Buddy) { } buddyWelcome := fmt.Sprintf("%d %s ", buddyAmount, buddyTxt) - fmt.Println(buddyWelcome) + logger.Info(buddyWelcome) waitfor := 10 if config.CheckBuddyIntervalSec > 1 { @@ -157,7 +157,7 @@ func findBuddy(buddyConfig []Buddy) { } } - if !dashGoatReady() { + if !isDashGoatReady() { setDashGoatReady(true) } @@ -357,6 +357,8 @@ func tellServiceListAboutBuddy(buddyName string, up bool) { result.Message = "My buddy is down" } + iSnewState(result) + ss.mutex.Lock() defer ss.mutex.Unlock() ss.serviceStateList[serviceName] = result diff --git a/cmd/dashgoat/buddyconfig.go b/cmd/dashgoat/buddyconfig.go index 2a899e5..b36273f 100644 --- a/cmd/dashgoat/buddyconfig.go +++ b/cmd/dashgoat/buddyconfig.go @@ -41,17 +41,21 @@ func initBuddyConf(rawConfig []Buddy) { } } - if nsconfig != "" { + if buddy_nsconfig == "" { + buddy_nsconfig = config.BuddyNsConfig + } + + if buddy_nsconfig != "" { findDNSBuddies() } //look for headless service IPs - if isK8s() && nsconfig == "" { - nsconfig = "dashgoat-headless-svc" + if isK8s() && buddy_nsconfig == "" { + buddy_nsconfig = "dashgoat-headless-svc" findDNSBuddies() } - if nsconfig != "" { + if buddy_nsconfig != "" { go loopFindDNSBuddies() } } @@ -116,7 +120,7 @@ func ipLookup() (error, bool) { var err error var new_ip bool - ips, err := net.LookupIP(nsconfig) + ips, err := net.LookupIP(buddy_nsconfig) if err != nil { fmt.Println(err) return err, new_ip @@ -143,7 +147,7 @@ func ajustBuddyConfig(ips []net.IP) { //Remove old nsconfig-Buddies not found last lookup for _, buddy := range listBuddies() { - if buddy.NSconfig == nsconfig && buddy.LastSeen != time_now { + if buddy.NSconfig == buddy_nsconfig && buddy.LastSeen != time_now { delBuddy(buddy) } } @@ -160,7 +164,7 @@ func compileBuddyConfig(ip string, time_now int64) (Buddy, bool) { } result.LastSeen = time_now - result.NSconfig = nsconfig + result.NSconfig = buddy_nsconfig result.Key = config.UpdateKey result.Url = "http://" + ip + ":" + strings.Split(config.IPport, ":")[1] //TODO - needs some work diff --git a/cmd/dashgoat/config.go b/cmd/dashgoat/config.go index 5ceeb55..f5221a1 100644 --- a/cmd/dashgoat/config.go +++ b/cmd/dashgoat/config.go @@ -8,51 +8,34 @@ package main import ( "fmt" - "net" "os" - "runtime" "strings" - "sync" - "time" "gopkg.in/yaml.v2" ) +var severitys = []string{"ok", "info", "warning", "error", "critical"} + type ( Configer struct { DashName string `yaml:"dashName"` IPport string `yaml:"ipport"` - WebLog string `yaml:"weblog"` WebPath string `yaml:"webpath"` UpdateKey string `yaml:"updatekey"` CheckBuddyIntervalSec int `yaml:"checkBuddyIntervalSec"` BuddyDownStatusMsg string `yaml:"buddyDownStatusMsg"` BuddyHosts []Buddy `yaml:"buddy"` + BuddyNsConfig string `yaml:"buddynsconfig"` IgnorePrefix []string `yaml:"ignorePrefix"` TtlBehavior string `yaml:"ttlbehavior"` TtlOkDelete int `yaml:"ttlokdelete"` DisableDependOn bool `yaml:"disableDependOn"` DisableMetrics bool `yaml:"disableMetrics"` Prometheusurl string `yaml:"prometheusurl"` - } - HostFact struct { - Hostnames []string - DashName string - Ready bool - UpAt time.Time - UpAtEpoch int64 - DashGoatVersion string - GoVersion string - MetricsHistory bool - } - HostFacts struct { - Items HostFact - mutex sync.RWMutex + PagerdutyConfig PdConfig `yaml:"pagerdutyconfig"` } ) -var host_facts HostFacts - func (conf *Configer) ReadEnv() { var tmp_buddy Buddy @@ -62,9 +45,6 @@ func (conf *Configer) ReadEnv() { if os.Getenv("IPPORT") != "" { config.IPport = os.Getenv("IPPORT") } - if os.Getenv("WEBLOG") != "" { - conf.WebLog = os.Getenv("WEBLOG") - } if os.Getenv("WEBPATH") != "" { conf.WebPath = os.Getenv("WEBPATH") } @@ -88,8 +68,8 @@ func (conf *Configer) ReadEnv() { if os.Getenv("IGNOREPREFIX") != "" { conf.IgnorePrefix = append(conf.IgnorePrefix, os.Getenv("IGNOREPREFIX")) } - if os.Getenv("NSCONFIG") != "" { - nsconfig = os.Getenv("NSCONFIG") + if os.Getenv("BUDDYNSCONFIG") != "" { + buddy_nsconfig = os.Getenv("BUDDYNSCONFIG") } if os.Getenv("TTLBEHAVIOR") != "" { conf.TtlBehavior = os.Getenv("TTLBEHAVIOR") @@ -106,24 +86,25 @@ func (conf *Configer) ReadEnv() { if os.Getenv("PROMETHEUSURL") != "" { conf.Prometheusurl = os.Getenv("PROMETHEUSURL") } + } // InitConfig initiates a new decoded Config struct Alex style -func (conf *Configer) InitConfig(configPath string) error { +func (conf *Configer) InitConfig(config_path string) error { var result error - if configPath == "" { - configPath = "dashgoat.yaml" + if config_path == "" { + config_path = "dashgoat.yaml" } - fileExists := isExists(configPath, "file") - if !fileExists { - result = fmt.Errorf("Cant find Config file " + configPath + ", moving on") - configPath = "" + file_exists := isExists(config_path, "file") + if !file_exists { + result = fmt.Errorf("Cant find Config file " + config_path + ", moving on") + config_path = "" } - if configPath != "" { - file, err := os.Open(configPath) + if config_path != "" { + file, err := os.Open(config_path) if err != nil { return err } @@ -135,15 +116,15 @@ func (conf *Configer) InitConfig(configPath string) error { if err := d.Decode(&config); err != nil { return err } - fmt.Println("Using settings from " + configPath + " ignoring cli args") + logger.Error("Using settings from " + config_path + " ignoring cli args") } if conf.DashName == "" { conf.DashName = "dashGoat" } - // buddy settings - if configPath == "" { + // Buddy settings + if config_path == "" { if buddy_cli.Url != "" && buddy_cli.Url != "0" { conf.BuddyHosts = append(conf.BuddyHosts, buddy_cli) } @@ -159,18 +140,28 @@ func (conf *Configer) InitConfig(configPath string) error { } } + // Default status when Buddy is down if conf.BuddyDownStatusMsg == "" { conf.BuddyDownStatusMsg = "warning" } + // Default TTL bahaviour if conf.TtlBehavior == "" { conf.TtlBehavior = "promotetook" } else { conf.TtlBehavior = strings.ToLower(conf.TtlBehavior) } + + // Default delete time on resolved TTL if conf.TtlOkDelete == 0 { conf.TtlOkDelete = 3600 } + + err := validatePagerdutyConf() + if err != nil { + return err + } + generateHostFacts() return result } @@ -192,90 +183,3 @@ func validateBuddyConf() error { return nil } - -func generateHostFacts() { - host_facts.mutex.Lock() - defer host_facts.mutex.Unlock() - - host_facts.Items.DashName = config.DashName - host_facts.Items.UpAtEpoch = time.Now().Unix() - host_facts.Items.UpAt = time.Now() - host_facts.Items.DashGoatVersion = Version - host_facts.Items.GoVersion = runtime.Version() - - hostname, _ := os.Hostname() - IPhost := "" - - for _, ip := range getHostIPs() { - IPhost = hostname + "_" + ip - host_facts.Items.Hostnames = append(host_facts.Items.Hostnames, IPhost) - } - if len(host_facts.Items.Hostnames) == 0 { - fmt.Println("Cant find an IP address, check ignorePrefix config") - os.Exit(1) - } - host_facts.Items.Hostnames = append(host_facts.Items.Hostnames, config.DashName) - fmt.Print("Hostnames found: ") - fmt.Println(host_facts.Items.Hostnames) - - if config.DisableMetrics && config.Prometheusurl == "" { - host_facts.Items.MetricsHistory = false - fmt.Println("MetricsHistory: off") - } else { - host_facts.Items.MetricsHistory = true - fmt.Println("MetricsHistory: on") - } -} - -func getHostIPs() []string { - var result []string - - // Get the list of IP addresses associated with the host - addrs, err := net.InterfaceAddrs() - if err != nil { - fmt.Println("Error:", err) - } - - ip_addr := "" - for _, addr := range addrs { - ip_addr = addr.String() - if !ignorePrefix(ip_addr) { //no localhost addr - result = append(result, strings.Split(addr.String(), "/")[0]) - } - } - - return result -} - -func ignorePrefix(ip_addr string) bool { - ignore := config.IgnorePrefix - - if len(ignore) == 0 { - ignore = []string{"8", "64", "128"} - } - - for _, ignoreStr := range ignore { - if strings.HasSuffix(ip_addr, ignoreStr) { - return true - } - } - return false -} - -func readHostFacts() HostFact { - host_facts.mutex.RLock() - defer host_facts.mutex.RUnlock() - return host_facts.Items -} - -func dashGoatReady() bool { - host_facts.mutex.RLock() - defer host_facts.mutex.RUnlock() - return host_facts.Items.Ready -} - -func setDashGoatReady(ready bool) { - host_facts.mutex.Lock() - defer host_facts.mutex.Unlock() - host_facts.Items.Ready = ready -} diff --git a/cmd/dashgoat/dependon.go b/cmd/dashgoat/dependon.go new file mode 100644 index 0000000..18a7bd2 --- /dev/null +++ b/cmd/dashgoat/dependon.go @@ -0,0 +1,68 @@ +/* + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. + */ + +package main + +import ( + "fmt" + "strings" + + dg "github.com/iobear/dashgoat/common" +) + +// Looks for service or tag that a second service depends on +// beware - Only call this method if you have ss.mutex lock +func isDependOnError(search_host_key string) string { + + if config.DisableDependOn { + return "" + } + + count_ok := 0 + count_error := 0 + + search := strings.ToLower(strings.TrimSpace(search_host_key)) + + for statekey := range ss.serviceStateList { + if ss.serviceStateList[statekey].Host == search || contains(ss.serviceStateList[statekey].Tags, search) { + if ss.serviceStateList[statekey].Status == "error" || ss.serviceStateList[statekey].Status == "critical" { + count_error++ + } else { + count_ok++ + } + } + } + + if count_error == 0 { + return "" + } + if count_error > 0 && count_ok == 0 { + return "down" + } + if count_error > 0 && count_ok > 0 { + return fmt.Sprintf("partly down %d/%d ", count_error, count_ok+count_error) + } + + return "" +} + +func runDependOn(ss dg.ServiceState) dg.ServiceState { + + if ss.Status != "ok" && ss.DependOn != "" { + msg := isDependOnError(ss.DependOn) + if msg == "down" { + ss.Severity = "warning" + ss.Status = "warning" + ss.Message = "( " + ss.DependOn + " down ) " + ss.Message + } else if msg != "" { + ss.Severity = "info" + ss.Status = "info" + ss.Message = "( " + ss.DependOn + " ) " + msg + ss.Message + } + } + + return ss +} diff --git a/cmd/dashgoat/handler.go b/cmd/dashgoat/handler.go index 73eb1fa..4856e6e 100644 --- a/cmd/dashgoat/handler.go +++ b/cmd/dashgoat/handler.go @@ -7,7 +7,6 @@ package main import ( - "fmt" "net/http" "strings" "time" @@ -16,42 +15,6 @@ import ( "github.com/labstack/echo/v4" ) -// Looks for service or tag that a second service depends on -// beware - Only call this method if you have ss.mutex lock -func isDependOnError(search_host_key string) string { - - if config.DisableDependOn { - return "" - } - - count_ok := 0 - count_error := 0 - - search := strings.ToLower(strings.TrimSpace(search_host_key)) - - for statekey := range ss.serviceStateList { - if ss.serviceStateList[statekey].Host == search || contains(ss.serviceStateList[statekey].Tags, search) { - if ss.serviceStateList[statekey].Status == "error" || ss.serviceStateList[statekey].Status == "critical" { - count_error++ - } else { - count_ok++ - } - } - } - - if count_error == 0 { - return "" - } - if count_error > 0 && count_ok == 0 { - return "down" - } - if count_error > 0 && count_ok > 0 { - return fmt.Sprintf("partly down %d/%d ", count_error, count_ok+count_error) - } - - return "" -} - // updateStatus - service update func updateStatus(c echo.Context) error { @@ -86,6 +49,12 @@ func updateStatus(c echo.Context) error { return c.JSON(http.StatusBadRequest, postService) } + if len(postService.From) == 0 { //From can't be empty + postService.From = append(postService.From, "127.0.0.1") + } + + iSnewState(postService) // Informs abount state change + if _, ok := ss.serviceStateList[strID]; ok { if postService.Change == 0 { @@ -200,7 +169,7 @@ func deleteServiceHandler(c echo.Context) error { // health of dashGoat app func health(c echo.Context) error { - if !dashGoatReady() { + if !isDashGoatReady() { return c.NoContent(http.StatusServiceUnavailable) } @@ -209,27 +178,5 @@ func health(c echo.Context) error { func checkUpdatekey(key string) bool { - if key != config.UpdateKey { - return false - } - - return true -} - -func runDependOn(ss dg.ServiceState) dg.ServiceState { - - if ss.Status != "ok" && ss.DependOn != "" { - msg := isDependOnError(ss.DependOn) - if msg == "down" { - ss.Severity = "info" - ss.Status = "info" - ss.Message = "( " + ss.DependOn + " down ) " + ss.Message - } else if msg != "" { - ss.Severity = "info" - ss.Status = "info" - ss.Message = "( " + ss.DependOn + " ) " + msg + ss.Message - } - } - - return ss + return key == config.UpdateKey } diff --git a/cmd/dashgoat/helper.go b/cmd/dashgoat/helper.go index 5448d5b..23699d1 100644 --- a/cmd/dashgoat/helper.go +++ b/cmd/dashgoat/helper.go @@ -7,7 +7,6 @@ package main import ( - "fmt" "io/fs" "net/http" "os" @@ -53,7 +52,7 @@ func add2url(path string, route string) string { return result.String() } -// isExists Does the given directory of filepath exist? +// isExists Does the given directory or filepath exist? func isExists(path string, task string) bool { fileStat, err := os.Stat(path) @@ -98,11 +97,11 @@ func str2bool(str_to_convert string) bool { func getFileSystem(useOS bool) http.FileSystem { if useOS { - fmt.Println("using live mode") + logger.Info("using live mode") return http.FS(os.DirFS("web")) } - fmt.Println("using embed mode") + logger.Info("using embed mode") fsys, err := fs.Sub(embededFiles, "web") if err != nil { panic(err) diff --git a/cmd/dashgoat/hostfacts.go b/cmd/dashgoat/hostfacts.go new file mode 100644 index 0000000..dd7172a --- /dev/null +++ b/cmd/dashgoat/hostfacts.go @@ -0,0 +1,117 @@ +package main + +import ( + "fmt" + "net" + "os" + "runtime" + "strings" + "sync" + "time" +) + +type ( + HostFact struct { + Hostnames []string + DashName string + Ready bool + UpAt time.Time + UpAtEpoch int64 + DashGoatVersion string + GoVersion string + MetricsHistory bool + } + + HostFacts struct { + Items HostFact + mutex sync.RWMutex + } +) + +var host_facts HostFacts + +func generateHostFacts() { + host_facts.mutex.Lock() + defer host_facts.mutex.Unlock() + + host_facts.Items.DashName = config.DashName + host_facts.Items.UpAtEpoch = time.Now().Unix() + host_facts.Items.UpAt = time.Now() + host_facts.Items.DashGoatVersion = Version + host_facts.Items.GoVersion = runtime.Version() + + hostname, _ := os.Hostname() + IPhost := "" + + for _, ip := range getHostIPs() { + IPhost = hostname + "_" + ip + host_facts.Items.Hostnames = append(host_facts.Items.Hostnames, IPhost) + } + if len(host_facts.Items.Hostnames) == 0 { + logger.Error("Cant find an IP address, check ignorePrefix config") + os.Exit(1) + } + host_facts.Items.Hostnames = append(host_facts.Items.Hostnames, config.DashName) + logger.Info("Welcome", "Hostnames found", host_facts.Items.Hostnames) + + if config.DisableMetrics && config.Prometheusurl == "" { + host_facts.Items.MetricsHistory = false + logger.Info("HostFacts", "MetricsHistory", "off") + } else { + host_facts.Items.MetricsHistory = true + logger.Info("HostFacts", "MetricsHistory", "on") + } +} + +func getHostIPs() []string { + var result []string + + // Get the list of IP addresses associated with the host + addrs, err := net.InterfaceAddrs() + if err != nil { + fmt.Println("Error:", err) + } + + ip_addr := "" + for _, addr := range addrs { + ip_addr = addr.String() + if !ignorePrefix(ip_addr) { //no localhost addr + result = append(result, strings.Split(addr.String(), "/")[0]) + } + } + + return result +} + +func ignorePrefix(ip_addr string) bool { + ignore := config.IgnorePrefix + + if len(ignore) == 0 { + ignore = []string{"8", "64", "128"} + } + + for _, ignoreStr := range ignore { + if strings.HasSuffix(ip_addr, ignoreStr) { + return true + } + } + return false +} + +func readHostFacts() HostFact { + host_facts.mutex.RLock() + defer host_facts.mutex.RUnlock() + return host_facts.Items +} + +func isDashGoatReady() bool { + host_facts.mutex.RLock() + defer host_facts.mutex.RUnlock() + return host_facts.Items.Ready +} + +func setDashGoatReady(ready bool) { + host_facts.mutex.Lock() + defer host_facts.mutex.Unlock() + host_facts.Items.Ready = ready +} diff --git a/cmd/dashgoat/main.go b/cmd/dashgoat/main.go index 8eda935..d1b7661 100644 --- a/cmd/dashgoat/main.go +++ b/cmd/dashgoat/main.go @@ -7,10 +7,12 @@ package main import ( + "context" "embed" "flag" - "fmt" + "log/slog" "net/http" + "os" "strings" "sync" @@ -33,12 +35,15 @@ type ( } ) +// var logger = slog.New(slog.NewJSONHandler(os.Stdout, nil)) +var logger = slog.New(slog.NewTextHandler(os.Stdout, nil)) + //go:embed web var embededFiles embed.FS var config Configer var buddy_cli Buddy -var nsconfig string +var buddy_nsconfig string var ss Services var backlog Backlog @@ -53,8 +58,29 @@ func main() { e := echo.New() + e.Use(middleware.RequestLoggerWithConfig(middleware.RequestLoggerConfig{ + LogStatus: true, + LogURI: true, + LogError: true, + HandleError: true, // forwards error to the global error handler, so it can decide appropriate status code + LogValuesFunc: func(c echo.Context, v middleware.RequestLoggerValues) error { + if v.Error == nil { + logger.LogAttrs(context.Background(), slog.LevelInfo, "REQUEST", + slog.String("uri", v.URI), + slog.Int("status", v.Status), + ) + } else { + logger.LogAttrs(context.Background(), slog.LevelError, "REQUEST_ERROR", + slog.String("uri", v.URI), + slog.Int("status", v.Status), + slog.String("err", v.Error.Error()), + ) + } + return nil + }, + })) + flag.StringVar(&config.IPport, "ipport", ":2000", "Specify :") - flag.StringVar(&config.WebLog, "weblog", "off", "HTTP log ") flag.StringVar(&config.WebPath, "webpath", "/", "Specify added url http://host:port/ Default: /") flag.StringVar(&config.UpdateKey, "updatekey", "changeme", "Specify key to API update") flag.StringVar(&config.DashName, "dashname", "", "Dashboard name") @@ -62,13 +88,13 @@ func main() { flag.StringVar(&buddy_cli.Url, "buddyurl", "", "Buddy url") flag.StringVar(&buddy_cli.Key, "buddykey", "", "Buddy update key, empty for same key") flag.StringVar(&buddy_cli.Name, "buddyname", "", "Buddy name") - flag.StringVar(&nsconfig, "nsconfig", "", "Configure buddies via DNS/k8s namespace") + flag.StringVar(&buddy_nsconfig, "buddynsconfig", "", "Configure buddies via DNS/k8s namespace") flag.Parse() config.ReadEnv() err := config.InitConfig(configfile) if err != nil { - fmt.Println(err) + logger.Error("cant initialize config", err) } pathStartsWith := strings.HasPrefix(config.WebPath, "/") @@ -87,9 +113,6 @@ func main() { e.HideBanner = true - if config.WebLog == "on" { - e.Use(middleware.Logger()) - } e.Use(middleware.Recover()) if !config.DisableMetrics { @@ -110,20 +133,15 @@ func main() { e.DELETE(add2url(config.WebPath, "/service/:id"), deleteServiceHandler) e.GET(add2url(config.WebPath, "/health"), health) - printWelcome() + logger.Error("welcome", "Starting dashGoat", readHostFacts().DashGoatVersion) + logger.Info("welcome details", "Go", readHostFacts().GoVersion, "Labstack Echo", echo.Version, "Dashboard name", readHostFacts().DashName) go lostProbeTimer() go ttlHousekeeping() go findBuddy(config.BuddyHosts) + go initPagerDuty() // Start server e.Logger.Fatal(e.Start(config.IPport)) } - -func printWelcome() { - fmt.Println("Starting dashGoat " + readHostFacts().DashGoatVersion) - fmt.Println("Dashboard name: " + readHostFacts().DashName + " ") - fmt.Println("Go: " + readHostFacts().GoVersion + " ") - fmt.Println("Labstack Echo: " + echo.Version + " ") -} diff --git a/cmd/dashgoat/pagerduty.go b/cmd/dashgoat/pagerduty.go new file mode 100644 index 0000000..cc39225 --- /dev/null +++ b/cmd/dashgoat/pagerduty.go @@ -0,0 +1,216 @@ +/* + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. + */ + +package main + +import ( + "encoding/json" + "fmt" + "io" + "net/http" + "strings" + "time" + + dg "github.com/iobear/dashgoat/common" +) + +type PagerDutyEvent struct { + Payload struct { + Summary string `json:"summary"` + Severity string `json:"severity"` + Source string `json:"source"` + Component string `json:"component"` + Timestamp string `json:"timestamp"` + } `json:"payload"` + RoutingKey string `json:"routing_key"` + DedupKey string `json:"dedup_key"` + EventAction string `json:"event_action"` + Client string `json:"client"` +} + +type PdConfig struct { + URL string `yaml:"url"` + Timeout time.Duration `yaml:"timeout"` + PdMode string `yaml:"pagerdutymode"` + TriggerLevel string `yaml:"triggerlevel"` + PdServiceMaps []PdServiceMap `yaml:"pagerdutyservicemaps"` +} + +type PdServiceMap struct { + HostService string `yaml:"hostservice"` + Tag string `yaml:"tag"` + EapiKey string `yaml:"eapikey"` +} + +type PdClient struct { + config PdConfig +} + +var pdClient = &PdClient{} + +func validatePagerdutyConf() error { + var result error + + // Is PagerDuty enabled? + if config.PagerdutyConfig.PdMode == "off" { + return result + } + + if len(config.PagerdutyConfig.PdServiceMaps) == 0 { + logger.Info("no pagerdutyservicemaps, setting pagerdutymode off") + config.PagerdutyConfig.PdMode = "off" + return result + } + + // Default PdMode + pdClient.config.PdMode = "push" + + // Default timeout value + if config.PagerdutyConfig.Timeout == 0 { + config.PagerdutyConfig.Timeout = 10 + } + + // Default PagerDuty Url US + if config.PagerdutyConfig.URL == "" { + config.PagerdutyConfig.URL = "https://events.pagerduty.com/v2/enqueue" + } + + // Default PagerDuty trigger level + if config.PagerdutyConfig.TriggerLevel == "" { + config.PagerdutyConfig.TriggerLevel = "error" + } + + for key, val := range config.PagerdutyConfig.PdServiceMaps { + if val.EapiKey == "" { + return fmt.Errorf("pagerDuty eapikey missing") + } + if val.HostService == "" && val.Tag == "" { + config.PagerdutyConfig.PdServiceMaps[key].HostService = "0catchall0" + } + } + + return result +} + +func initPagerDuty() { + logger.Info("PagerDuty mode is " + config.PagerdutyConfig.PdMode) + + if config.PagerdutyConfig.PdMode == "off" { + return + } + + pdClient.config = PdConfig{ + URL: config.PagerdutyConfig.URL, + Timeout: config.PagerdutyConfig.Timeout, + PdServiceMaps: config.PagerdutyConfig.PdServiceMaps, + } + +} + +func shouldPagerDutyTrigger(severity_to_check string) bool { + trigger_level := indexOf(severitys, config.PagerdutyConfig.TriggerLevel) + to_check := indexOf(severitys, severity_to_check) + + return to_check >= trigger_level + +} + +func (c *PdClient) CompilePdEvent(fromstate string, dgss dg.ServiceState) { + var pdevent PagerDutyEvent + + logger.Info("pdevent", "Severity", dgss.Severity) + + pdkey, _ := findKey(dgss) + if pdkey == "" { + logger.Info("No key found " + dgss.Host) + return + } + + action := "resolve" + + if shouldPagerDutyTrigger(dgss.Status) { + action = "trigger" + } + + changetimeTime := time.Unix(dgss.Probe, 0).UTC() + formattedTime := changetimeTime.Format("2006-01-02T15:04:05.000+0000") + + pdevent.Payload.Timestamp = formattedTime + pdevent.Payload.Severity = dgss.Severity + pdevent.Payload.Source = readHostFacts().DashName + pdevent.Payload.Summary = dgss.Host + " " + dgss.Service + " " + dgss.Message + pdevent.Payload.Component = dgss.Service + pdevent.EventAction = action + pdevent.DedupKey = dgss.Host + dgss.Service + "dashgoat" + pdevent.RoutingKey = pdkey + pdevent.Client = "dashgoat" + + err := pdClient.TellPagerDuty(pdevent) + if err != nil { + logger.Error("Error sending to PagerDuty:", err) + + } +} + +func findKey(dgss dg.ServiceState) (pdkey string, pdmatch string) { + + var result string + var match string + + for _, item := range config.PagerdutyConfig.PdServiceMaps { + if item.HostService == dgss.Host+dgss.Service || item.HostService == "0catchall0" { // look for match or catch all token + return item.EapiKey, item.HostService + } + + for _, tag := range dgss.Tags { + if tag == item.Tag { + return item.EapiKey, item.Tag + } + } + } + + return result, match +} + +// TellPagerDuty updates PagerDuty via HTTP +func (c *PdClient) TellPagerDuty(pdevent PagerDutyEvent) error { + + client := &http.Client{ + Timeout: c.config.Timeout, + } + + json_data, err := json.Marshal(pdevent) + if err != nil { + logger.Error("Error marshaling to JSON", err) + return err + } + + var payload = strings.NewReader(string(json_data)) + + req, err := http.NewRequest("POST", c.config.URL, payload) + if err != nil { + logger.Error("PagerDuty POST failed", err) + return err + } + + req.Header.Add("Content-Type", "application/json") + + res, err := client.Do(req) + if err != nil { + logger.Error("PagerDuty error client", err) + } + defer res.Body.Close() + + body, err := io.ReadAll(res.Body) + if err != nil { + logger.Error("Failed reading PagerDuty response", err) + return err + } + + logger.Info("PagerDuty", "response", string(body), "statuscode", res.StatusCode) + + return nil +} diff --git a/cmd/dashgoat/state.go b/cmd/dashgoat/state.go new file mode 100644 index 0000000..397ab53 --- /dev/null +++ b/cmd/dashgoat/state.go @@ -0,0 +1,40 @@ +/* + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. + */ + +package main + +import ( + dg "github.com/iobear/dashgoat/common" +) + +// iSnewState checks if state is changing +// Only call this method if you have ss.mutex lock +func iSnewState(checkss dg.ServiceState) (state string, new bool) { + hostservice := checkss.Host + checkss.Service + + if _, ok := ss.serviceStateList[hostservice]; ok { + + if ss.serviceStateList[hostservice].Status == checkss.Status { + return "", false + } + go reportStateChange(ss.serviceStateList[hostservice].Status, checkss) + return checkss.Status, false + } + + go reportStateChange("", checkss) + return checkss.Status, true +} + +// ReportStateChange +func reportStateChange(fromstate string, reportss dg.ServiceState) { + logger.Info("statechange", "hostservice", reportss.Host+reportss.Service, "from", fromstate, "to", reportss.Status) + + if config.PagerdutyConfig.PdMode != "off" { + if len(reportss.From) == 1 { // Check if I'm the first to know + pdClient.CompilePdEvent(fromstate, reportss) + } + } +} diff --git a/cmd/dashgoat/timeoutprobe.go b/cmd/dashgoat/timeoutprobe.go index b35fd82..9ccbcc8 100644 --- a/cmd/dashgoat/timeoutprobe.go +++ b/cmd/dashgoat/timeoutprobe.go @@ -73,6 +73,8 @@ func updateEventLostProbe(hostService string) { tmpStruct.Severity = "error" tmpStruct.Status = "error" + iSnewState(tmpStruct) + ss.serviceStateList[hostService] = tmpStruct } diff --git a/cmd/dashgoat/ttl.go b/cmd/dashgoat/ttl.go index 78534a6..0d99582 100644 --- a/cmd/dashgoat/ttl.go +++ b/cmd/dashgoat/ttl.go @@ -34,6 +34,7 @@ func updateServiceState(key string, serviceState dg.ServiceState) { }() ss.mutex.Lock() + iSnewState(serviceState) ss.serviceStateList[key] = serviceState } diff --git a/dashgoat.yaml.example b/dashgoat.yaml.example index 598dc78..012924d 100644 --- a/dashgoat.yaml.example +++ b/dashgoat.yaml.example @@ -1,10 +1,9 @@ dashName: dashGoat buddyDownStatusMsg: info ipport: :2000 -weblog: off webpath: / updatekey: my-precious! ttlbehavior: PromoteToOk # Remove, PromoteOnce, PromoteOneStep,PromoteToOk ttlokdelete: 300 # seconds to delete -nsconfig: dashgoat-headless-svc -prometheusurl: http://localhost:9090 \ No newline at end of file +buddynsconfig: dashgoat-headless-svc +prometheusurl: http://localhost:9090 diff --git a/go.mod b/go.mod index 2f237ab..d2d15eb 100644 --- a/go.mod +++ b/go.mod @@ -1,13 +1,13 @@ module github.com/iobear/dashgoat -go 1.22.0 +go 1.22.1 require ( - github.com/iobear/dashgoat/common v0.0.0-20240227172109-914ca4fa630b + github.com/iobear/dashgoat/common v0.0.0-20240314112511-1a9d15ac1e05 github.com/labstack/echo-contrib v0.15.0 github.com/labstack/echo/v4 v4.11.4 - github.com/prometheus/client_golang v1.18.0 - github.com/prometheus/common v0.48.0 + github.com/prometheus/client_golang v1.19.0 + github.com/prometheus/common v0.50.0 gopkg.in/yaml.v2 v2.4.0 ) @@ -22,15 +22,15 @@ require ( github.com/mattn/go-isatty v0.0.20 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect - github.com/prometheus/client_model v0.5.0 // indirect - github.com/prometheus/procfs v0.12.0 // indirect + github.com/prometheus/client_model v0.6.0 // indirect + github.com/prometheus/procfs v0.13.0 // indirect github.com/rogpeppe/go-internal v1.12.0 // indirect github.com/valyala/bytebufferpool v1.0.0 // indirect github.com/valyala/fasttemplate v1.2.2 // indirect - golang.org/x/crypto v0.19.0 // indirect - golang.org/x/net v0.21.0 // indirect - golang.org/x/sys v0.17.0 // indirect + golang.org/x/crypto v0.21.0 // indirect + golang.org/x/net v0.22.0 // indirect + golang.org/x/sys v0.18.0 // indirect golang.org/x/text v0.14.0 // indirect golang.org/x/time v0.5.0 // indirect - google.golang.org/protobuf v1.32.0 // indirect + google.golang.org/protobuf v1.33.0 // indirect ) diff --git a/go.sum b/go.sum index a83f7f8..88ac669 100644 --- a/go.sum +++ b/go.sum @@ -13,8 +13,8 @@ github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiu github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/iobear/dashgoat/common v0.0.0-20240227172109-914ca4fa630b h1:VkRFNfNM4gNDzdh6MKuMmC2auHsLL/uZT1M2RdB77A4= -github.com/iobear/dashgoat/common v0.0.0-20240227172109-914ca4fa630b/go.mod h1:+7p4F+iuFghBhSi8TZBN0j00SyvFVKiPHjLRtMsSR0Q= +github.com/iobear/dashgoat/common v0.0.0-20240314112511-1a9d15ac1e05 h1:G5MrMDJBswg6EBjvz5JIZBo1HkZnPml1IMkPkNicaiI= +github.com/iobear/dashgoat/common v0.0.0-20240314112511-1a9d15ac1e05/go.mod h1:+7p4F+iuFghBhSi8TZBN0j00SyvFVKiPHjLRtMsSR0Q= github.com/jpillora/backoff v1.0.0 h1:uvFg412JmmHBHw7iwprIxkPMI+sGQ4kzOWsMeHnm2EA= github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= @@ -43,14 +43,14 @@ github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f h1:KUppIJq7/+ github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/prometheus/client_golang v1.18.0 h1:HzFfmkOzH5Q8L8G+kSJKUx5dtG87sewO+FoDDqP5Tbk= -github.com/prometheus/client_golang v1.18.0/go.mod h1:T+GXkCk5wSJyOqMIzVgvvjFDlkOQntgjkJWKrN5txjA= -github.com/prometheus/client_model v0.5.0 h1:VQw1hfvPvk3Uv6Qf29VrPF32JB6rtbgI6cYPYQjL0Qw= -github.com/prometheus/client_model v0.5.0/go.mod h1:dTiFglRmd66nLR9Pv9f0mZi7B7fk5Pm3gvsjB5tr+kI= -github.com/prometheus/common v0.48.0 h1:QO8U2CdOzSn1BBsmXJXduaaW+dY/5QLjfB8svtSzKKE= -github.com/prometheus/common v0.48.0/go.mod h1:0/KsvlIEfPQCQ5I2iNSAWKPZziNCvRs5EC6ILDTlAPc= -github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo= -github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo= +github.com/prometheus/client_golang v1.19.0 h1:ygXvpU1AoN1MhdzckN+PyD9QJOSD4x7kmXYlnfbA6JU= +github.com/prometheus/client_golang v1.19.0/go.mod h1:ZRM9uEAypZakd+q/x7+gmsvXdURP+DABIEIjnmDdp+k= +github.com/prometheus/client_model v0.6.0 h1:k1v3CzpSRUTrKMppY35TLwPvxHqBu0bYgxZzqGIgaos= +github.com/prometheus/client_model v0.6.0/go.mod h1:NTQHnmxFpouOD0DpvP4XujX3CdOAGQPoaGhyTchlyt8= +github.com/prometheus/common v0.50.0 h1:YSZE6aa9+luNa2da6/Tik0q0A5AbR+U003TItK57CPQ= +github.com/prometheus/common v0.50.0/go.mod h1:wHFBCEVWVmHMUpg7pYcOm2QUR/ocQdYSJVQJKnHc3xQ= +github.com/prometheus/procfs v0.13.0 h1:GqzLlQyfsPbaEHaQkO7tbDlriv/4o5Hudv6OXHGKX7o= +github.com/prometheus/procfs v0.13.0/go.mod h1:cd4PFCR54QLnGKPaKGA6l+cfuNXtht43ZKY6tow0Y1g= github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -61,24 +61,24 @@ github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6Kllzaw github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= github.com/valyala/fasttemplate v1.2.2 h1:lxLXG0uE3Qnshl9QyaK6XJxMXlQZELvChBOCmQD0Loo= github.com/valyala/fasttemplate v1.2.2/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ= -golang.org/x/crypto v0.19.0 h1:ENy+Az/9Y1vSrlrvBSyna3PITt4tiZLf7sgCjZBX7Wo= -golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= -golang.org/x/net v0.21.0 h1:AQyQV4dYCvJ7vGmJyKki9+PBdyvhkSd8EIx/qb0AYv4= -golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= -golang.org/x/oauth2 v0.16.0 h1:aDkGMBSYxElaoP81NpoUoz2oo2R2wHdZpGToUxfyQrQ= -golang.org/x/oauth2 v0.16.0/go.mod h1:hqZ+0LWXsiVoZpeld6jVt06P3adbS2Uu911W1SsJv2o= +golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA= +golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs= +golang.org/x/net v0.22.0 h1:9sGLhx7iRIHEiX0oAJ3MRZMUCElJgy7Br1nO+AMN3Tc= +golang.org/x/net v0.22.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= +golang.org/x/oauth2 v0.18.0 h1:09qnuIAgzdx1XplqJvW6CQqMCtGZykZWcXzPMPUusvI= +golang.org/x/oauth2 v0.18.0/go.mod h1:Wf7knwG0MPoWIMMBgFlEaSUDaKskp0dCfrlJRJXbBi8= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.17.0 h1:25cE3gD+tdBA7lp7QfhuV+rJiE9YXTcS3VG1SqssI/Y= -golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= +golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/protobuf v1.32.0 h1:pPC6BG5ex8PDFnkbrGU3EixyhKcQ2aDuBS36lqK/C7I= -google.golang.org/protobuf v1.32.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= +google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= diff --git a/tests/pagerduty-test.sh b/tests/pagerduty-test.sh new file mode 100755 index 0000000..8a5a179 --- /dev/null +++ b/tests/pagerduty-test.sh @@ -0,0 +1,37 @@ +#!/bin/bash + +echo +echo "-- pagerduty test --" + +BASE_URL="http://localhost:2000" +CONTENT_TYPE="application/json" +UPDATE_KEY="changeme" + +# Define services with tags +services_with_tags=( + '"service": "nginx", "tags": ["web", "production"]' + '"service": "database", "tags": ["db", "test"]' + '"service": "cache", "tags": ["cache", "development"]' +) + + +for service_with_tags in "${services_with_tags[@]}"; do + + curl -X POST "$BASE_URL/update" \ + -H "Content-Type: $CONTENT_TYPE" \ + --data "{\"host\": \"host-1\", $service_with_tags, \"status\": \"error\", \"message\": \"Slow service\",\"updatekey\": \"$UPDATE_KEY\"}" + +done + + +echo "-- 2x Host-1 should be seen in PagerDuty" + +for service_with_tags in "${services_with_tags[@]}"; do + + curl -X POST "$BASE_URL/update" \ + -H "Content-Type: $CONTENT_TYPE" \ + --data "{\"host\": \"host-2\", $service_with_tags, \"status\": \"warning\", \"message\": \"Not so slow service\",\"updatekey\": \"$UPDATE_KEY\"}" + +done + +echo "-- Host-2 should NOT be seen in PagerDuty" \ No newline at end of file