fix: get metrics failed if values contain NaN

feat: set retention time to 2h to reduce memory usage of prometheus Signed-off-by: Feng Kun <[email protected]>
sujipo · Mar 26, 2020 · 2adf668 · 2adf668
1 parent 8609e2f
commit 2adf668
Show file tree

Hide file tree

Showing 2 changed files with 5 additions and 1 deletion.
diff --git a/pkg/monitor/storage/thanos/metric/metric.go b/pkg/monitor/storage/thanos/metric/metric.go
@@ -21,6 +21,7 @@ package metric
 import (
  "context"
  "fmt"
+ "math"
  "strings"
  "time"
  "tkestack.io/tke/api/monitor"
@@ -222,7 +223,9 @@ func MergeResult(results model.Matrix, timestamp string, groupByWithoutTimestamp
  }
  }
  values[index].([]interface{})[0] = sp.Timestamp.Unix() * 1000
- values[index].([]interface{})[fieldIndex+1] = float64(sp.Value)
+ if !math.IsNaN(float64(sp.Value)) {
+ values[index].([]interface{})[fieldIndex+1] = float64(sp.Value)
+ }
  for j, tag := range tags {
  values[index].([]interface{})[len(fieldIndexes)+1+j] = tag
  }

diff --git a/pkg/platform/controller/addon/prometheus/controller.go b/pkg/platform/controller/addon/prometheus/controller.go
@@ -1166,6 +1166,7 @@ func createPrometheusCRD(components images.Components, clusterName string, remot
  ScrapeInterval: "60s",
  RemoteRead: remoteReadSpecs,
  RemoteWrite: remoteWriteSpecs,
+ Retention: "2h",
  EvaluationInterval: "1m",
  AdditionalScrapeConfigs: &corev1.SecretKeySelector{
  LocalObjectReference: corev1.LocalObjectReference{Name: prometheusSecret},