Report both system.disk.io_time (wall clock) & system.disk.operation_…

…time (#1887)

* Report both disk io_time & disk operation_time

* Fix hostmetrics receiver test

CHANGELOG.md

@@ -2,8 +2,13 @@
 
 ## Unreleased
 
+## 💡 Enhancements 💡
+
+- Host Metrics receiver now reports both system.disk.io_time and system.disk.operation_time
+
 ## 🛑 Breaking changes 🛑
 
+- Host metric system.disk.time renamed to system.disk.operation_time
 - Use consumer for sender interface, remove unnecessary receiver address from Runner #1941
 - Enable sending queue by default in all exporters configured to use it #1924
 

receiver/hostmetricsreceiver/hostmetrics_receiver_test.go

@@ -45,9 +45,10 @@ var standardMetrics = []string{
  "system.cpu.time",
  "system.memory.usage",
  "system.disk.io",
+ "system.disk.io_time",
  "system.disk.ops",
+ "system.disk.operation_time",
  "system.disk.pending_operations",
- "system.disk.time",
  "system.filesystem.usage",
  "system.cpu.load_average.1m",
  "system.cpu.load_average.5m",

receiver/hostmetricsreceiver/internal/perfcounters/perfcounter_notwindows.go → receiver/hostmetricsreceiver/internal/perfcounters/doc.go

@@ -12,6 +12,8 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-// +build !windows
-
+// Package perfcounters is a thin wrapper around
+// https://godoc.org/github.com/leoluk/perflib_exporter/perflib that
+// provides functions to scrape raw performance counter data, without
+// calculating rates or formatting them, from the registry.
 package perfcounters

receiver/hostmetricsreceiver/internal/perfcounters/perfcounter_scraper.go

@@ -159,7 +159,7 @@ func (obj perfDataObject) GetValues(counterNames ...string) ([]*CounterValues, e
  // "Base" values give the value of a related counter that pdh.dll uses to compute the derived
  // value for this counter. We only care about raw values so ignore base values. See
  // https://docs.microsoft.com/en-us/windows/win32/perfctrs/retrieving-counter-data.
- if counter.IsBaseValue {
+ if counter.IsBaseValue && !counter.IsNanosecondCounter {
  continue
  }
 

receiver/hostmetricsreceiver/internal/scraper/diskscraper/disk_metadata.go

@@ -62,10 +62,24 @@ var diskOpsDescriptor = func() pdata.Metric {
  return metric
 }()
 
-var diskTimeDescriptor = func() pdata.Metric {
+var diskIOTimeDescriptor = func() pdata.Metric {
  metric := pdata.NewMetric()
  metric.InitEmpty()
- metric.SetName("system.disk.time")
+ metric.SetName("system.disk.io_time")
+ metric.SetDescription("Time disk spent activated. On Windows, this is calculated as the inverse of disk idle time.")
+ metric.SetUnit("s")
+ metric.SetDataType(pdata.MetricDataTypeDoubleSum)
+ sum := metric.DoubleSum()
+ sum.InitEmpty()
+ sum.SetIsMonotonic(true)
+ sum.SetAggregationTemporality(pdata.AggregationTemporalityCumulative)
+ return metric
+}()
+
+var diskOperationTimeDescriptor = func() pdata.Metric {
+ metric := pdata.NewMetric()
+ metric.InitEmpty()
+ metric.SetName("system.disk.operation_time")
  metric.SetDescription("Time spent in disk operations.")
  metric.SetUnit("s")
  metric.SetDataType(pdata.MetricDataTypeDoubleSum)

receiver/hostmetricsreceiver/internal/scraper/diskscraper/disk_scraper_others.go

@@ -94,12 +94,13 @@ func (s *scraper) ScrapeMetrics(_ context.Context) (pdata.MetricSlice, error) {
  ioCounters = s.filterByDevice(ioCounters)
 
  if len(ioCounters) > 0 {
- metrics.Resize(4 + systemSpecificMetricsLen)
+ metrics.Resize(5 + systemSpecificMetricsLen)
  initializeDiskIOMetric(metrics.At(0), s.startTime, now, ioCounters)
  initializeDiskOpsMetric(metrics.At(1), s.startTime, now, ioCounters)
- initializeDiskTimeMetric(metrics.At(2), s.startTime, now, ioCounters)
- initializeDiskPendingOperationsMetric(metrics.At(3), now, ioCounters)
- appendSystemSpecificMetrics(metrics, 4, s.startTime, now, ioCounters)
+ initializeDiskIOTimeMetric(metrics.At(2), s.startTime, now, ioCounters)
+ initializeDiskOperationTimeMetric(metrics.At(3), s.startTime, now, ioCounters)
+ initializeDiskPendingOperationsMetric(metrics.At(4), now, ioCounters)
+ appendSystemSpecificMetrics(metrics, 5, s.startTime, now, ioCounters)
  }
 
  return metrics, nil
@@ -133,8 +134,21 @@ func initializeDiskOpsMetric(metric pdata.Metric, startTime, now pdata.Timestamp
  }
 }
 
-func initializeDiskTimeMetric(metric pdata.Metric, startTime, now pdata.TimestampUnixNano, ioCounters map[string]disk.IOCountersStat) {
- diskTimeDescriptor.CopyTo(metric)
+func initializeDiskIOTimeMetric(metric pdata.Metric, startTime, now pdata.TimestampUnixNano, ioCounters map[string]disk.IOCountersStat) {
+ diskIOTimeDescriptor.CopyTo(metric)
+
+ ddps := metric.DoubleSum().DataPoints()
+ ddps.Resize(len(ioCounters))
+
+ idx := 0
+ for device, ioCounter := range ioCounters {
+ initializeDoubleDataPoint(ddps.At(idx+0), startTime, now, device, "", float64(ioCounter.IoTime)/1e3)
+ idx++
+ }
+}
+
+func initializeDiskOperationTimeMetric(metric pdata.Metric, startTime, now pdata.TimestampUnixNano, ioCounters map[string]disk.IOCountersStat) {
+ diskOperationTimeDescriptor.CopyTo(metric)
 
  ddps := metric.DoubleSum().DataPoints()
  ddps.Resize(2 * len(ioCounters))
@@ -163,7 +177,9 @@ func initializeDiskPendingOperationsMetric(metric pdata.Metric, now pdata.Timest
 func initializeInt64DataPoint(dataPoint pdata.IntDataPoint, startTime, now pdata.TimestampUnixNano, deviceLabel string, directionLabel string, value int64) {
  labelsMap := dataPoint.LabelsMap()
  labelsMap.Insert(deviceLabelName, deviceLabel)
- labelsMap.Insert(directionLabelName, directionLabel)
+ if directionLabel != "" {
+ labelsMap.Insert(directionLabelName, directionLabel)
+ }
  dataPoint.SetStartTime(startTime)
  dataPoint.SetTimestamp(now)
  dataPoint.SetValue(value)
@@ -172,7 +188,9 @@ func initializeInt64DataPoint(dataPoint pdata.IntDataPoint, startTime, now pdata
 func initializeDoubleDataPoint(dataPoint pdata.DoubleDataPoint, startTime, now pdata.TimestampUnixNano, deviceLabel string, directionLabel string, value float64) {
  labelsMap := dataPoint.LabelsMap()
  labelsMap.Insert(deviceLabelName, deviceLabel)
- labelsMap.Insert(directionLabelName, directionLabel)
+ if directionLabel != "" {
+ labelsMap.Insert(directionLabelName, directionLabel)
+ }
  dataPoint.SetStartTime(startTime)
  dataPoint.SetTimestamp(now)
  dataPoint.SetValue(value)

receiver/hostmetricsreceiver/internal/scraper/diskscraper/disk_scraper_test.go

@@ -104,40 +104,45 @@ func TestScrapeMetrics(t *testing.T) {
 
  assert.GreaterOrEqual(t, metrics.Len(), 4)
 
- assertInt64DiskMetricValid(t, metrics.At(0), diskIODescriptor, test.expectedStartTime)
- assertInt64DiskMetricValid(t, metrics.At(1), diskOpsDescriptor, test.expectedStartTime)
- assertDoubleDiskMetricValid(t, metrics.At(2), diskTimeDescriptor, test.expectedStartTime)
- assertDiskPendingOperationsMetricValid(t, metrics.At(3))
+ assertInt64DiskMetricValid(t, metrics.At(0), diskIODescriptor, true, test.expectedStartTime)
+ assertInt64DiskMetricValid(t, metrics.At(1), diskOpsDescriptor, true, test.expectedStartTime)
+ assertDoubleDiskMetricValid(t, metrics.At(2), diskIOTimeDescriptor, false, test.expectedStartTime)
+ assertDoubleDiskMetricValid(t, metrics.At(3), diskOperationTimeDescriptor, true, test.expectedStartTime)
+ assertDiskPendingOperationsMetricValid(t, metrics.At(4))
 
  if runtime.GOOS == "linux" {
- assertInt64DiskMetricValid(t, metrics.At(4), diskMergedDescriptor, test.expectedStartTime)
+ assertInt64DiskMetricValid(t, metrics.At(5), diskMergedDescriptor, true, test.expectedStartTime)
  }
 
  internal.AssertSameTimeStampForAllMetrics(t, metrics)
  })
  }
 }
 
-func assertInt64DiskMetricValid(t *testing.T, metric pdata.Metric, expectedDescriptor pdata.Metric, startTime pdata.TimestampUnixNano) {
+func assertInt64DiskMetricValid(t *testing.T, metric pdata.Metric, expectedDescriptor pdata.Metric, expectDirectionLabels bool, startTime pdata.TimestampUnixNano) {
  internal.AssertDescriptorEqual(t, expectedDescriptor, metric)
  if startTime != 0 {
  internal.AssertIntSumMetricStartTimeEquals(t, metric, startTime)
  }
  assert.GreaterOrEqual(t, metric.IntSum().DataPoints().Len(), 2)
  internal.AssertIntSumMetricLabelExists(t, metric, 0, deviceLabelName)
- internal.AssertIntSumMetricLabelHasValue(t, metric, 0, directionLabelName, readDirectionLabelValue)
- internal.AssertIntSumMetricLabelHasValue(t, metric, 1, directionLabelName, writeDirectionLabelValue)
+ if expectDirectionLabels {
+ internal.AssertIntSumMetricLabelHasValue(t, metric, 0, directionLabelName, readDirectionLabelValue)
+ internal.AssertIntSumMetricLabelHasValue(t, metric, 1, directionLabelName, writeDirectionLabelValue)
+ }
 }
 
-func assertDoubleDiskMetricValid(t *testing.T, metric pdata.Metric, expectedDescriptor pdata.Metric, startTime pdata.TimestampUnixNano) {
+func assertDoubleDiskMetricValid(t *testing.T, metric pdata.Metric, expectedDescriptor pdata.Metric, expectDirectionLabels bool, startTime pdata.TimestampUnixNano) {
  internal.AssertDescriptorEqual(t, expectedDescriptor, metric)
  if startTime != 0 {
  internal.AssertDoubleSumMetricStartTimeEquals(t, metric, startTime)
  }
  assert.GreaterOrEqual(t, metric.DoubleSum().DataPoints().Len(), 2)
  internal.AssertDoubleSumMetricLabelExists(t, metric, 0, deviceLabelName)
- internal.AssertDoubleSumMetricLabelHasValue(t, metric, 0, directionLabelName, readDirectionLabelValue)
- internal.AssertDoubleSumMetricLabelHasValue(t, metric, metric.DoubleSum().DataPoints().Len()-1, directionLabelName, writeDirectionLabelValue)
+ if expectDirectionLabels {
+ internal.AssertDoubleSumMetricLabelHasValue(t, metric, 0, directionLabelName, readDirectionLabelValue)
+ internal.AssertDoubleSumMetricLabelHasValue(t, metric, metric.DoubleSum().DataPoints().Len()-1, directionLabelName, writeDirectionLabelValue)
+ }
 }
 
 func assertDiskPendingOperationsMetricValid(t *testing.T, metric pdata.Metric) {

receiver/hostmetricsreceiver/internal/scraper/diskscraper/disk_scraper_windows.go

@@ -36,6 +36,8 @@ const (
  readBytesPerSec = "Disk Read Bytes/sec"
  writeBytesPerSec = "Disk Write Bytes/sec"
 
+ idleTime = "% Idle Time"
+
  avgDiskSecsPerRead = "Avg. Disk sec/Read"
  avgDiskSecsPerWrite = "Avg. Disk sec/Write"
 
@@ -114,17 +116,18 @@ func (s *scraper) ScrapeMetrics(ctx context.Context) (pdata.MetricSlice, error)
  // filter devices by name
  logicalDiskObject.Filter(s.includeFS, s.excludeFS, false)
 
- logicalDiskCounterValues, err := logicalDiskObject.GetValues(readsPerSec, writesPerSec, readBytesPerSec, writeBytesPerSec, avgDiskSecsPerRead, avgDiskSecsPerWrite, queueLength)
+ logicalDiskCounterValues, err := logicalDiskObject.GetValues(readsPerSec, writesPerSec, readBytesPerSec, writeBytesPerSec, idleTime, avgDiskSecsPerRead, avgDiskSecsPerWrite, queueLength)
  if err != nil {
  return metrics, err
  }
 
  if len(logicalDiskCounterValues) > 0 {
- metrics.Resize(4)
+ metrics.Resize(5)
  initializeDiskIOMetric(metrics.At(0), s.startTime, now, logicalDiskCounterValues)
  initializeDiskOpsMetric(metrics.At(1), s.startTime, now, logicalDiskCounterValues)
- initializeDiskTimeMetric(metrics.At(2), s.startTime, now, logicalDiskCounterValues)
- initializeDiskPendingOperationsMetric(metrics.At(3), now, logicalDiskCounterValues)
+ initializeDiskIOTimeMetric(metrics.At(2), s.startTime, now, logicalDiskCounterValues)
+ initializeDiskOperationTimeMetric(metrics.At(3), s.startTime, now, logicalDiskCounterValues)
+ initializeDiskPendingOperationsMetric(metrics.At(4), now, logicalDiskCounterValues)
  }
 
  return metrics, nil
@@ -152,8 +155,19 @@ func initializeDiskOpsMetric(metric pdata.Metric, startTime, now pdata.Timestamp
  }
 }
 
-func initializeDiskTimeMetric(metric pdata.Metric, startTime, now pdata.TimestampUnixNano, logicalDiskCounterValues []*perfcounters.CounterValues) {
- diskTimeDescriptor.CopyTo(metric)
+func initializeDiskIOTimeMetric(metric pdata.Metric, startTime, now pdata.TimestampUnixNano, logicalDiskCounterValues []*perfcounters.CounterValues) {
+ diskIOTimeDescriptor.CopyTo(metric)
+
+ ddps := metric.DoubleSum().DataPoints()
+ ddps.Resize(len(logicalDiskCounterValues))
+ for idx, logicalDiskCounter := range logicalDiskCounterValues {
+ // disk active time = system boot time - disk idle time
+ initializeDoubleDataPoint(ddps.At(idx), startTime, now, logicalDiskCounter.InstanceName, "", float64(now-startTime)/1e9-float64(logicalDiskCounter.Values[idleTime])/1e7)
+ }
+}
+
+func initializeDiskOperationTimeMetric(metric pdata.Metric, startTime, now pdata.TimestampUnixNano, logicalDiskCounterValues []*perfcounters.CounterValues) {
+ diskOperationTimeDescriptor.CopyTo(metric)
 
  ddps := metric.DoubleSum().DataPoints()
  ddps.Resize(2 * len(logicalDiskCounterValues))
@@ -176,7 +190,9 @@ func initializeDiskPendingOperationsMetric(metric pdata.Metric, now pdata.Timest
 func initializeInt64DataPoint(dataPoint pdata.IntDataPoint, startTime, now pdata.TimestampUnixNano, deviceLabel string, directionLabel string, value int64) {
  labelsMap := dataPoint.LabelsMap()
  labelsMap.Insert(deviceLabelName, deviceLabel)
- labelsMap.Insert(directionLabelName, directionLabel)
+ if directionLabel != "" {
+ labelsMap.Insert(directionLabelName, directionLabel)
+ }
  dataPoint.SetStartTime(startTime)
  dataPoint.SetTimestamp(now)
  dataPoint.SetValue(value)
@@ -185,7 +201,9 @@ func initializeInt64DataPoint(dataPoint pdata.IntDataPoint, startTime, now pdata
 func initializeDoubleDataPoint(dataPoint pdata.DoubleDataPoint, startTime, now pdata.TimestampUnixNano, deviceLabel string, directionLabel string, value float64) {
  labelsMap := dataPoint.LabelsMap()
  labelsMap.Insert(deviceLabelName, deviceLabel)
- labelsMap.Insert(directionLabelName, directionLabel)
+ if directionLabel != "" {
+ labelsMap.Insert(directionLabelName, directionLabel)
+ }
  dataPoint.SetStartTime(startTime)
  dataPoint.SetTimestamp(now)
  dataPoint.SetValue(value)