proxy: Add tls_config_last_reload_seconds metric (linkerd#1204)

Depends on linkerd#1141.

This PR adds a `tls_config_last_reload_seconds` Prometheus metric
that reports the last time the TLS configuration files were reloaded.

Proof that it works:

Started the proxy with no certs, then generated them:
```
➜ http GET localhost:4191/metrics
HTTP/1.1 200 OK
content-encoding: gzip
content-length: 323
content-type: text/plain
date: Mon, 25 Jun 2018 23:02:52 GMT

# HELP tls_config_reload_total Total number of times the proxy's TLS config files were reloaded.
# TYPE tls_config_reload_total counter
tls_config_reload_total{status="io_error",path="example-example.crt",error_code="2"} 9
tls_config_reload_total{status="reloaded"} 3
# HELP tls_config_last_reload_seconds Timestamp of when the TLS configuration files were last reloaded successfully (in seconds since the UNIX epoch)
# TYPE tls_config_last_reload_seconds gauge
tls_config_last_reload_seconds 1529967764
# HELP process_start_time_seconds Time that the process started (in seconds since the UNIX epoch)
# TYPE process_start_time_seconds gauge
process_start_time_seconds 1529967754
```

Started the proxy with certs already present:
```
➜ http GET localhost:4191/metrics
HTTP/1.1 200 OK
content-encoding: gzip
content-length: 285
content-type: text/plain
date: Mon, 25 Jun 2018 23:04:39 GMT

# HELP tls_config_reload_total Total number of times the proxy's TLS config files were reloaded.
# TYPE tls_config_reload_total counter
tls_config_reload_total{status="reloaded"} 4
# HELP tls_config_last_reload_seconds Timestamp of when the TLS configuration files were last reloaded successfully (in seconds since the UNIX epoch)
# TYPE tls_config_last_reload_seconds gauge
tls_config_last_reload_seconds 1529967876
# HELP process_start_time_seconds Time that the process started (in seconds since the UNIX epoch)
# TYPE process_start_time_seconds gauge
process_start_time_seconds 1529967874
```

Signed-off-by: Eliza Weisman <[email protected]>

proxy/src/telemetry/event.rs

@@ -1,5 +1,5 @@
 use std::sync::Arc;
-use std::time::{Duration, Instant};
+use std::time::{Duration, Instant, SystemTime};
 
 use h2;
 
@@ -18,7 +18,7 @@ pub enum Event {
  StreamResponseFail(Arc<ctx::http::Response>, StreamResponseFail),
  StreamResponseEnd(Arc<ctx::http::Response>, StreamResponseEnd),
 
- TlsConfigReloaded,
+ TlsConfigReloaded(SystemTime),
  TlsConfigReloadFailed(::transport::tls::ConfigError),
 }
 

proxy/src/telemetry/metrics/labels/mod.rs

@@ -301,7 +301,6 @@ impl fmt::Display for Direction {
  }
 }
 
-
 // ===== impl DstLabels ====
 
 impl DstLabels {

proxy/src/telemetry/metrics/mod.rs

@@ -114,6 +114,7 @@ struct Root {
  transport_closes: transport::CloseScopes,
 
  tls_config: TlsConfigScopes,
+ tls_config_last_reload_seconds: Option<Gauge>,
 
  process_metrics: Option<process::Sensor>,
 
@@ -184,6 +185,10 @@ impl Root {
  metrics! {
  process_start_time_seconds: Gauge {
  "Time that the process started (in seconds since the UNIX epoch)"
+ },
+ tls_config_last_reload_seconds: Gauge {
+ "Timestamp of when the TLS configuration files were last reloaded \
+ successfully (in seconds since the UNIX epoch)"
  }
  }
 
@@ -249,6 +254,11 @@ impl fmt::Display for Root {
  self.transport_closes.fmt(f)?;
  self.tls_config.fmt(f)?;
 
+ if let Some(timestamp) = self.tls_config_last_reload_seconds {
+ Self::tls_config_last_reload_seconds.fmt_help(f)?;
+ Self::tls_config_last_reload_seconds.fmt_metric(f, timestamp)?;
+ }
+
  if let Some(ref process_metrics) = self.process_metrics {
  match process_metrics.metrics() {
  Ok(process) => process.fmt(f)?,
@@ -284,7 +294,6 @@ impl fmt::Display for TlsConfigScopes {
  }
 }
 
-
 // ===== impl Stamped =====
 
 impl<T> Stamped<T> {

proxy/src/telemetry/metrics/record.rs

@@ -9,6 +9,7 @@ use super::labels::{
  TransportCloseLabels,
  TlsConfigLabels,
 };
+use std::time::UNIX_EPOCH;
 
 /// Tracks Prometheus metrics
 #[derive(Debug)]
@@ -84,8 +85,14 @@ impl Record {
  })
  },
 
- Event::TlsConfigReloaded =>
+ Event::TlsConfigReloaded(ref when) =>
  self.update(|metrics| {
+ let timestamp_secs = when
+ .duration_since(UNIX_EPOCH)
+ .expect("SystemTime before UNIX_EPOCH!")
+ .as_secs()
+ .into();
+ metrics.tls_config_last_reload_seconds = Some(timestamp_secs);
  metrics.tls_config(TlsConfigLabels::success()).incr();
  }),
 

proxy/src/telemetry/sensor/mod.rs

@@ -105,7 +105,8 @@ impl Sensors {
 impl TlsConfig {
 
  pub fn reloaded(&mut self) {
- self.0.send(|| event::Event::TlsConfigReloaded)
+ use std::time::SystemTime;
+ self.0.send(|| event::Event::TlsConfigReloaded(SystemTime::now()))
  }
 
  pub fn failed(&mut self, err: tls::ConfigError) {