Merge pull request #8899 from Icinga/feature/icingadb-connect_timeout

Introduce IcingaDB#connect_timeout

doc/09-object-types.md

@@ -1418,6 +1418,7 @@ Configuration Attributes:
  cipher\_list | String | **Optional.** Cipher list that is allowed. For a list of available ciphers run `openssl ciphers`. Defaults to `ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384:AES256-GCM-SHA384:AES128-GCM-SHA256`.
  tls\_protocolmin | String | **Optional.** Minimum TLS protocol version. Defaults to `TLSv1.2`.
  insecure\_noverify | Boolean | **Optional.** Whether not to verify the peer.
+ connect\_timeout | Number | **Optional.** Timeout for establishing new connections. Within this time, the TCP, TLS (if enabled) and Redis handshakes must complete. Defaults to `15s`.
 
 ### IdoMySqlConnection <a id="objecttype-idomysqlconnection"></a>
 

lib/base/tlsutility.hpp

@@ -28,6 +28,7 @@ namespace icinga
 const char * const DEFAULT_TLS_CIPHERS = "ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384:AES256-GCM-SHA384:AES128-GCM-SHA256";
 
 const char * const DEFAULT_TLS_PROTOCOLMIN = "TLSv1.2";
+const unsigned int DEFAULT_CONNECT_TIMEOUT = 15;
 
 void InitializeOpenSSL();
 

lib/icingadb/icingadb.cpp

@@ -66,7 +66,7 @@ void IcingaDB::Start(bool runtimeCreated)
 
  m_Rcon = new RedisConnection(GetHost(), GetPort(), GetPath(), GetPassword(), GetDbIndex(),
  GetEnableTls(), GetInsecureNoverify(), GetCertPath(), GetKeyPath(), GetCaPath(), GetCrlPath(),
- GetTlsProtocolmin(), GetCipherList(), GetDebugInfo());
+ GetTlsProtocolmin(), GetCipherList(), GetConnectTimeout(), GetDebugInfo());
 
  auto connectedCallback ([this](boost::asio::yield_context& yc) {
  m_WorkQueue.Enqueue([this]() { OnConnectedHandler(); });
@@ -89,7 +89,7 @@ void IcingaDB::Start(bool runtimeCreated)
 
  m_Rcons[ctype] = new RedisConnection(GetHost(), GetPort(), GetPath(), GetPassword(), GetDbIndex(),
  GetEnableTls(), GetInsecureNoverify(), GetCertPath(), GetKeyPath(), GetCaPath(), GetCrlPath(),
- GetTlsProtocolmin(), GetCipherList(), GetDebugInfo(), m_Rcon);
+ GetTlsProtocolmin(), GetCipherList(), GetConnectTimeout(), GetDebugInfo(), m_Rcon);
  }
 
  m_StatsTimer = new Timer();
@@ -175,6 +175,15 @@ void IcingaDB::ValidateTlsProtocolmin(const Lazy<String>& lvalue, const Validati
  }
 }
 
+void IcingaDB::ValidateConnectTimeout(const Lazy<double>& lvalue, const ValidationUtils& utils)
+{
+ ObjectImpl<IcingaDB>::ValidateConnectTimeout(lvalue, utils);
+
+ if (lvalue() <= 0) {
+ BOOST_THROW_EXCEPTION(ValidationError(this, { "connect_timeout" }, "Value must be greater than 0."));
+ }
+}
+
 void IcingaDB::AssertOnWorkQueue()
 {
  ASSERT(m_WorkQueue.IsWorkerThread());

lib/icingadb/icingadb.hpp

@@ -41,6 +41,7 @@ class IcingaDB : public ObjectImpl<IcingaDB>
 
 protected:
  void ValidateTlsProtocolmin(const Lazy<String>& lvalue, const ValidationUtils& utils) override;
+ void ValidateConnectTimeout(const Lazy<double>& lvalue, const ValidationUtils& utils) override;
 
 private:
  class DumpedGlobals

lib/icingadb/icingadb.ti

@@ -40,6 +40,10 @@ class IcingaDB : ConfigObject
  [config] String tls_protocolmin {
  default {{{ return DEFAULT_TLS_PROTOCOLMIN; }}}
  };
+
+ [config] double connect_timeout {
+ default {{{ return DEFAULT_CONNECT_TIMEOUT; }}}
+ };
 };
 
 }

lib/icingadb/redisconnection.cpp

@@ -32,19 +32,19 @@ boost::regex RedisConnection::m_ErrAuth ("\\AERR AUTH ");
 
 RedisConnection::RedisConnection(const String& host, int port, const String& path, const String& password, int db,
  bool useTls, bool insecure, const String& certPath, const String& keyPath, const String& caPath, const String& crlPath,
- const String& tlsProtocolmin, const String& cipherList, DebugInfo di, const RedisConnection::Ptr& parent)
+ const String& tlsProtocolmin, const String& cipherList, double connectTimeout, DebugInfo di, const RedisConnection::Ptr& parent)
  : RedisConnection(IoEngine::Get().GetIoContext(), host, port, path, password, db,
- useTls, insecure, certPath, keyPath, caPath, crlPath, tlsProtocolmin, cipherList, std::move(di), parent)
+ useTls, insecure, certPath, keyPath, caPath, crlPath, tlsProtocolmin, cipherList, connectTimeout, std::move(di), parent)
 {
 }
 
 RedisConnection::RedisConnection(boost::asio::io_context& io, String host, int port, String path, String password,
  int db, bool useTls, bool insecure, String certPath, String keyPath, String caPath, String crlPath,
- String tlsProtocolmin, String cipherList, DebugInfo di, const RedisConnection::Ptr& parent)
+ String tlsProtocolmin, String cipherList, double connectTimeout, DebugInfo di, const RedisConnection::Ptr& parent)
  : m_Host(std::move(host)), m_Port(port), m_Path(std::move(path)), m_Password(std::move(password)),
  m_DbIndex(db), m_CertPath(std::move(certPath)), m_KeyPath(std::move(keyPath)), m_Insecure(insecure),
  m_CaPath(std::move(caPath)), m_CrlPath(std::move(crlPath)), m_TlsProtocolmin(std::move(tlsProtocolmin)),
- m_CipherList(std::move(cipherList)), m_DebugInfo(std::move(di)), m_Connecting(false), m_Connected(false),
+ m_CipherList(std::move(cipherList)), m_ConnectTimeout(connectTimeout), m_DebugInfo(std::move(di)), m_Connecting(false), m_Connected(false),
  m_Started(false), m_Strand(io), m_QueuedWrites(io), m_QueuedReads(io), m_LogStatsTimer(io), m_Parent(parent)
 {
  if (useTls && m_Path.IsEmpty()) {
@@ -271,6 +271,8 @@ void RedisConnection::Connect(asio::yield_context& yc)
 
  auto conn (Shared<AsioTlsStream>::Make(m_Strand.context(), *m_TLSContext, m_Host));
  auto& tlsConn (conn->next_layer());
+ auto connectTimeout (MakeTimeout(conn));
+ Defer cancelTimeout ([&connectTimeout]() { connectTimeout->Cancel(); });
 
  if (!m_Insecure) {
  auto native (tlsConn.native_handle());
@@ -305,6 +307,9 @@ void RedisConnection::Connect(asio::yield_context& yc)
  << "Trying to connect to Redis server (async) on host '" << m_Host << ":" << m_Port << "'";
 
  auto conn (Shared<TcpConn>::Make(m_Strand.context()));
+ auto connectTimeout (MakeTimeout(conn));
+ Defer cancelTimeout ([&connectTimeout]() { connectTimeout->Cancel(); });
+
  icinga::Connect(conn->next_layer(), m_Host, Convert::ToString(m_Port), yc);
  Handshake(conn, yc);
  m_TcpConn = std::move(conn);
@@ -314,6 +319,9 @@ void RedisConnection::Connect(asio::yield_context& yc)
  << "Trying to connect to Redis server (async) on unix socket path '" << m_Path << "'";
 
  auto conn (Shared<UnixConn>::Make(m_Strand.context()));
+ auto connectTimeout (MakeTimeout(conn));
+ Defer cancelTimeout ([&connectTimeout]() { connectTimeout->Cancel(); });
+
  conn->next_layer().async_connect(Unix::endpoint(m_Path.CStr()), yc);
  Handshake(conn, yc);
  m_UnixConn = std::move(conn);

lib/icingadb/redisconnection.hpp

@@ -76,7 +76,7 @@ namespace icinga
 
  RedisConnection(const String& host, int port, const String& path, const String& password, int db,
  bool useTls, bool insecure, const String& certPath, const String& keyPath, const String& caPath, const String& crlPath,
- const String& tlsProtocolmin, const String& cipherList, DebugInfo di, const Ptr& parent = nullptr);
+ const String& tlsProtocolmin, const String& cipherList, double connectTimeout, DebugInfo di, const Ptr& parent = nullptr);
 
  void UpdateTLSContext();
 
@@ -157,7 +157,7 @@ namespace icinga
 
  RedisConnection(boost::asio::io_context& io, String host, int port, String path, String password,
  int db, bool useTls, bool insecure, String certPath, String keyPath, String caPath, String crlPath,
- String tlsProtocolmin, String cipherList, DebugInfo di, const Ptr& parent);
+ String tlsProtocolmin, String cipherList, double connectTimeout, DebugInfo di, const Ptr& parent);
 
  void Connect(boost::asio::yield_context& yc);
  void ReadLoop(boost::asio::yield_context& yc);
@@ -179,6 +179,9 @@ namespace icinga
  template<class StreamPtr>
  void Handshake(StreamPtr& stream, boost::asio::yield_context& yc);
 
+ template<class StreamPtr>
+ Timeout::Ptr MakeTimeout(StreamPtr& stream);
+
  String m_Path;
  String m_Host;
  int m_Port;
@@ -192,6 +195,7 @@ namespace icinga
  String m_CrlPath;
  String m_TlsProtocolmin;
  String m_CipherList;
+ double m_ConnectTimeout;
  DebugInfo m_DebugInfo;
 
  boost::asio::io_context::strand m_Strand;
@@ -454,6 +458,27 @@ void RedisConnection::Handshake(StreamPtr& strm, boost::asio::yield_context& yc)
  }
 }
 
+/**
+ * Creates a Timeout which cancels stream's I/O after m_ConnectTimeout
+ *
+ * @param stream Redis server connection
+ */
+template<class StreamPtr>
+Timeout::Ptr RedisConnection::MakeTimeout(StreamPtr& stream)
+{
+ Ptr keepAlive (this);
+
+ return new Timeout(
+ m_Strand.context(),
+ m_Strand,
+ boost::posix_time::microseconds(intmax_t(m_ConnectTimeout * 1000000)),
+ [keepAlive, stream](boost::asio::yield_context yc) {
+ boost::system::error_code ec;
+ stream->lowest_layer().cancel(ec);
+ }
+ );
+}
+
 /**
  * Read a Redis protocol value from stream
  *

lib/remote/apilistener.ti

@@ -45,7 +45,7 @@ class ApiListener : ConfigObject
  };
 
  [config] double connect_timeout {
- default {{{ return 15.0; }}}
+ default {{{ return DEFAULT_CONNECT_TIMEOUT; }}}
  };
 
  [config, no_user_view, no_user_modify] String ticket_salt;