Fix openssl engine not being able to load private key.

Closes #2066. Thanks to Boris Vanhoof.
eclipse · Feb 3, 2021 · 4a738f1 · 4a738f1
1 parent 7a3b69f
commit 4a738f1
Show file tree

Hide file tree

Showing 2 changed files with 55 additions and 61 deletions.
diff --git a/ChangeLog.txt b/ChangeLog.txt
@@ -2,6 +2,7 @@ Broker:
 - Fix exporting of executable symbols on BSD when building via makefile.
 - Fix some minor memory leaks on exit only.
 - Fix possible memory leak on connect. Closes #2057.
+- Fix openssl engine not being able to load private key. Closes #2066.
 
 Clients:
 - Fix config files truncating options after the first space. Closes #2059.

diff --git a/src/net.c b/src/net.c
@@ -458,11 +458,6 @@ int net__load_crl_file(struct mosquitto__listener *listener)
 int net__load_certificates(struct mosquitto__listener *listener)
 {
 #ifdef WITH_TLS
- ENGINE *engine = NULL;
-# if !defined(OPENSSL_NO_ENGINE)
- UI_METHOD *ui_method;
- EVP_PKEY *pkey;
-# endif
  int rc;
 
  if(listener->require_certificate){
@@ -474,25 +469,69 @@ int net__load_certificates(struct mosquitto__listener *listener)
  if(rc != 1){
  log__printf(NULL, MOSQ_LOG_ERR, "Error: Unable to load server certificate \"%s\". Check certfile.", listener->certfile);
  net__print_ssl_error(NULL);
-#if !defined(OPENSSL_NO_ENGINE)
- ENGINE_FINISH(engine);
+ return MOSQ_ERR_TLS;
+ }
+ if(listener->tls_engine == NULL){
+ rc = SSL_CTX_use_PrivateKey_file(listener->ssl_ctx, listener->keyfile, SSL_FILETYPE_PEM);
+ if(rc != 1){
+ log__printf(NULL, MOSQ_LOG_ERR, "Error: Unable to load server key file \"%s\". Check keyfile.", listener->keyfile);
+ net__print_ssl_error(NULL);
+ return MOSQ_ERR_TLS;
+ }
+ }
+ rc = SSL_CTX_check_private_key(listener->ssl_ctx);
+ if(rc != 1){
+ log__printf(NULL, MOSQ_LOG_ERR, "Error: Server certificate/key are inconsistent.");
+ net__print_ssl_error(NULL);
+ return MOSQ_ERR_TLS;
+ }
+ /* Load CRLs if they exist. */
+ if(listener->crlfile){
+ rc = net__load_crl_file(listener);
+ if(rc){
+ return rc;
+ }
+ }
 #endif
+ return MOSQ_ERR_SUCCESS;
+}
+
+
+static int net__load_engine(struct mosquitto__listener *listener)
+{
+#if defined(WITH_TLS) && !defined(OPENSSL_NO_ENGINE)
+ ENGINE *engine = NULL;
+ UI_METHOD *ui_method;
+ EVP_PKEY *pkey;
+
+ if(!listener->tls_engine){
+ return MOSQ_ERR_SUCCESS;
+ }
+
+ engine = ENGINE_by_id(listener->tls_engine);
+ if(!engine){
+ log__printf(NULL, MOSQ_LOG_ERR, "Error loading %s engine\n", listener->tls_engine);
+ net__print_ssl_error(NULL);
  return MOSQ_ERR_TLS;
  }
- if(listener->tls_engine && listener->tls_keyform == mosq_k_engine){
-#if !defined(OPENSSL_NO_ENGINE)
+ if(!ENGINE_init(engine)){
+ log__printf(NULL, MOSQ_LOG_ERR, "Failed engine initialisation\n");
+ net__print_ssl_error(NULL);
+ return MOSQ_ERR_TLS;
+ }
+ ENGINE_set_default(engine, ENGINE_METHOD_ALL);
+
+ if(listener->tls_keyform == mosq_k_engine){
  ui_method = net__get_ui_method();
  if(listener->tls_engine_kpass_sha1){
  if(!ENGINE_ctrl_cmd(engine, ENGINE_SECRET_MODE, ENGINE_SECRET_MODE_SHA, NULL, NULL, 0)){
  log__printf(NULL, MOSQ_LOG_ERR, "Error: Unable to set engine secret mode sha");
  net__print_ssl_error(NULL);
- ENGINE_FINISH(engine);
  return MOSQ_ERR_TLS;
  }
  if(!ENGINE_ctrl_cmd(engine, ENGINE_PIN, 0, listener->tls_engine_kpass_sha1, NULL, 0)){
  log__printf(NULL, MOSQ_LOG_ERR, "Error: Unable to set engine pin");
  net__print_ssl_error(NULL);
- ENGINE_FINISH(engine);
  return MOSQ_ERR_TLS;
  }
  ui_method = NULL;
@@ -501,55 +540,24 @@ int net__load_certificates(struct mosquitto__listener *listener)
  if(!pkey){
  log__printf(NULL, MOSQ_LOG_ERR, "Error: Unable to load engine private key file \"%s\".", listener->keyfile);
  net__print_ssl_error(NULL);
- ENGINE_FINISH(engine);
  return MOSQ_ERR_TLS;
  }
  if(SSL_CTX_use_PrivateKey(listener->ssl_ctx, pkey) <= 0){
  log__printf(NULL, MOSQ_LOG_ERR, "Error: Unable to use engine private key file \"%s\".", listener->keyfile);
  net__print_ssl_error(NULL);
- ENGINE_FINISH(engine);
- return MOSQ_ERR_TLS;
- }
-#endif
- }else{
- rc = SSL_CTX_use_PrivateKey_file(listener->ssl_ctx, listener->keyfile, SSL_FILETYPE_PEM);
- if(rc != 1){
- log__printf(NULL, MOSQ_LOG_ERR, "Error: Unable to load server key file \"%s\". Check keyfile.", listener->keyfile);
- net__print_ssl_error(NULL);
-#if !defined(OPENSSL_NO_ENGINE)
- ENGINE_FINISH(engine);
-#endif
  return MOSQ_ERR_TLS;
  }
  }
- rc = SSL_CTX_check_private_key(listener->ssl_ctx);
- if(rc != 1){
- log__printf(NULL, MOSQ_LOG_ERR, "Error: Server certificate/key are inconsistent.");
- net__print_ssl_error(NULL);
-#if !defined(OPENSSL_NO_ENGINE)
- ENGINE_FINISH(engine);
-#endif
- return MOSQ_ERR_TLS;
- }
- /* Load CRLs if they exist. */
- if(listener->crlfile){
- rc = net__load_crl_file(listener);
- if(rc){
-#if !defined(OPENSSL_NO_ENGINE)
- ENGINE_FINISH(engine);
-#endif
- return rc;
- }
- }
+ ENGINE_free(engine); /* release the structural reference from ENGINE_by_id() */
 #endif
+
  return MOSQ_ERR_SUCCESS;
 }
 
 
 int net__tls_load_verify(struct mosquitto__listener *listener)
 {
 #ifdef WITH_TLS
- ENGINE *engine = NULL;
  int rc;
 
 #if OPENSSL_VERSION_NUMBER < 0x30000000L
@@ -584,23 +592,8 @@ int net__tls_load_verify(struct mosquitto__listener *listener)
  }
 #endif
 
- if(listener->tls_engine){
-#if !defined(OPENSSL_NO_ENGINE)
- engine = ENGINE_by_id(listener->tls_engine);
- if(!engine){
- log__printf(NULL, MOSQ_LOG_ERR, "Error loading %s engine\n", listener->tls_engine);
- net__print_ssl_error(NULL);
- return MOSQ_ERR_TLS;
- }
- if(!ENGINE_init(engine)){
- log__printf(NULL, MOSQ_LOG_ERR, "Failed engine initialisation\n");
- net__print_ssl_error(NULL);
- ENGINE_free(engine);
- return MOSQ_ERR_TLS;
- }
- ENGINE_set_default(engine, ENGINE_METHOD_ALL);
- ENGINE_free(engine); /* release the structural reference from ENGINE_by_id() */
-#endif
+ if(net__load_engine(listener)){
+ return MOSQ_ERR_TLS;
  }
 #endif
  return net__load_certificates(listener);