Fix openssl engine not being able to load private key.

Closes #2066. Thanks to Boris Vanhoof.

ChangeLog.txt

@@ -2,6 +2,7 @@ Broker:
 - Fix exporting of executable symbols on BSD when building via makefile.
 - Fix some minor memory leaks on exit only.
 - Fix possible memory leak on connect. Closes #2057.
+- Fix openssl engine not being able to load private key. Closes #2066.
 
 Clients:
 - Fix config files truncating options after the first space. Closes #2059.

src/net.c

@@ -458,11 +458,6 @@ int net__load_crl_file(struct mosquitto__listener *listener)
 int net__load_certificates(struct mosquitto__listener *listener)
 {
 #ifdef WITH_TLS
-	ENGINE *engine = NULL;
-#  if !defined(OPENSSL_NO_ENGINE)
-	UI_METHOD *ui_method;
-	EVP_PKEY *pkey;
-#  endif
 	int rc;
 
 	if(listener->require_certificate){
@@ -474,25 +469,69 @@ int net__load_certificates(struct mosquitto__listener *listener)
 	if(rc != 1){
 		log__printf(NULL, MOSQ_LOG_ERR, "Error: Unable to load server certificate \"%s\". Check certfile.", listener->certfile);
 		net__print_ssl_error(NULL);
-#if !defined(OPENSSL_NO_ENGINE)
-		ENGINE_FINISH(engine);
+		return MOSQ_ERR_TLS;
+	}
+	if(listener->tls_engine == NULL){
+		rc = SSL_CTX_use_PrivateKey_file(listener->ssl_ctx, listener->keyfile, SSL_FILETYPE_PEM);
+		if(rc != 1){
+			log__printf(NULL, MOSQ_LOG_ERR, "Error: Unable to load server key file \"%s\". Check keyfile.", listener->keyfile);
+			net__print_ssl_error(NULL);
+			return MOSQ_ERR_TLS;
+		}
+	}
+	rc = SSL_CTX_check_private_key(listener->ssl_ctx);
+	if(rc != 1){
+		log__printf(NULL, MOSQ_LOG_ERR, "Error: Server certificate/key are inconsistent.");
+		net__print_ssl_error(NULL);
+		return MOSQ_ERR_TLS;
+	}
+	/* Load CRLs if they exist. */
+	if(listener->crlfile){
+		rc = net__load_crl_file(listener);
+		if(rc){
+			return rc;
+		}
+	}
 #endif
+	return MOSQ_ERR_SUCCESS;
+}
+
+
+static int net__load_engine(struct mosquitto__listener *listener)
+{
+#if defined(WITH_TLS) && !defined(OPENSSL_NO_ENGINE)
+	ENGINE *engine = NULL;
+	UI_METHOD *ui_method;
+	EVP_PKEY *pkey;
+
+	if(!listener->tls_engine){
+		return MOSQ_ERR_SUCCESS;
+	}
+
+	engine = ENGINE_by_id(listener->tls_engine);
+	if(!engine){
+		log__printf(NULL, MOSQ_LOG_ERR, "Error loading %s engine\n", listener->tls_engine);
+		net__print_ssl_error(NULL);
 		return MOSQ_ERR_TLS;
 	}
-	if(listener->tls_engine && listener->tls_keyform == mosq_k_engine){
-#if !defined(OPENSSL_NO_ENGINE)
+	if(!ENGINE_init(engine)){
+		log__printf(NULL, MOSQ_LOG_ERR, "Failed engine initialisation\n");
+		net__print_ssl_error(NULL);
+		return MOSQ_ERR_TLS;
+	}
+	ENGINE_set_default(engine, ENGINE_METHOD_ALL);
+
+	if(listener->tls_keyform == mosq_k_engine){
 		ui_method = net__get_ui_method();
 		if(listener->tls_engine_kpass_sha1){
 			if(!ENGINE_ctrl_cmd(engine, ENGINE_SECRET_MODE, ENGINE_SECRET_MODE_SHA, NULL, NULL, 0)){
 				log__printf(NULL, MOSQ_LOG_ERR, "Error: Unable to set engine secret mode sha");
 				net__print_ssl_error(NULL);
-				ENGINE_FINISH(engine);
 				return MOSQ_ERR_TLS;
 			}
 			if(!ENGINE_ctrl_cmd(engine, ENGINE_PIN, 0, listener->tls_engine_kpass_sha1, NULL, 0)){
 				log__printf(NULL, MOSQ_LOG_ERR, "Error: Unable to set engine pin");
 				net__print_ssl_error(NULL);
-				ENGINE_FINISH(engine);
 				return MOSQ_ERR_TLS;
 			}
 			ui_method = NULL;
@@ -501,47 +540,17 @@ int net__load_certificates(struct mosquitto__listener *listener)
 		if(!pkey){
 			log__printf(NULL, MOSQ_LOG_ERR, "Error: Unable to load engine private key file \"%s\".", listener->keyfile);
 			net__print_ssl_error(NULL);
-			ENGINE_FINISH(engine);
 			return MOSQ_ERR_TLS;
 		}
 		if(SSL_CTX_use_PrivateKey(listener->ssl_ctx, pkey) <= 0){
 			log__printf(NULL, MOSQ_LOG_ERR, "Error: Unable to use engine private key file \"%s\".", listener->keyfile);
 			net__print_ssl_error(NULL);
-			ENGINE_FINISH(engine);
-			return MOSQ_ERR_TLS;
-		}
-#endif
-	}else{
-		rc = SSL_CTX_use_PrivateKey_file(listener->ssl_ctx, listener->keyfile, SSL_FILETYPE_PEM);
-		if(rc != 1){
-			log__printf(NULL, MOSQ_LOG_ERR, "Error: Unable to load server key file \"%s\". Check keyfile.", listener->keyfile);
-			net__print_ssl_error(NULL);
-#if !defined(OPENSSL_NO_ENGINE)
-			ENGINE_FINISH(engine);
-#endif
 			return MOSQ_ERR_TLS;
 		}
 	}
-	rc = SSL_CTX_check_private_key(listener->ssl_ctx);
-	if(rc != 1){
-		log__printf(NULL, MOSQ_LOG_ERR, "Error: Server certificate/key are inconsistent.");
-		net__print_ssl_error(NULL);
-#if !defined(OPENSSL_NO_ENGINE)
-		ENGINE_FINISH(engine);
-#endif
-		return MOSQ_ERR_TLS;
-	}
-	/* Load CRLs if they exist. */
-	if(listener->crlfile){
-		rc = net__load_crl_file(listener);
-		if(rc){
-#if !defined(OPENSSL_NO_ENGINE)
-			ENGINE_FINISH(engine);
-#endif
-			return rc;
-		}
-	}
+	ENGINE_free(engine); /* release the structural reference from ENGINE_by_id() */
 #endif
+
 	return MOSQ_ERR_SUCCESS;
 }
 
@@ -549,7 +558,6 @@ int net__load_certificates(struct mosquitto__listener *listener)
 int net__tls_load_verify(struct mosquitto__listener *listener)
 {
 #ifdef WITH_TLS
-	ENGINE *engine = NULL;
 	int rc;
 
 #if OPENSSL_VERSION_NUMBER < 0x30000000L
@@ -584,23 +592,8 @@ int net__tls_load_verify(struct mosquitto__listener *listener)
 	}
 #endif
 
-	if(listener->tls_engine){
-#if !defined(OPENSSL_NO_ENGINE)
-		engine = ENGINE_by_id(listener->tls_engine);
-		if(!engine){
-			log__printf(NULL, MOSQ_LOG_ERR, "Error loading %s engine\n", listener->tls_engine);
-			net__print_ssl_error(NULL);
-			return MOSQ_ERR_TLS;
-		}
-		if(!ENGINE_init(engine)){
-			log__printf(NULL, MOSQ_LOG_ERR, "Failed engine initialisation\n");
-			net__print_ssl_error(NULL);
-			ENGINE_free(engine);
-			return MOSQ_ERR_TLS;
-		}
-		ENGINE_set_default(engine, ENGINE_METHOD_ALL);
-		ENGINE_free(engine); /* release the structural reference from ENGINE_by_id() */
-#endif
+	if(net__load_engine(listener)){
+		return MOSQ_ERR_TLS;
 	}
 #endif
 	return net__load_certificates(listener);