common/crypto_openssl: load legacy provider in decrypt_rfbdes()

Commit f135856 added the same code to encrypt_rfbdes() Signed-off-by: Christian Hitz <[email protected]>
LibVNC · Mar 28, 2024 · 6e777e6 · 6e777e6
1 parent 042a816
commit 6e777e6
Showing 1 changed file with 23 additions and 2 deletions.
diff --git a/src/common/crypto_openssl.c b/src/common/crypto_openssl.c
@@ -115,24 +115,45 @@ int encrypt_rfbdes(void *out, int *out_len, const unsigned char key[8], const vo
 int decrypt_rfbdes(void *out, int *out_len, const unsigned char key[8], const void *in, const size_t in_len)
 {
     int result = 0;
-    EVP_CIPHER_CTX *des;
+    EVP_CIPHER_CTX *des = NULL;
     unsigned char mungedkey[8];
     int i;
+#if (OPENSSL_VERSION_NUMBER >= 0x30000000L)
+    OSSL_PROVIDER *providerLegacy = NULL;
+    OSSL_PROVIDER *providerDefault = NULL;
+#endif
 
     for (i = 0; i < 8; i++)
       mungedkey[i] = reverseByte(key[i]);
 
+#if (OPENSSL_VERSION_NUMBER >= 0x30000000L)
+    /* Load Multiple providers into the default (NULL) library context */
+    if (!(providerLegacy = OSSL_PROVIDER_load(NULL, "legacy")))
+	goto out;
+    if (!(providerDefault = OSSL_PROVIDER_load(NULL, "default")))
+	goto out;
+#endif
+
     if(!(des = EVP_CIPHER_CTX_new()))
 	goto out;
     if(!EVP_DecryptInit_ex(des, EVP_des_ecb(), NULL, mungedkey, NULL))
 	goto out;
+    if(!EVP_CIPHER_CTX_set_padding(des, 0))
+	goto out;
     if(!EVP_DecryptUpdate(des, out, out_len, in, in_len))
 	goto out;
 
     result = 1;
 
  out:
-    EVP_CIPHER_CTX_free(des);
+    if (des)
+	EVP_CIPHER_CTX_free(des);
+#if (OPENSSL_VERSION_NUMBER >= 0x30000000L)
+    if (providerLegacy)
+	OSSL_PROVIDER_unload(providerLegacy);
+    if (providerDefault)
+	OSSL_PROVIDER_unload(providerDefault);
+#endif
     return result;
 }