diff --git a/src/Backend/OpenSSL.php b/src/Backend/OpenSSL.php
index 1fffc52a..a49fe2f8 100644
--- a/src/Backend/OpenSSL.php
+++ b/src/Backend/OpenSSL.php
@@ -6,7 +6,7 @@
 
 use SimpleSAML\XMLSecurity\Constants as C;
 use SimpleSAML\XMLSecurity\Exception\InvalidArgumentException;
-use SimpleSAML\XMLSecurity\Exception\RuntimeException;
+use SimpleSAML\XMLSecurity\Exception\OpenSSLException;
 use SimpleSAML\XMLSecurity\Key\AsymmetricKey;
 use SimpleSAML\XMLSecurity\Key\KeyInterface;
 use SimpleSAML\XMLSecurity\Key\PrivateKey;
@@ -17,7 +17,6 @@
 use function openssl_cipher_iv_length;
 use function openssl_decrypt;
 use function openssl_encrypt;
-use function openssl_error_string;
 use function openssl_sign;
 use function openssl_verify;
 use function ord;
@@ -73,7 +72,7 @@ public function __construct()
      * @param string $plaintext The original text to encrypt.
      *
      * @return string The encrypted plaintext (ciphertext).
-     * @throws \SimpleSAML\XMLSecurity\Exception\RuntimeException If there is an error while encrypting the plaintext.
+     * @throws \SimpleSAML\XMLSecurity\Exception\OpenSSLException If there is an error while encrypting the plaintext.
      */
     public function encrypt(KeyInterface $key, string $plaintext): string
     {
@@ -86,7 +85,7 @@ public function encrypt(KeyInterface $key, string $plaintext): string
 
             $ciphertext = '';
             if (!$fn($plaintext, $ciphertext, $key->getMaterial(), $this->padding)) {
-                throw new RuntimeException('Cannot encrypt data: ' . openssl_error_string());
+                throw new OpenSSLException('Cannot encrypt data');
             }
             return $ciphertext;
         }
@@ -112,7 +111,7 @@ public function encrypt(KeyInterface $key, string $plaintext): string
         );
 
         if (!$ciphertext) {
-            throw new RuntimeException('Cannot encrypt data: ' . openssl_error_string());
+            throw new OpenSSLException('Cannot encrypt data');
         }
         return $iv . $ciphertext . $authTag;
     }
@@ -126,7 +125,7 @@ public function encrypt(KeyInterface $key, string $plaintext): string
      *
      * @return string The decrypted ciphertext (plaintext).
      *
-     * @throws \SimpleSAML\XMLSecurity\Exception\RuntimeException If there is an error while decrypting the ciphertext.
+     * @throws \SimpleSAML\XMLSecurity\Exception\OpenSSLException If there is an error while decrypting the ciphertext.
      */
     public function decrypt(KeyInterface $key, string $ciphertext): string
     {
@@ -139,7 +138,7 @@ public function decrypt(KeyInterface $key, string $ciphertext): string
 
             $plaintext = '';
             if (!$fn($ciphertext, $plaintext, $key->getMaterial(), $this->padding)) {
-                throw new RuntimeException('Cannot decrypt data: ' . openssl_error_string());
+                throw new OpenSSLException('Cannot decrypt data');
             }
             return $plaintext;
         }
@@ -167,7 +166,7 @@ public function decrypt(KeyInterface $key, string $ciphertext): string
         );
 
         if ($plaintext === false) {
-            throw new RuntimeException('Cannot decrypt data: ' . openssl_error_string());
+            throw new OpenSSLException('Cannot decrypt data');
         }
         return $this->useAuthTag ? $plaintext : $this->unpad($plaintext);
     }
@@ -181,12 +180,12 @@ public function decrypt(KeyInterface $key, string $ciphertext): string
      *
      * @return string The (binary) signature corresponding to the given plaintext.
      *
-     * @throws \SimpleSAML\XMLSecurity\Exception\RuntimeException If there is an error while signing the plaintext.
+     * @throws \SimpleSAML\XMLSecurity\Exception\OpenSSLException If there is an error while signing the plaintext.
      */
     public function sign(KeyInterface $key, string $plaintext): string
     {
         if (!openssl_sign($plaintext, $signature, $key->getMaterial(), $this->digest)) {
-            throw new RuntimeException('Cannot sign data: ' . openssl_error_string());
+            throw new OpenSSLException('Cannot sign data');
         }
         return $signature;
     }
diff --git a/src/Exception/OpenSSLException.php b/src/Exception/OpenSSLException.php
new file mode 100644
index 00000000..fb2fdc1d
--- /dev/null
+++ b/src/Exception/OpenSSLException.php
@@ -0,0 +1,33 @@
+<?php
+
+declare(strict_types=1);
+
+namespace SimpleSAML\XMLSecurity\Exception;
+
+/**
+ * Class OpenSSLException
+ *
+ * This exception is thrown when an error occurs during a call to the openssl backend.
+ *
+ * @package simplesamlphp/xml-security
+ */
+class OpenSSLException extends RuntimeException
+{
+    /**
+     * @param string $message
+     */
+    public function __construct(string $message = 'Generic OpenSSL exception')
+    {
+        $stack = [];
+        while (($msg = openssl_error_string()) !== false) {
+            $stack[] = $msg;
+        }
+
+        foreach ($stack as $line) {
+            $message .= '; ' . $line;
+        }
+        $message .= '.';
+
+        parent::__construct($message);
+    }
+}
diff --git a/src/Key/PrivateKey.php b/src/Key/PrivateKey.php
index c12d12dc..e124afd8 100644
--- a/src/Key/PrivateKey.php
+++ b/src/Key/PrivateKey.php
@@ -6,9 +6,8 @@
 
 use SimpleSAML\Assert\Assert;
 use SimpleSAML\XMLSecurity\CryptoEncoding\PEM;
-use SimpleSAML\XMLSecurity\Exception\RuntimeException;
+use SimpleSAML\XMLSecurity\Exception\OpenSSLException;
 
-use function openssl_error_string;
 use function openssl_pkey_export;
 use function openssl_pkey_get_private;
 
@@ -49,19 +48,13 @@ final public function __construct(PEM $key)
     public static function fromFile(string $file, string $passphrase = ''): static
     {
         if (($key = openssl_pkey_get_private($file, $passphrase)) === false) {
-            throw new RuntimeException('Failed to read key: ' . openssl_error_string());
+            throw new OpenSSLException('Failed to read key');
         }
 
-        // Some OpenSSL functions will add errors to the list even if they succeed
-        while (openssl_error_string() !== false);
-
         if (openssl_pkey_export($key, $decrypted) === false) {
-            throw new RuntimeException('Failed to export key: ' . openssl_error_string());
+            throw new OpenSSLException('Failed to export key');
         }
 
-        // Some OpenSSL functions will add errors to the list even if they succeed
-        while (openssl_error_string() !== false);
-
         return new static(PEM::fromString($decrypted));
     }
 }
diff --git a/src/Key/X509Certificate.php b/src/Key/X509Certificate.php
index ef7f27e0..3b7de23b 100644
--- a/src/Key/X509Certificate.php
+++ b/src/Key/X509Certificate.php
@@ -7,10 +7,9 @@
 use SimpleSAML\Assert\Assert;
 use SimpleSAML\XMLSecurity\Constants as C;
 use SimpleSAML\XMLSecurity\CryptoEncoding\PEM;
-use SimpleSAML\XMLSecurity\Exception\RuntimeException;
+use SimpleSAML\XMLSecurity\Exception\OpenSSLException;
 use SimpleSAML\XMLSecurity\Exception\UnsupportedAlgorithmException;
 
-use function openssl_error_string;
 use function openssl_pkey_get_details;
 use function openssl_pkey_get_public;
 use function openssl_x509_fingerprint;
@@ -39,7 +38,7 @@ class X509Certificate
      * @param \SimpleSAML\XMLSecurity\CryptoEncoding\PEM $material
      *   The PEM-encoded certificate or the path to a file containing it.
      *
-     * @throws \SimpleSAML\XMLSecurity\Exception\RuntimeException If the certificate cannot be exported to PEM format.
+     * @throws \SimpleSAML\XMLSecurity\Exception\OpenSSLException If the certificate cannot be exported to PEM format.
      */
     final public function __construct(
         protected PEM $material,
@@ -47,19 +46,13 @@ final public function __construct(
         Assert::oneOf($material->type(), [PEM::TYPE_CERTIFICATE], "PEM structure has the wrong type %s.");
 
         if (($key = openssl_pkey_get_public($material->string())) === false) {
-            throw new RuntimeException('Failed to read key: ' . openssl_error_string());
+            throw new OpenSSLException('Failed to read key');
         }
 
-        // Some OpenSSL functions will add errors to the list even if they succeed
-        while (openssl_error_string() !== false);
-
         if (($details = openssl_pkey_get_details($key)) === false) {
-            throw new RuntimeException('Failed to export key: ' . openssl_error_string());
+            throw new OpenSSLException('Failed to export key');
         }
 
-        // Some OpenSSL functions will add errors to the list even if they succeed
-        while (openssl_error_string() !== false); // @phpstan-ignore-line
-
         $this->publicKey = new PublicKey(PEM::fromString($details['key']));
 
         $this->thumbprint[C::DIGEST_SHA1] = $this->getRawThumbprint();
diff --git a/tests/Backend/OpenSSLTest.php b/tests/Backend/OpenSSLTest.php
index b91444d6..d190f218 100644
--- a/tests/Backend/OpenSSLTest.php
+++ b/tests/Backend/OpenSSLTest.php
@@ -188,7 +188,7 @@ public function testEncryptRSA15DecryptOAEP(): void
         $ciphertext = self::$backend->encrypt(self::$pubKey, 'Plaintext');
         self::$backend->setCipher(C::KEY_TRANSPORT_OAEP);
         $this->expectException(RuntimeException::class);
-        $this->expectExceptionMessageMatches('/^Cannot decrypt data:/');
+        $this->expectExceptionMessageMatches('/^Cannot decrypt data;/');
         self::$backend->decrypt(self::$privKey, $ciphertext);
     }