Validate signatures with RSA PSS key type

.github/workflows/build.yml

@@ -132,7 +132,7 @@ jobs:
     container: ubuntu:${{ matrix.container }}
     strategy:
       matrix:
-        container: ['22.04', '24.04', '24.10', '25.04']
+        container: ['22.04', '24.04', '25.04']
         arch: ['amd64', 'arm64']
     env:
       DEBIAN_FRONTEND: noninteractive
@@ -164,15 +164,12 @@ jobs:
     runs-on: ${{ matrix.image }}
     strategy:
       matrix:
-        toolset: [143, 142]
+        toolset: [143]
         platform: [x86, x64, arm64]
         include:
         - toolset: 143
           image: windows-2022
           vcvars: "C:\\Program Files\\Microsoft Visual Studio\\2022\\Enterprise\\VC\\Auxiliary\\Build\\vcvarsall.bat"
-        - toolset: 142
-          image: windows-2019
-          vcvars: "C:\\Program Files (x86)\\Microsoft Visual Studio\\2019\\Enterprise\\VC\\Auxiliary\\Build\\vcvarsall.bat"
         - platform: x86
           setenv: amd64_x86
         - platform: x64

prepare_osx_build_environment.sh

@@ -57,6 +57,7 @@ function xmlsec {
     tar xf ${XMLSEC_DIR}.tar.gz
     cd ${XMLSEC_DIR}
     patch -Np1 -i ../vcpkg-ports/xmlsec/xmlsec1-1.3.5.legacy.patch
+    patch -Np1 -i ../vcpkg-ports/xmlsec/xmlsec1-1.3.7.rsapss.patch
     case "${ARGS}" in
     *iphone*) CONFIGURE="--host=aarch64-apple-darwin --enable-static --disable-shared --without-libxslt" ;;
     *) CONFIGURE="--disable-static --enable-shared" ;;

vcpkg-ports/xmlsec/CMakeLists.txt

@@ -18,21 +18,21 @@ FILE(GLOB SOURCESXMLSECOPENSSL
 # Generate xmlexports with fixed definition of XMLSEC_STATIC
 file(READ include/xmlsec/exports.h EXPORTS_H)
 if(BUILD_SHARED_LIBS)
-    string(REPLACE "!defined(XMLSEC_STATIC)" "1" EXPORTS_H "${EXPORTS_H}")
+	string(REPLACE "!defined(XMLSEC_STATIC)" "1" EXPORTS_H "${EXPORTS_H}")
 else()
-    string(REPLACE "!defined(XMLSEC_STATIC)" "0" EXPORTS_H "${EXPORTS_H}")
+	string(REPLACE "!defined(XMLSEC_STATIC)" "0" EXPORTS_H "${EXPORTS_H}")
 endif()
 file(WRITE ${CMAKE_CURRENT_BINARY_DIR}/exports.h "${EXPORTS_H}")
 
 message(STATUS "Reading version info from configure.ac")
 
 file(STRINGS "configure.ac"
-    _xmlsec_version_defines REGEX "XMLSEC_VERSION_(MAJOR|MINOR|SUBMINOR)=([0-9]+)$")
+	_xmlsec_version_defines REGEX "XMLSEC_VERSION_(MAJOR|MINOR|SUBMINOR)=([0-9]+)$")
 
 foreach(ver ${_xmlsec_version_defines})
-    if(ver MATCHES "XMLSEC_VERSION_(MAJOR|MINOR|SUBMINOR)=([0-9]+)$")
-        set(XMLSEC_VERSION_${CMAKE_MATCH_1} "${CMAKE_MATCH_2}" CACHE INTERNAL "")
-    endif()
+	if(ver MATCHES "XMLSEC_VERSION_(MAJOR|MINOR|SUBMINOR)=([0-9]+)$")
+		set(XMLSEC_VERSION_${CMAKE_MATCH_1} "${CMAKE_MATCH_2}" CACHE INTERNAL "")
+	endif()
 endforeach()
 
 set(XMLSEC_VERSION ${XMLSEC_VERSION_MAJOR}.${XMLSEC_VERSION_MINOR}.${XMLSEC_VERSION_SUBMINOR})
@@ -61,9 +61,9 @@ add_library(xmlsec1 ${SOURCESXMLSEC})
 add_library(xmlsec1-openssl ${SOURCESXMLSECOPENSSL})
 
 target_include_directories(xmlsec1 PUBLIC
-    $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}/include>
-    $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>
-    $<INSTALL_INTERFACE:include>
+	$<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}/include>
+	$<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>
+	$<INSTALL_INTERFACE:include>
 )
 target_link_libraries(xmlsec1 PUBLIC LibXml2::LibXml2)
 target_link_libraries(xmlsec1-openssl PUBLIC xmlsec1 OpenSSL::Crypto)
@@ -89,26 +89,26 @@ set_target_properties(xmlsec1 xmlsec1-openssl PROPERTIES VERSION ${XMLSEC_VERSIO
 
 set(XMLSEC_CORE_CFLAGS XMLSEC_NO_XSLT XMLSEC_CRYPTO_OPENSSL XMLSEC_NO_FTP XMLSEC_NO_HTTP)
 if(NOT BUILD_SHARED_LIBS)
-    list(APPEND XMLSEC_CORE_CFLAGS XMLSEC_STATIC XMLSEC_NO_CRYPTO_DYNAMIC_LOADING)
+	list(APPEND XMLSEC_CORE_CFLAGS XMLSEC_STATIC XMLSEC_NO_CRYPTO_DYNAMIC_LOADING)
 endif()
 set(XMLSEC_OPENSSL_CFLAGS XMLSEC_NO_MD5 XMLSEC_NO_RIPEMD160 XMLSEC_NO_GOST XMLSEC_NO_GOST2012)
 
 target_compile_definitions(xmlsec1
-    PRIVATE $<$<PLATFORM_ID:Windows>:XMLSEC_DL_WIN32>
-    PUBLIC ${XMLSEC_CORE_CFLAGS} 
+	PRIVATE $<IF:$<PLATFORM_ID:Windows>,XMLSEC_DL_WIN32,XMLSEC_DL_LIBLTDL>
+	PUBLIC ${XMLSEC_CORE_CFLAGS} 
 )
 target_compile_definitions(xmlsec1-openssl PUBLIC ${XMLSEC_OPENSSL_CFLAGS})
 
 install(TARGETS xmlsec1 xmlsec1-openssl
-    EXPORT unofficial-xmlsec-targets
-    RUNTIME DESTINATION bin
-    LIBRARY DESTINATION lib
-    ARCHIVE DESTINATION lib
+	EXPORT unofficial-xmlsec-targets
+	RUNTIME DESTINATION bin
+	LIBRARY DESTINATION lib
+	ARCHIVE DESTINATION lib
 )
 
 install(EXPORT unofficial-xmlsec-targets
-    NAMESPACE unofficial::xmlsec::
-    DESTINATION share/unofficial-xmlsec
+	NAMESPACE unofficial::xmlsec::
+	DESTINATION share/unofficial-xmlsec
 )
 
 if(INSTALL_HEADERS_TOOLS)
@@ -157,7 +157,7 @@ set(VERSION ${XMLSEC_VERSION})
 set(LIBXML_MIN_VERSION ${LIBXML2_VERSION_STRING})
 list(JOIN XMLSEC_CORE_CFLAGS " -D" XMLSEC_CORE_CFLAGS)
 set(XMLSEC_CORE_CFLAGS "-D${XMLSEC_CORE_CFLAGS} -I\${includedir}/xmlsec1")
-set(XMLSEC_CORE_LIBS "-lxmlsec1 -lltdl")
+set(XMLSEC_CORE_LIBS "-lxmlsec1")
 list(JOIN XMLSEC_OPENSSL_CFLAGS " -D" XMLSEC_OPENSSL_CFLAGS)
 set(XMLSEC_OPENSSL_CFLAGS "${XMLSEC_CORE_CFLAGS} -D${XMLSEC_OPENSSL_CFLAGS}")
 set(XMLSEC_OPENSSL_LIBS "-L\${libdir} -lxmlsec1-openssl ${XMLSEC_CORE_LIBS} -lcrypto")

vcpkg-ports/xmlsec/portfile.cmake

@@ -10,6 +10,7 @@ vcpkg_from_github(
     PATCHES 
         pkgconfig_fixes.patch
         xmlsec1-1.3.5.legacy.patch
+        xmlsec1-1.3.7.rsapss.patch
 )
 
 file(COPY "${CMAKE_CURRENT_LIST_DIR}/CMakeLists.txt" DESTINATION "${SOURCE_PATH}")

vcpkg-ports/xmlsec/xmlsec1-1.3.7.rsapss.patch

@@ -0,0 +1,194 @@
+From a7e8464f2a2826820b94cc641ac0aae345641fc6 Mon Sep 17 00:00:00 2001
+From: Raul Metsma <raul@metsma.ee>
+Date: Thu, 26 Jun 2025 17:47:54 +0300
+Subject: [PATCH] (xmlsec-openssl) add supoprt for RSA PSS key type (#933)
+
+---
+ src/openssl/evp.c     | 35 ++++++++++++++++++++++++++++++-----
+ src/openssl/kt_rsa.c  | 11 ++++++-----
+ src/openssl/private.h | 11 ++++++++++-
+ src/openssl/x509vfy.c |  1 -
+ 4 files changed, 46 insertions(+), 12 deletions(-)
+
+diff --git a/src/openssl/evp.c b/src/openssl/evp.c
+index 97b7b6c2..a3e00006 100644
+--- a/src/openssl/evp.c
++++ b/src/openssl/evp.c
+@@ -32,7 +32,6 @@
+ #include <xmlsec/openssl/app.h>
+ #include <xmlsec/openssl/crypto.h>
+ #include <xmlsec/openssl/evp.h>
+-#include "openssl_compat.h"
+
+ #ifdef XMLSEC_OPENSSL_API_300
+ #include <openssl/core_names.h>
+@@ -41,6 +40,8 @@
+
+ #include "../cast_helpers.h"
+ #include "../keysdata_helpers.h"
++#include "openssl_compat.h"
++#include "private.h"
+
+ static int
+ xmlSecOpenSSLGetBNValue(const xmlSecBufferPtr buf, BIGNUM **bigNum) {
+@@ -325,6 +326,8 @@ xmlSecOpenSSLEvpKeyAdopt(EVP_PKEY *pKey) {
+     switch(EVP_PKEY_base_id(pKey)) {
+ #ifndef XMLSEC_NO_RSA
+     case EVP_PKEY_RSA:
++    case EVP_PKEY_RSA2:
++    case EVP_PKEY_RSA_PSS:
+         data = xmlSecKeyDataCreate(xmlSecOpenSSLKeyDataRsaId);
+         if(data == NULL) {
+             xmlSecInternalError("xmlSecKeyDataCreate(xmlSecOpenSSLKeyDataRsaId)", NULL);
+@@ -3258,6 +3261,26 @@ done:
+
+ #ifndef XMLSEC_NO_RSA
+
++/**
++ * xmlSecOpenSSLKeyValueRsaCheckKeyType:
++ * @pKey: the EVP key to check
++ *
++ * Returns 0 if @pKey is a valid RSA key type, 1 if it is not, or a negative value if an error occurs.
++ */
++int
++xmlSecOpenSSLKeyValueRsaCheckKeyType(EVP_PKEY* pKey) {
++    xmlSecAssert2(pKey != NULL, -1);
++
++    switch(EVP_PKEY_base_id(pKey)) {
++    case EVP_PKEY_RSA:
++    case EVP_PKEY_RSA2:
++    case EVP_PKEY_RSA_PSS:
++        return(0);
++    default:
++        return(1);
++    }
++}
++
+ /*
+  * @xmlSecOpenSSLKeyValueRsa: holds the parts of OpenSSL RSA key
+  */
+@@ -3430,7 +3453,7 @@ int
+ xmlSecOpenSSLKeyDataRsaAdoptEvp(xmlSecKeyDataPtr data, EVP_PKEY* pKey) {
+     xmlSecAssert2(xmlSecKeyDataCheckId(data, xmlSecOpenSSLKeyDataRsaId), -1);
+     xmlSecAssert2(pKey != NULL, -1);
+-    xmlSecAssert2(EVP_PKEY_base_id(pKey) == EVP_PKEY_RSA, -1);
++    xmlSecAssert2(xmlSecOpenSSLKeyValueRsaCheckKeyType(pKey) == 0, -1);
+
+     return(xmlSecOpenSSLEvpKeyDataAdoptEvp(data, pKey));
+ }
+@@ -3535,9 +3558,11 @@ xmlSecOpenSSLKeyDataRsaGetRsa(xmlSecKeyDataPtr data) {
+     xmlSecAssert2(xmlSecKeyDataCheckId(data, xmlSecOpenSSLKeyDataRsaId), NULL);
+
+     pKey = xmlSecOpenSSLKeyDataRsaGetEvp(data);
+-    xmlSecAssert2((pKey == NULL) || (EVP_PKEY_base_id(pKey) == EVP_PKEY_RSA), NULL);
+-
+-    return((pKey != NULL) ? EVP_PKEY_get0_RSA(pKey) : NULL);
++    if (pKey == NULL) {
++        return(NULL);
++    }
++    xmlSecAssert2(xmlSecOpenSSLKeyValueRsaCheckKeyType(pKey) == 0, NULL);
++    return(EVP_PKEY_get0_RSA(pKey));
+ }
+
+ static int
+diff --git a/src/openssl/kt_rsa.c b/src/openssl/kt_rsa.c
+index a1153499..d25c001e 100644
+--- a/src/openssl/kt_rsa.c
++++ b/src/openssl/kt_rsa.c
+@@ -34,7 +34,6 @@
+
+ #include <xmlsec/openssl/crypto.h>
+ #include <xmlsec/openssl/evp.h>
+-#include "openssl_compat.h"
+
+ #ifdef XMLSEC_OPENSSL_API_300
+ #include <openssl/core_names.h>
+@@ -43,6 +42,8 @@
+
+ #include "../cast_helpers.h"
+ #include "../transform_helpers.h"
++#include "openssl_compat.h"
++#include "private.h"
+
+ #ifndef XMLSEC_NO_RSA_PKCS15
+
+@@ -162,7 +163,7 @@ xmlSecOpenSSLRsaPkcs1ProcessImpl(xmlSecOpenSSLRsaPkcs1CtxPtr ctx, const xmlSecBy
+
+     xmlSecAssert2(ctx != NULL, -1);
+     xmlSecAssert2(ctx->pKey != NULL, -1);
+-    xmlSecAssert2(EVP_PKEY_base_id(ctx->pKey) == EVP_PKEY_RSA, -1);
++    xmlSecAssert2(xmlSecOpenSSLKeyValueRsaCheckKeyType(ctx->pKey) == 0, -1);
+     xmlSecAssert2(inBuf != NULL, -1);
+     xmlSecAssert2(inSize > 0, -1);
+     xmlSecAssert2(outBuf != NULL, -1);
+@@ -364,7 +365,7 @@ xmlSecOpenSSLRsaPkcs1SetKey(xmlSecTransformPtr transform, xmlSecKeyPtr key) {
+                             xmlSecTransformGetName(transform));
+         return(-1);
+     }
+-    xmlSecAssert2(EVP_PKEY_base_id(pKey) == EVP_PKEY_RSA, -1);
++    xmlSecAssert2(xmlSecOpenSSLKeyValueRsaCheckKeyType(pKey) == 0, -1);
+
+     if (transform->operation == xmlSecTransformOperationEncrypt) {
+         encrypt = 1;
+@@ -693,7 +694,7 @@ xmlSecOpenSSLRsaOaepProcessImpl(xmlSecOpenSSLRsaOaepCtxPtr ctx, const xmlSecByte
+
+     xmlSecAssert2(ctx != NULL, -1);
+     xmlSecAssert2(ctx->pKey != NULL, -1);
+-    xmlSecAssert2(EVP_PKEY_base_id(ctx->pKey) == EVP_PKEY_RSA, -1);
++    xmlSecAssert2(xmlSecOpenSSLKeyValueRsaCheckKeyType(pKey) == 0, -1);
+     xmlSecAssert2(inBuf != NULL, -1);
+     xmlSecAssert2(inSize > 0, -1);
+     xmlSecAssert2(outBuf != NULL, -1);
+@@ -1223,7 +1224,7 @@ xmlSecOpenSSLRsaOaepSetKey(xmlSecTransformPtr transform, xmlSecKeyPtr key) {
+             xmlSecTransformGetName(transform));
+         return(-1);
+     }
+-    xmlSecAssert2(EVP_PKEY_base_id(pKey) == EVP_PKEY_RSA, -1);
++    xmlSecAssert2(xmlSecOpenSSLKeyValueRsaCheckKeyType(pKey) == 0, -1);
+
+     if (transform->operation == xmlSecTransformOperationEncrypt) {
+         encrypt = 1;
+diff --git a/src/openssl/private.h b/src/openssl/private.h
+index 6940f5bd..cc58a0b2 100644
+--- a/src/openssl/private.h
++++ b/src/openssl/private.h
+@@ -28,6 +28,16 @@ extern "C" {
+ #endif /* __cplusplus */
+
+
++/******************************************************************************
++ *
++ * RSA Util functions
++ *
++ ******************************************************************************/
++#ifndef XMLSEC_NO_RSA
++
++int             xmlSecOpenSSLKeyValueRsaCheckKeyType            (EVP_PKEY* pKey);
++
++#endif /* XMLSEC_NO_RSA */
+
+ /******************************************************************************
+  *
+@@ -85,7 +95,6 @@ int             xmlSecOpenSSLX509Asn1TimeToTime                 (const ASN1_TIME
+ STACK_OF(X509)*        xmlSecOpenSSLKeyDataX509GetCerts         (xmlSecKeyDataPtr data);
+ STACK_OF(X509_CRL)*    xmlSecOpenSSLKeyDataX509GetCrls          (xmlSecKeyDataPtr data);
+
+-
+ #endif /* XMLSEC_NO_X509 */
+
+ #ifdef __cplusplus
+diff --git a/src/openssl/x509vfy.c b/src/openssl/x509vfy.c
+index 322f6661..fdede0fc 100644
+--- a/src/openssl/x509vfy.c
++++ b/src/openssl/x509vfy.c
+@@ -34,7 +34,6 @@
+ #include <xmlsec/openssl/crypto.h>
+ #include <xmlsec/openssl/evp.h>
+ #include <xmlsec/openssl/x509.h>
+-#include "openssl_compat.h"
+
+ #include <openssl/evp.h>
+ #include <openssl/x509.h>
+-- 
+2.46.0
+