Support for "RSASSA-PSS" algorithm
Hi,

While we are trying to validate a user certificate that is generated using the RSASSA-PSS algorithm. Netty OpenSSL engine is throwing an exception "sun.security.validator.ValidatorException: Certificate signature algorithm disabled"

### Actual behavior
As the OpenSSL is supporting this algorithm now. So in Netty also should allow this algorithm.

### Netty version
4.1.65.Final
Netty-tcnative = 2.0.36.Final

### JVM version (e.g. `java -version`)
openjdk 11.0.10 2021-01-19 LTS
OpenJDK Runtime Environment Zulu11.45+52-SA (build 11.0.10+9-LTS)
OpenJDK 64-Bit Server VM Zulu11.45+52-SA (build 11.0.10+9-LTS, mixed mode)

### OS version (e.g. `uname -a`)
1-photon SMP Wed May 26 15:55:27 UTC 2021 x86_64 GNU/Linux
handler/src/main/java/io/netty/handler/ssl/ExtendedOpenSslSession.java
handler/src/main/java/io/netty/handler/ssl/ExtendedOpenSslSession.java handler/src/test/resources/io/netty/handler/ssl/rsapss-signing-ext.txt
handler/src/test/java/io/netty/handler/ssl/ConscryptOpenSslEngineInteropTest.java handler/src/test/java/io/netty/handler/ssl/ConscryptSslEngineTest.java handler/src/test/java/io/netty/handler/ssl/JdkConscryptSslEngineInteropTest.java handler/src/test/java/io/netty/handler/ssl/JdkOpenSslEngineInteroptTest.java handler/src/test/java/io/netty/handler/ssl/OpenSslEngineTest.java handler/src/test/java/io/netty/handler/ssl/OpenSslJdkSslEngineInteroptTest.java handler/src/test/java/io/netty/handler/ssl/SSLEngineTest.java handler/src/test/resources/io/netty/handler/ssl/generate-certificate.sh handler/src/test/resources/io/netty/handler/ssl/rsapss-ca-cert.cert
diff --git a/handler/src/main/java/io/netty/handler/ssl/ExtendedOpenSslSession.java b/handler/src/main/java/io/netty/handler/ssl/ExtendedOpenSslSession.java
index 963a9b616ab..5924325893d 100644
--- a/handler/src/main/java/io/netty/handler/ssl/ExtendedOpenSslSession.java
+++ b/handler/src/main/java/io/netty/handler/ssl/ExtendedOpenSslSession.java
@@ -42,6 +42,7 @@ abstract class ExtendedOpenSslSession extends ExtendedSSLSession implements Open
     private static final String[] LOCAL_SUPPORTED_SIGNATURE_ALGORITHMS = {
             "SHA512withRSA", "SHA512withECDSA", "SHA384withRSA", "SHA384withECDSA", "SHA256withRSA",
             "SHA256withECDSA", "SHA224withRSA", "SHA224withECDSA", "SHA1withRSA", "SHA1withECDSA",
+            "RSASSA-PSS",
     };
 
     private final OpenSslSession wrapped;
diff --git a/handler/src/test/resources/io/netty/handler/ssl/rsapss-signing-ext.txt b/handler/src/test/resources/io/netty/handler/ssl/rsapss-signing-ext.txt
new file mode 100644
index 00000000000..9716541c0d9
--- /dev/null
+++ b/handler/src/test/resources/io/netty/handler/ssl/rsapss-signing-ext.txt
@@ -0,0 +1,21 @@
+[ ext ]
+extendedKeyUsage = clientAuth
+keyUsage = nonRepudiation, digitalSignature, keyEncipherment
+#subjectKeyIdentifier = hash
+authorityKeyIdentifier = keyid,issuer
+
+
+[ exts ]
+extendedKeyUsage = serverAuth
+keyUsage = nonRepudiation, digitalSignature, keyEncipherment
+#subjectKeyIdentifier = hash
+authorityKeyIdentifier = keyid,issuer
+#subjectAltName = @alt_names
+
+[ extca ]
+authorityKeyIdentifier = keyid,issuer
+basicConstraints=CA:TRUE
+subjectKeyIdentifier = hash
+
+[alt_names]
+DNS.1 = aws-dev-node.skylo.local

diff --git a/handler/src/test/java/io/netty/handler/ssl/ConscryptOpenSslEngineInteropTest.java b/handler/src/test/java/io/netty/handler/ssl/ConscryptOpenSslEngineInteropTest.java
index 8084220ca30..9800c21e76a 100644
--- a/handler/src/test/java/io/netty/handler/ssl/ConscryptOpenSslEngineInteropTest.java
+++ b/handler/src/test/java/io/netty/handler/ssl/ConscryptOpenSslEngineInteropTest.java
@@ -18,6 +18,8 @@
 import org.junit.jupiter.api.BeforeAll;
 import org.junit.jupiter.api.Disabled;
 import org.junit.jupiter.api.condition.DisabledIf;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.MethodSource;
 
 import javax.net.ssl.SSLEngine;
 import javax.net.ssl.SSLSessionContext;
diff --git a/handler/src/test/java/io/netty/handler/ssl/ConscryptSslEngineTest.java b/handler/src/test/java/io/netty/handler/ssl/ConscryptSslEngineTest.java
index 6e825f6c7e6..7ecb8fa83fc 100644
--- a/handler/src/test/java/io/netty/handler/ssl/ConscryptSslEngineTest.java
+++ b/handler/src/test/java/io/netty/handler/ssl/ConscryptSslEngineTest.java
@@ -69,8 +69,15 @@ protected void invalidateSessionsAndAssert(SSLSessionContext context) {
     }
 
     @Disabled("Possible Conscrypt bug")
+    @Override
     public void testSessionCacheTimeout(SSLEngineTestParam param) throws Exception {
         // Skip
         // https://github.com/google/conscrypt/issues/851
     }
+
+    @Disabled("Not supported")
+    @Override
+    public void testRSASSAPSS(SSLEngineTestParam param) {
+        // skip
+    }
 }
diff --git a/handler/src/test/java/io/netty/handler/ssl/JdkConscryptSslEngineInteropTest.java b/handler/src/test/java/io/netty/handler/ssl/JdkConscryptSslEngineInteropTest.java
index 5a2f91c86c5..aa44fce7401 100644
--- a/handler/src/test/java/io/netty/handler/ssl/JdkConscryptSslEngineInteropTest.java
+++ b/handler/src/test/java/io/netty/handler/ssl/JdkConscryptSslEngineInteropTest.java
@@ -81,8 +81,15 @@ protected void invalidateSessionsAndAssert(SSLSessionContext context) {
     }
 
     @Disabled("Possible Conscrypt bug")
+    @Override
     public void testSessionCacheTimeout(SSLEngineTestParam param) {
         // Skip
         // https://github.com/google/conscrypt/issues/851
     }
+
+    @Disabled("Not supported")
+    @Override
+    public void testRSASSAPSS(SSLEngineTestParam param) {
+        // skip
+    }
 }
diff --git a/handler/src/test/java/io/netty/handler/ssl/JdkOpenSslEngineInteroptTest.java b/handler/src/test/java/io/netty/handler/ssl/JdkOpenSslEngineInteroptTest.java
index cf4565635ce..2d2830ba397 100644
--- a/handler/src/test/java/io/netty/handler/ssl/JdkOpenSslEngineInteroptTest.java
+++ b/handler/src/test/java/io/netty/handler/ssl/JdkOpenSslEngineInteroptTest.java
@@ -18,6 +18,8 @@
 import org.junit.jupiter.api.BeforeAll;
 import org.junit.jupiter.api.condition.DisabledOnOs;
 import org.junit.jupiter.api.condition.OS;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.MethodSource;
 
 import javax.net.ssl.SSLEngine;
 import java.util.ArrayList;
@@ -192,6 +194,14 @@ public void testSessionCacheSize(SSLEngineTestParam param) throws Exception {
         super.testSessionCacheSize(param);
     }
 
+    @MethodSource("newTestParams")
+    @ParameterizedTest
+    @Override
+    public void testRSASSAPSS(SSLEngineTestParam param) throws Exception {
+        checkShouldUseKeyManagerFactory();
+        super.testRSASSAPSS(param);
+    }
+
     @Override
     protected SSLEngine wrapEngine(SSLEngine engine) {
         return Java8SslTestUtils.wrapSSLEngineForTesting(engine);
diff --git a/handler/src/test/java/io/netty/handler/ssl/OpenSslEngineTest.java b/handler/src/test/java/io/netty/handler/ssl/OpenSslEngineTest.java
index 709cdcd3a82..2b5c2d04cf7 100644
--- a/handler/src/test/java/io/netty/handler/ssl/OpenSslEngineTest.java
+++ b/handler/src/test/java/io/netty/handler/ssl/OpenSslEngineTest.java
@@ -25,6 +25,7 @@
 import io.netty.util.CharsetUtil;
 import io.netty.util.internal.EmptyArrays;
 import io.netty.util.internal.PlatformDependent;
+import io.netty.util.internal.ResourcesUtil;
 import org.junit.AssumptionViolatedException;
 import org.junit.jupiter.api.AfterEach;
 import org.junit.jupiter.api.BeforeAll;
@@ -35,6 +36,7 @@
 import javax.crypto.Cipher;
 import javax.crypto.spec.IvParameterSpec;
 import javax.crypto.spec.SecretKeySpec;
+import javax.net.ssl.KeyManagerFactory;
 import javax.net.ssl.SSLEngine;
 import javax.net.ssl.SSLEngineResult;
 import javax.net.ssl.SSLEngineResult.HandshakeStatus;
@@ -42,12 +44,14 @@
 import javax.net.ssl.SSLHandshakeException;
 import javax.net.ssl.SSLParameters;
 import javax.net.ssl.X509ExtendedKeyManager;
+import java.io.File;
 import java.net.Socket;
 import java.nio.ByteBuffer;
 import java.security.AlgorithmConstraints;
 import java.security.AlgorithmParameters;
 import java.security.CryptoPrimitive;
 import java.security.Key;
+import java.security.KeyStore;
 import java.security.Principal;
 import java.security.PrivateKey;
 import java.security.cert.X509Certificate;
@@ -55,6 +59,7 @@
 import java.util.Arrays;
 import java.util.List;
 import java.util.Set;
+import java.util.concurrent.TimeUnit;
 
 import static io.netty.handler.ssl.OpenSslTestUtils.checkShouldUseKeyManagerFactory;
 import static io.netty.handler.ssl.ReferenceCountedOpenSslEngine.MAX_PLAINTEXT_LENGTH;
@@ -1559,4 +1564,12 @@ protected void assertSessionReusedForEngine(SSLEngine clientEngine, SSLEngine se
     protected boolean isSessionMaybeReused(SSLEngine engine) {
         return unwrapEngine(engine).isSessionReused();
     }
+
+    @MethodSource("newTestParams")
+    @ParameterizedTest
+    @Override
+    public void testRSASSAPSS(SSLEngineTestParam param) throws Exception {
+        checkShouldUseKeyManagerFactory();
+        super.testRSASSAPSS(param);
+    }
 }
diff --git a/handler/src/test/java/io/netty/handler/ssl/OpenSslJdkSslEngineInteroptTest.java b/handler/src/test/java/io/netty/handler/ssl/OpenSslJdkSslEngineInteroptTest.java
index f4d43e4b4c9..6d4e175b439 100644
--- a/handler/src/test/java/io/netty/handler/ssl/OpenSslJdkSslEngineInteroptTest.java
+++ b/handler/src/test/java/io/netty/handler/ssl/OpenSslJdkSslEngineInteroptTest.java
@@ -17,6 +17,8 @@
 
 import org.junit.jupiter.api.BeforeAll;
 import org.junit.jupiter.api.Disabled;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.MethodSource;
 
 import javax.net.ssl.SSLEngine;
 
@@ -148,6 +150,14 @@ public void testSessionCacheSize(SSLEngineTestParam param) throws Exception {
         super.testSessionCacheSize(param);
     }
 
+    @MethodSource("newTestParams")
+    @ParameterizedTest
+    @Override
+    public void testRSASSAPSS(SSLEngineTestParam param) throws Exception {
+        checkShouldUseKeyManagerFactory();
+        super.testRSASSAPSS(param);
+    }
+
     @Override
     protected SSLEngine wrapEngine(SSLEngine engine) {
         return Java8SslTestUtils.wrapSSLEngineForTesting(engine);
diff --git a/handler/src/test/java/io/netty/handler/ssl/SSLEngineTest.java b/handler/src/test/java/io/netty/handler/ssl/SSLEngineTest.java
index c2995b7fd11..a6fa867697d 100644
--- a/handler/src/test/java/io/netty/handler/ssl/SSLEngineTest.java
+++ b/handler/src/test/java/io/netty/handler/ssl/SSLEngineTest.java
@@ -780,11 +780,11 @@ protected boolean mySetupMutualAuthServerIsValidException(Throwable cause) {
     protected void mySetupMutualAuthServerInitSslHandler(SslHandler handler) {
     }
 
-    private void mySetupMutualAuth(final SSLEngineTestParam param, KeyManagerFactory serverKMF,
-                                   final File serverTrustManager,
-                                   KeyManagerFactory clientKMF, File clientTrustManager,
-                                   ClientAuth clientAuth, final boolean failureExpected,
-                                   final boolean serverInitEngine)
+    protected void mySetupMutualAuth(final SSLEngineTestParam param, KeyManagerFactory serverKMF,
+                                     final File serverTrustManager,
+                                     KeyManagerFactory clientKMF, File clientTrustManager,
+                                     ClientAuth clientAuth, final boolean failureExpected,
+                                     final boolean serverInitEngine)
             throws SSLException, InterruptedException {
         serverSslCtx =
                 wrapContext(param, SslContextBuilder.forServer(serverKMF)
@@ -908,7 +908,7 @@ public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) throws E
         clientChannel = ccf.channel();
     }
 
-    private static void rethrowIfNotNull(Throwable error) {
+    protected static void rethrowIfNotNull(Throwable error) {
         if (error != null) {
             throw new AssertionFailedError("Expected no error", error);
         }
@@ -4162,6 +4162,36 @@ public void testDefaultProtocolsIncludeTLSv13(SSLEngineTestParam param) throws E
                 arrayContains(serverProtocols, SslProtocols.TLS_v1_3));
     }
 
+    @MethodSource("newTestParams")
+    @ParameterizedTest
+    public void testRSASSAPSS(SSLEngineTestParam param) throws Exception {
+        char[] password = "password".toCharArray();
+
+        final KeyStore serverKeyStore = KeyStore.getInstance("PKCS12");
+        serverKeyStore.load(getClass().getResourceAsStream("rsaValidations-server-keystore.p12"), password);
+
+        final KeyStore clientKeyStore = KeyStore.getInstance("PKCS12");
+        clientKeyStore.load(getClass().getResourceAsStream("rsaValidation-user-certs.p12"), password);
+
+        final KeyManagerFactory serverKeyManagerFactory =
+                KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm());
+        serverKeyManagerFactory.init(serverKeyStore, password);
+        final KeyManagerFactory clientKeyManagerFactory =
+                KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm());
+        clientKeyManagerFactory.init(clientKeyStore, password);
+
+        File commonChain = ResourcesUtil.getFile(getClass(), "rsapss-ca-cert.cert");
+        ClientAuth auth = ClientAuth.REQUIRE;
+
+        mySetupMutualAuth(param, serverKeyManagerFactory, commonChain, clientKeyManagerFactory, commonChain,
+                auth, false, true);
+
+        assertTrue(clientLatch.await(10, TimeUnit.SECONDS));
+        rethrowIfNotNull(clientException);
+        assertTrue(serverLatch.await(5, TimeUnit.SECONDS));
+        rethrowIfNotNull(serverException);
+    }
+
     protected SSLEngine wrapEngine(SSLEngine engine) {
         return engine;
     }
diff --git a/handler/src/test/resources/io/netty/handler/ssl/generate-certificate.sh b/handler/src/test/resources/io/netty/handler/ssl/generate-certificate.sh
new file mode 100755
index 00000000000..b9c5bd119b6
--- /dev/null
+++ b/handler/src/test/resources/io/netty/handler/ssl/generate-certificate.sh
@@ -0,0 +1,18 @@
+# Generate CA key and certificate.
+openssl req -x509 -newkey rsa:2048 -days 3650 -keyout rsapss-ca-key.pem -out rsapss-ca-cert.cert -subj "/C=GB/O=Netty/OU=netty-parent/CN=west.int" -sigopt rsa_padding_mode:pss -sha256 -sigopt rsa_pss_saltlen:20
+
+# Generate user key nand.
+openssl req -newkey rsa:2048 -keyout rsapss-user-key.pem -out rsaValidation-req.pem -subj "/C=GB/O=Netty/OU=netty-parent/CN=c1" -sigopt rsa_padding_mode:pss -sha256 -sigopt rsa_pss_saltlen:20
+
+# Sign user cert request using CA certificate.
+openssl x509 -req -in rsaValidation-req.pem -days 365 -extensions ext -extfile rsapss-signing-ext.txt -CA rsapss-ca-cert.cert -CAkey rsapss-ca-key.pem -CAcreateserial -out rsapss-user-singed.cert -sigopt rsa_padding_mode:pss -sha256 -sigopt rsa_pss_saltlen:20
+
+# Create user certificate keystore.
+openssl pkcs12 -export -out rsaValidation-user-certs.p12 -inkey rsapss-user-key.pem -in rsapss-user-singed.cert
+
+# create keystore for the 
+openssl pkcs12 -in rsapss-ca-cert.cert -inkey rsapss-ca-key.pem -passin pass:password -certfile rsapss-ca-cert.cert -export -out rsaValidations-server-keystore.p12 -passout pass:password -name localhost
+
+
+# Create Trustore to verify the EndEntity certificate we have created.
+keytool -importcert -storetype PKCS12 -keystore rsaValidations-truststore.p12 -storepass password -alias ca -file rsapss-ca-cert.cert -noprompt
diff --git a/handler/src/test/resources/io/netty/handler/ssl/rsapss-ca-cert.cert b/handler/src/test/resources/io/netty/handler/ssl/rsapss-ca-cert.cert
new file mode 100644
index 00000000000..f767e738d3b
--- /dev/null
+++ b/handler/src/test/resources/io/netty/handler/ssl/rsapss-ca-cert.cert
@@ -0,0 +1,23 @@
+-----BEGIN CERTIFICATE-----
+MIIDxTCCAoKgAwIBAgIUJ2aZ084kIATHBPDJFXVu7SJ4uVcwOAYJKoZIhvcNAQEK
+MCugDTALBglghkgBZQMEAgGhGjAYBgkqhkiG9w0BAQgwCwYJYIZIAWUDBAIBMEcx
+CzAJBgNVBAYTAkdCMQ4wDAYDVQQKDAVOZXR0eTEVMBMGA1UECwwMbmV0dHktcGFy
+ZW50MREwDwYDVQQDDAh3ZXN0LmludDAeFw0yMTA4MjkwNjAxMTNaFw0zMTA4Mjcw
+NjAxMTNaMEcxCzAJBgNVBAYTAkdCMQ4wDAYDVQQKDAVOZXR0eTEVMBMGA1UECwwM
+bmV0dHktcGFyZW50MREwDwYDVQQDDAh3ZXN0LmludDCCASIwDQYJKoZIhvcNAQEB
+BQADggEPADCCAQoCggEBAL+xcxKjWgbYHIRqnP3Sw91SNTwY85ocb+3D4xh7/F6w
+cYgXwxgaHRKlk97HUzYZGFEb34BG89EOdDa1DvwxAMaN8sirefjrMLpvmfUD3Yti
+kGKj+CM3gh5wFSb9mPPoY/S61+KoRSAeMKeYyFQh5IIJyVqN5mrziu0+t04X4YEw
+9nATkmoS1V27Ucmo3OTkNNamqlXqVeiLKhvHtMViRGua8HwfEmjvFOTfyFHudcAz
+NFFH9JR9C2g9wuokcWFD3sdFfOZ4DJVN35NrXCO4FhxxcjHOXKRdbtsucFHqCPaE
+fVL0qrlkAm3pd9jKnBujC5sQbritg0uvmVuoxzy1jIUCAwEAAaNTMFEwHQYDVR0O
+BBYEFAzguQlpxd/3TPhYZqEryBQ6lUdJMB8GA1UdIwQYMBaAFAzguQlpxd/3TPhY
+ZqEryBQ6lUdJMA8GA1UdEwEB/wQFMAMBAf8wOAYJKoZIhvcNAQEKMCugDTALBglg
+hkgBZQMEAgGhGjAYBgkqhkiG9w0BAQgwCwYJYIZIAWUDBAIBA4IBAQB3jsUwdyFO
+9u/abLBGuETWbyuLX7NA9yvQL7cei40fJdsZZpZkHDJvNnrblpdaeFjuAI4vmAqz
+odiHzZodSaFCwODFX8oYyBcMTHW99UYiGywskF1NnJKq13r4kP7+7w7ZaE/5YukW
+VSeCXTHp1c0umuieluG87MZH4dCZgrvzpZwBeGoLLNyMyo4qHwYfkZiG2rTRpVX3
++VsWnMOaRVMYrzTB2tPZyAZyRMEfTd0fNi7ufSu6ywrOdziTu6Y1qVh18qDKpPsG
+eaSCNQoO5D9vUbiFjxKPJe8hZ0bDWTbVKRpeIrQMeHXnXGPEV5rPOcJUzwnDsGqI
+gqr6XlcEs+lp
+-----END CERTIFICATE-----
