JNI: add RSA-PSS sign/verify and RSA sign check PK callbacks

Author: cconlon

native/com_wolfssl_WolfCryptRSA.c

@@ -26,6 +26,8 @@
     #include <wolfssl/options.h>
 #endif
 #include <wolfssl/wolfcrypt/rsa.h>
+#include <wolfssl/wolfcrypt/hash.h>
+#include <wolfssl/wolfcrypt/error-crypt.h>
 
 #include "com_wolfssl_WolfCryptRSA.h"
 
@@ -214,6 +216,189 @@ JNIEXPORT jint JNICALL Java_com_wolfssl_WolfCryptRSA_doEnc
     return ret;
 }
 
+JNIEXPORT jint JNICALL Java_com_wolfssl_WolfCryptRSA_doPssSign
+  (JNIEnv* jenv, jobject jcl, jobject in, jlong inSz, jobject out, jintArray outSz, jint hash, jint mgf, jobject keyDer, jlong keySz)
+{
+#ifdef WC_RSA_PSS
+    int     ret;
+    WC_RNG  rng;
+    RsaKey  myKey;
+    int     rngInit = 0;
+    int     keyInit = 0;
+    unsigned int idx = 0;
+    unsigned int tmpOut;
+    unsigned char* inBuf = NULL;
+    unsigned char* outBuf = NULL;
+    unsigned char* keyBuf = NULL;
+    enum wc_HashType hashType;
+    (void)jcl;
+
+    if ((inSz  < 0) || (keySz < 0)) {
+        return -1;
+    }
+
+    inBuf = (*jenv)->GetDirectBufferAddress(jenv, in);
+    if (inBuf == NULL) {
+        printf("problem getting in buffer address\n");
+        return -1;
+    }
+
+    outBuf = (*jenv)->GetDirectBufferAddress(jenv, out);
+    if (outBuf == NULL) {
+        printf("problem getting out buffer address\n");
+        return -1;
+    }
+
+    keyBuf = (*jenv)->GetDirectBufferAddress(jenv, keyDer);
+    if (keyBuf == NULL) {
+        printf("problem getting key buffer address\n");
+        return -1;
+    }
+
+    hashType = wc_OidGetHash(hash);
+    if (hashType == WC_HASH_TYPE_NONE) {
+        printf("doPssSign: unsupported hash OID %d\n", hash);
+        return -1;
+    }
+
+    /* get output buffer size */
+    (*jenv)->GetIntArrayRegion(jenv, outSz, 0, 1, (jint*)&tmpOut);
+
+    ret = wc_InitRng(&rng);
+    if (ret != 0) {
+        printf("wc_InitRng failed, ret = %d\n", ret);
+        return ret;
+    }
+    rngInit = 1;
+
+    ret = wc_InitRsaKey(&myKey, NULL);
+    if (ret != 0) {
+        printf("wc_InitRsaKey failed, ret = %d\n", ret);
+        wc_FreeRng(&rng);
+        return ret;
+    }
+    keyInit = 1;
+
+    ret = wc_RsaPrivateKeyDecode(keyBuf, &idx, &myKey, (unsigned int)keySz);
+    if (ret == 0) {
+
+        ret = wc_RsaPSS_Sign(inBuf, (unsigned int)inSz, outBuf, tmpOut,
+            hashType, mgf, &myKey, &rng);
+        if (ret > 0) {
+            tmpOut = ret;
+            (*jenv)->SetIntArrayRegion(jenv, outSz, 0, 1, (jint*)&tmpOut);
+            ret = 0;
+        }
+    } else {
+        printf("wc_RsaPrivateKeyDecode failed, ret = %d\n", ret);
+    }
+
+    if (keyInit) {
+        wc_FreeRsaKey(&myKey);
+    }
+    if (rngInit) {
+        wc_FreeRng(&rng);
+    }
+
+    return ret;
+#else
+    (void)jenv;
+    (void)jcl;
+    (void)in;
+    (void)inSz;
+    (void)out;
+    (void)outSz;
+    (void)hash;
+    (void)mgf;
+    (void)keyDer;
+    (void)keySz;
+    return (jint)NOT_COMPILED_IN;
+#endif /* WC_RSA_PSS */
+}
+
+JNIEXPORT jint JNICALL Java_com_wolfssl_WolfCryptRSA_doPssVerify
+  (JNIEnv* jenv, jobject jcl, jobject sig, jlong sigSz, jobject out, jlong outSz, jint hash, jint mgf, jobject keyDer, jlong keySz)
+{
+#ifdef WC_RSA_PSS
+    int     ret;
+    RsaKey  myKey;
+    unsigned int idx = 0;
+    unsigned char* sigBuf = NULL;
+    unsigned char* outBuf = NULL;
+    unsigned char* keyBuf = NULL;
+    enum wc_HashType hashType;
+    (void)jcl;
+
+    if ((sigSz < 0) || (keySz < 0) || (outSz < 0)) {
+        return -1;
+    }
+
+    sigBuf = (*jenv)->GetDirectBufferAddress(jenv, sig);
+    if (sigBuf == NULL) {
+        printf("problem getting sig buffer address\n");
+        return -1;
+    }
+
+    outBuf = (*jenv)->GetDirectBufferAddress(jenv, out);
+    if (outBuf == NULL) {
+        printf("problem getting out buffer address\n");
+        return -1;
+    }
+
+    keyBuf = (*jenv)->GetDirectBufferAddress(jenv, keyDer);
+    if (keyBuf == NULL) {
+        printf("problem getting key buffer address\n");
+        return -1;
+    }
+
+    hashType = wc_OidGetHash(hash);
+    if (hashType == WC_HASH_TYPE_NONE) {
+        printf("doPssVerify: unsupported hash OID %d\n", hash);
+        return -1;
+    }
+
+    ret = wc_InitRsaKey(&myKey, NULL);
+    if (ret != 0) {
+        printf("wc_InitRsaKey failed, ret = %d\n", ret);
+        return ret;
+    }
+
+    /* Try private key decode first (sign check receives the server private),
+     * fall back to public key decode (verify receives the peer public) */
+    ret = wc_RsaPrivateKeyDecode(keyBuf, &idx, &myKey, (unsigned int)keySz);
+    if (ret != 0) {
+        idx = 0;
+        ret = wc_RsaPublicKeyDecode(keyBuf, &idx, &myKey, (unsigned int)keySz);
+    }
+
+    if (ret == 0) {
+        ret = wc_RsaPSS_Verify(sigBuf, (unsigned int)sigSz, outBuf,
+            (unsigned int)outSz, hashType, mgf, &myKey);
+        if (ret < 0) {
+            printf("wc_RsaPSS_Verify failed, ret = %d\n", ret);
+        }
+    } else {
+        printf("RSA key decode failed, ret = %d\n", ret);
+    }
+
+    wc_FreeRsaKey(&myKey);
+
+    return ret;
+#else
+    (void)jenv;
+    (void)jcl;
+    (void)sig;
+    (void)sigSz;
+    (void)out;
+    (void)outSz;
+    (void)hash;
+    (void)mgf;
+    (void)keyDer;
+    (void)keySz;
+    return (jint)NOT_COMPILED_IN;
+#endif /* WC_RSA_PSS */
+}
+
 JNIEXPORT jint JNICALL Java_com_wolfssl_WolfCryptRSA_doDec
   (JNIEnv* jenv, jobject jcl, jobject in, jlong inSz, jobject out,
    jlong outSz, jobject keyDer, jlong keySz)