fix: EVP_DigestSignUpdate accept size_t not unsigned int

EVP_DigestSignUpdate was declared with unsigned int for the count
parameter, unlike OpenSSL's size_t.  EVP_DigestVerifyUpdate had the
right declaration but cast the size_t to unsigned int before passing
it to the internal helper, silently truncating counts > UINT_MAX.

Fix all three sites:
- wolfssl_evp_digest_pk_update internal helper: unsigned int -> size_t
- wolfSSL_EVP_DigestSignUpdate: unsigned int -> size_t in decl + defn
- wolfSSL_EVP_DigestVerifyUpdate: remove (unsigned int) cast
- Add overflow guard before narrowing to word32 for wc_HmacUpdate

Refs: ZD-21734

Author: MarkAtwood

tests/api/test_evp_pkey.c

@@ -2429,3 +2429,77 @@ int test_wolfSSL_EVP_PKEY_print_public(void)
     return EXPECT_RESULT();
 }
 
+/*
+ * Regression test: EVP_DigestSignUpdate and EVP_DigestVerifyUpdate must both
+ * accept size_t for the byte count.
+ *
+ * Before the fix:
+ *   - DigestSignUpdate declared unsigned int (not size_t), breaking FFI parity.
+ *   - DigestVerifyUpdate declared size_t but immediately cast to unsigned int
+ *     internally, silently truncating any count > UINT_MAX.
+ *
+ * Test vector: RFC 4231 §4.2 HMAC-SHA256 (independent oracle).
+ *   Key  : "Jefe"
+ *   Data : "what do ya want for nothing?" (28 bytes)
+ *   HMAC : 5bdcc146bf60754e6a042426089575c75a003f089d2739839dec58b964ec3843
+ */
+int test_wolfSSL_EVP_DigestSign_size_t_cnt(void)
+{
+    EXPECT_DECLS;
+#if defined(OPENSSL_EXTRA) && !defined(NO_HMAC) && !defined(NO_SHA256)
+    static const byte kKey[] = "Jefe";
+    static const byte kMsg[] = "what do ya want for nothing?";
+    static const byte kExpected[] = {
+        0x5b, 0xdc, 0xc1, 0x46, 0xbf, 0x60, 0x75, 0x4e,
+        0x6a, 0x04, 0x24, 0x26, 0x08, 0x95, 0x75, 0xc7,
+        0x5a, 0x00, 0x3f, 0x08, 0x9d, 0x27, 0x39, 0x83,
+        0x9d, 0xec, 0x58, 0xb9, 0x64, 0xec, 0x38, 0x43
+    };
+    WOLFSSL_EVP_PKEY  *key = NULL;
+    WOLFSSL_EVP_MD_CTX mdCtx;
+    unsigned char      sig[WC_MAX_DIGEST_SIZE];
+    size_t             sigSz = sizeof(sig);
+    /* Deliberately size_t — not unsigned int — to verify both APIs accept it
+     * without a cast.  This was the type mismatch caught by ZD-21734. */
+    size_t             msgSz = sizeof(kMsg) - 1;
+
+    ExpectNotNull(key = wolfSSL_EVP_PKEY_new_mac_key(EVP_PKEY_HMAC, NULL,
+                                                     kKey,
+                                                     (int)sizeof(kKey) - 1));
+    wolfSSL_EVP_MD_CTX_init(&mdCtx);
+
+    /* Sign: passes size_t count directly — regression for unsigned int decl */
+    ExpectIntEQ(wolfSSL_EVP_DigestSignInit(&mdCtx, NULL, EVP_sha256(),
+                                           NULL, key), 1);
+    ExpectIntEQ(wolfSSL_EVP_DigestSignUpdate(&mdCtx, kMsg, msgSz), 1);
+    ExpectIntEQ(wolfSSL_EVP_DigestSignFinal(&mdCtx, sig, &sigSz), 1);
+    ExpectIntEQ((int)sigSz, (int)sizeof(kExpected));
+    ExpectIntEQ(XMEMCMP(sig, kExpected, sizeof(kExpected)), 0);
+    ExpectIntEQ(wolfSSL_EVP_MD_CTX_cleanup(&mdCtx), 1);
+
+    /* Verify: passes size_t count directly — regression for silent truncation */
+    wolfSSL_EVP_MD_CTX_init(&mdCtx);
+    ExpectIntEQ(wolfSSL_EVP_DigestVerifyInit(&mdCtx, NULL, EVP_sha256(),
+                                             NULL, key), 1);
+    ExpectIntEQ(wolfSSL_EVP_DigestVerifyUpdate(&mdCtx, kMsg, msgSz), 1);
+    ExpectIntEQ(wolfSSL_EVP_DigestVerifyFinal(&mdCtx, kExpected,
+                                              sizeof(kExpected)), 1);
+    ExpectIntEQ(wolfSSL_EVP_MD_CTX_cleanup(&mdCtx), 1);
+
+    /* Overflow guard: cnt > UINT_MAX must fail, not silently truncate.
+     * Only reachable on 64-bit platforms where size_t exceeds word32. */
+    if (sizeof(size_t) > sizeof(word32)) {
+        size_t oversized = (size_t)(word32)-1 + 1; /* UINT_MAX + 1 */
+        wolfSSL_EVP_MD_CTX_init(&mdCtx);
+        ExpectIntEQ(wolfSSL_EVP_DigestSignInit(&mdCtx, NULL, EVP_sha256(),
+                                               NULL, key), 1);
+        ExpectIntEQ(wolfSSL_EVP_DigestSignUpdate(&mdCtx, kMsg, oversized),
+                    WOLFSSL_FAILURE);
+        ExpectIntEQ(wolfSSL_EVP_MD_CTX_cleanup(&mdCtx), 1);
+    }
+
+    wolfSSL_EVP_PKEY_free(key);
+#endif
+    return EXPECT_RESULT();
+} /* END test_wolfSSL_EVP_DigestSign_size_t_cnt */
+

tests/api/test_evp_pkey.h

@@ -61,6 +61,7 @@ int test_wolfSSL_EVP_MD_ecc_signing(void);
 int test_wolfSSL_EVP_PKEY_encrypt(void);
 int test_wolfSSL_EVP_PKEY_derive(void);
 int test_wolfSSL_EVP_PKEY_print_public(void);
+int test_wolfSSL_EVP_DigestSign_size_t_cnt(void);
 
 #define TEST_EVP_PKEY_DECLS                                                    \
     TEST_DECL_GROUP("evp_pkey", test_wolfSSL_EVP_PKEY_CTX_new_id),             \
@@ -100,6 +101,7 @@ int test_wolfSSL_EVP_PKEY_print_public(void);
     TEST_DECL_GROUP("evp_pkey", test_wolfSSL_EVP_MD_ecc_signing),              \
     TEST_DECL_GROUP("evp_pkey", test_wolfSSL_EVP_PKEY_encrypt),                \
     TEST_DECL_GROUP("evp_pkey", test_wolfSSL_EVP_PKEY_derive),                 \
-    TEST_DECL_GROUP("evp_pkey", test_wolfSSL_EVP_PKEY_print_public)
+    TEST_DECL_GROUP("evp_pkey", test_wolfSSL_EVP_PKEY_print_public),           \
+    TEST_DECL_GROUP("evp_pkey", test_wolfSSL_EVP_DigestSign_size_t_cnt)
 
 #endif /* WOLFCRYPT_TEST_EVP_PKEY_H */

wolfcrypt/src/evp.c

@@ -4697,10 +4697,12 @@ static int wolfSSL_evp_digest_pk_init(WOLFSSL_EVP_MD_CTX *ctx,
  * Update a digest for RSA and ECC keys, or HMAC for HMAC key.
  */
 static int wolfssl_evp_digest_pk_update(WOLFSSL_EVP_MD_CTX *ctx,
-                                        const void *d, unsigned int cnt)
+                                        const void *d, size_t cnt)
 {
     if (ctx->isHMAC) {
-        if (wc_HmacUpdate(&ctx->hash.hmac, (const byte *)d, cnt) != 0)
+        if (cnt > (word32)-1)
+            return WOLFSSL_FAILURE;
+        if (wc_HmacUpdate(&ctx->hash.hmac, (const byte *)d, (word32)cnt) != 0)
             return WOLFSSL_FAILURE;
 
         return WOLFSSL_SUCCESS;
@@ -4851,7 +4853,7 @@ int wolfSSL_EVP_DigestSignInit(WOLFSSL_EVP_MD_CTX *ctx,
 
 
 int wolfSSL_EVP_DigestSignUpdate(WOLFSSL_EVP_MD_CTX *ctx, const void *d,
-                                 unsigned int cnt)
+                                 size_t cnt)
 {
     WOLFSSL_ENTER("EVP_DigestSignUpdate");
 
@@ -4988,7 +4990,7 @@ int wolfSSL_EVP_DigestVerifyUpdate(WOLFSSL_EVP_MD_CTX *ctx, const void *d,
     if (ctx == NULL || d == NULL)
         return WOLFSSL_FAILURE;
 
-    return wolfssl_evp_digest_pk_update(ctx, d, (unsigned int)cnt);
+    return wolfssl_evp_digest_pk_update(ctx, d, cnt);
 }
 
 

wolfssl/openssl/evp.h

@@ -841,7 +841,7 @@ WOLFSSL_API int wolfSSL_EVP_DigestFinal_ex(WOLFSSL_EVP_MD_CTX* ctx,
 WOLFSSL_API int wolfSSL_EVP_DigestFinalXOF(WOLFSSL_EVP_MD_CTX* ctx,
                                             unsigned char* md, size_t sz);
 WOLFSSL_API int wolfSSL_EVP_DigestSignUpdate(WOLFSSL_EVP_MD_CTX *ctx,
-                                             const void *d, unsigned int cnt);
+                                             const void *d, size_t cnt);
 WOLFSSL_API int wolfSSL_EVP_DigestSignFinal(WOLFSSL_EVP_MD_CTX *ctx,
                                             unsigned char *sig, size_t *siglen);
 WOLFSSL_API int wolfSSL_EVP_DigestVerifyUpdate(WOLFSSL_EVP_MD_CTX *ctx,