fix: always send USERAUTH_FAILURE on reject

RFC 4252 section 5.1 has the server answer a request it does not accept
with USERAUTH_FAILURE. A callback returning WOLFSSH_USERAUTH_REJECTED
now always gets that reply.

NO_FAILURE_ON_REJECTED suppressed it. The macro was never set by
configure, named in a header, or documented, and it guarded all four
method handlers alike.

- drop the macro and its four guards
- add test_UserAuthRejectedSendsFailure(), asserting on the wire, since
  the handler returns WS_USER_AUTH_E either way

Issue: F-11672

Author: ejohnstown

src/internal.c

@@ -8506,11 +8506,9 @@ static int DoUserAuthRequestNone(WOLFSSH* ssh, WS_UserAuthData* authData,
             }
             else if (ret == WOLFSSH_USERAUTH_REJECTED) {
                 WLOG(WS_LOG_DEBUG, "DUARN: none rejected");
-                #ifndef NO_FAILURE_ON_REJECTED
                 /* Count before failing: a send that blocks returns
                  * WS_WANT_WRITE, which isn't fatal on its own. */
                 (void)SendUserAuthFailureCount(ssh, 0, countIt);
-                #endif
                 ret = WS_USER_AUTH_E;
             }
             else if (ret == WOLFSSH_USERAUTH_WOULD_BLOCK) {
@@ -8639,9 +8637,7 @@ static int DoUserAuthInfoResponse(WOLFSSH* ssh,
             }
             else if (ret == WOLFSSH_USERAUTH_REJECTED) {
                 WLOG(WS_LOG_DEBUG, "DUARKB: keyboard rejected");
-                #ifndef NO_FAILURE_ON_REJECTED
-                    authFailure = 1;
-                #endif
+                authFailure = 1;
                 authRejected = 1;
                 ret = WS_USER_AUTH_E;
             }
@@ -8754,9 +8750,7 @@ static int DoUserAuthRequestPassword(WOLFSSH* ssh, WS_UserAuthData* authData,
                 }
                 else if (ret == WOLFSSH_USERAUTH_REJECTED) {
                     WLOG(WS_LOG_DEBUG, "DUARPW: password rejected");
-                    #ifndef NO_FAILURE_ON_REJECTED
-                        authFailure = 1;
-                    #endif
+                    authFailure = 1;
                     authRejected = 1;
                     ret = WS_USER_AUTH_E;
                 }
@@ -10186,9 +10180,7 @@ static int DoUserAuthRequestPublicKey(WOLFSSH* ssh, WS_UserAuthData* authData,
                 ret = WS_AUTH_PENDING;
             }
             else if (ret == WOLFSSH_USERAUTH_REJECTED) {
-                #ifndef NO_FAILURE_ON_REJECTED
-                    authFailure = 1;
-                #endif
+                authFailure = 1;
                 authRejected = 1;
                 ret = WS_USER_AUTH_E;
             }

tests/unit.c

@@ -7959,6 +7959,75 @@ static int CaptureIoSendAuthSvc(WOLFSSH* ssh, void* buf, word32 sz, void* ctx)
     return (int)sz;
 }
 
+/* Userauth callback that rejects outright. */
+static int UnitAuthAlwaysReject(byte authType, WS_UserAuthData* authData,
+        void* ctx)
+{
+    (void)authType;
+    (void)authData;
+    (void)ctx;
+    return WOLFSSH_USERAUTH_REJECTED;
+}
+
+
+/* RFC 4252 section 5.1: a rejected request is answered with
+ * USERAUTH_FAILURE. The handler returns WS_USER_AUTH_E either way, so the
+ * assertion is on what went out on the wire. */
+static int test_UserAuthRejectedSendsFailure(void)
+{
+    WOLFSSH_CTX* ctx = NULL;
+    WOLFSSH* ssh = NULL;
+    byte request[128];
+    word32 requestSz;
+    word32 idx = 0;
+    int ret;
+    int result = 0;
+
+    requestSz = BuildAuthPwRequest(request, (word32)sizeof(request));
+    if (requestSz == 0)
+        return -790;
+
+    ctx = wolfSSH_CTX_new(WOLFSSH_ENDPOINT_SERVER, NULL);
+    if (ctx == NULL)
+        return -791;
+    wolfSSH_SetUserAuth(ctx, UnitAuthAlwaysReject);
+    wolfSSH_SetIOSend(ctx, CaptureIoSendAuthSvc);
+
+    ssh = wolfSSH_new(ctx);
+    if (ssh == NULL) {
+        wolfSSH_CTX_free(ctx);
+        return -792;
+    }
+
+    s_authSvcCaptureSz = 0;
+    s_authSvcSendCount = 0;
+
+    ret = wolfSSH_TestDoUserAuthRequest(ssh, request, requestSz, &idx);
+
+    if (ret != WS_USER_AUTH_E) {
+        printf("UserAuthRejectedSendsFailure: ret=%d expected %d\n",
+               ret, WS_USER_AUTH_E);
+        result = -793;
+    }
+    else if (s_authSvcSendCount == 0) {
+        printf("UserAuthRejectedSendsFailure: nothing sent on rejection\n");
+        result = -794;
+    }
+    else if (CaptureMsgId(s_authSvcCapture, s_authSvcCaptureSz)
+            != MSGID_USERAUTH_FAILURE) {
+        printf("UserAuthRejectedSendsFailure: msgId=%d expected"
+               " USERAUTH_FAILURE\n",
+               CaptureMsgId(s_authSvcCapture, s_authSvcCaptureSz));
+        result = -795;
+    }
+
+    wolfSSH_free(ssh);
+    wolfSSH_CTX_free(ctx);
+
+    return result;
+}
+
+
 /* Verify DoUserAuthRequest rejects non-"ssh-connection" service names per
  * RFC 4252 Section 5.  For each case we assert:
  *   1. ret == WS_SUCCESS (connection stays open for retry)
@@ -17715,6 +17784,11 @@ int wolfSSH_UnitTest(int argc, char** argv)
            (unitResult == 0 ? "SUCCESS" : "FAILED"));
     testResult = testResult || unitResult;
 
+    unitResult = test_UserAuthRejectedSendsFailure();
+    printf("UserAuthRejectedSendsFailure: %s\n",
+           (unitResult == 0 ? "SUCCESS" : "FAILED"));
+    testResult = testResult || unitResult;
+
 #if !defined(WOLFSSH_NO_ECDSA_SHA2_NISTP256) && \
     !defined(WOLFSSH_NO_ECDSA_SHA2_NISTP384)
     unitResult = test_EccUserAuthCurveMismatch();