internal: send a disconnect when key exchange fails

- DoKexDhInit() sends SSH_MSG_DISCONNECT with KEY_EXCHANGE_FAILED on
  WS_CRYPTO_FAILED and WS_PUBKEY_REJECTED_E, DoKexDhGexGroup() on
  WS_CRYPTO_FAILED and WS_DH_SIZE_E.
- DoKexDhReply() sends KEY_EXCHANGE_FAILED on WS_CRYPTO_FAILED and
  HOST_KEY_NOT_VERIFIABLE on WS_PUBKEY_REJECTED_E.
- DuplexEndpoint records the reason code of a plaintext outbound
  disconnect, and InitKexReplyHarnessKex() takes an explicit KEX
  algorithm.
- New mutator modes shorten f and e, write a zero-length e, cut the
  GEX prime below the requested floor and set the GEX generator to 1;
  LocateSinglePacketPayload() finds the payload for all three
  single-packet rewriters.
- The harness KEX algorithm falls back to curve25519-sha256, then
  ecdh-sha2-nistp256, when no plain diffie-hellman-group is built.
- Tests assert the reason code on the wire for each new mode and for
  host key rejection, and assert no disconnect on a successful
  handshake.

Issue: F-8838

Author: yosuke-wolfssl

src/internal.c

@@ -5944,6 +5944,12 @@ static int DoKexDhInit(WOLFSSH* ssh, byte* buf, word32 len, word32* idx)
         ret = SendKexDhReply(ssh);
     }
 
+    /* RFC 8731 sec. 3: a rejected key exchange input aborts with a
+     * disconnect */
+    if (ret == WS_CRYPTO_FAILED || ret == WS_PUBKEY_REJECTED_E) {
+        (void)SendDisconnect(ssh, WOLFSSH_DISCONNECT_KEY_EXCHANGE_FAILED);
+    }
+
     return ret;
 }
 
@@ -7627,6 +7633,15 @@ static int DoKexDhReply(WOLFSSH* ssh, byte* buf, word32 len, word32* idx)
 
     if (sigKeyBlock_ptr)
         WFREE(sigKeyBlock_ptr, ssh->ctx->heap, DYNTYPE_PRIVKEY);
+    /* RFC 4253 11.1: WS_PUBKEY_REJECTED_E here is only the host key check,
+     * which is server authentication, so it gets its own reason. */
+    if (ret == WS_CRYPTO_FAILED) {
+        (void)SendDisconnect(ssh, WOLFSSH_DISCONNECT_KEY_EXCHANGE_FAILED);
+    }
+    else if (ret == WS_PUBKEY_REJECTED_E) {
+        (void)SendDisconnect(ssh,
+                WOLFSSH_DISCONNECT_HOST_KEY_NOT_VERIFIABLE);
+    }
     WLOG(WS_LOG_DEBUG, "Leaving DoKexDhReply(), ret = %d", ret);
     return ret;
 }
@@ -8027,6 +8042,12 @@ static int DoKexDhGexGroup(WOLFSSH* ssh,
         ret = SendKexDhInit(ssh);
     }
 
+    /* A group under the enforced floor (RFC 8270) or a bad generator ends
+     * the key exchange, so tell the peer why. */
+    if (ret == WS_CRYPTO_FAILED || ret == WS_DH_SIZE_E) {
+        (void)SendDisconnect(ssh, WOLFSSH_DISCONNECT_KEY_EXCHANGE_FAILED);
+    }
+
     return ret;
 }
 
@@ -16281,9 +16302,8 @@ int SendKexDhGexRequest(WOLFSSH* ssh)
         }
     }
 
-    /* RFC 4419 sec. 3 requires min <= preferred <= max on the wire. Clamp
-     * preferred into that range in both directions; riding it up with a raised
-     * min or down under a lowered max keeps the advertised triple ordered. */
+    /* Keep the advertised triple ordered: clamp preferred into [min, max]
+     * in both directions. */
     if (ret == WS_SUCCESS) {
         if (ssh->handshake->dhGexPreferredSz < ssh->handshake->dhGexMinSz)
             ssh->handshake->dhGexPreferredSz = ssh->handshake->dhGexMinSz;