wolfsshd.c

/* wolfsshd.c
 *
 * Copyright (C) 2014-2026 wolfSSL Inc.
 *
 * This file is part of wolfSSH.
 *
 * wolfSSH is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 3 of the License, or
 * (at your option) any later version.
 *
 * wolfSSH is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with wolfSSH.  If not, see <http://www.gnu.org/licenses/>.
 */

#ifdef HAVE_CONFIG_H
    #include <config.h>
#endif

#ifdef WOLFSSL_USER_SETTINGS
    #include <wolfssl/wolfcrypt/settings.h>
#else
    #include <wolfssl/options.h>
#endif

#ifdef WOLFSSH_SSHD

#include <wolfssh/ssh.h>
#include <wolfssh/internal.h>
#include <wolfssh/log.h>
#include <wolfssl/wolfcrypt/wc_port.h>
#include <wolfssl/wolfcrypt/error-crypt.h>
#include <wolfssl/wolfcrypt/logging.h>
#include <wolfssl/wolfcrypt/asn_public.h>

#define WOLFSSH_TEST_SERVER
#include <wolfssh/test.h>

#include "configuration.h"
#include "auth.h"

#include <signal.h>

#ifdef NO_INLINE
    #include <wolfssh/misc.h>
#else
    #define WOLFSSH_MISC_INCLUDED
    #include "src/misc.c"
#endif

#ifndef WOLFSSHD_TIMEOUT
    #define WOLFSSHD_TIMEOUT 1
#endif

#ifdef EXAMPLE_BUFFER_SZ
    #warning use WOLFSSHD_SHELL_BUFFER_SZ instead of EXAMPLE_BUFFER_SZ
    #define WOLFSSHD_SHELL_BUFFER_SZ EXAMPLE_BUFFER_SZ
#endif

#if defined(WOLFSSH_SHELL) && !defined(_WIN32)
    #ifdef HAVE_PTY_H
        #include <pty.h>
    #endif
    #ifdef HAVE_UTIL_H
        #include <util.h>
    #endif
    #ifdef HAVE_TERMIOS_H
        #include <termios.h>
    #endif
    #include <pwd.h>
    #include <signal.h>
    #include <sys/wait.h>
#if defined(__QNX__) || defined(__QNXNTO__)
    #include <errno.h>
    #include <unix.h>
#else
    #include <errno.h>
#endif

    static volatile int ChildRunning = 0;
    static void ChildSig(int sig)
    {
        (void)sig;
        ChildRunning = 0;
    }

    static void ConnClose(int sig)
    {
#ifndef WIN32
        pid_t p;
        int   ret;
        p = wait(&ret);
        if (p == 0 || p == -1)
            return; /* parent or error state*/
        (void)ret;
#endif
        (void)sig;
    }
#endif /* WOLFSSH_SHELL */

static volatile byte debugMode = 0; /* default to off */
static WFILE* logFile = NULL;

/* catch interrupts and close down gracefully */
static volatile byte quit = 0;

/* Initial connection information to pass on to threads/forks */
typedef struct WOLFSSHD_CONNECTION {
    WOLFSSH_CTX*   ctx;
    WOLFSSHD_AUTH* auth;
    int            fd;
    int            listenFd;
    char           ip[INET6_ADDRSTRLEN];
    byte           isThreaded;
#ifdef _WIN32
    /* set when the login grace time expires before authentication completes;
     * POSIX uses the file-scope loginGraceTimedOut flag instead */
    volatile byte  timeOut;
    PTP_TIMER      loginTimer; /* threadpool timer enforcing login grace time */
#endif
} WOLFSSHD_CONNECTION;

#ifdef __unix__

#include <syslog.h>

static void SyslogCb(enum wolfSSH_LogLevel level, const char *const msgStr)
{
    int priority;

    switch (level) {
        case WS_LOG_WARN:
            priority = LOG_WARNING;
            break;
        case WS_LOG_ERROR:
            priority = LOG_ERR;
            break;
        case WS_LOG_DEBUG:
            priority = LOG_DEBUG;
            break;
        case WS_LOG_INFO:
        case WS_LOG_USER:
        case WS_LOG_SFTP:
        case WS_LOG_SCP:
        case WS_LOG_AGENT:
        case WS_LOG_CERTMAN:
        default:
            priority = LOG_INFO;
            break;
    }
    openlog("sshd", LOG_PID, LOG_DAEMON);
    syslog(priority, "%s", msgStr);
    closelog();
}

#endif

#ifdef _WIN32
static void ServiceDebugCb(enum wolfSSH_LogLevel level, const char* const msgStr)
#ifdef UNICODE
{
    WCHAR* wc;
    size_t szWord = WSTRLEN(msgStr) + 3;  /* + 3 for null terminator and new
                                           * line */
    size_t sz     = szWord *sizeof(wchar_t);
    wc = (WCHAR*)WMALLOC(sz, NULL, DYNAMIC_TYPE_LOG);
    if (wc) {
        size_t con;

        if (mbstowcs_s(&con, wc, szWord, msgStr, szWord-1) == 0) {
            wc[con - 1] = L'\r';
            wc[con] = L'\n';
            wc[con + 1] = L'\0';
            OutputDebugString(wc);
        }
        WFREE(wc, NULL, DYNAMIC_TYPE_LOG);
    }
    WOLFSSH_UNUSED(level);
}
#else
{
    OutputDebugString(msgStr);
    WOLFSSH_UNUSED(level);
}
#endif
#endif /* _WIN32 */

static void ShowUsage(void)
{
    printf("wolfSSHd %s linked with wolfSSL %s\n", LIBWOLFSSH_VERSION_STRING,
        LIBWOLFSSL_VERSION_STRING);
    printf(" -?             display this help and exit\n");
    printf(" -f <file name> Configuration file to use, default is:\n"
           "                /etc/ssh/sshd_config\n");
    printf(" -p <int>       Port number to listen on\n");
    printf(" -d             Turn on debug mode\n");
    printf(" -D             Run in foreground (do not detach)\n");
    printf(" -h <file name> host private key file to use\n");
    printf(" -E <file name> append to log file\n");
}


/* catch if interrupted */
static void interruptCatch(int in)
{
    (void)in;
    quit = 1;
}

#ifdef WIN32
    #include <processthreadsapi.h>
    #define WGETPID GetCurrentProcessId
#else
    #define WGETPID getpid
#endif

/* redirect logging to a specific file and add the PID value */
static void wolfSSHDLoggingCb(enum wolfSSH_LogLevel lvl, const char *const str)
{
    /* always log errors and optionally log other info/debug level messages */
    if (lvl == WS_LOG_ERROR || debugMode) {
        fprintf(logFile, "[PID %d]: %s\n", WGETPID(), str);
        /* flush so each line is visible immediately, e.g. to a consumer
         * reading the log file while the daemon is still running */
        fflush(logFile);
    }
}


/* QNX fixes the host key's owner and modes in the system image, so
 * WOLFSSH_NO_HOSTKEY_PERMS hands that policy to the platform. Host key only, and
 * only ownership, modes and directories. Kept outside the NO_FILESYSTEM guard
 * below because SetupCTX() tests it and -Wundef is on. wolfssh-options.c repeats
 * this guard so the tests can skip the negative cases; keep the two in step. */
#if defined(WOLFSSH_NO_HOSTKEY_PERMS) && \
    (defined(__QNX__) || defined(__QNXNTO__))
    #define WOLFSSHD_HOSTKEY_RELAX_PERMS 1
#else
    #define WOLFSSHD_HOSTKEY_RELAX_PERMS 0
#endif

#ifndef NO_FILESYSTEM
static void freeBufferFromFile(byte* buf, void* heap)
{
    if (buf != NULL)
        WFREE(buf, heap, DYNTYPE_SSHD);
    (void)heap;
}


/* Load class for getBufferFromFile(). NORMAL files (e.g. the banner) are opened
 * directly. TRUST and SECRET files are trust anchors loaded through the secure
 * gate (no symlink, owned by root or the daemon, no group/world writable path
 * component); SECRET additionally rejects a group/world readable file, used for
 * the host private key. */
enum {
    WOLFSSHD_LOAD_NORMAL = 0,
    WOLFSSHD_LOAD_TRUST  = 1,
    WOLFSSHD_LOAD_SECRET = 2
};

/* set bufSz to size wanted if too small and buf is null */
static byte* getBufferFromFile(const char* fileName, word32* bufSz, void* heap,
        int loadClass)
{
    FILE* file;
    byte* buf = NULL;
    long fileSz;
    word32 readSz;
    int relaxPerms = 0;

    WOLFSSH_UNUSED(heap);

    if (fileName == NULL) return NULL;

#if WOLFSSHD_HOSTKEY_RELAX_PERMS
    if (loadClass == WOLFSSHD_LOAD_SECRET) {
        relaxPerms = 1;
    }
#endif

    if (loadClass == WOLFSSHD_LOAD_NORMAL) {
        if (WFOPEN(NULL, &file, fileName, "rb") != 0)
            return NULL;
    }
    else {
        /* Trust anchors always go through the secure gate, regardless of
         * StrictModes. The owner is the daemon's effective user (or root), and
         * the host private key (SECRET) is also refused if group/world
         * readable, unless WOLFSSHD_HOSTKEY_RELAX_PERMS. */
        if (wolfSSHD_OpenSecureFile(fileName,
#ifndef _WIN32
                geteuid(),
#else
                0,
#endif
                loadClass == WOLFSSHD_LOAD_SECRET /* rejectReadable */,
                relaxPerms, heap, &file) != WS_SUCCESS) {
            return NULL;
        }
    }

    if (!WFSEEK_SUCCESS(WFSEEK(NULL, file, 0, WSEEK_END))) {
        WFCLOSE(NULL, file);
        return NULL;
    }
    fileSz = WFTELL(NULL, file);
    if (fileSz < 0) {
        WFCLOSE(NULL, file);
        return NULL;
    }
    WREWIND(NULL, file);

    buf = (byte*)WMALLOC(fileSz + 1, heap, DYNTYPE_SSHD);
    if (buf != NULL) {
        readSz = (word32)WFREAD(NULL, buf, 1, fileSz, file);
        if (readSz < (size_t)fileSz) {
            WFCLOSE(NULL, file);
            WFREE(buf, heap, DYNTYPE_SSHD);
            return NULL;
        }
        /* NUL terminate so callers that treat the buffer as a C string (e.g.
         * the banner via wolfSSH_CTX_SetBanner/WSTRLEN) do not read past the
         * allocation. The extra byte was already accounted for above. */
        buf[readSz] = '\0';
        if (bufSz)
            *bufSz = readSz;
    }
    WFCLOSE(NULL, file);

    return buf;
}
#endif /* NO_FILESYSTEM */


static int UserAuthResult(byte result,
        WS_UserAuthData* authData, void* userAuthResultCtx);


/* Frees up the WOLFSSH_CTX struct */
static void CleanupCTX(WOLFSSHD_CONFIG* conf, WOLFSSH_CTX** ctx,
        byte** banner)
{
    if (banner != NULL && *banner != NULL) {
#ifndef NO_FILESYSTEM
        freeBufferFromFile(*banner, NULL);
#endif
        *banner = NULL;
    }
    if (ctx != NULL && *ctx != NULL) {
        wolfSSH_CTX_free(*ctx);
        *ctx = NULL;
    }
    (void)conf;
}

/* Initializes and sets up the WOLFSSH_CTX struct based on the configure options
 * return WS_SUCCESS on success
 */
static int SetupCTX(WOLFSSHD_CONFIG* conf, WOLFSSH_CTX** ctx,
        byte** banner)
{
    int ret = WS_SUCCESS;
    byte* keyDer = NULL;
    byte* privBuf = NULL;
    word32 privBufSz = 0;
    void* heap = NULL;

    if (ctx == NULL) {
        return WS_BAD_ARGUMENT;
    }

    /* create a new WOLFSSH_CTX */
    *ctx = wolfSSH_CTX_new(WOLFSSH_ENDPOINT_SERVER, NULL);
    if (*ctx == NULL) {
        wolfSSH_Log(WS_LOG_ERROR, "[SSHD] Couldn't allocate SSH CTX data.");
        ret = WS_MEMORY_E;
    }

    /* setup authority callback for checking peer connections */
    if (ret == WS_SUCCESS) {
        wolfSSH_SetUserAuth(*ctx, DefaultUserAuth);
        wolfSSH_SetUserAuthResult(*ctx, UserAuthResult);
    }

    /* set banner to display on connection */
    if (ret == WS_SUCCESS) {
#ifndef NO_FILESYSTEM
        *banner = getBufferFromFile(wolfSSHD_ConfigGetBanner(conf),
                NULL, heap, WOLFSSHD_LOAD_NORMAL);
#endif
        if (*banner) {
            wolfSSH_CTX_SetBanner(*ctx, (char*)*banner);
        }
    }

    /* Load in host private key */
    if (ret == WS_SUCCESS) {

        char* hostKey = wolfSSHD_ConfigGetHostKeyFile(conf);

        if (hostKey == NULL) {
            wolfSSH_Log(WS_LOG_ERROR, "[SSHD] No host private key set");
            ret = WS_BAD_ARGUMENT;
        }
        else {
            byte* data;
            word32 dataSz = 0;

            /* The host private key is a secret trust anchor: refuse a symlink,
             * an unsafe owner or path, or a group/world readable/writable
             * file. WOLFSSHD_HOSTKEY_RELAX_PERMS keeps only the symlink and
             * regular-file checks. */
        #if WOLFSSHD_HOSTKEY_RELAX_PERMS
            wolfSSH_Log(WS_LOG_INFO, "[SSHD] Built with "
                "WOLFSSH_NO_HOSTKEY_PERMS, host key ownership and permissions "
                "are left to the platform");
        #endif
            data = getBufferFromFile(hostKey, &dataSz, heap,
                WOLFSSHD_LOAD_SECRET);
            if (data == NULL) {
                /* NULL means the secure gate rejected the file (bad owner,
                 * symlink, group/world writable/readable; reason already
                 * logged) or the read failed, so report a file error rather
                 * than a memory error. */
                wolfSSH_Log(WS_LOG_ERROR,
                    "[SSHD] Error reading host key file.");
                ret = WS_BAD_FILE_E;

            }

            if (ret == WS_SUCCESS) {
                /* Host keys may be OpenSSH, PEM, or DER; detect the format
                 * and decode PEM/DER via wc_KeyPemToDer(), which handles
                 * PKCS#8 keys with no traditional DER form (e.g. ML-DSA). */
                if (dataSz == 0) {
                    /* An empty (0-byte) file passes the NULL check above but
                     * carries no key material. Handle it explicitly as a file
                     * error instead of falling into the PEM path, where
                     * WMALLOC(0) is implementation-defined (may return NULL and
                     * be misreported as WS_MEMORY_E). */
                    wolfSSH_Log(WS_LOG_ERROR, "[SSHD] Host key file is empty.");
                    ret = WS_BAD_FILE_E;
                }
                else {
                    int keyFormat = wolfSSHD_DetectPrivKeyFormat(data, dataSz,
                            heap, &keyDer, &privBuf, &privBufSz);

                    if (keyFormat == WS_MEMORY_E) {
                        wolfSSH_Log(WS_LOG_ERROR,
                            "[SSHD] Out of memory reading host private key.");
                        ret = WS_MEMORY_E;
                    }
                    else if (keyFormat < 0) {
                        wolfSSH_Log(WS_LOG_ERROR,
                            "[SSHD] Host private key file is invalid.");
                        ret = WS_BAD_FILE_E;
                    }
                    else if (keyFormat == WOLFSSH_FORMAT_OPENSSH) {
                        wolfSSH_Log(WS_LOG_DEBUG, "[SSHD] Loading host private "
                                    "key as OpenSSH format.");
                    }
                    else {
                        wolfSSH_Log(WS_LOG_DEBUG, "[SSHD] Loading host private "
                                    "key as DER format.");
                    }

                    if (ret == WS_SUCCESS &&
                            wolfSSH_CTX_UsePrivateKey_buffer(*ctx, privBuf,
                                    privBufSz, keyFormat) < 0) {
                        if (keyFormat == WOLFSSH_FORMAT_OPENSSH) {
                            /* Only composite ML-DSA keys support this format. */
                            wolfSSH_Log(WS_LOG_ERROR,
                                "[SSHD] Failed to use host private key: "
                                "OpenSSH format is only supported for "
                                "composite ML-DSA keys; convert with "
                                "\"ssh-keygen -p -m PEM\".");
                        }
                        else {
                            wolfSSH_Log(WS_LOG_ERROR,
                                "[SSHD] Failed to use host private key.");
                        }
                        ret = WS_BAD_ARGUMENT;
                    }
                }

                if (keyDer != NULL) {
                    WS_FORCEZERO(keyDer, dataSz);
                    WFREE(keyDer, heap, DYNTYPE_SSHD);
                }
                /* data is the key material itself for raw DER/OpenSSH
                 * input (privBuf aliases it directly, no copy). */
                WS_FORCEZERO(data, dataSz);
                freeBufferFromFile(data, heap);
            }
        }
    }

#if defined(WOLFSSH_OSSH_CERTS) || defined(WOLFSSH_CERTS)
    if (ret == WS_SUCCESS) {
        char* hostCert = wolfSSHD_ConfigGetHostCertFile(conf);

        if (hostCert != NULL) {
            byte*  data;
            word32 dataSz = 0;
        #ifdef WOLFSSH_CERTS
            byte* der = NULL;
            word32 derSz = 0;
            const byte* type = NULL;
            word32 typeSz = 0;
            byte flavor = WOLFSSH_CERT_FLAVOR_UNKNOWN;
        #endif

            data = getBufferFromFile(hostCert, &dataSz, heap,
                WOLFSSHD_LOAD_TRUST);
            if (data == NULL) {
                /* secure-gate rejection or read failure, not memory */
                wolfSSH_Log(WS_LOG_ERROR,
                    "[SSHD] Error reading host certificate file.");
                ret = WS_BAD_FILE_E;

            }

            if (ret == WS_SUCCESS) {
            #ifdef WOLFSSH_CERTS
                if (dataSz == 0) {
                    wolfSSH_Log(WS_LOG_ERROR,
                        "[SSHD] Host certificate file %s is empty.", hostCert);
                    ret = WS_BAD_FILE_E;
                }
                else {
                    /* Already read through the secure gate, so not _file. */
                    ret = wolfSSH_ReadCert_buffer(data, dataSz, &der, &derSz,
                        &type, &typeSz, &flavor, heap);
                    if (ret == WS_SUCCESS
                            && flavor != WOLFSSH_CERT_FLAVOR_X509) {
                        /* Verifying OpenSSH host certs is unimplemented. */
                        ret = WS_UNIMPLEMENTED_E;
                    }
                    if (ret == WS_SUCCESS) {
                        ret = wolfSSH_CTX_UseCert_buffer(*ctx, der, derSz,
                            WOLFSSH_FORMAT_ASN1);
                    }
                    if (ret != WS_SUCCESS) {
                        wolfSSH_Log(WS_LOG_ERROR,
                            "[SSHD] Failed to load in host certificate.");
                    }
                    if (der != NULL) {
                        WFREE(der, heap, DYNTYPE_CERT);
                    }
                }
            #else
                /* Only OpenSSH host certificates could apply here, and they are
                 * not implemented. Fail rather than ignore the directive. */
                wolfSSH_Log(WS_LOG_ERROR,
                    "[SSHD] HostCertificate is set but OpenSSH host "
                    "certificates are not supported in this build.");
                ret = WS_UNIMPLEMENTED_E;
            #endif

                freeBufferFromFile(data, heap);
            }
        }
    }
#endif /* WOLFSSH_OSSH_CERTS || WOLFSSH_CERTS */

#ifdef WOLFSSH_CERTS
    if (ret == WS_SUCCESS) {
        char* caCert = wolfSSHD_ConfigGetUserCAKeysFile(conf);
        if (caCert != NULL) {
            byte*  data;
            word32 dataSz = 0;

            wolfSSH_Log(WS_LOG_INFO, "[SSHD] Using CA keys file %s", caCert);
            data = getBufferFromFile(caCert, &dataSz, heap,
                WOLFSSHD_LOAD_TRUST);
            if (data == NULL) {
                /* secure-gate rejection or read failure, not memory */
                wolfSSH_Log(WS_LOG_ERROR,
                    "[SSHD] Error reading CA cert file.");
                ret = WS_BAD_FILE_E;

            }

            if (ret == WS_SUCCESS && dataSz == 0) {
                /* Empty means misconfigured, not a CA in another format. */
                wolfSSH_Log(WS_LOG_ERROR, "[SSHD] CA keys file %s is empty.",
                    caCert);
                ret = WS_BAD_FILE_E;
                freeBufferFromFile(data, heap);
            }
            else if (ret == WS_SUCCESS) {
                /* A CA is never named on the wire, so it takes raw bytes. */
                ret = wolfSSH_CTX_AddRootCert_buffer(*ctx, data, dataSz,
                    WOLFSSH_FORMAT_PEM);
                if (ret != WS_SUCCESS) {
                    ret = wolfSSH_CTX_AddRootCert_buffer(*ctx, data, dataSz,
                        WOLFSSH_FORMAT_ASN1);
                }
                if (ret != WS_SUCCESS) {
                #ifdef WOLFSSH_OSSH_CERTS
                    /* The file is not X.509; when OpenSSH certificates are also
                     * built, fall through and let the OpenSSH user-auth path
                     * consume TrustedUserCAKeys instead of failing startup. */
                    wolfSSH_Log(WS_LOG_INFO,
                        "[SSHD] CA keys file is not X.509; using it as an "
                        "OpenSSH-style CA.");
                    ret = WS_SUCCESS;
                #else
                    wolfSSH_Log(WS_LOG_ERROR,
                        "[SSHD] Failed to load in CA certificate.");
                #endif
                }

                freeBufferFromFile(data, heap);
            }
        }
    }
#endif

    if (ret == WS_SUCCESS) {
        wolfSSH_SetUserAuthTypes(*ctx, DefaultUserAuthTypes);
    }

    /* @TODO Load in host public key */

    /* Set allowed connection type, i.e. public key / password */

    (void)heap;
    return ret;
}

#ifndef _WIN32
/* return 1 if set, 0 if not set and negative values on error */
static int SetupChroot(WOLFSSHD_CONFIG* usrConf)
{
    int ret = 0;
    char* chrootPath;

    /* check for chroot set */
    chrootPath = wolfSSHD_ConfigGetChroot(usrConf);
    if (chrootPath != NULL) {
        ret = 1;
        wolfSSH_Log(WS_LOG_INFO, "[SSHD] chroot to path  %s", chrootPath);
        if (chdir(chrootPath) != 0) {
            wolfSSH_Log(WS_LOG_ERROR,
                "[SSHD] chdir to chroot path failed, %s", chrootPath);
            ret = WS_FATAL_ERROR;
        }
        /* Only chroot() once the chdir() into the target succeeded, and only
         * chdir("/") once inside the new root. Running a later step after an
         * earlier failure could leave the process chrooted with a working
         * directory outside the new root. */
        if (ret == 1 && chroot(chrootPath) != 0) {
            wolfSSH_Log(WS_LOG_ERROR,
                "[SSHD] chroot failed to path %s", chrootPath);
            ret = WS_FATAL_ERROR;
        }
        if (ret == 1 && chdir("/") != 0) {
            wolfSSH_Log(WS_LOG_ERROR,
                "[SSHD] chdir after chroot failed");
            ret = WS_FATAL_ERROR;
        }
   }
    return ret;
}
#endif

/* Resolve the force-command in effect for this session. A configured
 * ForceCommand takes precedence over a certificate's force-command. Returns
 * NULL when none is set. */
static const char* GetEffectiveForcedCmd(WOLFSSHD_CONNECTION* conn,
    WOLFSSHD_CONFIG* usrConf)
{
    const char* forcedCmd;

    forcedCmd = wolfSSHD_ConfigGetForcedCmd(usrConf);
#ifdef WOLFSSH_OSSH_CERTS
    forcedCmd = wolfSSHD_AuthMergeForcedCmd(forcedCmd,
        wolfSSHD_AuthGetForcedCmd(conn->auth));
#else
    (void)conn;
#endif
    return forcedCmd;
}


/* The force-command carried by an authenticated certificate, or NULL. A
 * configured ForceCommand does not mask it: the certificate restricts what the
 * session may do, while ForceCommand replaces what is run. */
static const char* GetCertForcedCmd(WOLFSSHD_CONNECTION* conn)
{
#ifdef WOLFSSH_OSSH_CERTS
    return wolfSSHD_AuthGetForcedCmd(conn->auth);
#else
    (void)conn;
    return NULL;
#endif
}

#ifdef WOLFSSH_SCP
static int SCP_Subsystem(WOLFSSHD_CONNECTION* conn, WOLFSSH* ssh,
    WPASSWD* pPasswd, WOLFSSHD_CONFIG* usrConf)
{
    int ret   = WS_SUCCESS;
    int error = WS_SUCCESS;
    int select_ret = 0;

#ifndef _WIN32
    /* temporarily elevate permissions to get users information */
    if (wolfSSHD_AuthRaisePermissions(conn->auth) != WS_SUCCESS) {
        wolfSSH_Log(WS_LOG_ERROR,
            "[SSHD] Failure to raise permissions for auth");
        return WS_FATAL_ERROR;
    }

    /* set additional groups if needed */
    if (wolfSSHD_AuthSetGroups(conn->auth, wolfSSH_GetUsername(ssh),
            pPasswd->pw_gid) != WS_SUCCESS) {
        wolfSSH_Log(WS_LOG_ERROR, "[SSHD] Error setting groups");
        ret = WS_FATAL_ERROR;
    }

    if (ret == WS_SUCCESS) {
        error = SetupChroot(usrConf);
        if (error < 0) {
            ret = error; /* error case with setup chroot */
        }
    }

    if (wolfSSHD_AuthReducePermissionsUser(conn->auth, pPasswd->pw_uid,
            pPasswd->pw_gid) != WS_SUCCESS) {
        wolfSSH_Log(WS_LOG_ERROR, "[SSHD] Error setting user ID");
        /* could not drop to the authenticated user; terminate the
         * per-connection process rather than continue at a higher
         * privilege level */
        exit(1);
    }
#else
    /* impersonate the logged on user for file permissions */
    if (ImpersonateLoggedOnUser(wolfSSHD_GetAuthToken(conn->auth)) == FALSE) {
        wolfSSH_Log(WS_LOG_ERROR,
            "[SSHD] Error impersonating logged on user");
        ret = WS_FATAL_ERROR;
    }
#endif

    if (ret == WS_SUCCESS) {
        ret = wolfSSH_accept(ssh);
        error = wolfSSH_get_error(ssh);
        while (ret != WS_SUCCESS && ret != WS_SCP_COMPLETE
                && (error == WS_WANT_READ || error == WS_WANT_WRITE)) {

            select_ret = tcp_select(conn->fd, 1);
            if (select_ret == WS_SELECT_RECV_READY  ||
                select_ret == WS_SELECT_ERROR_READY ||
                error      == WS_WANT_WRITE)
            {
                ret = wolfSSH_accept(ssh);
                error = wolfSSH_get_error(ssh);
            }
            else if (select_ret == WS_SELECT_TIMEOUT)
                error = WS_WANT_READ;
            else
                error = WS_FATAL_ERROR;
        }
    }

    if (ret != WS_SUCCESS && ret != WS_SCP_COMPLETE) {
        wolfSSH_Log(WS_LOG_ERROR,
            "[SSHD] Failed to finish SCP operation from IP %s",
            conn->ip);
    }

    (void)conn;
#ifdef _WIN32
    /* stop impersonating the user */
    RevertToSelf();
#endif
    return ret;
}
#endif /* WOLFSSH_SCP */

#ifdef WOLFSSH_SFTP
#define TEST_SFTP_TIMEOUT 1
#define TEST_SFTP_TIMEOUT_NONE 0

/* handle SFTP operations
 * returns WS_SUCCESS on success
 */
static int SFTP_Subsystem(WOLFSSHD_CONNECTION* conn, WOLFSSH* ssh,
    WPASSWD* pPasswd, WOLFSSHD_CONFIG* usrConf)
{
    int ret   = WS_SUCCESS;
    int error = WS_SUCCESS;
    WS_SOCKET_T sockfd;
    int select_ret = 0;
    int timeout = TEST_SFTP_TIMEOUT_NONE;
    byte peek_buf[1];

#ifndef _WIN32
    /* temporarily elevate permissions to get users information */
    if (wolfSSHD_AuthRaisePermissions(conn->auth) != WS_SUCCESS) {
        wolfSSH_Log(WS_LOG_ERROR,
            "[SSHD] Failure to raise permissions for auth");
        return WS_FATAL_ERROR;
    }

    /* set additional groups if needed */
    if (wolfSSHD_AuthSetGroups(conn->auth, wolfSSH_GetUsername(ssh),
            pPasswd->pw_gid) != WS_SUCCESS) {
        wolfSSH_Log(WS_LOG_ERROR, "[SSHD] Error setting groups");
        ret = WS_FATAL_ERROR;
    }

    if (ret == WS_SUCCESS) {
        error = SetupChroot(usrConf);
        if (error == 1) {
            /* chroot was executed */
            if (wolfSSH_SFTP_SetDefaultPath(ssh, "/") != WS_SUCCESS) {
                wolfSSH_Log(WS_LOG_ERROR,
                        "[SSHD] Error setting SFTP default path");
                ret = WS_FATAL_ERROR;
            }
        }
        else if (error < 0) {
            ret = error; /* error case with setup chroot */
        }
    }


    /* set starting SFTP directory */
    if (ret == WS_SUCCESS) {
        WDIR dir;

        /* if home directory exists than set it as the default */
        if (WOPENDIR(NULL, NULL, &dir, pPasswd->pw_dir) == 0) {
            if (wolfSSH_SFTP_SetDefaultPath(ssh, pPasswd->pw_dir)
                    != WS_SUCCESS) {
                wolfSSH_Log(WS_LOG_ERROR,
                    "[SSHD] Error setting SFTP default home path");
                ret = WS_FATAL_ERROR;
            }
            WCLOSEDIR(NULL, &dir);
        }
    }

    if (wolfSSHD_AuthReducePermissionsUser(conn->auth, pPasswd->pw_uid,
            pPasswd->pw_gid) != WS_SUCCESS) {
        wolfSSH_Log(WS_LOG_ERROR, "[SSHD] Error setting user ID");
        /* could not drop to the authenticated user; terminate the
         * per-connection process rather than continue at a higher
         * privilege level */
        exit(1);
    }
#else
    char r[MAX_PATH];
    size_t rSz = 0;
    WCHAR h[MAX_PATH];

    ret = wolfSSHD_GetHomeDirectory(conn->auth, ssh, h, MAX_PATH);

    /* convert home directory from wchar type to char */
    if (ret == WS_SUCCESS) {
        if (wcstombs_s(&rSz, r, MAX_PATH, h, MAX_PATH - 1) != 0) {
            ret = WS_FATAL_ERROR;
        }
    }

    if (ret == WS_SUCCESS) {
        wolfSSH_Log(WS_LOG_INFO,
            "[SSHD] Using directory %s for SFTP connection", r);
        if (wolfSSH_SFTP_SetDefaultPath(ssh, r) != WS_SUCCESS) {
            wolfSSH_Log(WS_LOG_ERROR,
                "[SSHD] Error setting SFTP default home path");
            ret = WS_FATAL_ERROR;
        }
    }

    /* impersonate the logged on user for file permissions */
    if (ImpersonateLoggedOnUser(wolfSSHD_GetAuthToken(conn->auth)) == FALSE) {
        wolfSSH_Log(WS_LOG_ERROR,
            "[SSHD] Error impersonating logged on user");
        ret = WS_FATAL_ERROR;
    }
#endif

    if (ret == WS_SUCCESS) {
        sockfd = (WS_SOCKET_T)wolfSSH_get_fd(ssh);
        do {
            if (wolfSSH_SFTP_PendingSend(ssh)) {
                /* Yes, process the SFTP data. */
                ret = wolfSSH_SFTP_read(ssh);
                error = wolfSSH_get_error(ssh);
                timeout = (ret == WS_REKEYING) ?
                    TEST_SFTP_TIMEOUT : TEST_SFTP_TIMEOUT_NONE;
                if (error == WS_WANT_READ || error == WS_WANT_WRITE ||
                    error == WS_CHAN_RXD || error == WS_REKEYING ||
                    error == WS_WINDOW_FULL)
                    ret = error;
                if (error == WS_WANT_WRITE && wolfSSH_SFTP_PendingSend(ssh)) {
                    continue; /* no need to spend time attempting to pull data
                                * if there is still pending sends */
                }
                if (error == WS_EOF) {
                    /* An ordinary session end, not a failure. */
                    ret = 0;
                    break;
                }
            }

            select_ret = tcp_select(sockfd, timeout);
            if (select_ret == WS_SELECT_ERROR_READY) {
                break;
            }

            if (ret == WS_WANT_READ || ret == WS_WANT_WRITE ||
                    select_ret == WS_SELECT_RECV_READY) {
                ret = wolfSSH_worker(ssh, NULL);
                error = wolfSSH_get_error(ssh);
                if (ret == WS_REKEYING) {
                    /* In a rekey, keeping turning the crank. */
                    timeout = TEST_SFTP_TIMEOUT;
                    continue;
                }

                if (error == WS_WANT_READ || error == WS_WANT_WRITE ||
                    error == WS_WINDOW_FULL) {
                    timeout = TEST_SFTP_TIMEOUT;
                    ret = error;
                    continue;
                }

                /* Drain what is buffered first. A rekey is not a drained
                 * channel: peek reports it without looking. */
                if (error == WS_EOF) {
                    int peekRet = wolfSSH_stream_peek(ssh, NULL, 1);

                    if (peekRet != WS_REKEYING && peekRet <= 0) {
                        /* An ordinary session end, not a failure. */
                        ret = 0;
                        break;
                    }
                }
                if (ret != WS_SUCCESS && ret != WS_CHAN_RXD
                        && ret != WS_EOF) {
                    /* If not successful and no channel data, leave. */
                    break;
                }
            }

            ret = wolfSSH_stream_peek(ssh, peek_buf, sizeof(peek_buf));
            if (ret > 0) {
                /* Yes, process the SFTP data. */
                ret = wolfSSH_SFTP_read(ssh);
                error = wolfSSH_get_error(ssh);
                timeout = (ret == WS_REKEYING) ?
                    TEST_SFTP_TIMEOUT : TEST_SFTP_TIMEOUT_NONE;
                if (error == WS_WANT_READ || error == WS_WANT_WRITE ||
                    error == WS_CHAN_RXD || error == WS_REKEYING ||
                    error == WS_WINDOW_FULL)
                    ret = error;
                if (error == WS_EOF) {
                    /* An ordinary session end, not a failure. */
                    ret = 0;
                    break;
                }
                continue;
            }
            else if (ret == WS_REKEYING) {
                timeout = TEST_SFTP_TIMEOUT;
                continue;
            }
            else if (ret < 0) {
                error = wolfSSH_get_error(ssh);
                if (error == WS_EOF) {
                    /* An ordinary session end, not a failure. */
                    ret = 0;
                    break;
                }
            }
            else {
                /* Channel is live with nothing buffered. Let the next select
                 * block instead of polling at its 100 us floor. */
                timeout = TEST_SFTP_TIMEOUT;
                continue;
            }

            if (ret == WS_FATAL_ERROR && error == 0) {
                WOLFSSH_CHANNEL* channel =
                    wolfSSH_ChannelNext(ssh, NULL);
                if (channel && wolfSSH_ChannelGetEof(channel)) {
                    ret = 0;
                    break;
                }
            }

        } while (ret != WS_FATAL_ERROR);
    }

    (void)conn;
#ifdef _WIN32
    /* stop impersonating the user */
    RevertToSelf();
#endif
    return ret;
}
#endif


#ifdef WOLFSSH_SHELL

#ifndef MAX_COMMAND_SZ
#define MAX_COMMAND_SZ 80
#endif

#ifdef WIN32

/* handles creating a new shell env. and maintains SSH connection for incoming
 * user input as well as output of the shell.
 * return WS_SUCCESS on success */
static int SHELL_Subsystem(WOLFSSHD_CONNECTION* conn, WOLFSSH* ssh,
    WPASSWD* pPasswd, WOLFSSHD_CONFIG* usrConf, const char* subCmd)
{
    BOOL ret;
    word32 shellChannelId = 0;
#ifndef WOLFSSHD_SHELL_BUFFER_SZ
    /* default to try and read max packet size */
    #define WOLFSSHD_SHELL_BUFFER_SZ 32768
#endif
    byte shellBuffer[WOLFSSHD_SHELL_BUFFER_SZ];
    int cnt_r, cnt_w;
    int windowFull = 0; /* bytes left in shellBuffer that a prior send could
                         * not pass on to wolfSSH yet. This happens with window
                         * full, rekey, or want-write; resent before reading
                         * more so the buffered data is not overwritten. */
    HANDLE ptyIn = NULL, ptyOut = NULL;
    HANDLE cnslIn = NULL, cnslOut = NULL;
    STARTUPINFOEX ext;
    PCWSTR sysCmd = L"c:\\windows\\system32\\cmd.exe";
#if 0
    /* start powershell instead */
    PCWSTR sysCmd = L"C:\\Windows\\System32\\WindowsPowerShell\\v1.0\\powershell.exe";
#endif
    PWSTR cmd = NULL;
    size_t cmdSz = 0;
    PWSTR conCmdPtr = NULL;
    size_t conCmdSz = 0;
    PROCESS_INFORMATION processInfo;
    int processCreated = 0;
    size_t sz = 0;
    WCHAR h[MAX_PATH];
    char* forcedCmd;
    WOLFSSH_CHANNEL* shellChannel;

    /* Off the channel list, not DEFAULT_NEXT_CHANNEL, which a build can
     * override. Same as the POSIX copy; this loop routes its reads and sends
     * off the same id. */
    shellChannel = wolfSSH_ChannelNext(ssh, NULL);
    if (shellChannel == NULL || wolfSSH_ChannelGetId(shellChannel,
            &shellChannelId, WS_CHANNEL_ID_SELF) != WS_SUCCESS) {
        wolfSSH_Log(WS_LOG_ERROR, "[SSHD] No session channel to service");
        return WS_FATAL_ERROR;
    }

    forcedCmd = wolfSSHD_ConfigGetForcedCmd(usrConf);

    /* @TODO check for conpty support LoadLibrary()and GetProcAddress(). */

    if (forcedCmd != NULL && WSTRCMP(forcedCmd, "internal-sftp") == 0) {
        wolfSSH_Log(WS_LOG_ERROR,
            "[SSHD] Only SFTP connections allowed for user "
            "%s", wolfSSH_GetUsername(ssh));
        return WS_FATAL_ERROR;
    }

    ret = wolfSSHD_GetHomeDirectory(conn->auth, ssh, h, MAX_PATH);
    if (ret == WS_SUCCESS) {
        ZeroMemory(&ext, sizeof(STARTUPINFOEX));
        ZeroMemory(&processInfo, sizeof(PROCESS_INFORMATION));

        /* use forced command if set over subCmd */
        if (forcedCmd == NULL && subCmd != NULL) {
            forcedCmd = (char*)subCmd;
        }

        if (forcedCmd != NULL) { /* copy over set command if present */
             /* +1 for terminator and +2 for quotes */
            cmdSz = WSTRLEN(forcedCmd) + wcslen(sysCmd) + WSTRLEN(" /C ") + 3;
            cmd = (PWSTR)WMALLOC(sizeof(wchar_t) * cmdSz, NULL, DYNTYPE_SSHD);
            if (cmd == NULL) {
                ret = WS_MEMORY_E;
            }
            else {
                WCHAR* tmp = (WCHAR*)WMALLOC(sizeof(wchar_t) * cmdSz, NULL,
                    DYNTYPE_SSHD);
                if (tmp == NULL) {
                    ret = WS_MEMORY_E;
                }

                if (ret == WS_SUCCESS) {
                    size_t wr = 0;
                    if (mbstowcs_s(&wr, tmp, cmdSz, forcedCmd, cmdSz - 1) != 0) {
                        ret = WS_FATAL_ERROR;
                    }
                }

                if (ret == WS_SUCCESS) {
                    swprintf(cmd, cmdSz, L"%s /C %s", sysCmd, tmp);
                }

                if (tmp != NULL) {
                    WFREE(tmp, NULL, DYNTYPE_SSHD);
                }

                /* read in case a window-change packet might be queued */
                {
                    int rc;
                    word32 lastChannel = 0;

                    do {
                        rc = wolfSSH_worker(ssh, &lastChannel);
                        if (rc < 0) {
                            rc = wolfSSH_get_error(ssh);
                        }
                    } while (rc == WS_WANT_WRITE);
                }
            }
        }
        else { /* when set command is not present start 'cmd.exe' */
            cmdSz = wcslen(sysCmd) + 1; /* +1 for terminator */
            cmd = (PWSTR)WMALLOC(sizeof(wchar_t) * cmdSz, NULL, DYNTYPE_SSHD);
            if (cmd == NULL) {
                ret = WS_MEMORY_E;
            }
            else {
                wcscpy_s(cmd, cmdSz, sysCmd);
            }
        }
    }

    if (ImpersonateLoggedOnUser(wolfSSHD_GetAuthToken(conn->auth)) == FALSE) {
        ret = WS_FATAL_ERROR;
    }

    if (ret == WS_SUCCESS) {
        SECURITY_ATTRIBUTES saAttr;

        ZeroMemory(&saAttr, sizeof(saAttr));
        saAttr.nLength = sizeof(SECURITY_ATTRIBUTES);
        saAttr.bInheritHandle = TRUE;
        saAttr.lpSecurityDescriptor = NULL;

        if (CreatePipe(&cnslIn, &ptyIn, &saAttr, 0) != TRUE) {
            ret = WS_FATAL_ERROR;
        }

        if (CreatePipe(&ptyOut, &cnslOut, &saAttr, 0) != TRUE) {
            ret = WS_FATAL_ERROR;
        }
    }

    if (ret == WS_SUCCESS) {
        STARTUPINFOW si;
        PCWSTR conCmd = L"wolfsshd.exe -r ";

        SetHandleInformation(ptyIn, HANDLE_FLAG_INHERIT, 0);
        SetHandleInformation(ptyOut, HANDLE_FLAG_INHERIT, 0);

        wolfSSH_SetTerminalResizeCtx(ssh, (void*)&ptyIn);

        conCmdSz = wcslen(conCmd) + cmdSz + 3;
            /* +1 for terminator, +2 for quotes */
        conCmdPtr = (PWSTR)WMALLOC(sizeof(wchar_t) * conCmdSz,
                NULL, DYNTYPE_SSHD);
        if (conCmdPtr == NULL) {
            ret = WS_MEMORY_E;
        }
        else {
            _snwprintf_s(conCmdPtr, conCmdSz, conCmdSz,
                L"wolfsshd.exe -r \"%s\"", cmd);
        }

        ZeroMemory(&si, sizeof(si));
        si.cb = sizeof(si);

        si.hStdInput  = cnslIn;
        si.hStdOutput = cnslOut;
        si.hStdError  = cnslOut;
        si.dwFlags    = STARTF_USESTDHANDLES;
        si.lpDesktop  = NULL;

        if (CreateProcessAsUserW(wolfSSHD_GetAuthToken(conn->auth), NULL, conCmdPtr,
            NULL, NULL, TRUE, DETACHED_PROCESS, NULL, h,
            &si, &processInfo) != TRUE) {
            wolfSSH_Log(WS_LOG_ERROR,
                "[SSHD] Issue creating process, Windows error %d", GetLastError());
            ret = WS_FATAL_ERROR;
            goto cleanup;
        }

        processCreated = 1;

        /* Release the parent's copies of the child's stdio handles now that
         * the child has inherited them. Holding cnslOut (write-end of stdout
         * pipe) open would prevent the OS from signalling EOF when the child
         * exits, making any future blocking ReadFile loop hang indefinitely. */
        CloseHandle(cnslIn);
        cnslIn = NULL;
        CloseHandle(cnslOut);
        cnslOut = NULL;
    }

    if (ret == WS_SUCCESS) {
        /* Worst case "\x1b[8;%u;%ut" with two 10-digit word32 values is 26
         * bytes plus the terminator; size generously. */
        char cmdWSize[32];
        int cmdWSizeSz;
        DWORD wrtn = 0;

        wolfSSH_Log(WS_LOG_INFO, "[SSHD] Successfully created process for "
            "console, waiting for it to start");

        WaitForInputIdle(processInfo.hProcess, 1000);

        /* Send initial terminal size to pseudo console with VT control sequence.
         * heightRows/widthChar are peer-supplied word32 values, so format them
         * with %u and clamp the return value before handing it to WriteFile to
         * avoid over-reading the stack buffer. */
        cmdWSizeSz = WSNPRINTF(cmdWSize, sizeof(cmdWSize), "\x1b[8;%u;%ut",
            ssh->heightRows, ssh->widthChar);
        if (cmdWSizeSz < 0 || cmdWSizeSz >= (int)sizeof(cmdWSize)) {
            cmdWSizeSz = (int)sizeof(cmdWSize) - 1;
        }
        if (WriteFile(ptyIn, cmdWSize, cmdWSizeSz, &wrtn, 0) != TRUE) {
            WLOG(WS_LOG_ERROR, "Issue with pseudo console resize");
            ret = WS_FATAL_ERROR;
        }
    }

    if (ret == WS_SUCCESS) {
        SOCKET sshFd;
        byte tmp[2];
        fd_set readFds;
        WS_SOCKET_T maxFd;
        int pending = 0;
        int readPending = 0;
        int rc = 0;
        DWORD processState;
        DWORD ava;
        struct timeval t;

        t.tv_sec = 0;
        t.tv_usec = 800;

        sshFd = wolfSSH_get_fd(ssh);
        maxFd = sshFd;

        FD_ZERO(&readFds);
        FD_SET(sshFd, &readFds);

        do {
            /* @TODO currently not blocking till data comes in */
            if (PeekNamedPipe(ptyOut, NULL, 0, NULL, &ava, NULL) == TRUE) {
                if (ava > 0) {
                    readPending = 1;
                }
            }

            if (readPending == 0) {
                /* check if process is still running before waiting to read */
                if (GetExitCodeProcess(processInfo.hProcess, &processState)
                    == TRUE) {
                    if (processState != STILL_ACTIVE) {
                        wolfSSH_Log(WS_LOG_INFO,
                            "[SSHD] Process has exited, exit state = %d, "
                            "close down SSH connection", processState);
                        Sleep(100); /* give the stdout/stderr of process a
                                     * little time to write to pipe */
                        if (PeekNamedPipe(ptyOut, NULL, 0, NULL, &ava, NULL)
                            == TRUE) {
                            if (ava > 0) {
                                /* if data still pending then continue
                                 * sending it over SSH */
                                readPending = 1;
                                continue;
                            }
                        }
                        /* keep looping while buffered data still needs to be
                         * flushed to the SSH channel */
                        if (!windowFull)
                            break;
                    }
                }
                if (wolfSSH_stream_peek(ssh, tmp, 1) <= 0) {
                    rc = select((int)maxFd + 1, &readFds, NULL, NULL, &t);
                    if (rc == -1) {
                        wolfSSH_Log(WS_LOG_INFO,
                            "[SSHD] select call waiting on socket failed");
                        break;
                    }
                    /* when select times out and no socket is set as ready
                       Windows overwrites readFds with 0. Reset the fd here
                       for next select call */
                    if (rc == 0) {
                        FD_SET(sshFd, &readFds);
                    }
                }
                else {
                    pending = 1;
                }
            }

            if (rc != 0 && (pending || FD_ISSET(sshFd, &readFds))) {
                word32 lastChannel = 0;

                /* The following tries to read from the first channel inside
                   the stream. If the pending data in the socket is for
                   another channel, this will return an error with id
                   WS_CHAN_RXD. That means the agent has pending data in its
                   channel. The additional channel is only used with the
                   agent. */
                cnt_r = wolfSSH_worker(ssh, &lastChannel);
                if (cnt_r < 0) {
                    rc = wolfSSH_get_error(ssh);
                    if (rc == WS_CHAN_RXD) {
                        if (lastChannel == shellChannelId) {
                            cnt_r = wolfSSH_ChannelIdRead(ssh,
                                shellChannelId, shellBuffer,
                                sizeof shellBuffer);
                            if (cnt_r <= 0)
                                break;
                            pending = 0;
                            if (WriteFile(ptyIn, shellBuffer, cnt_r, &cnt_r,
                                NULL) != TRUE) {
                                wolfSSH_Log(WS_LOG_INFO,
                                    "[SSHD] Error writing to pipe for "
                                    "console");
                                break;
                            }
                        }
                    }
                    else if (rc == WS_CHANNEL_CLOSED) {
                        continue;
                    }
                    else if (rc == WS_EOF) {
                        /* The peer is done sending. No EOF of ours here: it
                         * latches eofTxd and the child's remaining console
                         * output would then be refused, which both send sites
                         * below treat as fatal. wolfSSH_shutdown() sends it at
                         * teardown, as the POSIX copy relies on. Closing the
                         * write end of the child's stdin is still owed on this
                         * platform, and so is the per-pass drain: ptyIn is the
                         * terminal-resize context, and this copy reads into
                         * shellBuffer, which the windowFull resend owes the
                         * peer. Both want fixing where they can be tested. */
                        continue;
                    }
                    else if (rc != WS_WANT_READ && rc != WS_WANT_WRITE) {
                        break;
                    }
                }
            }

            /* if a previous send could not complete, resend the buffered data
             * before reading more so shellBuffer is not overwritten */
            if (windowFull) {
                cnt_w = wolfSSH_ChannelIdSend(ssh, shellChannelId,
                    shellBuffer, windowFull);
                if (cnt_w == WS_WINDOW_FULL || cnt_w == WS_REKEYING ||
                    cnt_w == WS_WANT_WRITE) {
                    continue;
                }
                else if (cnt_w < 0) {
                    break;
                }
                else {
                    windowFull -= cnt_w;
                    if (windowFull > 0) {
                        WMEMMOVE(shellBuffer, shellBuffer + cnt_w, windowFull);
                        continue;
                    }
                    if (windowFull < 0)
                        windowFull = 0;
                }
            }

            if (readPending && !windowFull) {
                WMEMSET(shellBuffer, 0, WOLFSSHD_SHELL_BUFFER_SZ);

                if (ReadFile(ptyOut, shellBuffer, WOLFSSHD_SHELL_BUFFER_SZ, &cnt_r,
                    NULL) != TRUE) {
                    wolfSSH_Log(WS_LOG_INFO,
                        "[SSHD] Error reading from pipe for console");
                    break;
                }
                else {
                    readPending = 0;
                    if (cnt_r > 0) {
                        cnt_w = wolfSSH_ChannelIdSend(ssh, shellChannelId,
                            shellBuffer, cnt_r);
                        if (cnt_w > 0 && cnt_w < cnt_r) { /* partial send */
                            windowFull = cnt_r - cnt_w;
                            WMEMMOVE(shellBuffer, shellBuffer + cnt_w,
                                windowFull);
                        }
                        else if (cnt_w == WS_WINDOW_FULL ||
                                 cnt_w == WS_REKEYING ||
                                 cnt_w == WS_WANT_WRITE) {
                            windowFull = cnt_r; /* save amount to be sent */
                        }
                        else if (cnt_w < 0) {
                            break;
                        }
                    }
                }
            }
        } while (1);

        wolfSSH_Log(WS_LOG_INFO,
            "[SSHD] Closing down process for console");

        if (ext.lpAttributeList != NULL) {
            HeapFree(GetProcessHeap(), 0, ext.lpAttributeList);
        }

        if (wolfSSH_SetExitStatus(ssh, processState) !=
            WS_SUCCESS) {
            wolfSSH_Log(WS_LOG_ERROR, "[SSHD] Issue sending child's exit "
                "status");
        }

    }

cleanup:
    if (conCmdPtr != NULL) {
        WFREE(conCmdPtr, NULL, DYNTYPE_SSHD);
    }
    if (cnslIn != NULL) {
        CloseHandle(cnslIn);
    }
    if (cnslOut != NULL) {
        CloseHandle(cnslOut);
    }
    if (ptyIn != NULL) {
        CloseHandle(ptyIn);
    }
    if (ptyOut != NULL) {
        CloseHandle(ptyOut);
    }
    if (processCreated) {
        CloseHandle(processInfo.hThread);
        CloseHandle(processInfo.hProcess);
        wolfSSHD_AuthCloseToken(conn->auth);
    }
    if (cmd != NULL) {
        WFREE(cmd, NULL, DYNTYPE_SSHD);
    }
    RevertToSelf();
    return ret;
}
#else
#if defined(HAVE_SYS_IOCTL_H)
    #include <sys/ioctl.h>
#endif

static int SHELL_IsPty(WOLFSSH* ssh)
{
    WOLFSSH_CHANNEL* channel;
    int ret = WS_SUCCESS;
    word32 channelId = 0;

    ret = wolfSSH_GetLastRxId(ssh, &channelId);
    if (ret == WS_SUCCESS) {
        channel = wolfSSH_ChannelFind(ssh, channelId, WS_CHANNEL_ID_SELF);
    }

    if (ret == WS_SUCCESS) {
        ret =  wolfSSH_ChannelIsPty(channel);
    }
    return ret;
}

/* set a descriptor to non blocking, returns 0 on success and -1 on failure */
static int SHELL_SetNonBlocking(int fd)
{
    int flags;

    flags = fcntl(fd, F_GETFL, 0);
    if (flags < 0) {
        wolfSSH_Log(WS_LOG_ERROR, "[SSHD] fcntl get failed");
        return -1;
    }

    if (fcntl(fd, F_SETFL, flags | O_NONBLOCK) < 0) {
        wolfSSH_Log(WS_LOG_ERROR, "[SSHD] fcntl set failed");
        return -1;
    }

    return 0;
}


#ifndef WOLFSSHD_SHELL_FLUSH_TRIES
    #define WOLFSSHD_SHELL_FLUSH_TRIES 10
#endif
#ifndef WOLFSSHD_SHELL_FLUSH_WAIT_US
    #define WOLFSSHD_SHELL_FLUSH_WAIT_US 50000
#endif

/* Send out the last of a shell's output once the child has been reaped. The
 * SSH socket is non blocking, so retry a bounded number of times on a full
 * window, a rekey or a would block. 'ext' selects the extended (stderr) data
 * stream. Returns 0 on success and -1 when the data could not all be sent. */
static int SHELL_FlushOut(WOLFSSH* ssh, WS_SOCKET_T sshFd, word32 channelId,
    byte* buf, int sz, int ext)
{
    int tries = 0;

    while (sz > 0 && tries < WOLFSSHD_SHELL_FLUSH_TRIES) {
        int cnt_w;

        if (ext) {
            cnt_w = wolfSSH_extended_data_send(ssh, buf, sz);
        }
        else {
            cnt_w = wolfSSH_ChannelIdSend(ssh, channelId, buf, sz);
        }

        if (cnt_w == WS_WINDOW_FULL || cnt_w == WS_REKEYING ||
                cnt_w == WS_WANT_WRITE) {
            fd_set fds;
            struct timeval to;

            FD_ZERO(&fds);
            FD_SET(sshFd, &fds);
            to.tv_sec  = 0;
            to.tv_usec = WOLFSSHD_SHELL_FLUSH_WAIT_US;
            if (cnt_w == WS_WANT_WRITE) {
                select((int)sshFd + 1, NULL, &fds, NULL, &to);
            }
            else {
                /* waiting on the peer's window adjust or kex packets */
                select((int)sshFd + 1, &fds, NULL, NULL, &to);
            }

            /* process what came in, otherwise the window never opens and
             * the rekey never finishes, and the send can not progress */
            if (wolfSSH_worker(ssh, NULL) < 0) {
                int err = wolfSSH_get_error(ssh);

                /* A peer EOF during the final flush is expected. */
                if (err != WS_WANT_READ && err != WS_WANT_WRITE &&
                        err != WS_CHAN_RXD && err != WS_REKEYING &&
                        err != WS_EOF) {
                    wolfSSH_Log(WS_LOG_ERROR,
                        "[SSHD] Issue draining connection on final flush");
                    return -1;
                }
            }
            tries++;
            continue;
        }

        if (cnt_w < 0) {
            wolfSSH_Log(WS_LOG_ERROR,
                "[SSHD] Issue sending final shell output");
            return -1;
        }

        sz -= cnt_w;
        if (sz > 0) {
            WMEMMOVE(buf, buf + cnt_w, sz);
            tries++;
        }
    }

    if (sz > 0) {
        wolfSSH_Log(WS_LOG_ERROR, "[SSHD] Unable to send all of the final "
            "shell output, %d bytes dropped", sz);
        return -1;
    }

    return 0;
}


/* handles creating a new shell env. and maintains SSH connection for incoming
 * user input as well as output of the shell.
 * return WS_SUCCESS on success */
static int SHELL_Subsystem(WOLFSSHD_CONNECTION* conn, WOLFSSH* ssh,
    WPASSWD* pPasswd, WOLFSSHD_CONFIG* usrConf, const char* subCmd)
{
    WS_SOCKET_T sshFd = 0;
    int rc;
    WS_SOCKET_T childFd = 0;
    int stdoutPipe[2], stderrPipe[2];
    int stdinPipe[2];
    pid_t childPid;

#ifndef WOLFSSHD_SHELL_BUFFER_SZ
    /* default to try and read max packet size */
    #define WOLFSSHD_SHELL_BUFFER_SZ 32768
#endif
#ifndef MAX_IDLE_COUNT
    #define MAX_IDLE_COUNT 2
#endif
    byte shellBuffer[WOLFSSHD_SHELL_BUFFER_SZ];
    byte channelBuffer[WOLFSSHD_SHELL_BUFFER_SZ];
    const char* forcedCmd;
    int   windowFull = 0; /* Contains size of bytes from shellBuffer that did
                           * not get passed on to wolfSSH yet. This happens
                           * with window full errors or when rekeying.  */
    int   windowFullExt = 0; /* Nonzero when the bytes held in shellBuffer
                              * belong to the extended (stderr) data stream
                              * and must be resent with
                              * wolfSSH_extended_data_send().  */
    int   wantWrite  = 0;
    int   peerConnected = 1;
    int   stdoutEmpty = 0;
    int   ptyReq = 0;
    int   childInSz = 0;  /* Bytes read off the channel into channelBuffer
                           * that the child has yet to take. The read is
                           * destructive, so what a short write leaves is
                           * carried to the next pass.  */
    int   childInIdx = 0; /* How much of those the child has taken.  */

    childFd = -1;
    stdoutPipe[0] = -1;
    stdoutPipe[1] = -1;
    stderrPipe[0] = -1;
    stderrPipe[1] = -1;
    stdinPipe[0] = -1;
    stdinPipe[1] = -1;

    forcedCmd = GetEffectiveForcedCmd(conn, usrConf);

    ptyReq = SHELL_IsPty(ssh);
    if (ptyReq < 0) {
        wolfSSH_Log(WS_LOG_ERROR, "[SSHD] Failure get channel PTY state");
        return WS_FATAL_ERROR;
    }

    /* do not overwrite a forced command with 'exec' sub shell. Only set the
     * 'exec' command when no forced command is set */
    if (forcedCmd == NULL) {
        forcedCmd = subCmd;
    }

    if (forcedCmd != NULL && WSTRCMP(forcedCmd, "internal-sftp") == 0) {
        wolfSSH_Log(WS_LOG_ERROR,
                                "[SSHD] Only SFTP connections allowed for user "
                                "%s", wolfSSH_GetUsername(ssh));
        return WS_FATAL_ERROR;
    }

    /* temporarily elevate permissions to get users information */
    if (wolfSSHD_AuthRaisePermissions(conn->auth) != WS_SUCCESS) {
        wolfSSH_Log(WS_LOG_ERROR, "[SSHD] Failure to raise permissions for auth");
        return WS_FATAL_ERROR;
    }


    /* create pipes for stdout and stderr */
    if (!ptyReq || forcedCmd) {
        if (pipe(stdoutPipe) != 0) {
            wolfSSH_Log(WS_LOG_ERROR, "[SSHD] Issue creating stdout pipe");
            if (wolfSSHD_AuthReducePermissions(conn->auth) != WS_SUCCESS) {
                wolfSSH_Log(WS_LOG_ERROR,
                        "[SSHD] Error lowering permissions level");
                exit(1);
            }
            return WS_FATAL_ERROR;
        }
        if (pipe(stderrPipe) != 0) {
            close(stdoutPipe[0]);
            close(stdoutPipe[1]);
            wolfSSH_Log(WS_LOG_ERROR, "[SSHD] Issue creating stderr pipe");
            if (wolfSSHD_AuthReducePermissions(conn->auth) != WS_SUCCESS) {
                wolfSSH_Log(WS_LOG_ERROR,
                        "[SSHD] Error lowering permissions level");
                exit(1);
            }
            return WS_FATAL_ERROR;
        }
        if (pipe(stdinPipe) != 0) {
            close(stdoutPipe[0]);
            close(stdoutPipe[1]);
            close(stderrPipe[0]);
            close(stderrPipe[1]);
            wolfSSH_Log(WS_LOG_ERROR, "[SSHD] Issue creating stdin pipe");
            if (wolfSSHD_AuthReducePermissions(conn->auth) != WS_SUCCESS) {
                wolfSSH_Log(WS_LOG_ERROR,
                        "[SSHD] Error lowering permissions level");
                exit(1);
            }
            return WS_FATAL_ERROR;
        }
    }

    ChildRunning = 1;
    childPid = forkpty(&childFd, NULL, NULL, NULL);
    if (childPid < 0) {
        /* forkpty failed, so return */
        ChildRunning = 0;
        wolfSSH_Log(WS_LOG_ERROR, "[SSHD] Issue creating new forkpty");
        if (!ptyReq || forcedCmd) {
            close(stdoutPipe[0]);
            close(stdoutPipe[1]);
            close(stderrPipe[0]);
            close(stderrPipe[1]);
            close(stdinPipe[0]);
            close(stdinPipe[1]);
            stdoutPipe[0] = -1;
            stdoutPipe[1] = -1;
            stderrPipe[0] = -1;
            stderrPipe[1] = -1;
            stdinPipe[0]  = -1;
            stdinPipe[1]  = -1;
        }
        if (wolfSSHD_AuthReducePermissions(conn->auth) != WS_SUCCESS) {
            wolfSSH_Log(WS_LOG_ERROR,
                    "[SSHD] Error lowering permissions level");
            exit(1);
        }
        return WS_FATAL_ERROR;
    }
    else if (childPid == 0) {
        /* Child process. Any setup failure below exit(1)s rather than returning
         * into the connection handler while still at a raised privilege level. */
        const char *args[] = {"-sh", NULL, NULL, NULL};
        char cmd[MAX_COMMAND_SZ];
        char shell[MAX_COMMAND_SZ];
        int ret;

        signal(SIGINT,  SIG_DFL);
        signal(SIGCHLD, SIG_DFL);

        if (!ptyReq || forcedCmd) {
            close(stdoutPipe[0]);
            close(stderrPipe[0]);
            close(stdinPipe[1]);
            stdoutPipe[0] = -1;
            stderrPipe[0] = -1;
            stdinPipe[1]  = -1;

            if (dup2(stdinPipe[0], STDIN_FILENO) == -1) {
                wolfSSH_Log(WS_LOG_ERROR,
                    "[SSHD] Error redirecting stdin pipe");
                exit(1);
            }
            if (dup2(stdoutPipe[1], STDOUT_FILENO) == -1) {
                wolfSSH_Log(WS_LOG_ERROR,
                    "[SSHD] Error redirecting stdout pipe");
                exit(1);
            }
            if (dup2(stderrPipe[1], STDERR_FILENO) == -1) {
                wolfSSH_Log(WS_LOG_ERROR,
                    "[SSHD] Error redirecting stderr pipe");
                exit(1);
            }
        }

        /* set additional groups if needed */
        if (wolfSSHD_AuthSetGroups(conn->auth, wolfSSH_GetUsername(ssh),
                pPasswd->pw_gid) != WS_SUCCESS) {
            wolfSSH_Log(WS_LOG_ERROR, "[SSHD] Error setting groups");
            exit(1);
        }

        rc = SetupChroot(usrConf);
        if (rc < 0) {
            wolfSSH_Log(WS_LOG_ERROR, "[SSHD] Error setting chroot");
            exit(1);
        }
        else if (rc == 1) {
            rc = chdir("/");
            if (rc != 0) {
                wolfSSH_Log(WS_LOG_ERROR,
                    "[SSHD] Error going to / after chroot");
                exit(1);
            }
        }

        if (wolfSSHD_AuthReducePermissionsUser(conn->auth, pPasswd->pw_uid,
            pPasswd->pw_gid) != WS_SUCCESS) {
            wolfSSH_Log(WS_LOG_ERROR, "[SSHD] Error setting user ID");
            exit(1);
        }

        setenv("HOME", pPasswd->pw_dir, 1);
        setenv("LOGNAME", pPasswd->pw_name, 1);
        setenv("SHELL", pPasswd->pw_shell, 1);

        if (WSTRLEN(pPasswd->pw_shell) < sizeof(shell)) {
            char* cursor;
            char* start;

            WSTRNCPY(shell, pPasswd->pw_shell, sizeof(shell)-1);
            shell[sizeof(shell)-1] = 0;
            cursor = shell;
            do {
                start = cursor;
                *cursor = '-';
                cursor = WSTRCHR(start, '/');
            } while (cursor && *cursor != '\0');
            args[0] = start;
        }

        rc = chdir(pPasswd->pw_dir);
        if (rc != 0) {
            /* not error'ing out if unable to find home directory */
            wolfSSH_Log(WS_LOG_ERROR, "[SSHD] Error going to user home dir %s",
            pPasswd->pw_dir);
        }

        /* default to /bin/sh if user shell is not set */
        if (pPasswd->pw_shell && WSTRLEN(pPasswd->pw_shell)) {
            WSNPRINTF(cmd, sizeof(cmd), "%s", pPasswd->pw_shell);
        }
        else {
            WSNPRINTF(cmd, sizeof(cmd), "%s", "/bin/sh");
        }

        errno = 0;
        if (forcedCmd) {
            args[1] = "-c";
            args[2] = forcedCmd;
            ret = execv(cmd, (char**)args);
            close(stdoutPipe[1]);
            close(stderrPipe[1]);
            close(stdinPipe[1]);
        }
        else if (!ptyReq) {
            ret = execv(cmd, (char**)args);
            close(stdoutPipe[1]);
            close(stderrPipe[1]);
            close(stdinPipe[1]);
        }
        else { /* open interactive shell */
            ret = execv(cmd, (char**)args);
        }
        if (ret) {
            wolfSSH_Log(WS_LOG_ERROR, "[SSHD] Issue opening shell");
            exit(1);
        }
        exit(ret); /* exit child process and close down SSH connection */
    }

    if (wolfSSHD_AuthReducePermissionsUser(conn->auth, pPasswd->pw_uid,
        pPasswd->pw_gid) != WS_SUCCESS) {
        wolfSSH_Log(WS_LOG_ERROR, "[SSHD] Error setting user ID");
        /* could not drop to the authenticated user; kill the already
         * forked shell child and terminate the per-connection process
         * rather than continue at a higher privilege level */
        kill(childPid, SIGKILL);
        exit(1);
    }
    sshFd = wolfSSH_get_fd(ssh);

    struct termios tios;
    word32 shellChannelId = 0;
    WOLFSSH_CHANNEL* shellChannel;

    /* Off the channel list, not DEFAULT_NEXT_CHANNEL, which a build can
     * override. The session channel is the only one open here. lastRxId is
     * no use: no request path sets it. A wrong id drops the peer's input. */
    shellChannel = wolfSSH_ChannelNext(ssh, NULL);
    if (shellChannel == NULL || wolfSSH_ChannelGetId(shellChannel,
            &shellChannelId, WS_CHANNEL_ID_SELF) != WS_SUCCESS) {
        /* The shell child is already forked, and nothing below can reach the
         * peer without an id to address, so take it down with us. */
        wolfSSH_Log(WS_LOG_ERROR, "[SSHD] No session channel to service");
        kill(childPid, SIGKILL);
        return WS_FATAL_ERROR;
    }

    signal(SIGCHLD, ChildSig);
    signal(SIGINT, SIG_DFL);

    rc = tcgetattr(childFd, &tios);
    if (rc != 0) {
        return WS_FATAL_ERROR;
    }
    rc = tcsetattr(childFd, TCSAFLUSH, &tios);
    if (rc != 0) {
        return WS_FATAL_ERROR;
    }

    /* set initial size of terminal based on saved size */
#if defined(HAVE_SYS_IOCTL_H)
    wolfSSH_DoModes(ssh->modes, ssh->modesSz, childFd);
    {
        struct winsize s;

        WMEMSET(&s, 0, sizeof(s));
        s.ws_col = ssh->widthChar;
        s.ws_row = ssh->heightRows;
        s.ws_xpixel = ssh->widthPixels;
        s.ws_ypixel = ssh->heightPixels;

        ioctl(childFd, TIOCSWINSZ, &s);
    }
#endif

    wolfSSH_SetTerminalResizeCtx(ssh, (void*)&childFd);
    if (!ptyReq || forcedCmd) {
        close(stdoutPipe[1]);
        close(stderrPipe[1]);
        close(stdinPipe[0]);
    }

    /* The loop below is the only reader of the child's output, so it must
     * never be the thing waiting on the child. A descriptor that cannot be
     * made non-blocking keeps the old behaviour: the write can stall, which
     * is still better than dropping the peer's input on the floor. The read
     * paths already treat EAGAIN as nothing to report. */
    (void)SHELL_SetNonBlocking((!ptyReq || forcedCmd) ?
            stdinPipe[1] : childFd);

    while (ChildRunning || windowFull || !stdoutEmpty || peerConnected) {
        byte tmp[2];
        fd_set readFds;
        fd_set writeFds;
        WS_SOCKET_T maxFd;
        int cnt_r;
        int cnt_w;
        WOLFSSH_CHANNEL* current;
        int pending = 0;
        int childStalled = 0; /* The child would not take the rest of what it
                               * is owed on this pass.  */

        FD_ZERO(&readFds);
        FD_SET(sshFd, &readFds);
        maxFd = sshFd;

        FD_ZERO(&writeFds);
        if (windowFull || wantWrite) {
            FD_SET(sshFd, &writeFds);
        }

        /* Buffered shell input is a wake condition in its own right.
         * wolfSSH_stream_peek() goes blind once the channel is at EOF, so
         * leaning on it alone would park undrained bytes in a select() that
         * nothing else can wake. */
        current = wolfSSH_ChannelFind(ssh, shellChannelId, WS_CHANNEL_ID_SELF);
        if (current != NULL
                && current->inputBuffer.length > current->inputBuffer.idx) {
            pending = 1;
        }

        if (!pending && wolfSSH_stream_peek(ssh, tmp, 1) > 0) {
            pending = 1; /* found some pending SSH data */
        }

        /* The child's output is watched on every pass, not only the ones
         * that have nothing else to do. Draining it is what lets a child
         * blocked writing carry on reading its stdin, so a pass that feeds
         * the child has to be a pass that empties it as well. */
        if (!ptyReq || forcedCmd) {
            FD_SET(stdoutPipe[0], &readFds);
            if (stdoutPipe[0] > maxFd)
                maxFd = stdoutPipe[0];

            FD_SET(stderrPipe[0], &readFds);
            if (stderrPipe[0] > maxFd)
                maxFd = stderrPipe[0];
        }
        else {
            FD_SET(childFd, &readFds);
            if (childFd > maxFd)
                maxFd = childFd;
        }

        /* Bytes the child has not taken yet: wait for it to make room. */
        if (childInIdx < childInSz) {
            int childIn = (!ptyReq || forcedCmd) ? stdinPipe[1] : childFd;

            if (childIn != -1) {
                FD_SET(childIn, &writeFds);
                if (childIn > maxFd)
                    maxFd = childIn;
            }
        }

        {
            struct timeval noWait;
            struct timeval* timeout = NULL;

            /* Work already in hand must not wait on the descriptors, but the
             * poll still runs so this pass sees the child's output too. Data
             * the child has not taken yet is not in hand: nothing can be
             * pulled off the channel until it drains, so a zero timeout
             * would spin. Its stdin is in the write set, so wait there. */
            if (pending && childInIdx == childInSz) {
                noWait.tv_sec = 0;
                noWait.tv_usec = 0;
                timeout = &noWait;
            }

            rc = select((int)maxFd + 1, &readFds, &writeFds, NULL, timeout);
            if (rc == -1) {
                /* Signal (e.g. SIGCHLD from child exit) interrupted select.
                 * Re-evaluate the loop condition so any pending windowFull
                 * data and remaining pipe contents still get drained. */
                if (errno == EINTR)
                    continue;
                break;
            }
        }

        if (wantWrite || windowFull || pending || childInIdx < childInSz
                || FD_ISSET(sshFd, &readFds)) {
            word32 avail;

            wantWrite = 0;
            /* The worker services the transport. What lands on the shell
               channel is handed to the child by the drain below, which names
               the channel itself, so the id the worker would report here is
               not needed. */
            cnt_r = wolfSSH_worker(ssh, NULL);
            if (cnt_r < 0) {
                rc = wolfSSH_get_error(ssh);
                if (rc == WS_CHAN_RXD) {
                    /* Arrival only; the drain below owns the read. */
                }
                else if (rc == WS_CHANNEL_CLOSED) {
                    /* The channel is retired, so nothing more can reach the
                     * child and the drain below is skipped on this pass.
                     * Close its stdin here or it blocks forever on input
                     * that cannot come. */
                    if (stdinPipe[1] != -1 && (!ptyReq || forcedCmd)) {
                        close(stdinPipe[1]);
                        stdinPipe[1] = -1;
                        /* Nothing left to write it to. */
                        childInIdx = 0;
                        childInSz = 0;
                    }
                    peerConnected = 0;
                    continue;
                }
                else if (rc == WS_EOF) {
                    /* Half-close, handled below. */
                }
                else if (rc == WS_WANT_WRITE) {
                    wantWrite = 1;
                    continue;
                }
                else if (rc == WS_REKEYING) {
                    wantWrite = 1;
                    continue;
                }
                else if (rc != WS_WANT_READ) {
                    /* unexpected error, kill off child process */
                    kill(childPid, SIGKILL);
                    break;
                }
            }

            /* The shell channel's own buffer, looked up again because the
             * worker above can retire it. */
            current = wolfSSH_ChannelFind(ssh, shellChannelId,
                WS_CHANNEL_ID_SELF);
            avail = (current != NULL) ?
                current->inputBuffer.length - current->inputBuffer.idx : 0;

            /* Hand over what the peer sent, this pass or left buffered
             * while the window was full; this is the only copy. Gated on
             * windowFull, and not because the buffers overlap -- shellBuffer
             * and channelBuffer are disjoint. While the peer will not take
             * the child's output, pulling more off the channel only parks it
             * somewhere the shutdown path cannot see. The backlog clears as
             * soon as the peer reads. One buffer at a time: what the child
             * will not take yet is carried to the next pass. */
            if (childInIdx == childInSz && avail > 0 && !windowFull) {
                cnt_r = wolfSSH_ChannelIdRead(ssh, shellChannelId,
                    channelBuffer, sizeof channelBuffer);
                if (cnt_r <= 0)
                    break;

                childInIdx = 0;
                childInSz = cnt_r;

                /* Data behind the peer's EOF, RFC 4254 section 5.3. Stdin
                 * is gone, so drop it rather than write to fd -1 and end the
                 * session on an EBADF. */
                if ((!ptyReq || forcedCmd) && stdinPipe[1] == -1)
                    childInIdx = childInSz;

                avail = current->inputBuffer.length - current->inputBuffer.idx;
            }

            /* The read took the bytes off the channel, so this is the only
             * copy: a short write has to be finished, not dropped. The
             * descriptor is non-blocking, so a child that has stopped
             * reading leaves the rest here for the next pass rather than
             * parking this loop inside write() -- which is the deadlock,
             * since the child stops reading exactly when its output has
             * nowhere to go and only this loop can empty it. */
            while (childInIdx < childInSz) {
                if (!ptyReq || forcedCmd) {
                    cnt_w = (int)write(stdinPipe[1], channelBuffer + childInIdx,
                        childInSz - childInIdx);
                }
                else {
                    cnt_w = (int)write(childFd, channelBuffer + childInIdx,
                        childInSz - childInIdx);
                }
                if (cnt_w <= 0) {
                    /* errno only speaks for a -1 return. */
                    if (cnt_w < 0 && errno == EINTR)
                        continue;
                    if (cnt_w < 0
                            && (errno == EAGAIN || errno == EWOULDBLOCK)) {
                        childStalled = 1;
                    }
                    break;
                }
                childInIdx += cnt_w;
            }
            if (childInIdx < childInSz && !childStalled)
                break;
            if (childInIdx == childInSz) {
                childInIdx = 0;
                childInSz = 0;
            }

            /* Peer done sending: close the child's stdin, but only once what
             * it already sent has been handed over. Closing early drops it
             * and the next write lands on fd -1. A channel that is gone is
             * the peer being done too -- DoChannelClose() retires it, and the
             * id was validated at entry, so a miss here cannot mean a wrong
             * id. Leaving the pipe open then would block the child forever on
             * a stdin nothing will ever close. */
            if (stdinPipe[1] != -1 && (!ptyReq || forcedCmd)) {
                if (current == NULL
                        || (wolfSSH_ChannelGetEof(current) && avail == 0
                            && childInIdx == childInSz)) {
                    /* SSH is done, close stdin pipe to child process */
                    close(stdinPipe[1]);
                    stdinPipe[1] = -1;
                }
            }
        }

        /* if the window was previously full, try resending the data */
        if (windowFull) {
            if (windowFullExt) {
                cnt_w = wolfSSH_extended_data_send(ssh, shellBuffer,
                        windowFull);
            }
            else {
                cnt_w = wolfSSH_ChannelIdSend(ssh, shellChannelId,
                        shellBuffer, windowFull);
            }
            if (cnt_w == WS_WINDOW_FULL || cnt_w == WS_REKEYING) {
                continue;
            }
            else if (cnt_w == WS_WANT_WRITE) {
                wantWrite = 1;
                continue;
            }
            else if (cnt_w < 0) {
                kill(childPid, SIGINT);
                break;
            }
            else {
                windowFull -= cnt_w;
                if (windowFull > 0) {
                    WMEMMOVE(shellBuffer, shellBuffer + cnt_w, windowFull);
                    continue;
                }
                if (windowFull < 0)
                    windowFull = 0;
                windowFullExt = 0;
            }
        }

        if (!ptyReq || forcedCmd) {
            if (FD_ISSET(stderrPipe[0], &readFds)) {
                cnt_r = (int)read(stderrPipe[0], shellBuffer,
                    sizeof shellBuffer);
                /* This read will return 0 on EOF */
                if (cnt_r <= 0) {
                    int err = errno;
                    if (err != EAGAIN && err != 0) {
                        break;
                    }
                }
                else {
                    if (cnt_r > 0) {
                        cnt_w = wolfSSH_extended_data_send(ssh, shellBuffer,
                            cnt_r);
                        if (cnt_w > 0 && cnt_w < cnt_r) { /* partial send */
                            windowFull = cnt_r - cnt_w;
                            windowFullExt = 1;
                            WMEMMOVE(shellBuffer, shellBuffer + cnt_w,
                                windowFull);
                            /* don't let the stdout read below trample the
                             * buffered stderr remainder */
                            continue;
                        }
                        else if (cnt_w == WS_WINDOW_FULL ||
                                 cnt_w == WS_REKEYING) {
                            windowFull = cnt_r; /* save amount to be sent */
                            windowFullExt = 1;
                            continue;
                        }
                        else if (cnt_w == WS_WANT_WRITE) {
                            windowFull = cnt_r;
                            windowFullExt = 1;
                            wantWrite = 1;
                            continue;
                        }
                        else if (cnt_w < 0)
                            break;
                    }
                }
            }

            /* handle stdout */
            if (FD_ISSET(stdoutPipe[0], &readFds)) {
                cnt_r = (int)read(stdoutPipe[0], shellBuffer,
                    sizeof shellBuffer);
                /* This read will return 0 on EOF */
                if (cnt_r < 0) {
                    int err = errno;
                    if (err != EAGAIN && err != 0) {
                        break;
                    }
                }
                else if (cnt_r == 0) {
                    stdoutEmpty = 1;
                }
                else {
                    if (cnt_r > 0) {
                        cnt_w = wolfSSH_ChannelIdSend(ssh, shellChannelId,
                                shellBuffer, cnt_r);
                        if (cnt_w > 0 && cnt_w < cnt_r) { /* partial send */
                            windowFull = cnt_r - cnt_w;
                            windowFullExt = 0;
                            WMEMMOVE(shellBuffer, shellBuffer + cnt_w,
                                windowFull);
                        }
                        else if (cnt_w == WS_WINDOW_FULL ||
                                 cnt_w == WS_REKEYING) {
                            windowFull = cnt_r; /* save amount to be sent */
                            windowFullExt = 0;
                            continue;
                        }
                        else if (cnt_w == WS_WANT_WRITE) {
                            windowFull = cnt_r;
                            windowFullExt = 0;
                            wantWrite = 1;
                            continue;
                        }
                        else if (cnt_w < 0) {
                            kill(childPid, SIGINT);
                            break;
                        }
                    }
                }
            }
        }
        else {
            if (FD_ISSET(childFd, &readFds)) {
                cnt_r = (int)read(childFd, shellBuffer, sizeof shellBuffer);
                /* Treat a 0 return as EOF so the loop can shut down. */
                if (cnt_r < 0) {
                    int err = errno;
                    if (err != EAGAIN && err != 0) {
                        break;
                    }
                }
                else if (cnt_r == 0) {
                    stdoutEmpty = 1;
                }
                else {
                    if (cnt_r > 0) {
                        cnt_w = wolfSSH_ChannelIdSend(ssh, shellChannelId,
                                shellBuffer, cnt_r);
                        if (cnt_w > 0 && cnt_w < cnt_r) { /* partial send */
                            windowFull = cnt_r - cnt_w;
                            windowFullExt = 0;
                            WMEMMOVE(shellBuffer, shellBuffer + cnt_w,
                                windowFull);
                        }
                        else if (cnt_w == WS_WINDOW_FULL ||
                                 cnt_w == WS_REKEYING) {
                            windowFull = cnt_r;
                            windowFullExt = 0;
                            continue;
                        }
                        else if (cnt_w == WS_WANT_WRITE) {
                            windowFull = cnt_r;
                            windowFullExt = 0;
                            wantWrite = 1;
                            continue;
                        }
                        else if (cnt_w < 0) {
                            kill(childPid, SIGINT);
                            break;
                        }
                    }
                }
            }
        }

        if (!ChildRunning && peerConnected && stdoutEmpty && !windowFull) {
            peerConnected = 0;
        }
    }

    /* get return value of child process */
    {
        int waitStatus;

        do {
            rc = waitpid(childPid, &waitStatus, 0);
            /* if the waitpid experienced an interrupt then try again */
        } while (rc < 0 && errno == EINTR);

        if (rc < 0) {
            wolfSSH_Log(WS_LOG_ERROR, "[SSHD] Issue waiting for child's exit "
                    "status");
        }
        else {
            if (wolfSSH_SetExitStatus(ssh, (word32)WEXITSTATUS(waitStatus)) !=
                WS_SUCCESS) {
                wolfSSH_Log(WS_LOG_ERROR, "[SSHD] Issue setting child's exit "
                    "status");
            }
        }
    }

    /* check for any left over data in pipes then close them */
    if (!ptyReq || forcedCmd) {
        int readSz;

        /* when the pipe can not be made non blocking skip the drain, a
         * blocking read here could hang the connection process */
        if (SHELL_SetNonBlocking(stdoutPipe[0]) == 0) {
            readSz = (int)read(stdoutPipe[0], shellBuffer, sizeof shellBuffer);
            if (readSz > 0) {
                SHELL_FlushOut(ssh, sshFd, shellChannelId, shellBuffer, readSz,
                    0);
            }
        }

        if (SHELL_SetNonBlocking(stderrPipe[0]) == 0) {
            readSz = (int)read(stderrPipe[0], shellBuffer, sizeof shellBuffer);
            if (readSz > 0) {
                SHELL_FlushOut(ssh, sshFd, shellChannelId, shellBuffer, readSz,
                    1);
            }
        }

        close(stdoutPipe[0]);
        close(stderrPipe[0]);
        if (stdinPipe[1] != -1) {
            close(stdinPipe[1]);
        }
    }

    WS_FORCEZERO(channelBuffer, sizeof channelBuffer);
    WS_FORCEZERO(shellBuffer, sizeof shellBuffer);
    (void)conn;
    return WS_SUCCESS;
}
#endif
#endif

#ifdef _WIN32
/* Cancels and frees the login grace time threadpool timer for a connection. */
static void CancelLoginTimer(WOLFSSHD_CONNECTION* conn)
{
    if (conn != NULL && conn->loginTimer != NULL) {
        /* setting the due time to NULL cancels any pending timer */
        SetThreadpoolTimer(conn->loginTimer, NULL, 0, 0);
        WaitForThreadpoolTimerCallbacks(conn->loginTimer, TRUE);
        CloseThreadpoolTimer(conn->loginTimer);
        conn->loginTimer = NULL;
    }
}

/* threadpool callback marking a connection as timed out at the grace deadline */
static VOID CALLBACK GraceTimeoutCb(PTP_CALLBACK_INSTANCE instance, PVOID ctx,
        PTP_TIMER timer)
{
    WOLFSSHD_CONNECTION* conn = (WOLFSSHD_CONNECTION*)ctx;

    if (conn != NULL) {
        /* published with an interlocked write so the connection thread reading
         * the flag in LoginGraceExpired() sees it without a data race */
        InterlockedExchange8((volatile CHAR*)&conn->timeOut, 1);
    }
    (void)instance;
    (void)timer;
}
#else
/* Set by the SIGALRM handler when the login grace time expires. A signal
 * handler may only access a volatile sig_atomic_t object, so the handler sets
 * just this flag and the accept loop observes it. */
static volatile sig_atomic_t loginGraceTimedOut = 0;

static void alarmCatch(int signum)
{
    loginGraceTimedOut = 1;
    (void)signum;
}
#endif

/* Returns non-zero once the login grace time has expired for this connection.
 * On Windows the threadpool callback records it on the connection; on POSIX the
 * SIGALRM handler sets a sig_atomic_t flag that the accept loop reads here. */
static int LoginGraceExpired(WOLFSSHD_CONNECTION* conn)
{
#ifdef _WIN32
    /* interlocked read pairs with the InterlockedExchange8 in GraceTimeoutCb;
     * the OR with 0 is a read-modify-write, hence the non-const parameter */
    return InterlockedOr8((volatile CHAR*)&conn->timeOut, 0);
#else
    (void)conn;
    return (int)loginGraceTimedOut;
#endif
}

static int UserAuthResult(byte result,
        WS_UserAuthData* authData, void* userAuthResultCtx)
{
    int ret = WS_SUCCESS;
#if defined(WOLFSSH_OSSH_CERTS) && !defined(_WIN32)
    WOLFSSHD_CONNECTION* conn = (WOLFSSHD_CONNECTION*)userAuthResultCtx;
#else
    (void)authData;
#endif

    if (result == WOLFSSH_USERAUTH_SUCCESS) {
        /* authentication finished in time, cancel the login grace timer */
    #ifdef _WIN32
        CancelLoginTimer((WOLFSSHD_CONNECTION*)userAuthResultCtx);
    #else
        (void)userAuthResultCtx;
        alarm(0);
    #endif
    #if defined(WOLFSSH_OSSH_CERTS) && !defined(_WIN32)
        /* Auth (incl. the user signature) has verified, so stash the cert's
         * force-command now, bound to the authenticated credential. Fail closed
         * if the copy fails so the session cannot run unrestricted. */
        if (conn != NULL && authData != NULL &&
                authData->type == WOLFSSH_USERAUTH_PUBLICKEY &&
                authData->sf.publicKey.isOsshCert &&
                authData->sf.publicKey.forceCommand != NULL &&
                authData->sf.publicKey.forceCommandSz > 0) {
            ret = wolfSSHD_AuthSetCertForcedCmd(conn->auth,
                    authData->sf.publicKey.forceCommand,
                    authData->sf.publicKey.forceCommandSz);
        }
    #endif
    }

    return ret;
}

#if defined(WOLFSSH_SFTP) || defined(WOLFSSH_SCP)
/* Returns 1 if root's forced-commands-only ForceCommand isn't
 * "internal-sftp": SFTP/SCP never honor ForceCommand (only SHELL_Subsystem
 * does), so this blocks the confinement bypass. No-op on Windows
 * (pPasswd is NULL there). */
static int IsRootTransferBlocked(WPASSWD* pPasswd, WOLFSSHD_CONFIG* usrConf)
{
#ifdef _WIN32
    (void)pPasswd;
    (void)usrConf;
    return 0;
#else
    char* forcedCmd;

    if (pPasswd == NULL || pPasswd->pw_uid != 0) {
        return 0;
    }
    if (wolfSSHD_ConfigGetPermitRoot(usrConf) !=
            WOLFSSHD_PERMIT_ROOT_FORCED_CMD) {
        return 0;
    }

    forcedCmd = wolfSSHD_ConfigGetForcedCmd(usrConf);
    return (forcedCmd == NULL || WSTRCMP(forcedCmd, "internal-sftp") != 0);
#endif
}

/* Logs and returns 1 if 'what' (e.g. "SFTP subsystem", "SCP session") is
 * blocked for root by IsRootTransferBlocked(); 0 otherwise. */
static int RejectRootTransfer(WPASSWD* pPasswd, WOLFSSHD_CONFIG* usrConf,
        const char* what)
{
    if (IsRootTransferBlocked(pPasswd, usrConf)) {
        wolfSSH_Log(WS_LOG_ERROR, "[SSHD] %s for root requires ForceCommand "
            "internal-sftp under forced-commands-only", what);
        return 1;
    }
    return 0;
}
#endif /* WOLFSSH_SFTP || WOLFSSH_SCP */

/* handle wolfSSH accept and directing to correct subsystem */
#ifdef _WIN32
static DWORD HandleConnection(void* arg)
#else
static void* HandleConnection(void* arg)
#endif
{
    int ret = WS_SUCCESS;
    int error;

    WOLFSSHD_CONNECTION* conn = NULL;
    WOLFSSH* ssh = NULL;

    if (arg == NULL) {
        ret = WS_BAD_ARGUMENT;
    }

    if (ret == WS_SUCCESS) {
        conn = (WOLFSSHD_CONNECTION*)arg;
        ssh = wolfSSH_new(conn->ctx);
        if (ssh == NULL) {
            wolfSSH_Log(WS_LOG_ERROR,
                "[SSHD] Failed to create new WOLFSSH struct");
            ret = -1;
        }
    }

    if (ret == WS_SUCCESS) {
        int select_ret = 0;
        int timedOut = 0;
        long graceTime;

        wolfSSH_set_fd(ssh, conn->fd);
        wolfSSH_SetUserAuthCtx(ssh, conn->auth);
        /* let UserAuthResult reach this connection to cancel the grace timer
         * and to reach conn->auth for the cert force-command */
        wolfSSH_SetUserAuthResultCtx(ssh, conn);
    #if defined(WOLFSSH_OSSH_CERTS) && !defined(_WIN32)
        /* Unix-only: each connection is a forked child with its own copy of the
         * auth struct. Windows does not enforce OpenSSH certs. */
        wolfSSHD_AuthSetPeerIp(conn->auth, conn->ip);
    #endif

        /* arm the login grace timer */
        graceTime = wolfSSHD_AuthGetGraceTime(conn->auth);
        if (graceTime > 0) {
    #ifdef _WIN32
            FILETIME due;
            ULARGE_INTEGER rel;

            /* negative relative time, in 100 ns units */
            rel.QuadPart = (ULONGLONG)(-((LONGLONG)graceTime * 10000000LL));
            due.dwLowDateTime  = rel.LowPart;
            due.dwHighDateTime = rel.HighPart;

            conn->loginTimer = CreateThreadpoolTimer(GraceTimeoutCb, conn, NULL);
            if (conn->loginTimer == NULL) {
                /* fail closed: mark the connection as timed out so the accept
                 * loop exits immediately rather than enforcing no grace time */
                InterlockedExchange8((volatile CHAR*)&conn->timeOut, 1);
                wolfSSH_Log(WS_LOG_ERROR, "[SSHD] Unable to create login grace "
                            "timer, closing connection");
            }
            else {
                SetThreadpoolTimer(conn->loginTimer, &due, 0, 0);
            }
    #else
            signal(SIGALRM, alarmCatch);
            loginGraceTimedOut = 0; /* clear any stale state before arming */
            alarm((unsigned int)graceTime);
    #endif
        }

        ret = wolfSSH_accept(ssh);
        error = wolfSSH_get_error(ssh);
        while (LoginGraceExpired(conn) == 0 && (ret != WS_SUCCESS
                && ret != WS_SCP_INIT && ret != WS_SFTP_COMPLETE)
                && (error == WS_WANT_READ || error == WS_WANT_WRITE)) {

            select_ret = tcp_select(conn->fd, 1);
            if (select_ret == WS_SELECT_RECV_READY  ||
                select_ret == WS_SELECT_ERROR_READY ||
                error      == WS_WANT_WRITE)
            {
                ret = wolfSSH_accept(ssh);
                error = wolfSSH_get_error(ssh);
            }
            else if (select_ret == WS_SELECT_TIMEOUT)
                error = WS_WANT_READ;
            else
                error = WS_FATAL_ERROR;
        }

        /* read the flag once through the accessor; on Windows this is an
         * interlocked read of the value the threadpool callback published, on
         * POSIX it is the SIGALRM handler's loginGraceTimedOut flag */
        timedOut = LoginGraceExpired(conn);
        wolfSSH_Log(WS_LOG_INFO,
                    "[SSHD] grace time = %ld timeout = %d", graceTime,
                    timedOut);
        if (graceTime > 0) {
            /* only report a grace failure when authentication did not also
             * complete in the same accept iteration */
            if (timedOut && ret != WS_SUCCESS &&
                ret != WS_SFTP_COMPLETE && ret != WS_SCP_INIT) {
                wolfSSH_Log(WS_LOG_ERROR,
                    "[SSHD] Failed login within grace period");
             }

    #ifdef _WIN32
            CancelLoginTimer(conn); /* cancel any pending timer */
    #else
            alarm(0); /* cancel any alarm */
    #endif
        }

        if (ret != WS_SUCCESS && ret != WS_SFTP_COMPLETE &&
            ret != WS_SCP_INIT) {
            wolfSSH_Log(WS_LOG_ERROR,
                "[SSHD] Failed to accept WOLFSSH connection from %s error %d",
                conn->ip, ret);
        }
    }

    if (ret == WS_SUCCESS || ret == WS_SFTP_COMPLETE || ret == WS_SCP_INIT) {
        WPASSWD* pPasswd = NULL;
        WOLFSSHD_CONFIG* usrConf;
        char* usr;
        const char* forcedCmd = NULL;

        /* get configuration for user */
        usr     = wolfSSH_GetUsername(ssh);
        usrConf = wolfSSHD_AuthGetUserConf(conn->auth, usr, NULL, NULL,
            NULL, NULL, NULL);
        if (usrConf == NULL) {
            wolfSSH_Log(WS_LOG_ERROR,
                "[SSHD] Error getting user configuration");
            ret = WS_FATAL_ERROR;
        }

    #ifndef WIN32
        if (ret == WS_SUCCESS || ret == WS_SFTP_COMPLETE ||
            ret == WS_SCP_INIT) {
            pPasswd = getpwnam((const char *)usr);
            if (pPasswd == NULL) {
                wolfSSH_Log(WS_LOG_ERROR, "[SSHD] Error getting user info");
                ret = WS_FATAL_ERROR;
            }
        }
    #endif

        if (ret != WS_FATAL_ERROR) {
            switch (wolfSSH_GetSessionType(ssh)) {
                case WOLFSSH_SESSION_SHELL:
                #ifdef WOLFSSH_SHELL
                    if (ret == WS_SUCCESS) {
                        wolfSSH_Log(WS_LOG_INFO, "[SSHD] Entering new shell");
                        ret = SHELL_Subsystem(conn, ssh, pPasswd, usrConf,
                                NULL);
                    }
                #else
                    wolfSSH_Log(WS_LOG_ERROR,
                        "[SSHD] Shell support is disabled");
                    ret = WS_NOT_COMPILED;
                #endif
                    break;

                case WOLFSSH_SESSION_SUBSYSTEM:
                    /* A certificate force-command overrides the requested
                     * subsystem. "internal-sftp" still permits SFTP; any other
                     * command denies file transfer (fail closed). */
                    forcedCmd = GetCertForcedCmd(conn);
                    if (forcedCmd != NULL &&
                            WSTRCMP(forcedCmd, "internal-sftp") != 0) {
                        wolfSSH_Log(WS_LOG_ERROR,
                            "[SSHD] Cert force-command set for user %s; "
                            "denying subsystem request",
                            wolfSSH_GetUsername(ssh));
                        ret = WS_FATAL_ERROR;
                        break;
                    }

                    /* test for known subsystems */
                    switch (ret) {
                        case WS_SFTP_COMPLETE:
                        #ifdef WOLFSSH_SFTP
                            if (RejectRootTransfer(pPasswd, usrConf,
                                    "SFTP subsystem")) {
                                ret = WS_FATAL_ERROR;
                            }
                            else {
                                ret = SFTP_Subsystem(conn, ssh, pPasswd,
                                        usrConf);
                            }
                        #else
                            err_sys("SFTP not compiled in. Please use "
                                    "--enable-sftp");
                        #endif
                            break;

                        case WS_SCP_INIT:
                        #ifdef WOLFSSH_SCP
                            /* any certificate force-command, including
                             * internal-sftp, denies SCP */
                            if (forcedCmd != NULL) {
                                wolfSSH_Log(WS_LOG_ERROR,
                                    "[SSHD] Cert force-command set for user "
                                    "%s; denying SCP",
                                    wolfSSH_GetUsername(ssh));
                                ret = WS_FATAL_ERROR;
                            }
                            else if (RejectRootTransfer(pPasswd, usrConf,
                                    "SCP session")) {
                                ret = WS_FATAL_ERROR;
                            }
                            else {
                                /* Intentional: SCP grants no more than the
                                 * SFTP access already permitted above. */
                                ret = SCP_Subsystem(conn, ssh, pPasswd,
                                        usrConf);
                            }
                        #else
                            err_sys("SCP not compiled in. Please use "
                                    "--enable-scp");
                        #endif
                            break;

                        default:
                            wolfSSH_Log(WS_LOG_ERROR,
                                "[SSHD] Unknown or build not supporting sub"
                                "system found [%s]",
                                wolfSSH_GetSessionCommand(ssh));
                            ret = WS_NOT_COMPILED;
                    }
                    break;

                case WOLFSSH_SESSION_UNKNOWN:
                case WOLFSSH_SESSION_EXEC:
                #if defined(WOLFSSH_SHELL)
                    if (ret == WS_SUCCESS) {
                        wolfSSH_Log(WS_LOG_INFO,
                            "[SSHD] Entering exec session [%s]",
                                wolfSSH_GetSessionCommand(ssh));
                        ret = SHELL_Subsystem(conn, ssh, pPasswd, usrConf,
                                wolfSSH_GetSessionCommand(ssh));
                    }
                #endif /* WOLFSSH_SHELL */

                    /* SCP can be an exec type */
                    if (ret == WS_SCP_INIT) {
                    #ifdef WOLFSSH_SCP
                        /* any certificate force-command denies SCP */
                        forcedCmd = GetCertForcedCmd(conn);
                        if (forcedCmd != NULL) {
                            wolfSSH_Log(WS_LOG_ERROR,
                                "[SSHD] Cert force-command set for user %s; "
                                "denying SCP", wolfSSH_GetUsername(ssh));
                            ret = WS_FATAL_ERROR;
                        }
                        else if (RejectRootTransfer(pPasswd, usrConf,
                                "SCP session")) {
                            ret = WS_FATAL_ERROR;
                        }
                        else {
                            /* Intentional: SCP grants no more than the
                             * SFTP access already permitted above. */
                            ret = SCP_Subsystem(conn, ssh, pPasswd, usrConf);
                        }
                    #else
                        err_sys("SCP not compiled in. Please use "
                                "--enable-scp");
                    #endif
                    }
                    break;

                case WOLFSSH_SESSION_TERMINAL:
                default:
                    wolfSSH_Log(WS_LOG_ERROR,
                        "[SSHD] Unknown or build not supporting session type "
                        "found");
                    ret = WS_NOT_COMPILED;
            }
        }

        wolfSSHD_ConfigFree(usrConf);
    }

    error = wolfSSH_get_error(ssh);
    if (error != WS_SOCKET_ERROR_E && error != WS_FATAL_ERROR) {
        wolfSSH_Log(WS_LOG_INFO, "[SSHD] Attempting to close down connection");
        ret = wolfSSH_shutdown(ssh);

        /* peer hung up, stop shutdown */
        if (ret == WS_SOCKET_ERROR_E) {
            ret = 0;
        }

        error = wolfSSH_get_error(ssh);
        if (error != WS_SOCKET_ERROR_E &&
                (error == WS_WANT_READ || error == WS_WANT_WRITE)) {
            int maxAttempt = 10; /* make 10 attempts max before giving up */
            int attempt;

            for (attempt = 0; attempt < maxAttempt; attempt++) {
                ret = wolfSSH_worker(ssh, NULL);
                error = wolfSSH_get_error(ssh);

                /* peer successfully closed down gracefully */
                if (ret == WS_CHANNEL_CLOSED || ret == WS_EOF) {
                    ret = 0;
                    break;
                }

                /* peer hung up, stop shutdown */
                if (ret == WS_SOCKET_ERROR_E) {
                    ret = 0;
                    break;
                }

                if (ret == WS_FATAL_ERROR &&
                   (error != WS_WANT_READ &&
                    error != WS_WANT_WRITE)) {
                    break;
                }
            #ifdef _WIN32
                Sleep(1);
            #else
                usleep(100000);
            #endif
            }

            if (attempt == maxAttempt) {
                wolfSSH_Log(WS_LOG_INFO,
                    "[SSHD] Gave up on graceful shutdown, closing the socket");
            }
        }
    }

    /* check if there is a response to the shutdown */
    wolfSSH_free(ssh);
    if (conn != NULL) {
        byte sc[1024];
        shutdown(conn->fd, 1);
        /* Spin until socket closes. */
        do {
            ret = (int)recv(conn->fd, sc, 1024, 0);
        } while (ret > 0);

        WCLOSESOCKET(conn->fd);
    }
    wolfSSH_Log(WS_LOG_INFO, "[SSHD] Return from closing connection = %d", ret);
#ifdef _WIN32
    /* free the per-connection auth allocated in StartSSHD */
    if (conn != NULL) {
        wolfSSHD_AuthFreeUser(conn->auth);
    }
#endif
    WFREE(conn, NULL, DYNTYPE_SSHD);

#ifdef _WIN32
    return 0;
#else
    return NULL;
#endif
}


/* returns WS_SUCCESS on success */
static int NewConnection(WOLFSSHD_CONNECTION* conn)
{

    int ret = WS_SUCCESS;
#ifndef WIN32
    int pd = 0;

    pd = fork();
    if (pd < 0) {
        wolfSSH_Log(WS_LOG_ERROR, "[SSHD] Issue spawning new process");
        ret = -1;
    }

    if (ret == WS_SUCCESS) {
        if (pd == 0) {
            /* child process */
            WCLOSESOCKET(conn->listenFd);
            signal(SIGINT, SIG_DFL);
            (void)HandleConnection((void*)conn);
            exit(0);
        }
        else {
            /* do not wait for status of child process, and signal that the
               child can be reaped to avoid zombie processes when running in
               the foreground */
            signal(SIGCHLD, SIG_IGN);

            wolfSSH_Log(WS_LOG_INFO, "[SSHD] Spawned new process %d\n", pd);
            WCLOSESOCKET(conn->fd);

            /* the child process has its own copy of the connection, free the
               parent's copy here */
            WFREE(conn, NULL, DYNTYPE_SSHD);
        }
    }
#else
    HANDLE t;
    DWORD id;

    if (conn->isThreaded) {
        t = CreateThread(NULL, 0, HandleConnection, (void*)conn, 0, &id);
        if (t == NULL) {
            wolfSSH_Log(WS_LOG_ERROR, "[SSHD] Issue creating new thread");
            ret = WS_FATAL_ERROR;
        }
        else {
            wolfSSH_Log(WS_LOG_INFO, "[SSHD] Spawned new thread %d\n", id);
            CloseHandle(t);
        }
    }
    else {
        HandleConnection((void*)conn);
    }
#endif

    return ret;
}


/* return non zero value for a pending connection */
static int PendingConnection(WS_SOCKET_T fd)
{
    int ret;
    struct timeval t;
    fd_set r, w, e;
    WS_SOCKET_T nfds = fd + 1;

    t.tv_usec = 0;
    t.tv_sec  = WOLFSSHD_TIMEOUT;

    FD_ZERO(&r);
    FD_ZERO(&w);
    FD_ZERO(&e);

    FD_SET(fd, &r);
    errno = 0;
    ret = select((int)nfds, &r, &w, &e, &t);
    if (ret < 0) {
        /* a socket level issue happened, could just be a system call int. */
        if (errno != EINTR) {
            wolfSSH_Log(WS_LOG_ERROR, "[SSHD] TCP socket error on select()");
            quit = 1;
        }
        ret  = 0;
    }
    else if (ret > 0) {
        if (FD_ISSET(fd, &r)) {
            wolfSSH_Log(WS_LOG_INFO, "[SSHD] Incoming TCP data found");
        }
        else {
            wolfSSH_Log(WS_LOG_INFO, "[SSHD] Found TCP write or error data");
            ret = 0; /* nothing to read */
        }
    }
    return ret;
}

int   myoptind = 0;
char* myoptarg = NULL;

#ifdef _WIN32
#include <tchar.h>
#include <shellapi.h>

SERVICE_STATUS        serviceStatus = { 0 };
SERVICE_STATUS_HANDLE serviceStatusHandle = NULL;
HANDLE                serviceStop = INVALID_HANDLE_VALUE;

#define WOLFSSHD_SERVICE_NAME  _T("wolfSSHd")


static void wolfSSHD_ServiceCb(DWORD CtrlCode)
{
    switch (CtrlCode) {
        case SERVICE_CONTROL_STOP:
            if (serviceStatus.dwCurrentState != SERVICE_RUNNING)
                break;
            serviceStatus.dwControlsAccepted = 0;
            serviceStatus.dwCurrentState = SERVICE_STOP_PENDING;
            serviceStatus.dwWin32ExitCode = 0;
            serviceStatus.dwCheckPoint = 4;

            if (SetServiceStatus(serviceStatusHandle, &serviceStatus) == FALSE) {
                wolfSSH_Log(WS_LOG_ERROR, "[SSHD] Issue setting service status");
            }

            /* send out signal that the service is stopping */
            SetEvent(serviceStop);
            break;

        default:
            break;
    }
}


static char* _convertHelper(WCHAR* in, void* heap) {
    int retSz;
    char* ret;
    
    retSz = (int)wcslen(in) * 2;
    ret   = (char*)WMALLOC(retSz + 1, heap, DYNTYPE_SSHD);
    if (ret != NULL) {
        size_t numConv = 0;
        if (wcstombs_s(&numConv, ret, retSz, in, retSz) != 0) {
            WFREE(ret, heap, DYNTYPE_SSHD);
            ret = NULL;
        }
    }
    return ret;
}

/* free the argv/cmdArgs buffers built from the wide command line. Safe to
 * call on partially initialized state: NULL argv or cmdArgs is ignored. */
static void _freeWinArgs(char** argv, DWORD argc, LPWSTR* cmdArgs)
{
    DWORD z;

    if (argv != NULL) {
        for (z = 0; z < argc; z++) {
            WFREE(argv[z], NULL, DYNTYPE_SSHD);
        }
        WFREE(argv, NULL, DYNTYPE_SSHD);
    }
    if (cmdArgs != NULL) {
        LocalFree(cmdArgs);
    }
}

static void StartSSHD(DWORD argc, LPTSTR* wargv)
#else
static int StartSSHD(int argc, char** argv)
#endif
{
    int ret = WS_SUCCESS;
    word16 port = 0;
    WS_SOCKET_T listenFd = 0;
    int ch;
    WOLFSSHD_CONFIG* conf = NULL;
    WOLFSSHD_AUTH* auth = NULL;
    WOLFSSH_CTX* ctx = NULL;
    byte isDaemon = 1;
    byte testMode = 0;

    const char* configFile = "/etc/ssh/sshd_config";
    const char* hostKeyFile = NULL;
    byte* banner = NULL;

    logFile = stderr;
    wolfSSH_SetLoggingCb(wolfSSHDLoggingCb);

#ifdef _WIN32
    char** argv = NULL;
    DWORD i;
    LPWSTR* cmdArgs = NULL;
    LPWSTR cmdLn;
    int cmdArgC = 0;

    /* get what the command line was and parse arguments from it */
    cmdLn   = GetCommandLineW();
    cmdArgs = CommandLineToArgvW(cmdLn, &cmdArgC);
    if (cmdArgs == NULL) {
        ret = WS_FATAL_ERROR;
    }

    if (ret == WS_SUCCESS) {
        for (i = 0; i < (DWORD)cmdArgC; i++) {
            /* cmdArgs entries are wide strings (CommandLineToArgvW); compare
             * as such instead of reinterpreting as narrow char data. */
            if (wcscmp(cmdArgs[i], L"-D") == 0) {
                isDaemon = 0;
            }
        }
    }

    if (isDaemon) {
        /* Set the logging to go to OutputDebugString */
        wolfSSH_SetLoggingCb(ServiceDebugCb);
    }

    if (ret == WS_SUCCESS) {
        /* Rebuild argv from cmdArgs, not the caller's wargv: wargv is
         * narrow data when called from main(), so it may not be a real
         * wide string. cmdArgs is always correct either way. */
        argc = (DWORD)cmdArgC;

        /* we want the arguments to be normal char strings not wchar_t */
        argv = (char**)WMALLOC(argc * sizeof(char*), NULL, DYNTYPE_SSHD);
        if (argv == NULL) {
            ret = WS_MEMORY_E;
        }
        else {
            unsigned int z;

            /* Zero first: _freeWinArgs() walks all argc slots. */
            WMEMSET(argv, 0, argc * sizeof(char*));
            for (z = 0; z < argc; z++) {
                argv[z] = _convertHelper(cmdArgs[z], NULL);
                if (argv[z] == NULL) {
                    /* mygetopt() dereferences every entry it walks. */
                    ret = WS_MEMORY_E;
                    break;
                }
            }
        }
    }
    (void)wargv;
#endif

    signal(SIGINT, interruptCatch);
    WSTARTTCP();

    if (ret == WS_SUCCESS) {
        wolfSSH_Init();
    }

    if (ret == WS_SUCCESS) {
        conf = wolfSSHD_ConfigNew(NULL);
        if (conf == NULL) {
            ret = WS_MEMORY_E;
        }
    }

    while (ret == WS_SUCCESS &&
            (ch = mygetopt(argc, argv, "?f:p:h:dDE:o:t")) != -1) {
        switch (ch) {
        case 'f':
            configFile = myoptarg;
            break;

        case 'p':
            if (ret == WS_SUCCESS) {
                if (myoptarg == NULL) {
                    ret = WS_BAD_ARGUMENT;
                    break;
                }

                ret = XATOI(myoptarg);
                if (ret < 0) {
                    fprintf(stderr, "Issue parsing port number %s\n",
                        myoptarg);
                    ret = WS_BAD_ARGUMENT;
                }
                else {
                    if (ret <= (word16)-1) {
                        port = (word16)ret;
                        ret = WS_SUCCESS;
                    }
                    else {
                        fprintf(stderr, "Port number %d too big.\n", ret);
                        ret = WS_BAD_ARGUMENT;
                    }
                }
            }
            break;

        case 'h':
            hostKeyFile = myoptarg;
            break;

        case 'd':
            debugMode = 1; /* turn on debug mode */
            wolfSSL_Debugging_ON();
            wolfSSH_Debugging_ON();
            break;

        case 'D':
            isDaemon = 0;
            break;

        case 'E':
            ret = WFOPEN(NULL, &logFile, myoptarg, "ab");
            if (ret != 0 || logFile == WBADFILE) {
                fprintf(stderr, "Unable to open log file %s\n", myoptarg);
                ret = WS_FATAL_ERROR;
            }
            break;

        case 'o':
        #ifdef WOLFSSH_IGNORE_UNKNOWN_CONFIG
            wolfSSH_Log(WS_LOG_DEBUG, "[SSHD] ignoring -o.");
            break;
        #else
            ShowUsage();
            #ifndef _WIN32
            return WS_FATAL_ERROR;
            #else
            _freeWinArgs(argv, argc, cmdArgs);
            return;
            #endif
        #endif

        case 't':
            testMode = 1;
            break;

        case '?':
            ShowUsage();
        #ifndef _WIN32
            return WS_SUCCESS;
        #else
            _freeWinArgs(argv, argc, cmdArgs);
            return;
        #endif

        default:
            ShowUsage();
        #ifndef _WIN32
            return WS_SUCCESS;
        #else
            _freeWinArgs(argv, argc, cmdArgs);
            return;
        #endif
        }
    }

    /* Must run before privilege drop so the shadow file is accessible.
     * Degrades to a fixed-cost fake hash if the shadow read fails. */
    if (ret == WS_SUCCESS && !testMode) {
        wolfSSHD_AuthInit();
    }

    if (ret == WS_SUCCESS) {
        ret = wolfSSHD_ConfigLoad(conf, configFile);
        if (ret != WS_SUCCESS) {
            wolfSSH_Log(WS_LOG_ERROR, "Error reading in configure file %s\n",
                configFile);
        }
    }

    /* port was not overridden with argument, read from config file */
    if (ret == WS_SUCCESS && port == 0) {
        port = wolfSSHD_ConfigGetPort(conf);
    }

    /* check if host key file was passed in */
    if (hostKeyFile != NULL) {
        wolfSSHD_ConfigSetHostKeyFile(conf, hostKeyFile);
    }

    if (ret == WS_SUCCESS) {
        wolfSSH_Log(WS_LOG_INFO, "[SSHD] Starting wolfSSH SSHD application");
        ret = SetupCTX(conf, &ctx, &banner);
    }

    if (ret == WS_SUCCESS) {
        auth = wolfSSHD_AuthCreateUser(NULL, conf);
        if (auth == NULL) {
            wolfSSH_Log(WS_LOG_ERROR, "[SSHD] Issue creating auth struct");
            ret = WS_MEMORY_E;
        }
    }

    if (logFile == NULL) {
        logFile = stderr;
    }

    /* run as a daemon or service */
#ifndef WIN32
    if (ret == WS_SUCCESS && isDaemon) {
        pid_t p;

#ifdef __unix__
        /* Daemonizing in POSIX, so set a syslog based log */
        if (logFile == stderr) {
            wolfSSH_SetLoggingCb(SyslogCb);
        }
#endif
        p = fork();
        if (p < 0) {
            fprintf(stderr, "Failed to fork process\n");
            exit(EXIT_FAILURE);
        }
        if (p > 0) {
            exit(EXIT_SUCCESS); /* stop parent process */
        }

        if (setsid() < 0) {
            fprintf(stderr, "Failed to set a new session");
            ret = WS_FATAL_ERROR;
        }
        else {
            signal(SIGCHLD, ConnClose);
            p = fork();
            if (p < 0) {
                fprintf(stderr, "Failed to fork process\n");
                exit(EXIT_FAILURE);
            }
            if (p > 0) {
                exit(EXIT_SUCCESS);
            }

            umask(0);
            if (chdir("/") < 0) {
                ret = WS_FATAL_ERROR;
            }

            if (ret == WS_SUCCESS) {
                int fd;

                fd = open("/dev/null", O_RDWR);
                if (fd < 0) {
                    ret = WS_FATAL_ERROR;
                }
                else {
                    if (dup2(fd, STDIN_FILENO) < 0 ||
                        dup2(fd, STDOUT_FILENO) < 0 ||
                        dup2(fd, STDERR_FILENO) < 0) {
                        ret = WS_FATAL_ERROR;
                    }
                    close(fd);
                }
            }
        }
    }
#else
    if (isDaemon) {
        /* Set function to handle service query and commands */
        serviceStatusHandle = RegisterServiceCtrlHandler(WOLFSSHD_SERVICE_NAME, wolfSSHD_ServiceCb);
        if (serviceStatusHandle == NULL) {
            ret = WS_FATAL_ERROR;
        }
        else {
            /* Update service status as 'start pending' */
            ZeroMemory(&serviceStatus, sizeof(serviceStatus));
            serviceStatus.dwServiceType = SERVICE_WIN32_OWN_PROCESS;
            serviceStatus.dwCurrentState = SERVICE_START_PENDING;
            if (SetServiceStatus(serviceStatusHandle, &serviceStatus) == FALSE) {
                wolfSSH_Log(WS_LOG_ERROR, "[SSHD] Issue updating service status");
            }
        }
        if (ret == WS_SUCCESS) {
            /* Create a stop event to watch on */
            serviceStop = CreateEvent(NULL, TRUE, FALSE, NULL);
            if (serviceStop == NULL) {
                serviceStatus.dwControlsAccepted = 0;
                serviceStatus.dwCurrentState = SERVICE_STOPPED;
                serviceStatus.dwWin32ExitCode = GetLastError();
                serviceStatus.dwCheckPoint = 1;

                if (SetServiceStatus(serviceStatusHandle, &serviceStatus) == FALSE) {
                    wolfSSH_Log(WS_LOG_ERROR, "[SSHD] Issue updating service status");
                }
                _freeWinArgs(argv, argc, cmdArgs);
                return;
            }
        }
    }
#endif

    if (ret == WS_SUCCESS && !testMode) {
        wolfSSHD_ConfigSavePID(conf);
        wolfSSH_Log(WS_LOG_INFO, "[SSHD] Starting to listen on port %d", port);
        tcp_listen(&listenFd, &port, 1);
        wolfSSH_Log(WS_LOG_INFO, "[SSHD] Listening on port %d", port);
        if (wolfSSHD_AuthReducePermissions(auth) != WS_SUCCESS) {
            wolfSSH_Log(WS_LOG_INFO, "[SSHD] Error lowering permissions level");
            ret = WS_FATAL_ERROR;
        }

    #ifdef WIN32
        if (ret == WS_SUCCESS && isDaemon) {
            /* update service status as started */
            serviceStatus.dwControlsAccepted = SERVICE_ACCEPT_STOP;
            serviceStatus.dwCurrentState = SERVICE_RUNNING;
            serviceStatus.dwWin32ExitCode = 0;
            serviceStatus.dwCheckPoint = 0;

            if (SetServiceStatus(serviceStatusHandle, &serviceStatus) == FALSE) {
                wolfSSH_Log(WS_LOG_ERROR, "[SSHD] Issue updating service status");
            }
        }
    #endif
        /* wait for incoming connections and fork them off */
        while (ret == WS_SUCCESS && quit == 0) {
            WOLFSSHD_CONNECTION* conn;
#ifdef WOLFSSL_NUCLEUS
            struct addr_struct clientAddr;
#else
            struct sockaddr_in6 clientAddr;
            socklen_t           clientAddrSz = sizeof(clientAddr);
#endif
            conn = (WOLFSSHD_CONNECTION*)WMALLOC(sizeof(WOLFSSHD_CONNECTION),
                NULL, DYNTYPE_SSHD);
            if (conn == NULL) {
                wolfSSH_Log(WS_LOG_ERROR,
                    "[SSHD] Failed to malloc memory for connection");
                break;
            }
            WMEMSET(conn, 0, sizeof(WOLFSSHD_CONNECTION));

            conn->auth = auth;
            conn->listenFd = (int)listenFd;
            conn->isThreaded = isDaemon;
#ifdef _WIN32
            conn->timeOut = 0;
            conn->loginTimer = NULL;
#endif

            /* wait for a connection */
            if (PendingConnection(listenFd)) {
                conn->ctx = ctx;
#ifdef WOLFSSL_NUCLEUS
                conn->fd = NU_Accept(listenFd, &clientAddr, 0);
#else
                conn->fd = (int)accept(listenFd, (struct sockaddr*)&clientAddr,
                    &clientAddrSz);
                if (conn->fd >= 0) {
                    if (clientAddr.sin6_family == AF_INET) {
                        struct sockaddr_in* addr4 =
                            (struct sockaddr_in*)&clientAddr;
                        inet_ntop(AF_INET, &addr4->sin_addr, conn->ip,
                              INET_ADDRSTRLEN);
                    }
                    else if (clientAddr.sin6_family == AF_INET6) {
                        inet_ntop(AF_INET6, &clientAddr.sin6_addr, conn->ip,
                              INET6_ADDRSTRLEN);
                    }
                }
#endif

                {
#ifdef USE_WINDOWS_API
                    unsigned long blocking = 1;
                    if (ioctlsocket(conn->fd, FIONBIO, &blocking)
                        == SOCKET_ERROR) {
                        WLOG(WS_LOG_DEBUG, "wolfSSH non-fatal error: "
                                "ioctlsocket failed");
                        WCLOSESOCKET(conn->fd);
                        WFREE(conn, NULL, DYNTYPE_SSHD);
                        continue;
                    }
#elif defined(WOLFSSL_MDK_ARM) || defined(WOLFSSL_KEIL_TCP_NET) \
                        || defined (WOLFSSL_TIRTOS)|| defined(WOLFSSL_VXWORKS) || \
                        defined(WOLFSSL_NUCLEUS)
                    /* non blocking not supported, for now */
#else
                    int flags = fcntl(conn->fd, F_GETFL, 0);
                    if (flags < 0) {
                        WLOG(WS_LOG_DEBUG, "wolfSSH non-fatal error: "
                                "fcntl get failed");
                        WCLOSESOCKET(conn->fd);
                        WFREE(conn, NULL, DYNTYPE_SSHD);
                        continue;
                    }
                    flags = fcntl(conn->fd, F_SETFL, flags | O_NONBLOCK);
                    if (flags < 0) {
                        WLOG(WS_LOG_DEBUG, "wolfSSH non-fatal error: "
                                "fcntl set failed");
                        WCLOSESOCKET(conn->fd);
                        WFREE(conn, NULL, DYNTYPE_SSHD);
                        continue;
                    }
#endif
                }
#ifdef _WIN32
                {
                    WOLFSSHD_AUTH* connAuth =
                        wolfSSHD_AuthCreateUser(NULL, conf);
                    if (connAuth == NULL) {
                        wolfSSH_Log(WS_LOG_ERROR, "[SSHD] Failed to create "
                            "auth struct for connection");
                        WCLOSESOCKET(conn->fd);
                        WFREE(conn, NULL, DYNTYPE_SSHD);
                        continue;
                    }
                    conn->auth = connAuth;
                }
#endif
                ret = NewConnection(conn);
                if (ret != WS_SUCCESS) {
                    /* on failure the connection was not handed off, it is
                       still owned here and needs cleaned up */
                    WCLOSESOCKET(conn->fd);
                #ifdef _WIN32
                    wolfSSHD_AuthFreeUser(conn->auth);
                #endif
                    WFREE(conn, NULL, DYNTYPE_SSHD);
                }
            }
            else {
                WFREE(conn, NULL, DYNTYPE_SSHD);
            }
#ifdef _WIN32
            /* check if service has been shutdown */
            if (isDaemon && WaitForSingleObject(serviceStop, 0) ==
                    WAIT_OBJECT_0) {
                quit = 1;
            }
#endif
        }
        if (quit && logFile) {
            fprintf(logFile, "Closing down wolfSSHD\n");
        }
    }

#ifdef _WIN32
    /* close down windows service */
    if (isDaemon) {
        CloseHandle(serviceStop);

        serviceStatus.dwControlsAccepted = 0;
        serviceStatus.dwCurrentState = SERVICE_STOPPED;
        serviceStatus.dwWin32ExitCode = 0;
        serviceStatus.dwCheckPoint = 3;

        if (serviceStatusHandle != NULL &&
            SetServiceStatus(serviceStatusHandle, &serviceStatus) == FALSE) {
            wolfSSH_Log(WS_LOG_ERROR, "[SSHD] Issue updating service status");
        }
    }
#endif

    CleanupCTX(conf, &ctx, &banner);
    if (banner) {
        WFREE(banner, NULL, DYNTYPE_STRING);
    }
    wolfSSHD_ConfigFree(conf);
    wolfSSHD_AuthFreeUser(auth);
    wolfSSH_Cleanup();

    if (debugMode) {
        wolfSSH_Debugging_OFF();
        wolfSSL_Debugging_OFF();
    }

#ifdef _WIN32
    /* free up temporary memory used for conversion of args from wchar_t.
     * Not gated on isDaemon: argv/cmdArgs are allocated in both daemon and
     * -D modes. _freeWinArgs() tolerates NULL argv/cmdArgs from an early
     * failure, and argc tracks argv's length once it is allocated. */
    _freeWinArgs(argv, argc, cmdArgs);
#else
    return 0;
#endif
}

#ifdef _WIN32
/* Used to setup a console and run command as a user.
 * returns the process exit value */
static int SetupConsole(char* inCmd)
{
    HANDLE sOut;
    HANDLE sIn;
    HPCON pCon = 0;
    COORD cord = { 80,24 }; /* Default to 80x24. Updated later. */
    STARTUPINFOEXW ext;
    int ret = WS_SUCCESS;
    PWSTR cmd    = NULL;
    size_t cmdSz = 0;
    PROCESS_INFORMATION processInfo;
    size_t sz = 0;
    DWORD processState = 0;
    PCSTR shellCmd = "c:\\windows\\system32\\cmd.exe";

    if (inCmd == NULL) {
        return -1;
    }

    sIn  = GetStdHandle(STD_INPUT_HANDLE);

    if (WSTRCMP(shellCmd, inCmd) != 0) {
        /* if not opening a shell, pipe virtual terminal sequences to 'nul' */
        if (CreatePseudoConsole(cord, sIn, INVALID_HANDLE_VALUE, 0, &pCon) != S_OK) {
            ret = WS_FATAL_ERROR;
        }
        else {
            CloseHandle(sIn);
        }
    }
    else
    {
        /* if opening a shell, pipe virtual terminal sequences back to calling process */
        sOut = GetStdHandle(STD_OUTPUT_HANDLE);
        if (CreatePseudoConsole(cord, sIn, sOut, 0, &pCon) != S_OK) {
            ret = WS_FATAL_ERROR;
        }
        else {
            CloseHandle(sIn);
            CloseHandle(sOut);
        }
    }

    /* setup startup extended info for pseudo terminal */
    ZeroMemory(&ext, sizeof(ext));
    if (ret == WS_SUCCESS) {
        ext.StartupInfo.cb = sizeof(STARTUPINFOEX);
        (void)InitializeProcThreadAttributeList(NULL, 1, 0, &sz);
        if (sz == 0) {
            ret = WS_FATAL_ERROR;
        }

        if (ret == WS_SUCCESS) {
            /* Using HeapAlloc for better support when possibly passing
               memory between Windows Modules */
            ext.lpAttributeList =
                (PPROC_THREAD_ATTRIBUTE_LIST)HeapAlloc(GetProcessHeap(), 0, sz);
            if (ext.lpAttributeList == NULL) {
                ret = WS_FATAL_ERROR;
            }
        }

        if (ret == WS_SUCCESS) {
            if (InitializeProcThreadAttributeList(ext.lpAttributeList, 1, 0,
                &sz) != TRUE) {
                ret = WS_FATAL_ERROR;
            }
        }

        if (ret == WS_SUCCESS) {
            if (UpdateProcThreadAttribute(ext.lpAttributeList, 0,
                PROC_THREAD_ATTRIBUTE_PSEUDOCONSOLE,
                pCon, sizeof(HPCON), NULL, NULL) != TRUE) {
                ret = WS_FATAL_ERROR;
            }
        }
    }

    if (ret == WS_SUCCESS) {
        cmdSz = WSTRLEN(inCmd) + 1; /* +1 for terminator */
        cmd   = (PWSTR)HeapAlloc(GetProcessHeap(), 0, sizeof(wchar_t) * cmdSz);
        if (cmd == NULL) {
            ret = WS_MEMORY_E;
        }
        else {
            size_t numConv = 0;

            WMEMSET(cmd, 0, sizeof(wchar_t) * cmdSz);
            mbstowcs_s(&numConv, cmd, cmdSz, inCmd, strlen(inCmd));
        }
    }

    ZeroMemory(&processInfo, sizeof(processInfo));
    if (ret == WS_SUCCESS) {
        if (CreateProcessW(NULL, cmd,
            NULL, NULL, FALSE, EXTENDED_STARTUPINFO_PRESENT, NULL, NULL,
            &ext.StartupInfo, &processInfo) != TRUE) {
            return WS_FATAL_ERROR;
        }
        else {
            DWORD ava = 0;

            WaitForInputIdle(processInfo.hProcess, 1000);

            do {
                /* wait indefinitely for console process to exit */
                if (ava == 0) {
                    if (WaitForSingleObject(processInfo.hProcess, INFINITE) == WAIT_FAILED) {
                        break;
                    }
                }

                /* check if process is still running and give time to drain pipes */
                if (GetExitCodeProcess(processInfo.hProcess, &processState)
                    == TRUE) {
                    if (processState != STILL_ACTIVE) {
                        Sleep(100); /* give the stdout/stderr of process a
                                     * little time to write to pipe */
                        if (PeekNamedPipe(GetStdHandle(STD_OUTPUT_HANDLE), NULL, 0, NULL, &ava, NULL)
                            == TRUE) {
                            if (ava > 0) {
                                /* if data still pending then continue
                                 * sending it over SSH */
                                continue;
                            }
                        }
                        break;
                    }
                }
            } while (1);
            CloseHandle(processInfo.hThread);
        }
    }

    if (cmd != NULL) {
        HeapFree(GetProcessHeap(), 0, cmd);
    }

    if (ext.lpAttributeList != NULL) {
        HeapFree(GetProcessHeap(), 0, ext.lpAttributeList);
    }

    if (pCon != 0) {
        ClosePseudoConsole(pCon);
    }

    return processState;
}
#endif /* _WIN32 */

int main(int argc, char** argv)
{
#ifdef _WIN32
    /* First look if this is a service being started */
    int i, isService = 1;
    for (i = 0; i < argc; i++) {
        if (WSTRCMP(argv[i], "-D") == 0) {
            isService = 0;
        }
        if (WSTRCMP(argv[i], "-r") == 0) {
            if (argc < i + 1) {
                /* was expecting command to run after -r argument */
                return -1;
            }
            return SetupConsole(argv[i + 1]);
        }
    }

    if (isService) {
        SERVICE_TABLE_ENTRY ServiceTable[] =
        {
            {_T("wolfSSHd"), (LPSERVICE_MAIN_FUNCTION)StartSSHD},
            {NULL, NULL}
        };

        if (StartServiceCtrlDispatcher(ServiceTable) == FALSE) {
            printf("StartServiceCtrlDispatcher failed\n");
            return GetLastError();
        }
    }
    else {
        StartSSHD(argc, (LPTSTR*)argv);
    }
    return 0;
#else
    return StartSSHD(argc, argv);
#endif
}

#else

#include <stdio.h>

/* helpful print out if compiling without SSHD feature enabled */
int main(int argc, char** argv)
{
    printf("Not compiled in. Please recompile wolfSSH with :\n");
    printf("--enable-sshd (user_settings.h macro define WOLFSSH_SSHD\n");
    return -1;
}
#endif /* WOLFSSH_SSHD */