parser lyb BUGFIX integer overflow and OOM (#2513)

lyb_read_string: when str_len == UINT32_MAX, (str_len + 1) wraps to 0,
malloc(0) returns non-NULL, and the subsequent write to (*str)[UINT32_MAX]
causes a WRITE SEGV (memory corruption).

lyb_read_value: when lyb_size_bits == UINT32_MAX with VARIABLE_BYTES,
LYPLG_BITS2BYTES() produces ~4 GiB, causing calloc to attempt a 4 GiB
allocation which triggers OOM / DoS.

Both paths are reachable by supplying a malformed LYB input with
length field set to 0xFFFFFFFF.

Reported-by: Dominik Blain <dominik@qreativelab.io>, Cobalt AI

Author: dom-omg

src/parser_lyb.c

@@ -284,6 +284,9 @@ lyb_read_string(char **str, struct lylyb_parse_ctx *lybctx)
     /* read length in bytes */
     lyb_read_size(&str_len, lybctx);
 
+    /* str_len + 1 wraps to 0 when str_len == UINT32_MAX, causing malloc(0) followed by an out-of-bounds write */
+    LY_CHECK_ERR_RET(str_len == UINT32_MAX, LOGERR(lybctx->ctx, LY_EINVAL, "LYB string length overflow."), LY_EINVAL);
+
     /* allocate mem */
     *str = malloc(((uint64_t)str_len + 1) * sizeof **str);
     LY_CHECK_ERR_RET(!*str, LOGMEM(lybctx->ctx), LY_EMEM);
@@ -340,6 +343,11 @@ lyb_read_value(const struct lysc_type *type, uint8_t **val, uint64_t *val_size_b
         }
     }
 
+    /* LYPLG_BITS2BYTES(val_size_bits) + 1 can reach 4 GiB when lyb_size_bits == UINT32_MAX and
+     * size_type == VARIABLE_BYTES, causing calloc to attempt a 4 GiB allocation (OOM / DoS) */
+    LY_CHECK_ERR_RET(LYPLG_BITS2BYTES(*val_size_bits) >= UINT32_MAX,
+            LOGERR(lybctx->ctx, LY_EINVAL, "LYB value size overflow."), LY_EINVAL);
+
     /* allocate zeroed memory with an addition zero byte */
     *val = calloc(LYPLG_BITS2BYTES(*val_size_bits) + 1, sizeof **val);
     LY_CHECK_ERR_RET(!*val, LOGMEM(lybctx->ctx), LY_EMEM);