feat: add bounded two-pass large-repo indexing

Signed-off-by: Zhiyu <zhiyuzhang001@gmail.com>

Author: zhiyuzhang001-a11y

internal/cbm/arena.c

@@ -9,6 +9,7 @@ void cbm_arena_init(CBMArena *a) {
     a->block_size = CBM_ARENA_DEFAULT_BLOCK_SIZE;
     a->blocks[0] = (char *)malloc(a->block_size);
     if (a->blocks[0]) {
+        a->block_sizes[0] = a->block_size;
         a->nblocks = SKIP_ONE;
     }
 }
@@ -26,6 +27,7 @@ static int arena_grow(CBMArena *a, size_t min_size) {
         return 0;
     }
     a->blocks[a->nblocks] = block;
+    a->block_sizes[a->nblocks] = new_size;
     a->nblocks++;
     a->block_size = new_size;
     a->used = 0;
@@ -39,6 +41,16 @@ void *cbm_arena_alloc(CBMArena *a, size_t n) {
     // 8-byte alignment
     n = (n + 7) & ~(size_t)7;
 
+    if (n <= 64) {
+        a->alloc_le_64 += n;
+    } else if (n <= 256) {
+        a->alloc_le_256 += n;
+    } else if (n <= 4096) {
+        a->alloc_le_4096 += n;
+    } else {
+        a->alloc_gt_4096 += n;
+    }
+
     if (a->nblocks == 0) {
         return NULL;
     }
@@ -51,6 +63,7 @@ void *cbm_arena_alloc(CBMArena *a, size_t n) {
 
     char *ptr = a->blocks[a->nblocks - SKIP_ONE] + a->used;
     a->used += n;
+    a->total_alloc += n;
     return ptr;
 }
 
@@ -60,6 +73,7 @@ char *cbm_arena_strdup(CBMArena *a, const char *s) {
     size_t len = strlen(s);
     char *dst = (char *)cbm_arena_alloc(a, len + SKIP_ONE);
     if (dst) {
+        a->strdup_alloc += len + SKIP_ONE;
         memcpy(dst, s, len + SKIP_ONE);
     }
     return dst;
@@ -70,6 +84,7 @@ char *cbm_arena_strndup(CBMArena *a, const char *s, size_t len) {
         return NULL;
     char *dst = (char *)cbm_arena_alloc(a, len + SKIP_ONE);
     if (dst) {
+        a->strdup_alloc += len + SKIP_ONE;
         memcpy(dst, s, len);
         dst[len] = '\0';
     }
@@ -91,6 +106,7 @@ char *cbm_arena_sprintf(CBMArena *a, const char *fmt, ...) {
     if (!dst) {
         return NULL;
     }
+    a->sprintf_alloc += (size_t)needed + SKIP_ONE;
 
     va_start(args, fmt);
     vsnprintf(dst, (size_t)needed + SKIP_ONE, fmt, args);

internal/cbm/arena.h

@@ -1,41 +1,13 @@
-#ifndef CBM_ARENA_H
-#define CBM_ARENA_H
-
-#include <stddef.h>
-
-// CBMArena is a simple bump allocator that allocates from fixed-size blocks.
-// All memory is freed at once via cbm_arena_destroy(). Individual frees are not
-// supported — this is by design for per-file extraction where all data has the
-// same lifetime.
-#define CBM_ARENA_MAX_BLOCKS 256
-#define CBM_ARENA_DEFAULT_BLOCK_SIZE (64 * 1024) // 64KB initial
-
-typedef struct {
-    char *blocks[CBM_ARENA_MAX_BLOCKS];
-    size_t block_sizes[CBM_ARENA_MAX_BLOCKS]; // per-block sizes (for stats)
-    int nblocks;
-    size_t block_size;
-    size_t used;        // bytes used in current block
-    size_t total_alloc; // cumulative bytes allocated (for stats)
-} CBMArena;
-
-// Initialize an arena with the default block size.
-void cbm_arena_init(CBMArena *a);
-
-// Allocate n bytes from the arena. Returns NULL on OOM or block exhaustion.
-// All returned pointers are 8-byte aligned.
-void *cbm_arena_alloc(CBMArena *a, size_t n);
-
-// Duplicate a string into arena memory. Returns arena-owned copy.
-char *cbm_arena_strdup(CBMArena *a, const char *s);
-
-// Duplicate a string of known length into arena memory. NUL-terminates.
-char *cbm_arena_strndup(CBMArena *a, const char *s, size_t len);
-
-// sprintf into arena memory. Returns arena-owned string.
-char *cbm_arena_sprintf(CBMArena *a, const char *fmt, ...) __attribute__((format(printf, 2, 3)));
-
-// Free all blocks. Arena is invalid after this call.
-void cbm_arena_destroy(CBMArena *a);
-
-#endif // CBM_ARENA_H
+/*
+ * Extraction compatibility include.
+ *
+ * CBMArena used to be duplicated here and in src/foundation/arena.h. The
+ * production binary links the foundation implementation, so any field drift
+ * between those definitions is an ABI violation. Keep one canonical layout.
+ */
+#ifndef CBM_EXTRACTION_ARENA_COMPAT_H
+#define CBM_EXTRACTION_ARENA_COMPAT_H
+
+#include "../../src/foundation/arena.h"
+
+#endif /* CBM_EXTRACTION_ARENA_COMPAT_H */

internal/cbm/cbm.c

@@ -105,14 +105,12 @@ void cbm_reset_profile(void) {
 
 #define GROW_ARRAY(arr, arena)                                                                   \
     do {                                                                                         \
+        (void)(arena);                                                                           \
         if ((arr)->count >= (arr)->cap) {                                                        \
             int new_cap = (arr)->cap == 0 ? CBM_SZ_32 : (arr)->cap * PAIR_LEN;                   \
-            void *new_items = cbm_arena_alloc((arena), (size_t)new_cap * sizeof(*(arr)->items)); \
+            void *new_items = realloc((arr)->items, (size_t)new_cap * sizeof(*(arr)->items));    \
             if (!new_items)                                                                      \
                 return;                                                                          \
-            if ((arr)->items && (arr)->count > 0) {                                              \
-                memcpy(new_items, (arr)->items, (size_t)(arr)->count * sizeof(*(arr)->items));   \
-            }                                                                                    \
             (arr)->items = new_items;                                                            \
             (arr)->cap = new_cap;                                                                \
         }                                                                                        \
@@ -1679,6 +1677,28 @@ void cbm_free_result(CBMFileResult *result) {
     free(result->owned_results);
     result->owned_results = NULL;
     result->owned_result_count = 0;
+#define FREE_RESULT_ARRAY(field)                                                                 \
+    do {                                                                                         \
+        free(result->field.items);                                                               \
+        result->field.items = NULL;                                                              \
+        result->field.count = 0;                                                                 \
+        result->field.cap = 0;                                                                   \
+    } while (0)
+    FREE_RESULT_ARRAY(defs);
+    FREE_RESULT_ARRAY(calls);
+    FREE_RESULT_ARRAY(imports);
+    FREE_RESULT_ARRAY(usages);
+    FREE_RESULT_ARRAY(throws);
+    FREE_RESULT_ARRAY(rw);
+    FREE_RESULT_ARRAY(type_refs);
+    FREE_RESULT_ARRAY(env_accesses);
+    FREE_RESULT_ARRAY(type_assigns);
+    FREE_RESULT_ARRAY(impl_traits);
+    FREE_RESULT_ARRAY(resolved_calls);
+    FREE_RESULT_ARRAY(string_refs);
+    FREE_RESULT_ARRAY(infra_bindings);
+    FREE_RESULT_ARRAY(channels);
+#undef FREE_RESULT_ARRAY
     cbm_arena_destroy(&result->arena);
     free(result);
 }

internal/cbm/extract_node_stack.h

@@ -4,10 +4,12 @@
  * Replaces fixed-size TSNode stack[] arrays that silently drop AST subtrees
  * when the stack overflows (GitHub issue #199).
  *
- * Uses the arena allocator for zero-fragmentation growth: old blocks are
- * abandoned (freed when the arena is destroyed at end of file extraction).
- * Initial capacity matches the previous fixed caps so small files allocate
- * no extra memory.
+ * The common capacity lives inline in the traversal's own stack frame. Only an
+ * unusually broad AST spills into the result arena. Call-site capacities
+ * historically mirrored fixed stack limits (usually 512, sometimes 4096), so
+ * allocating every traversal there made temporary work survive with durable
+ * extraction results across the whole repository. Inline storage preserves the
+ * same traversal order and geometric growth without that lifetime inversion.
  */
 #ifndef CBM_EXTRACT_NODE_STACK_H
 #define CBM_EXTRACT_NODE_STACK_H
@@ -20,19 +22,24 @@ typedef struct {
     TSNode *items;
     int count;
     int cap;
+    TSNode inline_items[128];
 } TSNodeStack;
 
-/* Initialize a stack with the given initial capacity, arena-allocated. */
+enum { TS_NSTACK_INLINE_CAP = 128 };
+
+/* Initialize with inline storage; arena is used only if the stack spills. */
 static inline void ts_nstack_init(TSNodeStack *s, CBMArena *arena, int initial_cap) {
-    s->items = (TSNode *)cbm_arena_alloc(arena, (size_t)initial_cap * sizeof(TSNode));
+    (void)arena;
+    (void)initial_cap;
+    s->items = s->inline_items;
     s->count = 0;
-    s->cap = s->items ? initial_cap : 0;
+    s->cap = TS_NSTACK_INLINE_CAP;
 }
 
 /* Push a node onto the stack, growing 2x if needed. */
 static inline void ts_nstack_push(TSNodeStack *s, CBMArena *arena, TSNode node) {
     if (s->count >= s->cap) {
-        int new_cap = s->cap ? s->cap * 2 : 512;
+        int new_cap = s->cap ? s->cap * 2 : TS_NSTACK_INLINE_CAP;
         TSNode *new_items = (TSNode *)cbm_arena_alloc(arena, (size_t)new_cap * sizeof(TSNode));
         if (!new_items)
             return; /* OOM: best-effort, stop growing */

internal/cbm/lsp/kotlin_lsp.c

@@ -412,28 +412,13 @@ static void kt_emit_resolved_kind(KotlinLSPContext *ctx, const char *callee_qn,
         return;
     }
 
-    CBMResolvedCallArray *arr = ctx->resolved_calls;
-    if (arr->count >= arr->cap) {
-        int new_cap = arr->cap == 0 ? 16 : arr->cap * 2;
-        CBMResolvedCall *new_items = (CBMResolvedCall *)cbm_arena_alloc(
-            ctx->arena, (size_t)new_cap * sizeof(CBMResolvedCall));
-        if (!new_items) {
-            return;
-        }
-        if (arr->items && arr->count > 0) {
-            memcpy(new_items, arr->items, (size_t)arr->count * sizeof(CBMResolvedCall));
-        }
-        arr->items = new_items;
-        arr->cap = new_cap;
-    }
-    CBMResolvedCall *rc = &arr->items[arr->count];
-    memset(rc, 0, sizeof(CBMResolvedCall));
-    rc->caller_qn = ctx->enclosing_func_qn;
-    rc->callee_qn = cbm_arena_strdup(ctx->arena, callee_qn);
-    rc->strategy = strategy;
-    rc->confidence = confidence;
-    rc->kind = kind;
-    arr->count++;
+    CBMResolvedCall rc = {0};
+    rc.caller_qn = ctx->enclosing_func_qn;
+    rc.callee_qn = cbm_arena_strdup(ctx->arena, callee_qn);
+    rc.strategy = strategy;
+    rc.confidence = confidence;
+    rc.kind = kind;
+    cbm_resolvedcall_push(ctx->resolved_calls, ctx->arena, rc);
 }
 
 static void kt_stamp_resolved_site(KotlinLSPContext *ctx, int first, TSNode site) {

src/foundation/arena.c

@@ -61,6 +61,15 @@ void *cbm_arena_alloc(CBMArena *a, size_t n) {
     }
     /* 8-byte alignment */
     n = (n + ARENA_ALIGN) & ~(size_t)ARENA_ALIGN;
+    if (n <= 64) {
+        a->alloc_le_64 += n;
+    } else if (n <= 256) {
+        a->alloc_le_256 += n;
+    } else if (n <= 4096) {
+        a->alloc_le_4096 += n;
+    } else {
+        a->alloc_gt_4096 += n;
+    }
     if (a->nblocks == 0) {
         return NULL;
     }
@@ -90,6 +99,7 @@ char *cbm_arena_strdup(CBMArena *a, const char *s) {
     size_t len = strlen(s);
     char *dst = (char *)cbm_arena_alloc(a, len + SKIP_ONE);
     if (dst) {
+        a->strdup_alloc += len + SKIP_ONE;
         memcpy(dst, s, len + SKIP_ONE);
     }
     return dst;
@@ -101,6 +111,7 @@ char *cbm_arena_strndup(CBMArena *a, const char *s, size_t len) {
     }
     char *dst = (char *)cbm_arena_alloc(a, len + SKIP_ONE);
     if (dst) {
+        a->strdup_alloc += len + SKIP_ONE;
         memcpy(dst, s, len);
         dst[len] = '\0';
     }
@@ -120,6 +131,7 @@ char *cbm_arena_sprintf(CBMArena *a, const char *fmt, ...) {
     if (!dst) {
         return NULL;
     }
+    a->sprintf_alloc += (size_t)needed + SKIP_ONE;
 
     va_start(args, fmt);
     vsnprintf(dst, (size_t)needed + SKIP_ONE, fmt, args);

src/foundation/arena.h

@@ -24,6 +24,12 @@ typedef struct {
     size_t block_size;  /* current block capacity */
     size_t used;        /* bytes used in current block */
     size_t total_alloc; /* cumulative bytes allocated (for stats) */
+    size_t strdup_alloc;  /* requested bytes returned by strdup/strndup helpers */
+    size_t sprintf_alloc; /* requested bytes returned by sprintf helper */
+    size_t alloc_le_64;
+    size_t alloc_le_256;
+    size_t alloc_le_4096;
+    size_t alloc_gt_4096;
 } CBMArena;
 
 /* Initialize arena with default block size. */