fix(coverage): narrow parse-error ranges, and stop the report hiding what it dropped (#963)

Makefile.cbm

@@ -98,9 +98,10 @@ endif
 KOTLIN_DEDUP_TEST_DEFINE = -DCBM_KOTLIN_DEDUP_TEST_API=1
 CALL_REFERENCE_LOOKUP_TEST_DEFINE = -DCBM_CALL_REFERENCE_LOOKUP_TEST_API=1
 INCREMENTAL_TEST_DEFINE = -DCBM_INCREMENTAL_TEST_API=1
+COVERAGE_MARKER_TEST_DEFINE = -DCBM_COVERAGE_MARKER_TEST_API=1
 CFLAGS_TEST = $(CFLAGS_COMMON) $(EDITOR_TEST_DEFINES) $(SANITIZED_DEFINE) \
 	$(KOTLIN_DEDUP_TEST_DEFINE) $(CALL_REFERENCE_LOOKUP_TEST_DEFINE) \
-	$(INCREMENTAL_TEST_DEFINE) -g -O1 $(SANITIZE)
+	$(INCREMENTAL_TEST_DEFINE) $(COVERAGE_MARKER_TEST_DEFINE) -g -O1 $(SANITIZE)
 CXXFLAGS_TEST = $(CXXFLAGS_COMMON) $(SANITIZED_DEFINE) -g -O1 $(SANITIZE) $(CXX_STDLIB_FLAGS)
 
 # TSan (can't combine with ASan)
@@ -118,6 +119,7 @@ TSAN_SANITIZE = -fsanitize=thread -fno-omit-frame-pointer
 # macro of ours.
 CFLAGS_TSAN = $(CFLAGS_COMMON) $(EDITOR_TEST_DEFINES) $(KOTLIN_DEDUP_TEST_DEFINE) \
 	$(CALL_REFERENCE_LOOKUP_TEST_DEFINE) $(INCREMENTAL_TEST_DEFINE) \
+	$(COVERAGE_MARKER_TEST_DEFINE) \
 	-DCBM_SANITIZED_BUILD=1 -g -O1 $(TSAN_SANITIZE)
 CXXFLAGS_TSAN = $(CXXFLAGS_COMMON) -DCBM_SANITIZED_BUILD=1 -g -O1 \
                 $(TSAN_SANITIZE)

internal/cbm/cbm.h

@@ -511,6 +511,21 @@ typedef struct CBMFileResult {
      * completeness guarantee. Callers should treat a flagged file as "prefer
      * grep here", never treat an unflagged file as provably complete. */
     bool parse_incomplete;
+    /* True when the ranges cover so much of the file that they are no longer
+     * useful advice — one range over 80% of the line count. The file WAS
+     * indexed, but pointing a reader at almost every line tells them nothing,
+     * so the report says "read the source" instead of listing the range.
+     *
+     * Its main customers are non-C languages. The refinement that narrows a
+     * whole-file range using the preprocessed parse only runs for C, C++ and
+     * CUDA, so a Python, Java or Ruby file whose root node is ERROR still
+     * reports 1-N.
+     *
+     * Note the naming: this field and the phase string it produces are both
+     * `parse_unusable`. The older `parse_incomplete` field emits the phase
+     * `parse_partial` instead. That mismatch is historical, not deliberate —
+     * do not copy it. */
+    bool parse_unusable;
     const char *error_ranges;
     int error_region_count;
     bool is_test_file;

src/pipeline/pass_definitions.c

@@ -18,6 +18,7 @@ enum { PD_RING = 4, PD_RING_MASK = 3, PD_JSON_MARGIN = 10, PD_ESC_MARGIN = 3, PD
 enum { PD_JSON_FIELD_OVERHEAD = 6 };
 #include "pipeline/pipeline.h"
 #include <stdint.h>
+#include <ctype.h>
 #include "pipeline/pipeline_internal.h"
 #include "graph_buffer/graph_buffer.h"
 #include "foundation/log.h"
@@ -544,27 +545,93 @@ static bool objectscript_export_append_secondary_arrays(CBMFileResult *aggregate
 /* Preserve every generated class's parse diagnostics. The generated UDL
  * snippets all map back to one physical Studio Export file, so their compact
  * range lists can be concatenated using the ordinary comma separator. */
+/* Read the trailing ",+<N>" truncation marker off a range string. Returns the
+ * number of dropped ranges the marker reports, or 0 when there is no marker,
+ * and writes the length of the part before the marker to `body_len`. */
+static int objectscript_export_split_range_marker(const char *ranges, size_t *body_len) {
+    size_t len = ranges ? strlen(ranges) : 0;
+    *body_len = len;
+    if (len == 0) {
+        return 0;
+    }
+    size_t i = len;
+    while (i > 0 && isdigit((unsigned char)ranges[i - 1])) {
+        i--;
+    }
+    if (i == len || i == 0 || ranges[i - 1] != '+') {
+        return 0;
+    }
+    size_t marker = i - 1; /* index of '+' */
+    if (marker > 0 && ranges[marker - 1] == ',') {
+        marker--; /* drop the separator too */
+    }
+    *body_len = marker;
+    return atoi(ranges + i);
+}
+
+/* Join one Studio Export part's ranges onto the aggregate.
+ *
+ * One export file can hold several <Class> elements, each parsed separately,
+ * so their range strings get concatenated. A ",+<N>" truncation marker must
+ * end up ONCE, at the very end: every reader stops at the first token that is
+ * not a range, so a marker left in the middle would silently hide every range
+ * after it. Strip the marker off both sides, join the plain ranges, then add
+ * one marker back carrying the summed count. */
 static bool objectscript_export_append_error_ranges(CBMFileResult *aggregate,
                                                     const CBMFileResult *part) {
     aggregate->parse_incomplete = aggregate->parse_incomplete || part->parse_incomplete;
+    aggregate->parse_unusable = aggregate->parse_unusable || part->parse_unusable;
     aggregate->error_region_count += part->error_region_count;
     if (!part->error_ranges || !part->error_ranges[0]) {
         return true;
     }
+
+    size_t agg_len = 0;
+    size_t part_len = 0;
+    int dropped = 0;
+    const char *agg_body = aggregate->error_ranges;
+    if (agg_body && agg_body[0]) {
+        dropped += objectscript_export_split_range_marker(agg_body, &agg_len);
+    } else {
+        agg_body = NULL;
+    }
+    dropped += objectscript_export_split_range_marker(part->error_ranges, &part_len);
+
     const char *combined = NULL;
-    if (aggregate->error_ranges && aggregate->error_ranges[0]) {
-        combined = cbm_arena_sprintf(&aggregate->arena, "%s,%s", aggregate->error_ranges,
-                                     part->error_ranges);
+    if (agg_body && agg_len > 0 && part_len > 0) {
+        combined = cbm_arena_sprintf(&aggregate->arena, "%.*s,%.*s", (int)agg_len, agg_body,
+                                     (int)part_len, part->error_ranges);
+    } else if (agg_body && agg_len > 0) {
+        combined = cbm_arena_sprintf(&aggregate->arena, "%.*s", (int)agg_len, agg_body);
+    } else if (part_len > 0) {
+        combined = cbm_arena_sprintf(&aggregate->arena, "%.*s", (int)part_len, part->error_ranges);
     } else {
-        combined = cbm_arena_strdup(&aggregate->arena, part->error_ranges);
+        combined = cbm_arena_strdup(&aggregate->arena, "");
     }
     if (!combined) {
         return false;
     }
+    if (dropped > 0) {
+        combined = cbm_arena_sprintf(&aggregate->arena, "%s%s+%d", combined, combined[0] ? "," : "",
+                                     dropped);
+        if (!combined) {
+            return false;
+        }
+    }
     aggregate->error_ranges = combined;
     return true;
 }
 
+#if defined(CBM_COVERAGE_MARKER_TEST_API) && CBM_COVERAGE_MARKER_TEST_API
+/* Test seam. This join only fires for a Studio Export file holding several
+ * <Class> elements where a class overruns the 256-region cap — hard to reach
+ * through the pipeline, easy to get wrong, and a wrong result hides ranges
+ * without saying so. Expose the join so the marker rules can be pinned. */
+bool cbm_pipeline_coverage_marker_test_join(CBMFileResult *aggregate, const CBMFileResult *part) {
+    return objectscript_export_append_error_ranges(aggregate, part);
+}
+#endif
+
 /* Studio Export files may contain multiple <Class> elements, while the
  * pipeline cache has one slot per physical file. Extract each generated UDL
  * class independently (preserving the upstream parser behavior), then compose
@@ -785,9 +852,9 @@ int cbm_pipeline_pass_definitions(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t
         } else if (result->parse_incomplete) {
             /* Best-effort parse-coverage signal (#963): indexed, but with
              * ERROR/MISSING regions — see pass_parallel.c (keep in sync). */
-            cbm_pipeline_add_file_error(ctx->pipeline, rel,
-                                        result->error_ranges ? result->error_ranges : "unknown",
-                                        "parse_partial");
+            cbm_pipeline_add_file_error(
+                ctx->pipeline, rel, result->error_ranges ? result->error_ranges : "unknown",
+                result->parse_unusable ? "parse_unusable" : "parse_partial");
         }
 
         /* Create nodes for each definition */

src/pipeline/pass_parallel.c

@@ -889,11 +889,12 @@ static void extract_worker(int worker_id, void *ctx_ptr) {
         } else if (result->parse_incomplete) {
             /* Best-effort parse-coverage signal (#963): the file WAS indexed,
              * but its tree contains ERROR/MISSING regions whose constructs are
-             * silently absent from the graph. Not a skip — recorded under the
-             * distinct "parse_partial" phase (reason = the line-range list) so
-             * the MCP layer reports it separately from skipped[]. */
+             * silently absent from the graph. Neither phase is a skip — both
+             * are recorded separately from skipped[] by the MCP layer.
+             * "parse_unusable" means one range covers so much of the file that
+             * naming the lines helps nobody; see parse_unusable in cbm.h. */
             pp_err_add(errs, fi->rel_path, result->error_ranges ? result->error_ranges : "unknown",
-                       "parse_partial");
+                       result->parse_unusable ? "parse_unusable" : "parse_partial");
         }
 
         /* Create definition nodes in local gbuf */

src/pipeline/pipeline_internal.h

@@ -864,6 +864,12 @@ void cbm_pp_bp_nap_cycles_reset(void);
 uint64_t cbm_pp_lsp_linear_fallback_rows(void);
 void cbm_pp_lsp_linear_fallback_rows_reset(void);
 
+#if defined(CBM_COVERAGE_MARKER_TEST_API) && CBM_COVERAGE_MARKER_TEST_API
+/* Test-only view of the Studio Export range join, so the ",+<N>" truncation
+ * marker rules can be checked without building a 256-region export file. */
+bool cbm_pipeline_coverage_marker_test_join(CBMFileResult *aggregate, const CBMFileResult *part);
+#endif
+
 #if defined(CBM_CALL_REFERENCE_LOOKUP_TEST_API) && CBM_CALL_REFERENCE_LOOKUP_TEST_API
 /* Deterministic test-only operation count for the shared semantic-reference
  * matcher used by both sequential and fused-parallel usage materialization. */

src/store/store.c

@@ -300,9 +300,25 @@ static int init_schema(cbm_store_t *s) {
         "  PRIMARY KEY (project, rel_path)"
         ");"
         /* Best-effort indexing-coverage signal (#963). One row per file the
-         * indexer could not fully cover: kind "parse_partial" (indexed, but the
-         * parse tree had ERROR/MISSING regions — detail = 1-based line ranges)
-         * or a skip phase ("read"/"extract"/"oversized" — detail = reason).
+         * indexer could not fully cover. `kind` says which of three things
+         * happened, and `detail` means something different in each:
+         *
+         *   "parse_partial"   the file WAS indexed, but the parse tree had
+         *                     ERROR/MISSING regions. detail = 1-based line
+         *                     ranges, "start-end,start-end", with an optional
+         *                     trailing "+<N>" saying N more ranges were dropped
+         *                     by the producer's cap. Read those lines.
+         *   "parse_unusable"  the file WAS indexed, but one range covers 80% or
+         *                     more of it, so naming the lines is useless advice.
+         *                     detail = the same range string. Read the source.
+         *   a skip phase      the file was NOT indexed at all: "read",
+         *                     "extract" or "oversized". detail = the reason.
+         *
+         * The first two are easy to confuse with the third, and the difference
+         * matters to a reader: a skipped file is absent from the graph, while
+         * the other two are present but incomplete. Name a new kind so that
+         * distinction stays obvious — "parse_failed" would read as a skip.
+         *
          * Deliberately SEPARATE from the graph tables: coverage is metadata
          * about the graph, not part of it. */
         "CREATE TABLE IF NOT EXISTS index_coverage ("

tests/test_index_resilience.c

@@ -337,8 +337,16 @@ TEST(index_parse_partial_reported) {
     ASSERT_STR_EQ("indexed", status);
     ASSERT_EQ(yyjson_get_int(yyjson_obj_get(sc, "skipped_count")), 0);
 
-    /* The coverage signal is surfaced with ranges + the best-effort note. */
+    /* The coverage signal is surfaced with ranges + the best-effort note.
+     * Both bounds matter. The floor catches the signal going missing. The
+     * ceiling catches the opposite failure: exactly one of the two files has
+     * a gap, so a count above 1 means the clean Python neighbour got flagged
+     * as well, which is how over-flagging looks from the outside. */
     ASSERT_GTE(yyjson_get_int(yyjson_obj_get(sc, "parse_partial_count")), 1);
+    ASSERT_EQ(yyjson_get_int(yyjson_obj_get(sc, "parse_partial_count")), 1);
+    /* A local gap is not a whole-file failure, so the other coverage kind
+     * must stay empty here. */
+    ASSERT_EQ(yyjson_get_int(yyjson_obj_get(sc, "parse_unusable_count")), 0);
     yyjson_val *pp = yyjson_obj_get(sc, "parse_partial");
     ASSERT_NOT_NULL(pp);
     yyjson_val *files = yyjson_obj_get(pp, "files");
@@ -354,7 +362,13 @@ TEST(index_parse_partial_reported) {
             found_split = 1;
             ASSERT_NOT_NULL(ranges);
             ASSERT_GT((int)strlen(ranges), 0);
+            /* The gap is the two-header block, not the whole file. An
+             * 8-line file reported as 1-8 would be the old whole-file
+             * blame coming back. */
+            ASSERT_NULL(strstr(ranges, "1-8"));
         }
+        /* The clean file must not appear in the list at all. */
+        ASSERT_NULL(fp ? strstr(fp, "good.py") : NULL);
     }
     ASSERT_TRUE(found_split);
     const char *note = yyjson_get_str(yyjson_obj_get(pp, "note"));
@@ -474,6 +488,94 @@ TEST(index_parse_partial_reported) {
     PASS();
 }
 
+/* The whole-file class as index_status prints it, and what its number means.
+ *
+ * Each parse_unusable entry reports the END of the file's one range. The field
+ * was called "lines", which reads as the length of the file, and the two are
+ * not the same number — a grammar can end an error node past the last line,
+ * which this repo has already seen (a 326-line PowerShell file whose range
+ * ended at 327). "lines" also already means a definition's line span in the
+ * rest of this response, so the old name collided as well.
+ *
+ * The end line is checked against the persisted coverage row rather than a
+ * constant, so the test states the property and not a measurement. */
+TEST(index_parse_unusable_names_the_range_end) {
+    RProj lp;
+    memset(&lp, 0, sizeof(lp));
+    snprintf(lp.tmpdir, sizeof(lp.tmpdir), "/tmp/cbm_resil_XXXXXX");
+    if (!cbm_mkdtemp(lp.tmpdir)) {
+        FAIL("mkdtemp failed");
+    }
+    rh_to_fwd_slashes(lp.tmpdir);
+
+    /* Python gets no C preprocessor refinement, so a root-level ERROR still
+     * reports one range over the whole file — the parse_unusable class. */
+    ri_write_text(lp.tmpdir, "unparseable.py", ")))\n((( \n]]] [[[\ndef x(:\n");
+    ri_write_text(lp.tmpdir, "good.py", "def alpha():\n    return 1\n");
+
+    char *resp = NULL;
+    cbm_store_t *store = ri_index_capture(&lp, &resp);
+    if (!resp) {
+        FAIL("no MCP response");
+    }
+    if (!store) {
+        free(resp);
+        FAIL("store did not open");
+    }
+
+    /* The end line the report should be naming, read from the persisted row. */
+    cbm_coverage_row_t *rows = NULL;
+    int cov_count = 0;
+    ASSERT_EQ(cbm_store_coverage_get(store, lp.project, &rows, &cov_count), CBM_STORE_OK);
+    int want_end = 0;
+    for (int i = 0; i < cov_count; i++) {
+        if (rows[i].rel_path && strstr(rows[i].rel_path, "unparseable.py") && rows[i].detail) {
+            const char *dash = strchr(rows[i].detail, '-');
+            if (dash) {
+                want_end = atoi(dash + 1);
+            }
+        }
+    }
+    cbm_store_free_coverage(rows, cov_count);
+    ASSERT_GT(want_end, 0);
+
+    yyjson_doc *d = yyjson_read(resp, strlen(resp), 0);
+    ASSERT_NOT_NULL(d);
+    yyjson_val *sc = yyjson_obj_get(yyjson_doc_get_root(d), "structuredContent");
+    ASSERT_NOT_NULL(sc);
+    ASSERT_EQ(yyjson_get_int(yyjson_obj_get(sc, "parse_unusable_count")), 1);
+
+    yyjson_val *pu = yyjson_obj_get(sc, "parse_unusable");
+    ASSERT_NOT_NULL(pu);
+    yyjson_val *files = yyjson_obj_get(pu, "files");
+    ASSERT_NOT_NULL(files);
+    int found = 0;
+    size_t idx = 0;
+    size_t fmax = 0;
+    yyjson_val *fe = NULL;
+    yyjson_arr_foreach(files, idx, fmax, fe) {
+        const char *fp = yyjson_get_str(yyjson_obj_get(fe, "path"));
+        /* The clean neighbour must not be listed at all. */
+        ASSERT_NULL(fp ? strstr(fp, "good.py") : NULL);
+        if (!fp || !strstr(fp, "unparseable.py")) {
+            continue;
+        }
+        found = 1;
+        yyjson_val *range_end = yyjson_obj_get(fe, "range_end");
+        ASSERT_NOT_NULL(range_end);
+        ASSERT_EQ(yyjson_get_int(range_end), want_end);
+        ASSERT_TRUE(yyjson_get_bool(yyjson_obj_get(fe, "whole_file")));
+        /* The old name is gone, not kept beside the new one. */
+        ASSERT_NULL(yyjson_obj_get(fe, "lines"));
+    }
+    ASSERT_TRUE(found);
+
+    yyjson_doc_free(d);
+    free(resp);
+    rh_cleanup(&lp, store);
+    PASS();
+}
+
 /* INV(parse-partial-clears-on-fix, #963): the persisted coverage signal must
  * stay FRESH — after the broken file is fixed and the project re-indexed
  * (incremental route: the DB already exists), its parse_partial row is gone
@@ -798,6 +900,7 @@ SUITE(index_resilience) {
     RUN_TEST(index_clean_run_no_logfile);
     RUN_TEST(index_parse_partial_reported);
     RUN_TEST(index_parse_partial_clears_on_fix);
+    RUN_TEST(index_parse_unusable_names_the_range_end);
     RUN_TEST(index_not_indexed_by_design_reported);
     RUN_TEST(index_relative_repo_path_canonicalized);
 }