From 3f831936b5f3012d13175e7c97326165efecefad Mon Sep 17 00:00:00 2001 From: Joshua Richter Date: Mon, 31 Aug 2026 19:57:11 -0400 Subject: [PATCH 1/3] fix(grammar): stop the Swift scanner shifting past the width of an int The Swift scanner keeps a 64-bit mask of the symbols that suppress a match -- the rule that stops `try!` emitting its `!` as a token of its own. It tests one bit per candidate: uint64_t suppressing_symbols = OP_SYMBOL_SUPPRESSOR[full_match]; for (uint64_t suppressor = 0; suppressor < TOKEN_COUNT; suppressor++) { if (!(suppressing_symbols & 1 << suppressor)) { The mask is uint64_t but the literal `1` is an int, so the shift is an int shift. TOKEN_COUNT is larger than 32, so once suppressor reaches 31 the shift runs past the width of the type. That is undefined behavior, and every bit above 31 is tested against a value the standard does not define. Nothing caught it because nothing in the tree reached the suppressor path. Any Swift force-unwrap does: `cached!` is enough. UBSan reports it as: scanner.c:514:47: runtime error: left shift of 1 by 31 places cannot be represented in type 'int' `1ULL` makes the literal as wide as the mask it is tested against. The new test in tests/test_extraction.c cannot go red on its own. The normal test build prints the UBSan message and carries on, which is why this survived. The Windows CLANGARM64 leg runs UBSan in trap mode, and there the same shift is an illegal-instruction crash -- so the test exists to make sure that leg keeps parsing a force-unwrap at all. scripts/vendored-checksums.txt records the new hash for the one changed file, as scripts/security-vendored.sh --update writes it. Layer 8 of the security gate compares vendored content against that manifest, so the edit and its recorded hash belong in the same commit. Found while adding Swift URL extraction in #1892 / #1976, and split out of that PR so the vendored change can be reviewed on its own. Signed-off-by: Joshua Richter --- .../cbm/vendored/grammars/swift/scanner.c | 2 +- scripts/vendored-checksums.txt | 2 +- tests/test_extraction.c | 19 +++++++++++++++++++ 3 files changed, 21 insertions(+), 2 deletions(-) diff --git a/internal/cbm/vendored/grammars/swift/scanner.c b/internal/cbm/vendored/grammars/swift/scanner.c index 2615afe43..bb2dcac58 100644 --- a/internal/cbm/vendored/grammars/swift/scanner.c +++ b/internal/cbm/vendored/grammars/swift/scanner.c @@ -511,7 +511,7 @@ static bool eat_operators( uint64_t suppressing_symbols = OP_SYMBOL_SUPPRESSOR[full_match]; if (suppressing_symbols) { for (uint64_t suppressor = 0; suppressor < TOKEN_COUNT; suppressor++) { - if (!(suppressing_symbols & 1 << suppressor)) { + if (!(suppressing_symbols & 1ULL << suppressor)) { continue; } diff --git a/scripts/vendored-checksums.txt b/scripts/vendored-checksums.txt index 4b15db072..5412a8fdb 100644 --- a/scripts/vendored-checksums.txt +++ b/scripts/vendored-checksums.txt @@ -791,7 +791,7 @@ c71d239df91726fc519c6eb72d318ec65820627232b2f796219e87dcf35d0ab4 internal/cbm/v 2db740ea1be6b71014d2be1385491f01bf2d15b607e61c5b30b4128535efe68a internal/cbm/vendored/grammars/sway/tree_sitter/parser.h 3533cec129bb4bba015c0d61d86dd7c3b7e82110e4d2ff7837a01eff5bad5ccc internal/cbm/vendored/grammars/swift/LICENSE 93e6b39fc5b16ef9d5869862ec8d56380a838355c362481942942d59ca666de2 internal/cbm/vendored/grammars/swift/parser.c -b835c1ded068e902944fe82c3770b9dcf85f67dcaab8fbf772f1606007da7373 internal/cbm/vendored/grammars/swift/scanner.c +f3d6271d64f58c39eed544104a70ca2cf9ecbf80c5d900620f1afd38836542cb internal/cbm/vendored/grammars/swift/scanner.c b29c1c9fb7cc82f58c84b376df1297d6e2737a1d655fd356db0859e3c29c2fea internal/cbm/vendored/grammars/swift/tree_sitter/alloc.h 5bdf6ed1a78e3409fd443e085ca967a64c188a5d082aaf7f819bccd53a471c94 internal/cbm/vendored/grammars/swift/tree_sitter/array.h a1f6ef161fbaf48a0e10fca90ef5290a062462b307b3898aa562993853b9f80a internal/cbm/vendored/grammars/swift/tree_sitter/parser.h diff --git a/tests/test_extraction.c b/tests/test_extraction.c index 5b8d16f61..cb7a1a8c8 100644 --- a/tests/test_extraction.c +++ b/tests/test_extraction.c @@ -1810,6 +1810,24 @@ TEST(swift_chained_call) { PASS(); } +/* A Swift force-unwrap is the one thing that reaches the scanner's suppressor + * path -- the rule that stops `try!` emitting its `!` as a token of its own. + * That path shifted an int by up to TOKEN_COUNT bits, which runs past the + * width of the type once the index reaches 31. + * + * This test cannot go red here. The normal test build prints the UBSan + * message and carries on, which is why the bug survived. The Windows + * CLANGARM64 leg runs UBSan in trap mode, where the same shift is an + * illegal-instruction crash, so parsing this file at all is the check. */ +TEST(swift_force_unwrap_scanner_shift) { + CBMFileResult *r = + extract("func load() { let u = cached! }\n", CBM_LANG_SWIFT, "t", "Load.swift"); + ASSERT_NOT_NULL(r); + ASSERT_FALSE(r->has_error); + cbm_free_result(r); + PASS(); +} + /* --- Objective-C --- */ TEST(objc_interface) { CBMFileResult *r = @@ -6661,6 +6679,7 @@ SUITE(extraction) { RUN_TEST(swift_method_call); RUN_TEST(swift_constructor_call); RUN_TEST(swift_chained_call); + RUN_TEST(swift_force_unwrap_scanner_shift); RUN_TEST(objc_interface); RUN_TEST(objc_implementation); RUN_TEST(dart_top_level_function); From a2ea711c29560511eb8467bea7ad309ebf500d6e Mon Sep 17 00:00:00 2001 From: Joshua Richter Date: Mon, 31 Aug 2026 23:01:44 -0400 Subject: [PATCH 2/3] fix(grammar): stop the Swift try-bang entry shifting past its own type scanner.c line 131 shifts a literal 1UL by 32 to build the entry that suppresses the bang of a try!. unsigned long is 64 bits on Linux and macOS but 32 bits on Windows, so there the shift count equals the width of the type, which is undefined. 1ULL is 64 bits on every target this project builds for. Nothing reports it today. The expression is a compile-time constant, so no sanitizer sees it run, and Makefile.cbm:719 builds vendored grammars with -w, which switches off -Wshift-count-overflow. Upstream tree-sitter-swift made the same change in 6ab8d1d74ebd, after the commit this grammar is pinned to. Refs #1892 Signed-off-by: Joshua Richter --- internal/cbm/vendored/grammars/swift/scanner.c | 2 +- scripts/vendored-checksums.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/cbm/vendored/grammars/swift/scanner.c b/internal/cbm/vendored/grammars/swift/scanner.c index bb2dcac58..357350282 100644 --- a/internal/cbm/vendored/grammars/swift/scanner.c +++ b/internal/cbm/vendored/grammars/swift/scanner.c @@ -128,7 +128,7 @@ const uint64_t OP_SYMBOL_SUPPRESSOR[OPERATOR_COUNT] = { 0, // EQ_EQ, 0, // PLUS_THEN_WS, 0, // MINUS_THEN_WS, - 1UL << FAKE_TRY_BANG, // BANG, + 1ULL << FAKE_TRY_BANG, // BANG, 0, // THROWS_KEYWORD, 0, // RETHROWS_KEYWORD, 0, // DEFAULT_KEYWORD, diff --git a/scripts/vendored-checksums.txt b/scripts/vendored-checksums.txt index 5412a8fdb..a54ab6440 100644 --- a/scripts/vendored-checksums.txt +++ b/scripts/vendored-checksums.txt @@ -791,7 +791,7 @@ c71d239df91726fc519c6eb72d318ec65820627232b2f796219e87dcf35d0ab4 internal/cbm/v 2db740ea1be6b71014d2be1385491f01bf2d15b607e61c5b30b4128535efe68a internal/cbm/vendored/grammars/sway/tree_sitter/parser.h 3533cec129bb4bba015c0d61d86dd7c3b7e82110e4d2ff7837a01eff5bad5ccc internal/cbm/vendored/grammars/swift/LICENSE 93e6b39fc5b16ef9d5869862ec8d56380a838355c362481942942d59ca666de2 internal/cbm/vendored/grammars/swift/parser.c -f3d6271d64f58c39eed544104a70ca2cf9ecbf80c5d900620f1afd38836542cb internal/cbm/vendored/grammars/swift/scanner.c +8a14f26c9011a02298a076674461beb48625a4ca7cea4a0f63947930d93631fd internal/cbm/vendored/grammars/swift/scanner.c b29c1c9fb7cc82f58c84b376df1297d6e2737a1d655fd356db0859e3c29c2fea internal/cbm/vendored/grammars/swift/tree_sitter/alloc.h 5bdf6ed1a78e3409fd443e085ca967a64c188a5d082aaf7f819bccd53a471c94 internal/cbm/vendored/grammars/swift/tree_sitter/array.h a1f6ef161fbaf48a0e10fca90ef5290a062462b307b3898aa562993853b9f80a internal/cbm/vendored/grammars/swift/tree_sitter/parser.h From e8204092333ee1c38cd681a81abb36f125288894 Mon Sep 17 00:00:00 2001 From: Joshua Richter Date: Tue, 1 Sep 2026 14:06:41 -0400 Subject: [PATCH 3/3] docs(vendored): record the Swift scanner shift patch in MANIFEST.md The vendoring table claims our copy of a grammar matches the pinned upstream commit. A local patch makes that claim false unless the patch is written down, so a future re-vendor would drop the fix without anyone noticing. The row covers both changed lines, because both are the same defect in the same file: OP_SYMBOL_SUPPRESSOR at line 131 and eat_operators at line 514. The row also names the two upstream commits that already carry these changes, and the table intro gains one clause for the case. Upstream fixed eat_operators in fb63a7004f07 on 2026-04-06 (their issue #558) and the OP_SYMBOL_SUPPRESSOR entry in 6ab8d1d74ebd on 2026-08-10. Our pin 8abb3e8b3325 is from 2026-03-20 and predates both, so these are backports rather than local inventions. That distinction changes what a re-vendor should do. The table's instruction is "must re-apply these", which is right for the other four rows -- upstream never took those. For this row it would mean hand-applying a change the newer file already has. The row now says to delete it instead, and the intro allows for that case. MANIFEST.md is itself listed in scripts/vendored-checksums.txt, so its own recorded checksum moves with the edit. Refs #1892 Signed-off-by: Joshua Richter --- internal/cbm/vendored/grammars/MANIFEST.md | 5 ++++- scripts/vendored-checksums.txt | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/internal/cbm/vendored/grammars/MANIFEST.md b/internal/cbm/vendored/grammars/MANIFEST.md index 7dd9688b4..4025b5f50 100644 --- a/internal/cbm/vendored/grammars/MANIFEST.md +++ b/internal/cbm/vendored/grammars/MANIFEST.md @@ -77,7 +77,9 @@ Guarded by the `contract_all_grammars_in_graph` graph-breadth test in The grammars below carry a small local patch to their vendored sources, on top of the pinned upstream commit recorded in the vendoring table below. -Re-vendoring from upstream must re-apply these. +Re-vendoring from upstream must re-apply these, unless the reason column +names an upstream commit that already carries the change — then drop the +row instead. | grammar | location | patch | reason | |---|---|---|---| @@ -85,6 +87,7 @@ Re-vendoring from upstream must re-apply these. | rescript | `rescript/scanner.c`, deserialize | guard `memcpy(state, buffer, n_bytes)` with `if (n_bytes > 0)` | UBSan: zero-length `memcpy` with a NULL `buffer` / `n_bytes == 0` on empty-state deserialize (formal UB, harmless). The sibling serialize copies a fixed `sizeof(ScannerState)` (always > 0, non-NULL src) and needs no guard. | | purescript | `purescript/scanner.c`, serialize | guard `memcpy(buffer, indents->data, to_copy)` with `if (to_copy > 0)` | UBSan: zero-length `memcpy` with a NULL/0-size source when the indent vector is empty (formal UB, harmless) | | plsql | `plsql/parser.c`, include | `#include ` → `#include "tree_sitter/parser.h"` | The older ABI-14 generator emits angle brackets; every other vendored grammar uses the quoted form, which resolves the per-grammar `tree_sitter/` header from the including file's directory | +| swift | `swift/scanner.c`, `OP_SYMBOL_SUPPRESSOR` + `eat_operators` | `1UL <<` / `1 <<` → `1ULL <<` | UBSan: `1 << suppressor` shifts an `int` by up to `TOKEN_COUNT` bits, undefined once the index reaches 31, while the mask it feeds is `uint64_t`. `1UL << FAKE_TRY_BANG` is the same defect on Windows, where `unsigned long` is 32 bits and `FAKE_TRY_BANG` is 32; the CLANGARM64 leg runs UBSan in trap mode, so there it is an illegal instruction rather than a log line. Upstream already carries both changes: `fb63a7004f07` (2026-04-06, upstream #558) for `eat_operators`, `6ab8d1d74ebd` (2026-08-10) for the `OP_SYMBOL_SUPPRESSOR` entry. Our pin `8abb3e8b3325` (2026-03-20) predates both, so this is a backport rather than a local invention — a re-vendor past 2026-08-10 should delete this row, not re-apply it | ## Vendored from verified upstream diff --git a/scripts/vendored-checksums.txt b/scripts/vendored-checksums.txt index a54ab6440..c4b809dc6 100644 --- a/scripts/vendored-checksums.txt +++ b/scripts/vendored-checksums.txt @@ -5,7 +5,7 @@ c5cfb43042b6b72045f4ba997834d0a7786d2793d91680868b5815b39f14fc78 internal/cbm/v b29c1c9fb7cc82f58c84b376df1297d6e2737a1d655fd356db0859e3c29c2fea internal/cbm/vendored/common/tree_sitter/alloc.h 31e60a1bff6f715afacce03b5b70efe42b58371b4f9595dd4af52a577ff9608c internal/cbm/vendored/common/tree_sitter/array.h 180b893c8734778fd32f372dfbc27bd6ad1cd2221f26150b31256ff6716320d2 internal/cbm/vendored/common/tree_sitter/parser.h -e1233d81f21d868b8f2d9e8fca10e15f2ec3ae4db4a7ba0301161cc011deef65 internal/cbm/vendored/grammars/MANIFEST.md +00a84162eb34fb58bcdd38c08922d4fea4c87b7c2219dd64b773dd15ba18de71 internal/cbm/vendored/grammars/MANIFEST.md ad8425038de519f8c4e3e9339feebf99dfad8a6002dcf79227d91402780a32cc internal/cbm/vendored/grammars/ada/LICENSE 02805ec13939b749c891567be36bf024b09034e04c80683a1ae667272458d549 internal/cbm/vendored/grammars/ada/parser.c 115a75d000bef9c70c4de7dfbb7f2a80b90cbb14ba265056dc7abb1ef3b9b6db internal/cbm/vendored/grammars/ada/tree_sitter/alloc.h