Merge branch 'main' into bing/elem-it

Author: spiral-ladder

bindings/napi/pool.zig

@@ -1,16 +1,23 @@
 const std = @import("std");
-const js = @import("zapi:zapi").js;
 const Node = @import("persistent_merkle_tree").Node;
 const RefCount = @import("state_transition").RefCount;
 
 /// Backs the `PoolRc` wrapper allocation only — `Node.Pool` uses its own
 /// `InitOptions` allocators.
 const allocator = std.heap.page_allocator;
 
-const default_pool_size: u32 = 0;
+const pool_size_environment_variable = "LODESTAR_Z_NODE_POOL_CAPACITY";
+// Arbitrary limit to avoid excessive memory usage.
+const default_pool_size: u32 = 10_000_000;
 
 const PoolRc = RefCount(Node.Pool);
 
+fn poolSizeFromEnvironment() !u32 {
+    const raw = std.c.getenv(pool_size_environment_variable) orelse return default_pool_size;
+    const value = std.mem.span(raw);
+    return std.fmt.parseInt(u32, value, 10) catch error.InvalidPoolCapacity;
+}
+
 /// Pool is wrapped in `RefCount` so binding objects holding pool refs at
 /// process exit keep the pool alive until their JS finalizer runs. NAPI
 /// env cleanup hook fires before module-level JS holders are finalized,
@@ -22,9 +29,9 @@ const State = struct {
     pub fn init(self: *State) !void {
         if (self.pool_rc != null) return;
 
-        // Small-object lane must stay non-page: page_allocator rounds each
-        // alloc to 4 KB and blows up once the binding preheats 10M nodes.
-        var pool_value = try Node.Pool.init(.{ .allocator = std.heap.c_allocator, .pool_size = default_pool_size });
+        const pool_size = try poolSizeFromEnvironment();
+
+        var pool_value = try Node.Pool.init(.{ .allocator = std.heap.c_allocator, .pool_size = pool_size });
         errdefer pool_value.deinit();
 
         self.pool_rc = try PoolRc.init(allocator, pool_value);
@@ -49,17 +56,3 @@ const State = struct {
 };
 
 pub var state: State = .{};
-
-/// JS: pool.ensureCapacity(newSize)
-pub fn ensureCapacity(new_size: js.Number) !void {
-    if (state.pool_rc == null) {
-        return error.PoolNotInitialized;
-    }
-
-    const requested = new_size.assertU32();
-    const old_size = state.pool().nodes.capacity;
-    if (requested <= old_size) {
-        return;
-    }
-    try state.pool().preheat(@intCast(requested - state.pool().nodes.capacity));
-}

bindings/napi/root.zig

@@ -1,7 +1,7 @@
 const std = @import("std");
 const builtin = @import("builtin");
 const js = @import("zapi:zapi").js;
-pub const pool = @import("./pool.zig");
+const pool = @import("./pool.zig");
 pub const shuffle = @import("./shuffle.zig");
 pub const config = @import("./config.zig");
 pub const metrics = @import("./metrics.zig");

bindings/perf/loadState.test.ts

@@ -12,7 +12,6 @@ const stateBytes = await reader.readSerializedState();
 await reader.close();
 const requiredPubkeyCapacity = getPubkeyCacheCapacityForState(stateBytes);
 
-bindings.pool.ensureCapacity(10_000_000);
 let loadedPkix = false;
 try {
   bindings.pubkeys.load("./mainnet.pkix", requiredPubkeyCapacity);

bindings/src/index.d.ts

@@ -370,9 +370,6 @@ export declare class BeaconStateView {
 }
 
 declare const bindings: {
-  pool: {
-    ensureCapacity: (capacity: number) => void;
-  };
   config: {
     set: (chainConfig: object, genesisValidatorsRoot: Uint8Array) => void;
   };

bindings/test/beaconStateView.test.ts

@@ -140,7 +140,6 @@ describe("BeaconStateView", () => {
     global.gc?.();
 
     // Phase 2: Create native BeaconStateView
-    bindings.pool.ensureCapacity(10_000_000);
     try {
       bindings.pubkeys.load("./mainnet.pkix", MAINNET_PUBKEY_CACHE_LIMIT);
     } catch (_e) {

bindings/test/demo.ts

@@ -32,8 +32,6 @@ const hasPkix = printDuration("check for pkix file", () => {
   }
 });
 
-bindings.pool.ensureCapacity(10_000_000);
-
 const reader = await printDurationAsync("load era reader", () => era.era.EraReader.open(config, getFirstEraFilePath()));
 
 const nextReader = await printDurationAsync("load era reader", () =>

bindings/test/teardown.test.ts

@@ -23,7 +23,6 @@ const reader = await era.era.EraReader.open(config, getFirstEraFilePath());
 const stateBytes = await reader.readSerializedState();
 await reader.close();
 
-bindings.pool.ensureCapacity(10_000_000);
 bindings.pubkeys.ensureCapacity(getSerializedFuluValidatorCount(stateBytes));
 
 const seedState = bindings.BeaconStateView.createFromBytes(stateBytes);

docs/security/IMPLEMENTATION_MAP.md

@@ -30,6 +30,7 @@ supported caller supplies a current hostile path.
 | Serialized block boundary | [`BeaconStateView.stateTransition`](../../bindings/napi/BeaconStateView.zig) accepts serialized signed-block bytes and passes the decoded block to the transition. This is a hostile-input boundary for integrated callers. |
 | BLS verifier validation | [`bls_verifier.zig`](../../bindings/napi/bls_verifier.zig) validates every signature for infinity and G2 membership before pairing. It also validates raw public keys for infinity and G1 membership. Indexed and aggregate sets trust cached affine keys. Direct append callers must supply a validated public key. State-transition appends follow successful deposit proof-of-possession checks. Bulk sync requires a trusted validator list. PKIX load requires trusted file provenance. |
 | Pubkey cache | [`pubkey_cache.zig`](../../src/state_transition/cache/pubkey_cache.zig) defines an application-wide, append-only cache with locked access and no escaping pointers into movable storage. [`bindings/napi/pubkeys.zig`](../../bindings/napi/pubkeys.zig) owns its process-wide instance. |
+| Persistent Merkle tree pool | [`Node.Pool`](../../src/persistent_merkle_tree/Node.zig) allocates a fixed number of user slots and returns `PoolExhausted` without resizing. The shared addon reads `LODESTAR_Z_NODE_POOL_CAPACITY` during initialization, defaults to 10,000,000 slots, and rejects invalid values. Chunked-leaf and container payloads use a separate dynamic allocator. |
 | Reused epoch cache | [`epoch_transition_cache.zig`](../../src/state_transition/cache/epoch_transition_cache.zig) stores process-global arrays borrowed by an `EpochTransitionCache`. The lock covers acquisition and resize, not the full borrowed lifetime. Current safe use requires non-overlapping transitions and no concurrent teardown. |
 | PKIX persistence | [`pkix.zig`](../../src/state_transition/cache/pkix.zig) checks framing, bounds, ABI compatibility, and corruption checksums. It does not authenticate the file or semantically revalidate affine entries, so file provenance remains trusted. |
 | Build and release provenance | [`build.zig.zon`](../../build.zig.zon) and [`pnpm-lock.yaml`](../../pnpm-lock.yaml) pin dependency inputs. [`publish-bindings.yml`](../../.github/workflows/publish-bindings.yml) pins actions, builds ReleaseSafe artifacts, and publishes them with npm provenance. |

examples/metrics.ts

@@ -45,9 +45,6 @@ const hasPkix = (() => {
   }
 })();
 
-console.log("Initializing pool...");
-bindings.pool.ensureCapacity(10_000_000);
-
 console.log("Initializing metrics...");
 bindings.metrics.init();
 

src/beacon_node/chain/state_cache/block_state_cache.zig

@@ -421,7 +421,7 @@ const BlockHarness = struct {
 
         h.allocator = allocator;
         h.io = std.testing.io;
-        h.pool = try Node.Pool.init(.{ .page_allocator = allocator, .allocator = allocator, .pool_size = 256 * 8 });
+        h.pool = try Node.Pool.init(.{ .page_allocator = allocator, .allocator = allocator, .pool_size = 180_000 });
         errdefer h.pool.deinit();
 
         h.factory = try TestStateFactory.init(allocator, &h.pool);
@@ -762,7 +762,7 @@ fn cloneDistinct(seed: *CachedBeaconState, alloc: Allocator, slot: u64) !*Cached
 test "BlockStateCache add - insert/prune/duplicate paths free the owned state exactly once" {
     const seed_alloc = testing.allocator;
     const io = std.testing.io;
-    const pool_size = 256 * 64;
+    const pool_size = 180_000;
     var pool = try Node.Pool.init(.{ .page_allocator = seed_alloc, .allocator = seed_alloc, .pool_size = pool_size });
     defer pool.deinit();