[Bug Report] GC Crash when using @codon.jit with NumPy operations in ThreadPoolExecutor

[LeeLee26]

Problem Description

GC Crash / Segmentation Fault when returning NumPy arrays from @codon.jit functions in ThreadPoolExecutor.

Environment

• OS: Debian GNU/Linux 12 (bookworm)
• Codon version: 0.19.3 && the latest version 0.19.6 build from source (develop codon branch)
• Python version: 3.11.10
• numpy version: 2.1.3

Minimal Reproducible Code

import codon
import numpy as np
from concurrent.futures import ThreadPoolExecutor

@codon.jit
def func():
    a = np.random.randn(1024,1024)
    return a

def worker(tid: int):
    for i in range(10):
        func()

if __name__ == "__main__":
    num_threads = 1  # Crash occurs even with single thread
    with ThreadPoolExecutor(max_workers=num_threads) as ex:
        futures = [ex.submit(worker, tid) for tid in range(num_threads)]
        for f in futures:
            f.result()

How to Run

python3 test.py

Crash Frame

Collecting from unknown thread
Aborted (core dumped)
#0  0x00007fff4a360f12 in GC_find_limit_with_bound () from /data00/home/leelee/.codon/lib/codon/libcodonrt.so
#1  0x00007fff4a360d30 in GC_init_linux_data_start () from /data00/home/leelee/.codon/lib/codon/libcodonrt.so
#2  0x00007fff4a35efef in GC_init () from /data00/home/leelee/.codon/lib/codon/libcodonrt.so
#3  0x00007fff4a362218 in GC_add_roots () from /data00/home/leelee/.codon/lib/codon/libcodonrt.so
#4  0x00007fff4d86afb6 in codon::BoehmGCJITLinkMemoryManager::allocate(llvm::jitlink::JITLinkDylib const*, llvm::jitlink::LinkGraph&, llvm::unique_function<void (llvm::Expected<std::unique_ptr<llvm::jitlink::JITLinkMemoryManager::InFlightAlloc, std::default_delete<llvm::jitlink::JITLinkMemoryManager::InFlightAlloc> > >)>) ()
   from /data00/home/leelee/.codon/lib/codon/libcodonc.so
#5  0x00007fff522874c2 in llvm::jitlink::JITLinkerBase::linkPhase1(std::unique_ptr<llvm::jitlink::JITLinkerBase, std::default_delete<llvm::jitlink::JITLinkerBase> >) ()
   from /data00/home/leelee/.codon/lib/codon/libcodonc.so
#6  0x00007fff52263f58 in llvm::jitlink::link_ELF_x86_64(std::unique_ptr<llvm::jitlink::LinkGraph, std::default_delete<llvm::jitlink::LinkGraph> >, std::unique_ptr<llvm::jitlink::JITLinkContext, std::default_delete<llvm::jitlink::JITLinkContext> >) () from /data00/home/leelee/.codon/lib/codon/libcodonc.so
#7  0x00007fff5222413a in llvm::jitlink::link_ELF(std::unique_ptr<llvm::jitlink::LinkGraph, std::default_delete<llvm::jitlink::LinkGraph> >, std::unique_ptr<llvm::jitlink::JITLinkContext, std::default_delete<llvm::jitlink::JITLinkContext> >) () from /data00/home/leelee/.codon/lib/codon/libcodonc.so
#8  0x00007fff521ef75d in llvm::jitlink::link(std::unique_ptr<llvm::jitlink::LinkGraph, std::default_delete<llvm::jitlink::LinkGraph> >, std::unique_ptr<llvm::jitlink::JITLinkContext, std::default_delete<llvm::jitlink::JITLinkContext> >) () from /data00/home/leelee/.codon/lib/codon/libcodonc.so
#9  0x00007fff50a31cc8 in llvm::orc::LinkGraphLinkingLayer::emit(std::unique_ptr<llvm::orc::MaterializationResponsibility, std::default_delete<llvm::orc::MaterializationResponsibility> >, std::unique_ptr<llvm::jitlink::LinkGraph, std::default_delete<llvm::jitlink::LinkGraph> >, std::unique_ptr<llvm::MemoryBuffer, std::default_delete<llvm::MemoryBuffer> >) ()
   from /data00/home/leelee/.codon/lib/codon/libcodonc.so
#10 0x00007fff50a81479 in llvm::orc::ObjectLinkingLayer::emit(std::unique_ptr<llvm::orc::MaterializationResponsibility, std::default_delete<llvm::orc::MaterializationResponsibility> >, std::unique_ptr<llvm::MemoryBuffer, std::default_delete<llvm::MemoryBuffer> >) () from /data00/home/leelee/.codon/lib/codon/libcodonc.so
#11 0x00007fff50a81849 in llvm::orc::ObjectTransformLayer::emit(std::unique_ptr<llvm::orc::MaterializationResponsibility, std::default_delete<llvm::orc::MaterializationResponsibility> >, std::unique_ptr<llvm::MemoryBuffer, std::default_delete<llvm::MemoryBuffer> >) () from /data00/home/leelee/.codon/lib/codon/libcodonc.so
#12 0x00007fff50ac76c3 in llvm::orc::IRCompileLayer::emit(std::unique_ptr<llvm::orc::MaterializationResponsibility, std::default_delete<llvm::orc::MaterializationResponsibility> >, llvm::orc::ThreadSafeModule) () from /data00/home/leelee/.codon/lib/codon/libcodonc.so
#13 0x00007fff50ac7f47 in llvm::orc::IRTransformLayer::emit(std::unique_ptr<llvm::orc::MaterializationResponsibility, std::default_delete<llvm::orc::MaterializationResponsibility> >, llvm::orc::ThreadSafeModule) () from /data00/home/leelee/.codon/lib/codon/libcodonc.so
#14 0x00007fff50ac7f47 in llvm::orc::IRTransformLayer::emit(std::unique_ptr<llvm::orc::MaterializationResponsibility, std::default_delete<llvm::orc::MaterializationResponsibility> >, llvm::orc::ThreadSafeModule) () from /data00/home/leelee/.codon/lib/codon/libcodonc.so
#15 0x00007fff50a28537 in llvm::orc::BasicIRLayerMaterializationUnit::materialize(std::unique_ptr<llvm::orc::MaterializationResponsibility, std::default_delete<llvm::orc::MaterializationResponsibility> >) () from /data00/home/leelee/.codon/lib/codon/libcodonc.so
#16 0x00007fff509f1a8b in llvm::orc::MaterializationTask::run() () from /data00/home/leelee/.codon/lib/codon/libcodonc.so
#17 0x00007fff509f1e41 in llvm::orc::ExecutionSession::dispatchOutstandingMUs() () from /data00/home/leelee/.codon/lib/codon/libcodonc.so
#18 0x00007fff509f2f3d in llvm::orc::ExecutionSession::OL_completeLookup(std::unique_ptr<llvm::orc::InProgressLookupState, std::default_delete<llvm::orc::InProgressLookupState> >, std::shared_ptr<llvm::orc::AsynchronousSymbolQuery>, std::function<void (llvm::DenseMap<llvm::orc::JITDylib*, llvm::DenseSet<llvm::orc::SymbolStringPtr, llvm::DenseMapInfo<llvm::orc::SymbolStringPtr, void> >, llvm::DenseMapInfo<llvm::orc::JITDylib*, void>, llvm::detail::DenseMapPair<llvm::orc::JITDylib*, llvm::DenseSet<llvm::orc::SymbolStringPtr, llvm::DenseMapInfo<llvm::orc::SymbolStringPtr, void> > > > const&)>) () from /data00/home/leelee/.codon/lib/codon/libcodonc.so
#19 0x00007fff509f3d33 in llvm::orc::InProgressFullLookupState::complete(std::unique_ptr<llvm::orc::InProgressLookupState, std::default_delete<llvm::orc::InProgressLookupState> >) ()
   from /data00/home/leelee/.codon/lib/codon/libcodonc.so
#20 0x00007fff509f6c93 in llvm::orc::ExecutionSession::OL_applyQueryPhase1(std::unique_ptr<llvm::orc::InProgressLookupState, std::default_delete<llvm::orc::InProgressLookupState> >, llvm::Error) () from /data00/home/leelee/.codon/lib/codon/libcodonc.so
#21 0x00007fff509f73f2 in llvm::orc::ExecutionSession::lookup(llvm::orc::LookupKind, std::vector<std::pair<llvm::orc::JITDylib*, llvm::orc::JITDylibLookupFlags>, std::allocator<std::p--Type <RET> for more, q to quit, c to continue without paging--
air<llvm::orc::JITDylib*, llvm::orc::JITDylibLookupFlags> > > const&, llvm::orc::SymbolLookupSet, llvm::orc::SymbolState, llvm::unique_function<void (llvm::Expected<llvm::DenseMap<llvm::orc::SymbolStringPtr, llvm::orc::ExecutorSymbolDef, llvm::DenseMapInfo<llvm::orc::SymbolStringPtr, void>, llvm::detail::DenseMapPair<llvm::orc::SymbolStringPtr, llvm::orc::ExecutorSymbolDef> > >)>, std::function<void (llvm::DenseMap<llvm::orc::JITDylib*, llvm::DenseSet<llvm::orc::SymbolStringPtr, llvm::DenseMapInfo<llvm::orc::SymbolStringPtr, void> >, llvm::DenseMapInfo<llvm::orc::JITDylib*, void>, llvm::detail::DenseMapPair<llvm::orc::JITDylib*, llvm::DenseSet<llvm::orc::SymbolStringPtr, llvm::DenseMapInfo<llvm::orc::SymbolStringPtr, void> > > > const&)>) () from /data00/home/leelee/.codon/lib/codon/libcodonc.so
#22 0x00007fff509f7601 in llvm::orc::ExecutionSession::lookup(std::vector<std::pair<llvm::orc::JITDylib*, llvm::orc::JITDylibLookupFlags>, std::allocator<std::pair<llvm::orc::JITDylib*, llvm::orc::JITDylibLookupFlags> > > const&, llvm::orc::SymbolLookupSet, llvm::orc::LookupKind, llvm::orc::SymbolState, std::function<void (llvm::DenseMap<llvm::orc::JITDylib*, llvm::DenseSet<llvm::orc::SymbolStringPtr, llvm::DenseMapInfo<llvm::orc::SymbolStringPtr, void> >, llvm::DenseMapInfo<llvm::orc::JITDylib*, void>, llvm::detail::DenseMapPair<llvm::orc::JITDylib*, llvm::DenseSet<llvm::orc::SymbolStringPtr, llvm::DenseMapInfo<llvm::orc::SymbolStringPtr, void> > > > const&)>) ()
   from /data00/home/leelee/.codon/lib/codon/libcodonc.so
#23 0x00007fff509f83ce in llvm::orc::ExecutionSession::lookup(std::vector<std::pair<llvm::orc::JITDylib*, llvm::orc::JITDylibLookupFlags>, std::allocator<std::pair<llvm::orc::JITDylib*, llvm::orc::JITDylibLookupFlags> > > const&, llvm::orc::SymbolStringPtr, llvm::orc::SymbolState) () from /data00/home/leelee/.codon/lib/codon/libcodonc.so
#24 0x00007fff50a45b66 in llvm::orc::LLJIT::lookupLinkerMangled(llvm::orc::JITDylib&, llvm::orc::SymbolStringPtr) () from /data00/home/leelee/.codon/lib/codon/libcodonc.so
#25 0x00007fff4d847421 in llvm::orc::LLJIT::lookup(llvm::orc::JITDylib&, llvm::StringRef) () from /data00/home/leelee/.codon/lib/codon/libcodonc.so
#26 0x00007fff4d845686 in codon::jit::Engine::lookup(llvm::StringRef) () from /data00/home/leelee/.codon/lib/codon/libcodonc.so
#27 0x00007fff4d84bdf1 in codon::jit::JIT::init(bool) () from /data00/home/leelee/.codon/lib/codon/libcodonc.so
#28 0x00007fff4d84fb16 in jit_init () from /data00/home/leelee/.codon/lib/codon/libcodonc.so

Key Observations

• Occurs even with num_threads = 1, but no crash when not use ThreadPoolExecutor
• Occurs any numpy operation in func