From 1147174eb68894cb1620a3d835d91b2a185bdd9e Mon Sep 17 00:00:00 2001 From: Hugh Perkins Date: Mon, 31 Aug 2026 13:40:33 -0700 Subject: [PATCH 01/19] [Caching] Ref 7: CPU per-task artifact cache (ORC object serialize/load) Extend the cross-process per-task artifact tier (6a) to the CPU backend. On a cache miss the per-task module is compiled to a host relocatable object via orc::SimpleCompiler and stored (object bytes + launch metadata) under the task's IR key; on a hit the cached object is loaded straight into the ORC object layer. Each task gets its own JITDylib (CPU analog of CUDA's one-CUmodule-per-task), so per-task objects never collide on shared symbols and no relink is needed. - codegen.cpp: widen the artifact-tier + per-task build gates to arch_is_cpu (CPU only when the tier is on; else keeps the whole-kernel module). - jit_cpu.cpp: JITSessionCPU::add_module_per_task fill+load site, JITModuleCPU multi-dylib resolve, host PIC object compile helper. - cpu/kernel_launcher.cpp: take the per-task path when per_construct_artifacts is non-empty (mirror of the CUDA launcher). - llvm_program.cpp: scope the per-task dir by host triple+CPU (as CUDA scopes by sm) so an NFS-shared cache never serves an incompatible host object. - test_per_offload_cache.py: CPU cross-process reuse + disabled-tier tests. --- quadrants/codegen/codegen.cpp | 17 +-- quadrants/runtime/cpu/jit_cpu.cpp | 112 +++++++++++++++++- quadrants/runtime/cpu/kernel_launcher.cpp | 9 +- .../program_impls/llvm/llvm_program.cpp | 16 +++ tests/python/test_per_offload_cache.py | 82 ++++++++++++- 5 files changed, 218 insertions(+), 18 deletions(-) diff --git a/quadrants/codegen/codegen.cpp b/quadrants/codegen/codegen.cpp index b3b72eae2c..f66db76de7 100644 --- a/quadrants/codegen/codegen.cpp +++ b/quadrants/codegen/codegen.cpp @@ -72,10 +72,11 @@ LLVMCompiledKernel KernelCodeGen::compile_kernel_to_module() { const int n = (int)offloads.size(); std::vector> data(n); - // Cross-process per-task artifact cache: a hit skips a task's entire compilation and carries cached PTX + launch - // metadata to the launcher. CUDA-only; an empty dir means offline cache is off (tier disabled). + // Cross-process per-task artifact cache: a hit skips a task's entire compilation and carries cached backend code + + // launch metadata to the launcher. CUDA + CPU; an empty dir means offline cache is off (tier disabled). const std::string art_dir = pertask_artifact_dir_ref(); - const bool artifact_tier = compile_config_.arch == Arch::cuda && !art_dir.empty(); + const bool artifact_tier = + (compile_config_.arch == Arch::cuda || arch_is_cpu(compile_config_.arch)) && !art_dir.empty(); const PerTaskArtifactCache artifact_cache(art_dir); const DeviceCapabilityConfig pertask_caps = prog->get_device_caps(); std::vector pertask_keys(n); @@ -140,10 +141,12 @@ LLVMCompiledKernel KernelCodeGen::compile_kernel_to_module() { } worker.flush(); - // Build one self-contained artifact per task BEFORE the whole-module link consumes `data`: a hit carries cached PTX - // (null module), a miss builds+optimizes a module for the JIT to compile and store. + // Build one self-contained artifact per task BEFORE the whole-module link consumes `data`: a hit carries cached + // backend code (null module), a miss builds+optimizes a module for the JIT to compile and store. CUDA always takes + // this path (the Option B composite module); CPU only when the artifact tier is on (otherwise it keeps its + // whole-kernel module), so gate CPU on `artifact_tier`. std::vector per_construct_artifacts; - if (compile_config_.arch == Arch::cuda) { + if (compile_config_.arch == Arch::cuda || artifact_tier) { for (int i = 0; i < n; i++) { if (!data[i]) continue; @@ -172,7 +175,7 @@ LLVMCompiledKernel KernelCodeGen::compile_kernel_to_module() { } // A cross-process hit leaves a task with no module, so skip the whole-kernel link (the launcher assembles the - // CUmodule from the per-task artifacts); still concatenate every task's metadata into `tasks`, which it runs off. + // kernel from the per-task artifacts); still concatenate every task's metadata into `tasks`, which it runs off. const bool code_only_tasks = std::any_of(data.begin(), data.end(), [](const auto &d) { return d && !d->module; }); LLVMCompiledKernel llvm_compiled_kernel; if (code_only_tasks) { diff --git a/quadrants/runtime/cpu/jit_cpu.cpp b/quadrants/runtime/cpu/jit_cpu.cpp index a3f750205d..ca48878a28 100644 --- a/quadrants/runtime/cpu/jit_cpu.cpp +++ b/quadrants/runtime/cpu/jit_cpu.cpp @@ -32,6 +32,7 @@ #include "llvm/Support/DynamicLibrary.h" #include "llvm/Support/raw_ostream.h" #include "llvm/Support/Error.h" +#include "llvm/Support/MemoryBuffer.h" #include "llvm/Target/TargetMachine.h" #include "llvm/Transforms/InstCombine/InstCombine.h" #include "llvm/Transforms/Scalar.h" @@ -49,6 +50,7 @@ #include "quadrants/jit/jit_session.h" #include "quadrants/util/file_sequence_writer.h" #include "quadrants/runtime/llvm/llvm_context.h" +#include "quadrants/codegen/llvm/per_task_artifact_cache.h" namespace quadrants::lang { @@ -80,10 +82,16 @@ class JITSessionCPU; class JITModuleCPU : public JITModule { private: JITSessionCPU *session_; - JITDylib *dylib_; + // The whole-module path resolves in one dylib; the per-task path (add_module_per_task) resolves across N, one per + // task. A task's entry symbol lives in exactly one of them, so a by-name lookup that searches them all is + // unambiguous (only the unique task entry names are ever looked up; duplicated helper symbols never are). + std::vector dylibs_; public: - JITModuleCPU(JITSessionCPU *session, JITDylib *dylib) : session_(session), dylib_(dylib) { + JITModuleCPU(JITSessionCPU *session, JITDylib *dylib) : session_(session), dylibs_{dylib} { + } + JITModuleCPU(JITSessionCPU *session, std::vector dylibs) + : session_(session), dylibs_(std::move(dylibs)) { } void *lookup_function(const std::string &name) override; @@ -104,6 +112,9 @@ class JITSessionCPU : public JITSession { std::vector all_libs_; int module_counter_; SectionMemoryManager *memory_manager_; + // Lazily built host PIC target machine for the per-task object serialize path (add_module_per_task); reused across + // all misses in a session. + std::unique_ptr pertask_target_machine_; public: JITSessionCPU(QuadrantsLLVMContext *tlctx, @@ -175,6 +186,55 @@ class JITSessionCPU : public JITSession { return new_module_raw_ptr; } + JITModule *add_module_per_task(std::vector artifacts, int max_reg) override { + QD_ASSERT(max_reg == 0); // No need to specify max_reg on CPUs + std::lock_guard _(mut_); + + // Cross-process fill site (mirror of runtime/cuda/jit_cuda.cpp): a hit already carries the cached host object in + // `code`, used verbatim; a miss compiles the per-task module to a host object and stores it -- object bytes plus + // the launch metadata that must travel with it -- under the task's IR key so a later process skips this + // compilation. No-ops when the dir is empty (offline cache off). + const PerTaskArtifactCache artifact_cache(pertask_artifact_dir_ref()); + + // One JITDylib per task (a task's entry symbol lives in exactly one), so per-task objects never collide on a + // shared helper / global symbol -- the CPU analog of CUDA's one-CUmodule-per-task, and why CPU needs no relink. + std::vector dylibs; + dylibs.reserve(artifacts.size()); + for (auto &art : artifacts) { + auto dylib_expect = es_.createJITDylib(fmt::format("pertask_{}", module_counter_)); + QD_ASSERT(dylib_expect); + auto &dylib = dylib_expect.get(); + dylib.addGenerator( + cantFail(llvm::orc::DynamicLibrarySearchGenerator::GetForCurrentProcess(dl_.getGlobalPrefix()))); + + std::unique_ptr obj; + if (!art.code.empty()) { + // Hit: the cached bytes are a relocatable host object; load them straight into the object layer. + obj = llvm::MemoryBuffer::getMemBufferCopy(llvm::StringRef(art.code.data(), art.code.size()), + fmt::format("pertask_{}", module_counter_)); + } else { + QD_ASSERT(art.module); + obj = compile_module_to_object(*art.module); + if (!art.key.empty()) { + PerTaskArtifact rec; + rec.tasks = art.tasks; + rec.used_tree_ids = art.used_tree_ids; + rec.struct_for_tls_sizes = art.struct_for_tls_sizes; + rec.code.assign(obj->getBufferStart(), obj->getBufferEnd()); + artifact_cache.store(art.key, rec); + } + } + cantFail(object_layer_.add(dylib, std::move(obj))); + dylibs.push_back(&dylib); + module_counter_++; + } + + auto new_module = std::make_unique(this, std::move(dylibs)); + auto *new_module_raw_ptr = new_module.get(); + modules.push_back(std::move(new_module)); + return new_module_raw_ptr; + } + void *lookup(const std::string Name) override { std::lock_guard _(mut_); #ifdef __APPLE__ @@ -187,21 +247,61 @@ class JITSessionCPU : public JITSession { return symbol->getAddress().toPtr(); } - void *lookup_in_module(JITDylib *lib, const std::string Name) { + void *lookup_in_modules(const std::vector &libs, const std::string Name) { std::lock_guard _(mut_); #ifdef __APPLE__ - auto symbol = es_.lookup({lib}, mangle_(Name)); + auto symbol = es_.lookup(libs, mangle_(Name)); #else - auto symbol = es_.lookup({lib}, es_.intern(Name)); + auto symbol = es_.lookup(libs, es_.intern(Name)); #endif if (!symbol) QD_ERROR("Function \"{}\" not found", Name); return symbol->getAddress().toPtr(); } + + private: + // Serialize a self-contained per-task module to a host relocatable object -- the "serialize" half of the per-task + // disk tier, called on a cache miss. Build the target machine the same way as KernelCodeGenCPU::optimize_module + // (host CPU, PIC so the object is loadable by the ORC object layer) so the emitted object matches the module that + // optimize_module already produced. + std::unique_ptr compile_module_to_object(llvm::Module &M) { + if (!pertask_target_machine_) { + auto expected_jtmb = llvm::orc::JITTargetMachineBuilder::detectHost(); + if (!expected_jtmb) { + QD_ERROR("LLVM TargetMachineBuilder has failed."); + } + auto triple = expected_jtmb->getTargetTriple(); + std::string err_str; + const llvm::Target *target = llvm::TargetRegistry::lookupTarget(triple.str(), err_str); + QD_ERROR_UNLESS(target, err_str); + llvm::TargetOptions options; + if (config_.fast_math) { + options.AllowFPOpFusion = llvm::FPOpFusion::Fast; + options.NoInfsFPMath = 1; + options.NoNaNsFPMath = 1; + } else { + options.AllowFPOpFusion = llvm::FPOpFusion::Strict; + options.NoInfsFPMath = 0; + options.NoNaNsFPMath = 0; + } + llvm::StringRef mcpu = llvm::sys::getHostCPUName(); + pertask_target_machine_.reset(target->createTargetMachine(triple, mcpu.str(), "", options, llvm::Reloc::PIC_, + llvm::CodeModel::Small, + llvm::CodeGenOptLevel::Aggressive)); + QD_ERROR_UNLESS(pertask_target_machine_.get(), "Could not allocate target machine!"); + } + M.setDataLayout(pertask_target_machine_->createDataLayout()); + llvm::orc::SimpleCompiler compiler(*pertask_target_machine_); + auto obj = compiler(M); + if (!obj) { + QD_ERROR("Per-task CPU object compilation failed"); + } + return std::move(*obj); + } }; void *JITModuleCPU::lookup_function(const std::string &name) { - return session_->lookup_in_module(dylib_, name); + return session_->lookup_in_modules(dylibs_, name); } std::unique_ptr create_llvm_jit_session_cpu(QuadrantsLLVMContext *tlctx, diff --git a/quadrants/runtime/cpu/kernel_launcher.cpp b/quadrants/runtime/cpu/kernel_launcher.cpp index 7d7fcfde86..4019235a8d 100644 --- a/quadrants/runtime/cpu/kernel_launcher.cpp +++ b/quadrants/runtime/cpu/kernel_launcher.cpp @@ -339,7 +339,14 @@ KernelLauncher::Handle KernelLauncher::register_llvm_kernel(const LLVM::Compiled auto *executor = get_runtime_executor(); auto data = compiled.get_internal_data().compiled_data.clone(); - auto *jit_module = executor->create_jit_module(std::move(data.module)); + JITModule *jit_module = nullptr; + if (!data.per_construct_artifacts.empty()) { + // Per-task path: assemble the kernel from per-task objects (the cross-process artifact cache) instead of the + // whole-kernel module. Mirrors runtime/cuda/kernel_launcher.cpp. + jit_module = executor->create_jit_module_per_task(std::move(data.per_construct_artifacts)); + } else { + jit_module = executor->create_jit_module(std::move(data.module)); + } std::vector task_funcs; std::vector checkpoint_id_per_task; diff --git a/quadrants/runtime/program_impls/llvm/llvm_program.cpp b/quadrants/runtime/program_impls/llvm/llvm_program.cpp index bdaf5d09a9..6569e17581 100644 --- a/quadrants/runtime/program_impls/llvm/llvm_program.cpp +++ b/quadrants/runtime/program_impls/llvm/llvm_program.cpp @@ -1,6 +1,8 @@ #include "quadrants/runtime/program_impls/llvm/llvm_program.h" #include "llvm/IR/Module.h" +#include "llvm/ExecutionEngine/Orc/JITTargetMachineBuilder.h" +#include "llvm/TargetParser/Host.h" #include "quadrants/codegen/cpu/codegen_cpu.h" #include "quadrants/codegen/llvm/llvm_compiled_data.h" @@ -43,6 +45,20 @@ LlvmProgramImpl::LlvmProgramImpl(CompileConfig &config_, KernelProfilerBase *pro pertask_dir += "_sm_" + std::to_string(CUDAContext::get_instance().get_compute_capability()); } #endif + if (arch_is_cpu(config_.arch)) { + // Artifacts hold host object code tied to the target machine, so scope the dir by host triple + CPU (as the + // CUDA branch scopes by sm); an NFS-shared cache path could otherwise serve an object built for one host to an + // incompatible CPU. Matches the target machine built in jit_cpu.cpp / KernelCodeGenCPU::optimize_module. + auto jtmb = llvm::orc::JITTargetMachineBuilder::detectHost(); + std::string tag = + (jtmb ? jtmb->getTargetTriple().str() : std::string("unknown")) + "_" + llvm::sys::getHostCPUName().str(); + for (char &c : tag) { + if (c == '/' || c == ':' || c == ' ' || c == '\\') { + c = '_'; + } + } + pertask_dir += "_cpu_" + tag; + } } pertask_artifact_dir_ref() = pertask_dir; } diff --git a/tests/python/test_per_offload_cache.py b/tests/python/test_per_offload_cache.py index 7879a8b760..baa206acb9 100644 --- a/tests/python/test_per_offload_cache.py +++ b/tests/python/test_per_offload_cache.py @@ -668,12 +668,12 @@ def kernel_struct(out1: qd.types.ndarray(), out2: qd.types.ndarray()) -> None: assert np.allclose(out2.to_numpy(), _C[0], atol=1.0), out2.to_numpy() -# --- Cross-process per-task artifact cache (CUDA-only backend reuse tier) --------------------------------------------- +# --- Cross-process per-task artifact cache (CUDA + CPU backend reuse tier) -------------------------------------------- # # The per-task artifact cache stores each offloaded task's fully compiled code + launch metadata on disk, keyed by the -# task's own IR (name-free), so a later process reuses an unchanged task instead of recompiling it. Only CUDA fills it, -# and it is gated on `offline_cache`. Reuse is reported on `PerOffloadCacheObservations.tasks_*` (-1 when the tier did -# not run). +# task's own IR (name-free), so a later process reuses an unchanged task instead of recompiling it. CUDA fills it with +# PTX and CPU (ref 7) with a host object; it is gated on `offline_cache`. Reuse is reported on +# `PerOffloadCacheObservations.tasks_*` (-1 when the tier did not run). @test_utils.test(arch=qd.cuda, offline_cache=False) @@ -750,3 +750,77 @@ def k_second(x: qd.types.ndarray(qd.f32, ndim=1)) -> None: finally: qd.reset() shutil.rmtree(cache_dir, ignore_errors=True) + + +@test_utils.test(arch=qd.cpu, offline_cache=False) +def test_per_task_artifact_cache_disabled_without_offline_cache_cpu() -> None: + # CPU sibling of the disabled-tier assertion above. `offline_cache` is the sole gate for the per-task disk tier, so + # with it off the per-task counts stay at the -1 sentinel while the (backend-agnostic) FRONTEND split still fires. + @qd.kernel + def kernel_two_loops(x: qd.types.ndarray(qd.f32, ndim=1)) -> None: + for i in x: + x[i] = x[i] * 2.0 + 1.0 + for i in x: + x[i] = x[i] - 3.0 + + arr = qd.ndarray(qd.f32, shape=(_N,)) + arr.from_numpy(np.arange(_N, dtype=np.float32)) + kernel_two_loops(arr) + + obs = kernel_two_loops._primal.per_offload_cache_observations + assert obs.frontend_constructs_total == 2, obs + assert obs.tasks_total == -1, obs + assert obs.tasks_cache_hit == -1, obs + assert obs.tasks_recompiled == -1, obs + assert np.allclose(arr.to_numpy(), np.arange(_N) * 2.0 + 1.0 - 3.0), arr.to_numpy() + + +def test_per_task_artifact_cache_reuses_shared_task_cross_process_cpu() -> None: + # CPU sibling of `test_per_task_artifact_cache_reuses_shared_task_cross_process` (ref 7). The per-task disk tier + # stores each task's compiled host object + launch metadata, so a fresh process (cold in-memory, warm disk) loads + # an unchanged task from disk instead of recompiling it. CPU fills the tier via the ORC object layer + # (runtime/cpu/jit_cpu.cpp). Uses a re-`init` with the same cache path to emulate a second process. + if qd.cpu not in test_utils.expected_archs(): + pytest.skip("this variant exercises the CPU per-task artifact cache") + + cache_dir = tempfile.mkdtemp() + try: + qd.init(arch=qd.cpu, offline_cache=True, offline_cache_file_path=cache_dir) + + @qd.kernel + def k_first(x: qd.types.ndarray(qd.f32, ndim=1)) -> None: + for i in x: + x[i] = x[i] * 2.0 + 1.0 + for i in x: + x[i] = x[i] - 3.0 + + a = qd.ndarray(qd.f32, shape=(_N,)) + a.from_numpy(np.arange(_N, dtype=np.float32)) + k_first(a) + obs1 = k_first._primal.per_offload_cache_observations + assert obs1.tasks_total >= 2, obs1 + assert obs1.tasks_cache_hit == 0, obs1 + + # Second "process": fresh runtime, same disk. `k_second` is a new kernel (whole-kernel entry misses, codegen + # runs), but its first loop matches `k_first`'s, so that task is served from disk. + qd.init(arch=qd.cpu, offline_cache=True, offline_cache_file_path=cache_dir) + + @qd.kernel + def k_second(x: qd.types.ndarray(qd.f32, ndim=1)) -> None: + for i in x: + x[i] = x[i] * 2.0 + 1.0 + for i in x: + x[i] = x[i] + 7.0 + + b = qd.ndarray(qd.f32, shape=(_N,)) + b.from_numpy(np.arange(_N, dtype=np.float32)) + k_second(b) + obs2 = k_second._primal.per_offload_cache_observations + assert obs2.tasks_cache_hit > 0, obs2 + assert obs2.tasks_recompiled >= 1, obs2 + assert obs2.tasks_cache_hit + obs2.tasks_recompiled == obs2.tasks_total, obs2 + + assert np.allclose(b.to_numpy(), np.arange(_N) * 2.0 + 1.0 + 7.0), b.to_numpy() + finally: + qd.reset() + shutil.rmtree(cache_dir, ignore_errors=True) From 7653c8f540db227b9b0590fe8e3fb451d5acfae6 Mon Sep 17 00:00:00 2001 From: Hugh Perkins Date: Mon, 31 Aug 2026 14:14:50 -0700 Subject: [PATCH 02/19] [Caching] Ref 7: clang-format fix (single-line JITModuleCPU ctor) --- quadrants/runtime/cpu/jit_cpu.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/quadrants/runtime/cpu/jit_cpu.cpp b/quadrants/runtime/cpu/jit_cpu.cpp index ca48878a28..ca16ce4e02 100644 --- a/quadrants/runtime/cpu/jit_cpu.cpp +++ b/quadrants/runtime/cpu/jit_cpu.cpp @@ -90,8 +90,7 @@ class JITModuleCPU : public JITModule { public: JITModuleCPU(JITSessionCPU *session, JITDylib *dylib) : session_(session), dylibs_{dylib} { } - JITModuleCPU(JITSessionCPU *session, std::vector dylibs) - : session_(session), dylibs_(std::move(dylibs)) { + JITModuleCPU(JITSessionCPU *session, std::vector dylibs) : session_(session), dylibs_(std::move(dylibs)) { } void *lookup_function(const std::string &name) override; From 8153378b713f1ba67ce804b0c1e80b6035a3e0a0 Mon Sep 17 00:00:00 2001 From: Hugh Perkins Date: Tue, 1 Sep 2026 06:31:59 -0700 Subject: [PATCH 03/19] [Caching] Ref 7: degrade corrupt per-task cache to a catchable error (codex) A decodable .qdb record whose object bytes are malformed (e.g. disk corruption) reaches object_layer_.add, which parses eagerly; the previous cantFail would abort the whole process on every launch through that cache path. Raise a catchable QD_ERROR instead (mirroring the CUDA per-task load's QD_ERROR_IF) and drop the offending entry so a later process recompiles and refills it. Adds PerTaskArtifactCache::erase for the invalidate step. --- quadrants/codegen/llvm/per_task_artifact_cache.h | 11 +++++++++++ quadrants/runtime/cpu/jit_cpu.cpp | 15 +++++++++++++-- 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/quadrants/codegen/llvm/per_task_artifact_cache.h b/quadrants/codegen/llvm/per_task_artifact_cache.h index b61fb94fba..799e30d8e5 100644 --- a/quadrants/codegen/llvm/per_task_artifact_cache.h +++ b/quadrants/codegen/llvm/per_task_artifact_cache.h @@ -85,6 +85,17 @@ class PerTaskArtifactCache { } } + // Drop a cached record so a later process recompiles and refills it. Used when a backend rejects the payload bytes as + // corrupt: try_load only catches framing-level corruption (truncation / bad length header), not a decodable record + // whose `code` is a malformed backend object / PTX, which only the backend loader can detect. + void erase(const std::string &ir_key) const { + if (dir_.empty()) { + return; + } + std::error_code ec; + std::filesystem::remove(path_for(ir_key), ec); + } + private: // The IR key is a hex digest plus a `#` suffix; '#' is legal but shell-awkward, so map it (and '/') to '_'. // The `.qdb` extension is mandatory -- the binary serializer refuses any other suffix. diff --git a/quadrants/runtime/cpu/jit_cpu.cpp b/quadrants/runtime/cpu/jit_cpu.cpp index ca16ce4e02..c123007388 100644 --- a/quadrants/runtime/cpu/jit_cpu.cpp +++ b/quadrants/runtime/cpu/jit_cpu.cpp @@ -207,7 +207,8 @@ class JITSessionCPU : public JITSession { cantFail(llvm::orc::DynamicLibrarySearchGenerator::GetForCurrentProcess(dl_.getGlobalPrefix()))); std::unique_ptr obj; - if (!art.code.empty()) { + const bool from_cache = !art.code.empty(); + if (from_cache) { // Hit: the cached bytes are a relocatable host object; load them straight into the object layer. obj = llvm::MemoryBuffer::getMemBufferCopy(llvm::StringRef(art.code.data(), art.code.size()), fmt::format("pertask_{}", module_counter_)); @@ -223,7 +224,17 @@ class JITSessionCPU : public JITSession { artifact_cache.store(art.key, rec); } } - cantFail(object_layer_.add(dylib, std::move(obj))); + // `object_layer_.add` parses the object eagerly, so malformed bytes (e.g. offline-cache corruption on the hit + // path) surface here. Don't `cantFail` -- that aborts the whole process on every launch through a corrupt cache. + // Drop the offending entry so a later process recompiles and refills it, and raise a catchable error instead + // (mirrors the CUDA per-task load, which QD_ERRORs on a bad module rather than terminating the process). + if (auto err = object_layer_.add(dylib, std::move(obj))) { + if (from_cache && !art.key.empty()) { + artifact_cache.erase(art.key); + } + QD_ERROR("Failed to load per-task CPU object into the JIT (offline cache may be corrupt): {}", + llvm::toString(std::move(err))); + } dylibs.push_back(&dylib); module_counter_++; } From 2e9ea9a02ae1480fdb4cff7092aaf9cfefc06aa4 Mon Sep 17 00:00:00 2001 From: Hugh Perkins Date: Tue, 1 Sep 2026 06:41:48 -0700 Subject: [PATCH 04/19] [Caching] Ref 7: document CPU per-task artifact cache in init_options (codex) The per-task artifact cache is no longer CUDA-only; note that it also applies to CPU (host object payload) and that it is scoped per target device. --- docs/source/user_guide/init_options.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/user_guide/init_options.md b/docs/source/user_guide/init_options.md index 55129923df..e28f14b5a5 100644 --- a/docs/source/user_guide/init_options.md +++ b/docs/source/user_guide/init_options.md @@ -13,7 +13,7 @@ Setting `offline_cache=False` is intended to emulate cold-start, i.e. a fresh Py When `offline_cache=True`, compilation artifacts persist on disk under `offline_cache_file_path` (default `~/.cache/quadrants/qdcache`), so a later Python process reuses them instead of recompiling. Setting `offline_cache=False` (or `QD_OFFLINE_CACHE=0`) forces a cold start: Quadrants recompiles kernels and neither reads nor writes its own on-disk cache. (On CUDA the driver keeps its own separate cache of compiled GPU code at `~/.nv/ComputeCache` that this flag does not disable; `offline_cache=False` only stops that cache from serving results across runs. Set `CUDA_CACHE_DISABLE=1` to turn it off entirely.) The flag gates two disk tiers, not just the whole-kernel one: -- On CUDA it also enables a **per-task artifact cache**: a kernel is compiled as several *tasks* (roughly one per parallel loop), so editing one task reuses the *other* tasks' compiled code across processes instead of recompiling the whole kernel. +- On CUDA and CPU it also enables a **per-task artifact cache**: a kernel is compiled as several *tasks* (roughly one per parallel loop), so editing one task reuses the *other* tasks' compiled code across processes instead of recompiling the whole kernel. (CUDA caches each task's PTX; CPU caches a host relocatable object file. The cache is scoped per target -- by GPU compute capability on CUDA, by host CPU on CPU -- so a shared `offline_cache_file_path` never serves code built for a different device.) - There is no eviction policy for these per-task artifacts yet, so this cache grows over time; wipe `offline_cache_file_path` occasionally if it gets large. The separate source-level cache used by [fastcache](./fastcache.md) kernels is controlled by `src_ll_cache` (on by default), not by `offline_cache`; with `offline_cache=False` it still writes its own bookkeeping files to disk, so set `src_ll_cache=False` as well to stop that too. From 57d654cb766676232dfab58dce5b82102dfec1f7 Mon Sep 17 00:00:00 2001 From: Hugh Perkins Date: Tue, 1 Sep 2026 06:52:46 -0700 Subject: [PATCH 05/19] [Caching] Ref 7: keep JIT session consistent when a per-task object is rejected (codex) On the corrupt-object error path, advance module_counter_ before creating the dylib so the lingering (empty) dylib left behind after the throw can't collide with a later per-task/whole-module name (which would fail createJITDylib). Also erase the on-disk entry on both the hit and miss paths (either may have written one), so a bad freshly-compiled object never poisons the cache. --- quadrants/runtime/cpu/jit_cpu.cpp | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/quadrants/runtime/cpu/jit_cpu.cpp b/quadrants/runtime/cpu/jit_cpu.cpp index c123007388..46578af4e4 100644 --- a/quadrants/runtime/cpu/jit_cpu.cpp +++ b/quadrants/runtime/cpu/jit_cpu.cpp @@ -200,7 +200,10 @@ class JITSessionCPU : public JITSession { std::vector dylibs; dylibs.reserve(artifacts.size()); for (auto &art : artifacts) { - auto dylib_expect = es_.createJITDylib(fmt::format("pertask_{}", module_counter_)); + // Advance the counter up front: the error path below throws mid-iteration and the created dylib lingers in the + // session, so reusing this id on a later call would collide on the dylib name (createJITDylib would fail). + const int mod_id = module_counter_++; + auto dylib_expect = es_.createJITDylib(fmt::format("pertask_{}", mod_id)); QD_ASSERT(dylib_expect); auto &dylib = dylib_expect.get(); dylib.addGenerator( @@ -211,7 +214,7 @@ class JITSessionCPU : public JITSession { if (from_cache) { // Hit: the cached bytes are a relocatable host object; load them straight into the object layer. obj = llvm::MemoryBuffer::getMemBufferCopy(llvm::StringRef(art.code.data(), art.code.size()), - fmt::format("pertask_{}", module_counter_)); + fmt::format("pertask_{}", mod_id)); } else { QD_ASSERT(art.module); obj = compile_module_to_object(*art.module); @@ -224,19 +227,19 @@ class JITSessionCPU : public JITSession { artifact_cache.store(art.key, rec); } } - // `object_layer_.add` parses the object eagerly, so malformed bytes (e.g. offline-cache corruption on the hit - // path) surface here. Don't `cantFail` -- that aborts the whole process on every launch through a corrupt cache. - // Drop the offending entry so a later process recompiles and refills it, and raise a catchable error instead - // (mirrors the CUDA per-task load, which QD_ERRORs on a bad module rather than terminating the process). + // `object_layer_.add` parses the object eagerly, so malformed bytes surface here -- offline-cache corruption on + // the hit path, or (defensively) a bad freshly-compiled object we just stored on the miss path. Don't `cantFail` + // -- that aborts the whole process on every launch through a corrupt cache. Drop the on-disk entry (either path + // may have written one) so a later process recompiles and refills it, and raise a catchable error instead of + // terminating (mirrors the CUDA per-task load, which QD_ERRORs on a bad module). if (auto err = object_layer_.add(dylib, std::move(obj))) { - if (from_cache && !art.key.empty()) { + if (!art.key.empty()) { artifact_cache.erase(art.key); } QD_ERROR("Failed to load per-task CPU object into the JIT (offline cache may be corrupt): {}", llvm::toString(std::move(err))); } dylibs.push_back(&dylib); - module_counter_++; } auto new_module = std::make_unique(this, std::move(dylibs)); From bda38070059637235639fae848e32c2921858061 Mon Sep 17 00:00:00 2001 From: Hugh Perkins Date: Tue, 1 Sep 2026 07:03:38 -0700 Subject: [PATCH 06/19] [Caching] Ref 7: catch deferred per-task link failures at the fill site (codex) object_layer_.add only registers a materialization unit, so it catches object parse errors but not a corrupt relocation / bad reference, which fails later during linking -- at launch-time lookup, where no key is available to invalidate the record. Force materialization right after add (resolve each task's entry symbol in its self-contained dylib) so a deferred link failure is caught while art.key is still in scope, erased, and raised catchably instead of poisoning every future process. --- quadrants/runtime/cpu/jit_cpu.cpp | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/quadrants/runtime/cpu/jit_cpu.cpp b/quadrants/runtime/cpu/jit_cpu.cpp index 46578af4e4..e6883b83f2 100644 --- a/quadrants/runtime/cpu/jit_cpu.cpp +++ b/quadrants/runtime/cpu/jit_cpu.cpp @@ -239,6 +239,25 @@ class JITSessionCPU : public JITSession { QD_ERROR("Failed to load per-task CPU object into the JIT (offline cache may be corrupt): {}", llvm::toString(std::move(err))); } + // `object_layer_.add` only registers a materialization unit, so it catches parse errors but not a corrupt + // relocation / undefined reference, which fails later during linking. That failure would otherwise surface at + // launch in lookup_in_modules -- with no key at hand to invalidate the record, poisoning every future process. + // Force materialization here (each task's entry symbol resolves in exactly this self-contained dylib) so a + // deferred link failure is caught while `art.key` is still available, then erased and raised catchably. + for (const auto &task : art.tasks) { +#ifdef __APPLE__ + auto sym = es_.lookup({&dylib}, mangle_(task.name)); +#else + auto sym = es_.lookup({&dylib}, es_.intern(task.name)); +#endif + if (!sym) { + if (!art.key.empty()) { + artifact_cache.erase(art.key); + } + QD_ERROR("Failed to materialize per-task CPU object for \"{}\" (offline cache may be corrupt): {}", task.name, + llvm::toString(sym.takeError())); + } + } dylibs.push_back(&dylib); } From 3863ade1ca79817bf05c6a8b4b76350660b53209 Mon Sep 17 00:00:00 2001 From: Hugh Perkins Date: Tue, 1 Sep 2026 09:30:36 -0700 Subject: [PATCH 07/19] [Caching] Ref 7: fix per-task object target features (SIGILL) + doc jargon compile_module_to_object built the TargetMachine from the host CPU *name* with an empty feature string, which selects that CPU model's default feature set -- a superset of what the running core may actually enable -- so the emitted per-task object could use instructions the host lacks and crash with SIGILL at kernel launch (seen on some CI runners). Build the target machine from detectHost()'s JITTargetMachineBuilder instead, carrying the explicit detected host features, exactly as the whole-kernel ConcurrentIRCompiler(JTMB) path does. Also reword the init_options per-task cache note to drop undefined jargon (PTX / compute capability) that failed the doc-quality check. --- docs/source/user_guide/init_options.md | 2 +- quadrants/runtime/cpu/jit_cpu.cpp | 23 +++++++++++++---------- 2 files changed, 14 insertions(+), 11 deletions(-) diff --git a/docs/source/user_guide/init_options.md b/docs/source/user_guide/init_options.md index e28f14b5a5..ab802aed19 100644 --- a/docs/source/user_guide/init_options.md +++ b/docs/source/user_guide/init_options.md @@ -13,7 +13,7 @@ Setting `offline_cache=False` is intended to emulate cold-start, i.e. a fresh Py When `offline_cache=True`, compilation artifacts persist on disk under `offline_cache_file_path` (default `~/.cache/quadrants/qdcache`), so a later Python process reuses them instead of recompiling. Setting `offline_cache=False` (or `QD_OFFLINE_CACHE=0`) forces a cold start: Quadrants recompiles kernels and neither reads nor writes its own on-disk cache. (On CUDA the driver keeps its own separate cache of compiled GPU code at `~/.nv/ComputeCache` that this flag does not disable; `offline_cache=False` only stops that cache from serving results across runs. Set `CUDA_CACHE_DISABLE=1` to turn it off entirely.) The flag gates two disk tiers, not just the whole-kernel one: -- On CUDA and CPU it also enables a **per-task artifact cache**: a kernel is compiled as several *tasks* (roughly one per parallel loop), so editing one task reuses the *other* tasks' compiled code across processes instead of recompiling the whole kernel. (CUDA caches each task's PTX; CPU caches a host relocatable object file. The cache is scoped per target -- by GPU compute capability on CUDA, by host CPU on CPU -- so a shared `offline_cache_file_path` never serves code built for a different device.) +- On CUDA and CPU it also enables a **per-task artifact cache**: a kernel is compiled as several *tasks* (roughly one per parallel loop), so editing one task reuses the *other* tasks' compiled code across processes instead of recompiling the whole kernel. (CUDA caches each task's compiled GPU code; CPU caches a compiled host object file. The cache is scoped per target device -- by GPU model on CUDA, by host CPU on CPU -- so a shared `offline_cache_file_path` never serves code built for a different device.) - There is no eviction policy for these per-task artifacts yet, so this cache grows over time; wipe `offline_cache_file_path` occasionally if it gets large. The separate source-level cache used by [fastcache](./fastcache.md) kernels is controlled by `src_ll_cache` (on by default), not by `offline_cache`; with `offline_cache=False` it still writes its own bookkeeping files to disk, so set `src_ll_cache=False` as well to stop that too. diff --git a/quadrants/runtime/cpu/jit_cpu.cpp b/quadrants/runtime/cpu/jit_cpu.cpp index e6883b83f2..2658da818c 100644 --- a/quadrants/runtime/cpu/jit_cpu.cpp +++ b/quadrants/runtime/cpu/jit_cpu.cpp @@ -302,11 +302,16 @@ class JITSessionCPU : public JITSession { if (!expected_jtmb) { QD_ERROR("LLVM TargetMachineBuilder has failed."); } - auto triple = expected_jtmb->getTargetTriple(); - std::string err_str; - const llvm::Target *target = llvm::TargetRegistry::lookupTarget(triple.str(), err_str); - QD_ERROR_UNLESS(target, err_str); - llvm::TargetOptions options; + // Build the target machine straight from the JTMB so it carries detectHost()'s *explicit* host feature vector, + // exactly as the whole-kernel path does via ConcurrentIRCompiler(JTMB). Passing the host CPU name with an empty + // feature string instead selects that CPU model's *default* features, which can be a superset of what the + // running core actually enables and emits illegal instructions at kernel launch on some hosts. PIC so the + // emitted object is loadable by the ORC object layer. + auto jtmb = std::move(*expected_jtmb); + jtmb.setRelocationModel(llvm::Reloc::PIC_); + jtmb.setCodeModel(llvm::CodeModel::Small); + jtmb.setCodeGenOptLevel(llvm::CodeGenOptLevel::Aggressive); + llvm::TargetOptions &options = jtmb.getOptions(); if (config_.fast_math) { options.AllowFPOpFusion = llvm::FPOpFusion::Fast; options.NoInfsFPMath = 1; @@ -316,11 +321,9 @@ class JITSessionCPU : public JITSession { options.NoInfsFPMath = 0; options.NoNaNsFPMath = 0; } - llvm::StringRef mcpu = llvm::sys::getHostCPUName(); - pertask_target_machine_.reset(target->createTargetMachine(triple, mcpu.str(), "", options, llvm::Reloc::PIC_, - llvm::CodeModel::Small, - llvm::CodeGenOptLevel::Aggressive)); - QD_ERROR_UNLESS(pertask_target_machine_.get(), "Could not allocate target machine!"); + auto expected_tm = jtmb.createTargetMachine(); + QD_ERROR_UNLESS(expected_tm, "Could not allocate target machine!"); + pertask_target_machine_ = std::move(*expected_tm); } M.setDataLayout(pertask_target_machine_->createDataLayout()); llvm::orc::SimpleCompiler compiler(*pertask_target_machine_); From 96b07d64d1ab4070d98e8645705299b95ca88b8b Mon Sep 17 00:00:00 2001 From: Hugh Perkins Date: Tue, 1 Sep 2026 12:02:41 -0700 Subject: [PATCH 08/19] [Caching] Ref 7: clang-format wrap of merge-resolved comment --- quadrants/codegen/codegen.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/quadrants/codegen/codegen.cpp b/quadrants/codegen/codegen.cpp index d2c5384c1b..1e149eb059 100644 --- a/quadrants/codegen/codegen.cpp +++ b/quadrants/codegen/codegen.cpp @@ -181,7 +181,8 @@ LLVMCompiledKernel KernelCodeGen::compile_kernel_to_module() { } // A cross-process hit leaves a task with no module, so skip the whole-kernel link; the launcher assembles the - // backend module from the per-task artifacts. Still concatenate every task's metadata into `tasks`, which it runs off. + // backend module from the per-task artifacts. Still concatenate every task's metadata into `tasks`, which it runs + // off. const bool code_only_tasks = std::any_of(data.begin(), data.end(), [](const auto &d) { return d && !d->module; }); LLVMCompiledKernel llvm_compiled_kernel; if (code_only_tasks) { From 4356f6b15f6924da629fd538099e2b4af34868d9 Mon Sep 17 00:00:00 2001 From: Hugh Perkins Date: Tue, 1 Sep 2026 13:32:24 -0700 Subject: [PATCH 09/19] [Caching] Ref 7: trim per-task cache doc note per review Drop the parenthetical detailing per-backend payloads and per-target scoping from the init_options per-task cache bullet. --- docs/source/user_guide/init_options.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/user_guide/init_options.md b/docs/source/user_guide/init_options.md index 2759aaf54a..a2d923bdb3 100644 --- a/docs/source/user_guide/init_options.md +++ b/docs/source/user_guide/init_options.md @@ -13,7 +13,7 @@ Setting `offline_cache=False` is intended to emulate cold-start, i.e. a fresh Py When `offline_cache=True`, compilation artifacts persist on disk under `offline_cache_file_path` (default `~/.cache/quadrants/qdcache`), so a later Python process reuses them instead of recompiling. Setting `offline_cache=False` (or `QD_OFFLINE_CACHE=0`) forces a cold start: Quadrants recompiles kernels and neither reads nor writes its own on-disk cache. (On CUDA the driver keeps its own separate cache of compiled GPU code at `~/.nv/ComputeCache` that this flag does not disable; `offline_cache=False` only stops that cache from serving results across runs. Set `CUDA_CACHE_DISABLE=1` to turn it off entirely.) The flag gates two disk tiers, not just the whole-kernel one: -- On CUDA, AMDGPU, and CPU it also enables a **per-task artifact cache**: a kernel is compiled as several *tasks* (roughly one per parallel loop), so editing one task reuses the *other* tasks' compiled code across processes instead of recompiling the whole kernel. (CUDA and AMDGPU cache each task's compiled GPU code; CPU caches a compiled host object file. The cache is scoped per target device -- by GPU model on CUDA and AMDGPU, by host CPU on CPU -- so a shared `offline_cache_file_path` never serves code built for a different device.) +- On CUDA, AMDGPU, and CPU it also enables a **per-task artifact cache**: a kernel is compiled as several *tasks* (roughly one per parallel loop), so editing one task reuses the *other* tasks' compiled code across processes instead of recompiling the whole kernel. - There is no eviction policy for these per-task artifacts yet, so this cache grows over time; wipe `offline_cache_file_path` occasionally if it gets large. The separate source-level cache used by [fastcache](./fastcache.md) kernels is controlled by `src_ll_cache` (on by default), not by `offline_cache`; with `offline_cache=False` it still writes its own bookkeeping files to disk, so set `src_ll_cache=False` as well to stop that too. From 7ce5fca0635cec06bde932f954adc23ede594b78 Mon Sep 17 00:00:00 2001 From: Hugh Perkins Date: Tue, 1 Sep 2026 13:40:50 -0700 Subject: [PATCH 10/19] [Caching] Ref 7: trim verbose comments on per-task CPU path Shorten the per-task cache comments to only note the non-obvious bits (one dylib per task, forced materialization, host-feature detection); drop restatements of the code and cross-backend history. --- quadrants/codegen/codegen.cpp | 16 ++----- .../codegen/llvm/per_task_artifact_cache.h | 5 +- quadrants/runtime/cpu/jit_cpu.cpp | 46 ++++++------------- quadrants/runtime/cpu/kernel_launcher.cpp | 3 +- .../program_impls/llvm/llvm_program.cpp | 5 +- tests/python/test_per_offload_cache.py | 13 ++---- 6 files changed, 28 insertions(+), 60 deletions(-) diff --git a/quadrants/codegen/codegen.cpp b/quadrants/codegen/codegen.cpp index 1e149eb059..e772bd9b1b 100644 --- a/quadrants/codegen/codegen.cpp +++ b/quadrants/codegen/codegen.cpp @@ -72,8 +72,7 @@ LLVMCompiledKernel KernelCodeGen::compile_kernel_to_module() { const int n = (int)offloads.size(); std::vector> data(n); - // Cross-process per-task artifact cache: a hit skips a task's entire compilation and carries cached backend code + - // launch metadata to the launcher. CUDA + AMDGPU + CPU; an empty dir means offline cache is off (tier disabled). + // Per-task artifact cache: a hit skips a task's compilation and carries cached code to the launcher. Empty dir = off. const std::string art_dir = pertask_artifact_dir_ref(); const bool artifact_tier = (compile_config_.arch == Arch::cuda || compile_config_.arch == Arch::amdgpu || arch_is_cpu(compile_config_.arch)) && @@ -142,12 +141,8 @@ LLVMCompiledKernel KernelCodeGen::compile_kernel_to_module() { } worker.flush(); - // Build one self-contained artifact per task before the whole-module link consumes `data`: a hit carries cached - // backend code (null module), a miss builds+optimizes a module for the JIT to compile and store. - // - // AMDGPU links each task with a separate `ld.lld`, and CPU serializes each task to a host object, so their per-task - // path only pays off when the tier is on; with it off they stay on the single whole-module link. CUDA's per-task - // load is cheap, so it always takes the per-task path. + // Build one artifact per task before the whole-module link consumes `data`. CUDA always takes this path; AMDGPU and + // CPU only when the tier is on, since their per-task link/serialize only pays off then. std::vector per_construct_artifacts; const bool build_per_construct_artifacts = compile_config_.arch == Arch::cuda || @@ -180,9 +175,8 @@ LLVMCompiledKernel KernelCodeGen::compile_kernel_to_module() { } } - // A cross-process hit leaves a task with no module, so skip the whole-kernel link; the launcher assembles the - // backend module from the per-task artifacts. Still concatenate every task's metadata into `tasks`, which it runs - // off. + // A cache hit leaves a task with no module, so skip the whole-kernel link and let the launcher assemble from the + // per-task artifacts. Still gather every task's metadata into `tasks`. const bool code_only_tasks = std::any_of(data.begin(), data.end(), [](const auto &d) { return d && !d->module; }); LLVMCompiledKernel llvm_compiled_kernel; if (code_only_tasks) { diff --git a/quadrants/codegen/llvm/per_task_artifact_cache.h b/quadrants/codegen/llvm/per_task_artifact_cache.h index 799e30d8e5..7dc8f6e96e 100644 --- a/quadrants/codegen/llvm/per_task_artifact_cache.h +++ b/quadrants/codegen/llvm/per_task_artifact_cache.h @@ -85,9 +85,8 @@ class PerTaskArtifactCache { } } - // Drop a cached record so a later process recompiles and refills it. Used when a backend rejects the payload bytes as - // corrupt: try_load only catches framing-level corruption (truncation / bad length header), not a decodable record - // whose `code` is a malformed backend object / PTX, which only the backend loader can detect. + // Drop a cached record so a later process refills it. try_load rejects framing-level corruption, but a malformed + // payload is only detectable by the backend loader. void erase(const std::string &ir_key) const { if (dir_.empty()) { return; diff --git a/quadrants/runtime/cpu/jit_cpu.cpp b/quadrants/runtime/cpu/jit_cpu.cpp index 2658da818c..e84ba18943 100644 --- a/quadrants/runtime/cpu/jit_cpu.cpp +++ b/quadrants/runtime/cpu/jit_cpu.cpp @@ -82,9 +82,8 @@ class JITSessionCPU; class JITModuleCPU : public JITModule { private: JITSessionCPU *session_; - // The whole-module path resolves in one dylib; the per-task path (add_module_per_task) resolves across N, one per - // task. A task's entry symbol lives in exactly one of them, so a by-name lookup that searches them all is - // unambiguous (only the unique task entry names are ever looked up; duplicated helper symbols never are). + // One dylib on the whole-module path, one per task on the per-task path. Lookups search them all; task entry names + // are unique across dylibs. std::vector dylibs_; public: @@ -111,8 +110,7 @@ class JITSessionCPU : public JITSession { std::vector all_libs_; int module_counter_; SectionMemoryManager *memory_manager_; - // Lazily built host PIC target machine for the per-task object serialize path (add_module_per_task); reused across - // all misses in a session. + // Built on first per-task cache miss, reused for the rest of the session. std::unique_ptr pertask_target_machine_; public: @@ -189,19 +187,13 @@ class JITSessionCPU : public JITSession { QD_ASSERT(max_reg == 0); // No need to specify max_reg on CPUs std::lock_guard _(mut_); - // Cross-process fill site (mirror of runtime/cuda/jit_cuda.cpp): a hit already carries the cached host object in - // `code`, used verbatim; a miss compiles the per-task module to a host object and stores it -- object bytes plus - // the launch metadata that must travel with it -- under the task's IR key so a later process skips this - // compilation. No-ops when the dir is empty (offline cache off). const PerTaskArtifactCache artifact_cache(pertask_artifact_dir_ref()); - // One JITDylib per task (a task's entry symbol lives in exactly one), so per-task objects never collide on a - // shared helper / global symbol -- the CPU analog of CUDA's one-CUmodule-per-task, and why CPU needs no relink. + // One dylib per task keeps each task's object in its own symbol namespace, so shared helper symbols never collide. std::vector dylibs; dylibs.reserve(artifacts.size()); for (auto &art : artifacts) { - // Advance the counter up front: the error path below throws mid-iteration and the created dylib lingers in the - // session, so reusing this id on a later call would collide on the dylib name (createJITDylib would fail). + // Advance up front: an error below throws with the dylib already created, so this id must not be reused. const int mod_id = module_counter_++; auto dylib_expect = es_.createJITDylib(fmt::format("pertask_{}", mod_id)); QD_ASSERT(dylib_expect); @@ -212,7 +204,6 @@ class JITSessionCPU : public JITSession { std::unique_ptr obj; const bool from_cache = !art.code.empty(); if (from_cache) { - // Hit: the cached bytes are a relocatable host object; load them straight into the object layer. obj = llvm::MemoryBuffer::getMemBufferCopy(llvm::StringRef(art.code.data(), art.code.size()), fmt::format("pertask_{}", mod_id)); } else { @@ -227,11 +218,8 @@ class JITSessionCPU : public JITSession { artifact_cache.store(art.key, rec); } } - // `object_layer_.add` parses the object eagerly, so malformed bytes surface here -- offline-cache corruption on - // the hit path, or (defensively) a bad freshly-compiled object we just stored on the miss path. Don't `cantFail` - // -- that aborts the whole process on every launch through a corrupt cache. Drop the on-disk entry (either path - // may have written one) so a later process recompiles and refills it, and raise a catchable error instead of - // terminating (mirrors the CUDA per-task load, which QD_ERRORs on a bad module). + // A corrupt cached object must not abort the process, so don't cantFail. Drop the record so a later run refills + // it and raise a catchable error. if (auto err = object_layer_.add(dylib, std::move(obj))) { if (!art.key.empty()) { artifact_cache.erase(art.key); @@ -239,11 +227,8 @@ class JITSessionCPU : public JITSession { QD_ERROR("Failed to load per-task CPU object into the JIT (offline cache may be corrupt): {}", llvm::toString(std::move(err))); } - // `object_layer_.add` only registers a materialization unit, so it catches parse errors but not a corrupt - // relocation / undefined reference, which fails later during linking. That failure would otherwise surface at - // launch in lookup_in_modules -- with no key at hand to invalidate the record, poisoning every future process. - // Force materialization here (each task's entry symbol resolves in exactly this self-contained dylib) so a - // deferred link failure is caught while `art.key` is still available, then erased and raised catchably. + // add() links lazily, so link errors (e.g. a corrupt relocation) only surface at lookup, by which point the key + // is gone. Force materialization now to catch them while we can still erase the record. for (const auto &task : art.tasks) { #ifdef __APPLE__ auto sym = es_.lookup({&dylib}, mangle_(task.name)); @@ -292,21 +277,16 @@ class JITSessionCPU : public JITSession { } private: - // Serialize a self-contained per-task module to a host relocatable object -- the "serialize" half of the per-task - // disk tier, called on a cache miss. Build the target machine the same way as KernelCodeGenCPU::optimize_module - // (host CPU, PIC so the object is loadable by the ORC object layer) so the emitted object matches the module that - // optimize_module already produced. + // Compile a per-task module to a host object file for the on-disk cache. std::unique_ptr compile_module_to_object(llvm::Module &M) { if (!pertask_target_machine_) { auto expected_jtmb = llvm::orc::JITTargetMachineBuilder::detectHost(); if (!expected_jtmb) { QD_ERROR("LLVM TargetMachineBuilder has failed."); } - // Build the target machine straight from the JTMB so it carries detectHost()'s *explicit* host feature vector, - // exactly as the whole-kernel path does via ConcurrentIRCompiler(JTMB). Passing the host CPU name with an empty - // feature string instead selects that CPU model's *default* features, which can be a superset of what the - // running core actually enables and emits illegal instructions at kernel launch on some hosts. PIC so the - // emitted object is loadable by the ORC object layer. + // Build from the JTMB so it uses detectHost()'s explicit feature vector. A bare CPU name with empty features + // would enable that model's default features, a superset of the running core, and emit illegal instructions. + // PIC so the object loads in the object layer. auto jtmb = std::move(*expected_jtmb); jtmb.setRelocationModel(llvm::Reloc::PIC_); jtmb.setCodeModel(llvm::CodeModel::Small); diff --git a/quadrants/runtime/cpu/kernel_launcher.cpp b/quadrants/runtime/cpu/kernel_launcher.cpp index 4019235a8d..f7e27adb21 100644 --- a/quadrants/runtime/cpu/kernel_launcher.cpp +++ b/quadrants/runtime/cpu/kernel_launcher.cpp @@ -341,8 +341,7 @@ KernelLauncher::Handle KernelLauncher::register_llvm_kernel(const LLVM::Compiled auto data = compiled.get_internal_data().compiled_data.clone(); JITModule *jit_module = nullptr; if (!data.per_construct_artifacts.empty()) { - // Per-task path: assemble the kernel from per-task objects (the cross-process artifact cache) instead of the - // whole-kernel module. Mirrors runtime/cuda/kernel_launcher.cpp. + // Per-task cache hit path: assemble the kernel from per-task objects instead of a whole-kernel module. jit_module = executor->create_jit_module_per_task(std::move(data.per_construct_artifacts)); } else { jit_module = executor->create_jit_module(std::move(data.module)); diff --git a/quadrants/runtime/program_impls/llvm/llvm_program.cpp b/quadrants/runtime/program_impls/llvm/llvm_program.cpp index 54724d45a8..8140edecfe 100644 --- a/quadrants/runtime/program_impls/llvm/llvm_program.cpp +++ b/quadrants/runtime/program_impls/llvm/llvm_program.cpp @@ -53,9 +53,8 @@ LlvmProgramImpl::LlvmProgramImpl(CompileConfig &config_, KernelProfilerBase *pro } #endif if (arch_is_cpu(config_.arch)) { - // Artifacts hold host object code tied to the target machine, so scope the dir by host triple + CPU (as the - // CUDA branch scopes by sm); an NFS-shared cache path could otherwise serve an object built for one host to an - // incompatible CPU. Matches the target machine built in jit_cpu.cpp / KernelCodeGenCPU::optimize_module. + // Host objects are CPU-specific, so scope the dir by host triple + CPU; a shared path must never serve an object + // built for an incompatible CPU. auto jtmb = llvm::orc::JITTargetMachineBuilder::detectHost(); std::string tag = (jtmb ? jtmb->getTargetTriple().str() : std::string("unknown")) + "_" + llvm::sys::getHostCPUName().str(); diff --git a/tests/python/test_per_offload_cache.py b/tests/python/test_per_offload_cache.py index c21987e4c2..c5d422dd1f 100644 --- a/tests/python/test_per_offload_cache.py +++ b/tests/python/test_per_offload_cache.py @@ -755,8 +755,8 @@ def k_second(x: qd.types.ndarray(qd.f32, ndim=1)) -> None: @test_utils.test(arch=qd.cpu, offline_cache=False) def test_per_task_artifact_cache_disabled_without_offline_cache_cpu() -> None: - # CPU sibling of the disabled-tier assertion above. `offline_cache` is the sole gate for the per-task disk tier, so - # with it off the per-task counts stay at the -1 sentinel while the (backend-agnostic) FRONTEND split still fires. + # With offline_cache off the per-task tier never runs, so its counts stay at the -1 sentinel; the frontend split + # still fires. @qd.kernel def kernel_two_loops(x: qd.types.ndarray(qd.f32, ndim=1)) -> None: for i in x: @@ -777,10 +777,8 @@ def kernel_two_loops(x: qd.types.ndarray(qd.f32, ndim=1)) -> None: def test_per_task_artifact_cache_reuses_shared_task_cross_process_cpu() -> None: - # CPU sibling of `test_per_task_artifact_cache_reuses_shared_task_cross_process` (ref 7). The per-task disk tier - # stores each task's compiled host object + launch metadata, so a fresh process (cold in-memory, warm disk) loads - # an unchanged task from disk instead of recompiling it. CPU fills the tier via the ORC object layer - # (runtime/cpu/jit_cpu.cpp). Uses a re-`init` with the same cache path to emulate a second process. + # A fresh process with a warm disk cache loads an unchanged task instead of recompiling it. Re-init with the same + # cache path emulates the second process. if qd.cpu not in test_utils.expected_archs(): pytest.skip("this variant exercises the CPU per-task artifact cache") @@ -802,8 +800,7 @@ def k_first(x: qd.types.ndarray(qd.f32, ndim=1)) -> None: assert obs1.tasks_total >= 2, obs1 assert obs1.tasks_cache_hit == 0, obs1 - # Second "process": fresh runtime, same disk. `k_second` is a new kernel (whole-kernel entry misses, codegen - # runs), but its first loop matches `k_first`'s, so that task is served from disk. + # Second "process": fresh runtime, same disk. k_second's first loop matches k_first's, so that task is a hit. qd.init(arch=qd.cpu, offline_cache=True, offline_cache_file_path=cache_dir) @qd.kernel From 7488fabbbc49f5d55b91ee7d50ad32e78eadd149 Mon Sep 17 00:00:00 2001 From: Hugh Perkins Date: Tue, 1 Sep 2026 13:59:34 -0700 Subject: [PATCH 11/19] [Caching] Ref 7: replace "tier" jargon with offline_cache in comments Say "offline_cache is enabled" instead of "the tier is on" so the comments are readable without knowing the internal cache vocabulary. --- quadrants/codegen/codegen.cpp | 2 +- tests/python/test_per_offload_cache.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/quadrants/codegen/codegen.cpp b/quadrants/codegen/codegen.cpp index e772bd9b1b..05017889b7 100644 --- a/quadrants/codegen/codegen.cpp +++ b/quadrants/codegen/codegen.cpp @@ -142,7 +142,7 @@ LLVMCompiledKernel KernelCodeGen::compile_kernel_to_module() { worker.flush(); // Build one artifact per task before the whole-module link consumes `data`. CUDA always takes this path; AMDGPU and - // CPU only when the tier is on, since their per-task link/serialize only pays off then. + // CPU only when offline_cache is enabled, since their per-task link/serialize only pays off then. std::vector per_construct_artifacts; const bool build_per_construct_artifacts = compile_config_.arch == Arch::cuda || diff --git a/tests/python/test_per_offload_cache.py b/tests/python/test_per_offload_cache.py index c5d422dd1f..fbdcbea3a5 100644 --- a/tests/python/test_per_offload_cache.py +++ b/tests/python/test_per_offload_cache.py @@ -668,12 +668,12 @@ def kernel_struct(out1: qd.types.ndarray(), out2: qd.types.ndarray()) -> None: assert np.allclose(out2.to_numpy(), _C[0], atol=1.0), out2.to_numpy() -# --- Cross-process per-task artifact cache (CUDA + AMDGPU + CPU backend reuse tier) ----------------------------------- +# --- Cross-process per-task artifact cache (CUDA + AMDGPU + CPU) ------------------------------------------------------ # # The per-task artifact cache stores each offloaded task's fully compiled code + launch metadata on disk, keyed by the # task's own IR (name-free), so a later process reuses an unchanged task instead of recompiling it. CUDA and AMDGPU fill # it with GPU code and CPU (ref 7) with a host object; it is gated on `offline_cache`. Reuse is reported on -# `PerOffloadCacheObservations.tasks_*` (-1 when the tier did not run). +# `PerOffloadCacheObservations.tasks_*` (-1 when offline_cache is disabled). @test_utils.test(arch=[qd.cuda, qd.amdgpu], offline_cache=False) @@ -755,7 +755,7 @@ def k_second(x: qd.types.ndarray(qd.f32, ndim=1)) -> None: @test_utils.test(arch=qd.cpu, offline_cache=False) def test_per_task_artifact_cache_disabled_without_offline_cache_cpu() -> None: - # With offline_cache off the per-task tier never runs, so its counts stay at the -1 sentinel; the frontend split + # With offline_cache off the per-task cache never runs, so its counts stay at the -1 sentinel; the frontend split # still fires. @qd.kernel def kernel_two_loops(x: qd.types.ndarray(qd.f32, ndim=1)) -> None: From 5dabb8d328d8798af44eef1cd7da1a784ba64830 Mon Sep 17 00:00:00 2001 From: Hugh Perkins Date: Tue, 1 Sep 2026 14:03:27 -0700 Subject: [PATCH 12/19] [Caching] Ref 7: rename artifact_tier -> per_task_cache_enabled Clearer name for the flag that gates the per-task artifact cache. --- quadrants/codegen/codegen.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/quadrants/codegen/codegen.cpp b/quadrants/codegen/codegen.cpp index 05017889b7..e2ddf67099 100644 --- a/quadrants/codegen/codegen.cpp +++ b/quadrants/codegen/codegen.cpp @@ -74,7 +74,7 @@ LLVMCompiledKernel KernelCodeGen::compile_kernel_to_module() { // Per-task artifact cache: a hit skips a task's compilation and carries cached code to the launcher. Empty dir = off. const std::string art_dir = pertask_artifact_dir_ref(); - const bool artifact_tier = (compile_config_.arch == Arch::cuda || compile_config_.arch == Arch::amdgpu || + const bool per_task_cache_enabled = (compile_config_.arch == Arch::cuda || compile_config_.arch == Arch::amdgpu || arch_is_cpu(compile_config_.arch)) && !art_dir.empty(); const PerTaskArtifactCache artifact_cache(art_dir); @@ -97,7 +97,7 @@ LLVMCompiledKernel KernelCodeGen::compile_kernel_to_module() { // - external-func: the body has the so/bc path + name, not its contents, so a stale update keeps the key; // - real-func (FuncCallStmt): the printer emits only the callee name, but codegen inlines its body; // - mem_access_opt (BLS/read-only hints): serialized from an unordered_map, so its order varies by process. - bool artifact_eligible = artifact_tier && offload->as()->mem_access_opt.get_all().empty(); + bool artifact_eligible = per_task_cache_enabled && offload->as()->mem_access_opt.get_all().empty(); if (artifact_eligible) { irpass::analysis::gather_statements(offload.get(), [&artifact_eligible](Stmt *s) { if (s->is() || s->is() || s->is()) { @@ -146,7 +146,7 @@ LLVMCompiledKernel KernelCodeGen::compile_kernel_to_module() { std::vector per_construct_artifacts; const bool build_per_construct_artifacts = compile_config_.arch == Arch::cuda || - ((compile_config_.arch == Arch::amdgpu || arch_is_cpu(compile_config_.arch)) && artifact_tier); + ((compile_config_.arch == Arch::amdgpu || arch_is_cpu(compile_config_.arch)) && per_task_cache_enabled); if (build_per_construct_artifacts) { for (int i = 0; i < n; i++) { if (!data[i]) @@ -206,7 +206,7 @@ LLVMCompiledKernel KernelCodeGen::compile_kernel_to_module() { } // Record per-task reuse counts for PerOffloadCacheObservations.tasks_* (read back by the compilation manager). Only // when the tier ran, so non-CUDA / cache-off compiles keep the -1 sentinel instead of a misleading 0. - if (artifact_tier && prog != nullptr) { + if (per_task_cache_enabled && prog != nullptr) { auto &cc = prog->per_construct_cache(); std::lock_guard g(cc.mu); cc.last_task_stats[kernel->get_name()] = {n, n_hit.load(), n_recompiled.load()}; From 892f35d5ac966320680f325509a710a264838614 Mon Sep 17 00:00:00 2001 From: Hugh Perkins Date: Tue, 1 Sep 2026 14:09:47 -0700 Subject: [PATCH 13/19] [Caching] Ref 7: rename artifact_eligible -> eligible_for_per_task_cache Clearer name for the per-task cache eligibility flag. --- quadrants/codegen/codegen.cpp | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/quadrants/codegen/codegen.cpp b/quadrants/codegen/codegen.cpp index e2ddf67099..520cc4f1fd 100644 --- a/quadrants/codegen/codegen.cpp +++ b/quadrants/codegen/codegen.cpp @@ -97,11 +97,12 @@ LLVMCompiledKernel KernelCodeGen::compile_kernel_to_module() { // - external-func: the body has the so/bc path + name, not its contents, so a stale update keeps the key; // - real-func (FuncCallStmt): the printer emits only the callee name, but codegen inlines its body; // - mem_access_opt (BLS/read-only hints): serialized from an unordered_map, so its order varies by process. - bool artifact_eligible = per_task_cache_enabled && offload->as()->mem_access_opt.get_all().empty(); - if (artifact_eligible) { - irpass::analysis::gather_statements(offload.get(), [&artifact_eligible](Stmt *s) { + bool eligible_for_per_task_cache = + per_task_cache_enabled && offload->as()->mem_access_opt.get_all().empty(); + if (eligible_for_per_task_cache) { + irpass::analysis::gather_statements(offload.get(), [&eligible_for_per_task_cache](Stmt *s) { if (s->is() || s->is() || s->is()) { - artifact_eligible = false; + eligible_for_per_task_cache = false; } return false; }); @@ -110,7 +111,7 @@ LLVMCompiledKernel KernelCodeGen::compile_kernel_to_module() { // Key on `#index` too: the module bakes the task's kernel-wide index into its symbol names, so two // byte-identical tasks at different indices must not alias (they would collide at link). std::string cache_key; - if (artifact_eligible) { + if (eligible_for_per_task_cache) { cache_key = get_hashed_per_task_cache_key(compile_config_, pertask_caps, offload->as(), kernel) + "#" + std::to_string(i); // Under kernel profiling, drop the name-free cross-kernel aliasing: the artifact carries OffloadedTask::name, From abff0fc3cec5d366c48ae94121e04806bda3252d Mon Sep 17 00:00:00 2001 From: Hugh Perkins Date: Tue, 1 Sep 2026 14:12:56 -0700 Subject: [PATCH 14/19] [Caching] Ref 7: clang-format realign per_task_cache_enabled continuation Rename widened the first line, so re-indent the aligned continuation. --- quadrants/codegen/codegen.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/quadrants/codegen/codegen.cpp b/quadrants/codegen/codegen.cpp index 520cc4f1fd..36020df3fe 100644 --- a/quadrants/codegen/codegen.cpp +++ b/quadrants/codegen/codegen.cpp @@ -75,8 +75,8 @@ LLVMCompiledKernel KernelCodeGen::compile_kernel_to_module() { // Per-task artifact cache: a hit skips a task's compilation and carries cached code to the launcher. Empty dir = off. const std::string art_dir = pertask_artifact_dir_ref(); const bool per_task_cache_enabled = (compile_config_.arch == Arch::cuda || compile_config_.arch == Arch::amdgpu || - arch_is_cpu(compile_config_.arch)) && - !art_dir.empty(); + arch_is_cpu(compile_config_.arch)) && + !art_dir.empty(); const PerTaskArtifactCache artifact_cache(art_dir); const DeviceCapabilityConfig pertask_caps = prog->get_device_caps(); std::vector pertask_keys(n); From 28dfae47939ba3230841dc47319587784590f118 Mon Sep 17 00:00:00 2001 From: Hugh Perkins Date: Tue, 1 Sep 2026 14:31:47 -0700 Subject: [PATCH 15/19] [Caching] Ref 7: drop internal "ref 7" reference from test comment The comment should read without knowing internal PR numbering. --- tests/python/test_per_offload_cache.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/python/test_per_offload_cache.py b/tests/python/test_per_offload_cache.py index fbdcbea3a5..569d342cf3 100644 --- a/tests/python/test_per_offload_cache.py +++ b/tests/python/test_per_offload_cache.py @@ -672,7 +672,7 @@ def kernel_struct(out1: qd.types.ndarray(), out2: qd.types.ndarray()) -> None: # # The per-task artifact cache stores each offloaded task's fully compiled code + launch metadata on disk, keyed by the # task's own IR (name-free), so a later process reuses an unchanged task instead of recompiling it. CUDA and AMDGPU fill -# it with GPU code and CPU (ref 7) with a host object; it is gated on `offline_cache`. Reuse is reported on +# it with GPU code and CPU with a host object; it is gated on `offline_cache`. Reuse is reported on # `PerOffloadCacheObservations.tasks_*` (-1 when offline_cache is disabled). From 110cb4b24119a15ae181b240e2a6bb883b2a172a Mon Sep 17 00:00:00 2001 From: Hugh Perkins Date: Wed, 2 Sep 2026 11:43:15 -0700 Subject: [PATCH 16/19] [Caching] Ref 7: use large code model for per-task CPU objects Each per-task object is loaded through its own SectionMemoryManager, so under memory pressure (many concurrent workers loading many cached objects) RTDyld can place a task's .text and .rodata more than 2GB apart. The small code model's 32-bit RIP-relative references then overflow and the JITed code reads constants from garbage, crashing with SIGSEGV at launch. Build the per-task target machine with the large code model so 64-bit addressing makes the section distance irrelevant, and drop the reloc/opt-level/fast-math overrides so the object matches the whole-module JIT compiler. --- quadrants/runtime/cpu/jit_cpu.cpp | 23 +++++++---------------- 1 file changed, 7 insertions(+), 16 deletions(-) diff --git a/quadrants/runtime/cpu/jit_cpu.cpp b/quadrants/runtime/cpu/jit_cpu.cpp index e84ba18943..2aa1fbad87 100644 --- a/quadrants/runtime/cpu/jit_cpu.cpp +++ b/quadrants/runtime/cpu/jit_cpu.cpp @@ -284,23 +284,14 @@ class JITSessionCPU : public JITSession { if (!expected_jtmb) { QD_ERROR("LLVM TargetMachineBuilder has failed."); } - // Build from the JTMB so it uses detectHost()'s explicit feature vector. A bare CPU name with empty features - // would enable that model's default features, a superset of the running core, and emit illegal instructions. - // PIC so the object loads in the object layer. + // detectHost() carries the running core's explicit feature vector, matching the whole-module JIT compiler; a bare + // CPU name with empty features would enable that model's default superset and emit illegal instructions. + // + // Large code model: each per-task object gets its own SectionMemoryManager, so under memory pressure RTDyld can + // place a task's .text and .rodata more than 2GB apart, overflowing the small model's 32-bit RIP-relative refs + // and reading constants from garbage at launch. 64-bit addressing makes the section distance irrelevant. auto jtmb = std::move(*expected_jtmb); - jtmb.setRelocationModel(llvm::Reloc::PIC_); - jtmb.setCodeModel(llvm::CodeModel::Small); - jtmb.setCodeGenOptLevel(llvm::CodeGenOptLevel::Aggressive); - llvm::TargetOptions &options = jtmb.getOptions(); - if (config_.fast_math) { - options.AllowFPOpFusion = llvm::FPOpFusion::Fast; - options.NoInfsFPMath = 1; - options.NoNaNsFPMath = 1; - } else { - options.AllowFPOpFusion = llvm::FPOpFusion::Strict; - options.NoInfsFPMath = 0; - options.NoNaNsFPMath = 0; - } + jtmb.setCodeModel(llvm::CodeModel::Large); auto expected_tm = jtmb.createTargetMachine(); QD_ERROR_UNLESS(expected_tm, "Could not allocate target machine!"); pertask_target_machine_ = std::move(*expected_tm); From dbb042bc4b939e8f4e7704440bc2b6d1d338c711 Mon Sep 17 00:00:00 2001 From: Hugh Perkins Date: Wed, 2 Sep 2026 14:08:17 -0700 Subject: [PATCH 17/19] [Caching] Ref 7: scope CPU per-task cache dir by host feature vector compile_module_to_object() builds host objects with detectHost()'s feature vector, so two hosts sharing an offline_cache_file_path whose target triple and CPU name match but whose enabled features differ (e.g. a feature-masked VM) would select the same cache dir and keys. One could then load an object using an instruction its core lacks and fault. Add a sorted, hashed digest of the detected feature vector to the CPU cache scope alongside triple and CPU name. --- .../runtime/program_impls/llvm/CMakeLists.txt | 1 + .../program_impls/llvm/llvm_program.cpp | 21 +++++++++++++++++-- 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/quadrants/runtime/program_impls/llvm/CMakeLists.txt b/quadrants/runtime/program_impls/llvm/CMakeLists.txt index 922c047b70..46b99a033d 100644 --- a/quadrants/runtime/program_impls/llvm/CMakeLists.txt +++ b/quadrants/runtime/program_impls/llvm/CMakeLists.txt @@ -11,6 +11,7 @@ target_include_directories(llvm_program_impl ${PROJECT_SOURCE_DIR} ${PROJECT_SOURCE_DIR}/external/eigen ${PROJECT_SOURCE_DIR}/external/spdlog/include + ${PROJECT_SOURCE_DIR}/external/PicoSHA2 ${LLVM_INCLUDE_DIRS} ) diff --git a/quadrants/runtime/program_impls/llvm/llvm_program.cpp b/quadrants/runtime/program_impls/llvm/llvm_program.cpp index 8140edecfe..67954cc139 100644 --- a/quadrants/runtime/program_impls/llvm/llvm_program.cpp +++ b/quadrants/runtime/program_impls/llvm/llvm_program.cpp @@ -29,6 +29,10 @@ #include "quadrants/codegen/llvm/compiled_kernel_data.h" #include "quadrants/codegen/llvm/per_task_artifact_cache.h" +#include "picosha2.h" + +#include + namespace quadrants::lang { LlvmProgramImpl::LlvmProgramImpl(CompileConfig &config_, KernelProfilerBase *profiler) : ProgramImpl(config_), compilation_workers("compile", config_.print_ir ? 1 : config_.num_compile_threads) { @@ -53,11 +57,24 @@ LlvmProgramImpl::LlvmProgramImpl(CompileConfig &config_, KernelProfilerBase *pro } #endif if (arch_is_cpu(config_.arch)) { - // Host objects are CPU-specific, so scope the dir by host triple + CPU; a shared path must never serve an object - // built for an incompatible CPU. + // compile_module_to_object() builds host objects from detectHost()'s triple, CPU name and feature vector, so the + // dir is scoped by all three. Matching triple + CPU but different enabled features (e.g. a feature-masked VM) + // must not share a path, or a loaded object could use an instruction the running core lacks. auto jtmb = llvm::orc::JITTargetMachineBuilder::detectHost(); std::string tag = (jtmb ? jtmb->getTargetTriple().str() : std::string("unknown")) + "_" + llvm::sys::getHostCPUName().str(); + if (jtmb) { + // Sort so the hash is independent of detectHost()'s feature order; the full vector is too long for a path. + std::vector features = jtmb->getFeatures().getFeatures(); + std::sort(features.begin(), features.end()); + std::string joined; + for (const auto &f : features) { + joined += f + ","; + } + std::string feat_hex; + picosha2::hash256_hex_string(joined, feat_hex); + tag += "_" + feat_hex.substr(0, 16); + } for (char &c : tag) { if (c == '/' || c == ':' || c == ' ' || c == '\\') { c = '_'; From 4b03dfd98fcd59f0ecd84f2080ce208a474bacdb Mon Sep 17 00:00:00 2001 From: Hugh Perkins Date: Wed, 2 Sep 2026 14:19:26 -0700 Subject: [PATCH 18/19] [Caching] Ref 7: restrict per-task large code model to x86 The >2GB section-distance overflow that the large code model works around is specific to x86's 32-bit RIP-relative addressing. Setting it unconditionally breaks AArch64 Mach-O, whose backend rejects the large model for this JIT configuration, so createTargetMachine() would fail on Apple Silicon. Gate the large model on an x86 target triple; other targets keep the default model (AArch64 already reaches +/-4GB). --- quadrants/runtime/cpu/jit_cpu.cpp | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/quadrants/runtime/cpu/jit_cpu.cpp b/quadrants/runtime/cpu/jit_cpu.cpp index 2aa1fbad87..8de9978f48 100644 --- a/quadrants/runtime/cpu/jit_cpu.cpp +++ b/quadrants/runtime/cpu/jit_cpu.cpp @@ -286,12 +286,14 @@ class JITSessionCPU : public JITSession { } // detectHost() carries the running core's explicit feature vector, matching the whole-module JIT compiler; a bare // CPU name with empty features would enable that model's default superset and emit illegal instructions. - // - // Large code model: each per-task object gets its own SectionMemoryManager, so under memory pressure RTDyld can - // place a task's .text and .rodata more than 2GB apart, overflowing the small model's 32-bit RIP-relative refs - // and reading constants from garbage at launch. 64-bit addressing makes the section distance irrelevant. auto jtmb = std::move(*expected_jtmb); - jtmb.setCodeModel(llvm::CodeModel::Large); + // Large code model on x86 only: each per-task object gets its own SectionMemoryManager, so under memory pressure + // RTDyld can place a task's .text and .rodata more than 2GB apart, overflowing x86's 32-bit RIP-relative refs and + // reading constants from garbage at launch; 64-bit addressing makes the distance irrelevant. Other targets keep + // the default model -- AArch64 already reaches +/-4GB and its Mach-O backend rejects the large model here. + if (jtmb.getTargetTriple().isX86()) { + jtmb.setCodeModel(llvm::CodeModel::Large); + } auto expected_tm = jtmb.createTargetMachine(); QD_ERROR_UNLESS(expected_tm, "Could not allocate target machine!"); pertask_target_machine_ = std::move(*expected_tm); From 7f4b226f3eac65cbb7a32add79b1d2441a8ec1b1 Mon Sep 17 00:00:00 2001 From: Hugh Perkins Date: Wed, 2 Sep 2026 14:38:10 -0700 Subject: [PATCH 19/19] [Caching] Ref 7: tighten per-task cache comments Shorten the code-model and CPU cache-scope comments to the non-obvious rationale and drop parenthetical asides. --- quadrants/runtime/cpu/jit_cpu.cpp | 13 +++++-------- .../runtime/program_impls/llvm/llvm_program.cpp | 7 +++---- 2 files changed, 8 insertions(+), 12 deletions(-) diff --git a/quadrants/runtime/cpu/jit_cpu.cpp b/quadrants/runtime/cpu/jit_cpu.cpp index 8de9978f48..7a2bd8e6c9 100644 --- a/quadrants/runtime/cpu/jit_cpu.cpp +++ b/quadrants/runtime/cpu/jit_cpu.cpp @@ -227,8 +227,8 @@ class JITSessionCPU : public JITSession { QD_ERROR("Failed to load per-task CPU object into the JIT (offline cache may be corrupt): {}", llvm::toString(std::move(err))); } - // add() links lazily, so link errors (e.g. a corrupt relocation) only surface at lookup, by which point the key - // is gone. Force materialization now to catch them while we can still erase the record. + // add() links lazily, so link errors only surface at lookup, by which point the key is gone. Force + // materialization now to catch them while we can still erase the record. for (const auto &task : art.tasks) { #ifdef __APPLE__ auto sym = es_.lookup({&dylib}, mangle_(task.name)); @@ -284,13 +284,10 @@ class JITSessionCPU : public JITSession { if (!expected_jtmb) { QD_ERROR("LLVM TargetMachineBuilder has failed."); } - // detectHost() carries the running core's explicit feature vector, matching the whole-module JIT compiler; a bare - // CPU name with empty features would enable that model's default superset and emit illegal instructions. + // Use detectHost()'s explicit features; a bare CPU name would enable features the running core may lack. auto jtmb = std::move(*expected_jtmb); - // Large code model on x86 only: each per-task object gets its own SectionMemoryManager, so under memory pressure - // RTDyld can place a task's .text and .rodata more than 2GB apart, overflowing x86's 32-bit RIP-relative refs and - // reading constants from garbage at launch; 64-bit addressing makes the distance irrelevant. Other targets keep - // the default model -- AArch64 already reaches +/-4GB and its Mach-O backend rejects the large model here. + // On x86 RTDyld can place a per-task object's .text and .rodata over 2GB apart, past the reach of 32-bit + // RIP-relative refs; the large code model's 64-bit addressing avoids that. if (jtmb.getTargetTriple().isX86()) { jtmb.setCodeModel(llvm::CodeModel::Large); } diff --git a/quadrants/runtime/program_impls/llvm/llvm_program.cpp b/quadrants/runtime/program_impls/llvm/llvm_program.cpp index 67954cc139..c273e5785b 100644 --- a/quadrants/runtime/program_impls/llvm/llvm_program.cpp +++ b/quadrants/runtime/program_impls/llvm/llvm_program.cpp @@ -57,14 +57,13 @@ LlvmProgramImpl::LlvmProgramImpl(CompileConfig &config_, KernelProfilerBase *pro } #endif if (arch_is_cpu(config_.arch)) { - // compile_module_to_object() builds host objects from detectHost()'s triple, CPU name and feature vector, so the - // dir is scoped by all three. Matching triple + CPU but different enabled features (e.g. a feature-masked VM) - // must not share a path, or a loaded object could use an instruction the running core lacks. + // Host objects match detectHost()'s triple, CPU and features, so a cache path shared between hosts is scoped by + // all three. auto jtmb = llvm::orc::JITTargetMachineBuilder::detectHost(); std::string tag = (jtmb ? jtmb->getTargetTriple().str() : std::string("unknown")) + "_" + llvm::sys::getHostCPUName().str(); if (jtmb) { - // Sort so the hash is independent of detectHost()'s feature order; the full vector is too long for a path. + // Sort so the hash ignores feature order; the full vector is too long to use directly. std::vector features = jtmb->getFeatures().getFeatures(); std::sort(features.begin(), features.end()); std::string joined;