From e868911cdfdf57e8ed8d57594f2a4c587305252c Mon Sep 17 00:00:00 2001 From: Fausto Milletari Date: Thu, 3 Sep 2026 18:44:26 +0000 Subject: [PATCH 1/2] OSS sync: esm 3.4.1 Sync from monorepo main (d875f5284b) at version 3.4.1. Adds ESMFold2 context-parallel inference (`esm/models/esmfold2/distributed/`) with the `cookbook/foldcp/` guides, `esm/utils/compression.py`, and the `fused` / `fold-cp` / `cueq12` / `cueq13` extras. Co-Authored-By: Claude Opus 5 (1M context) --- .github/scripts/airtable_issue_sync.py | 4 +- cookbook/foldcp/README.md | 66 ++ cookbook/foldcp/cuequivariance_cp.py | 95 +++ cookbook/foldcp/esmfold2-cp.py | 74 +++ cookbook/foldcp/esmfold2-cueq.py | 54 ++ cookbook/foldcp/esmfold2-fused.py | 54 ++ cookbook/foldcp/esmfold2-none.py | 54 ++ cookbook/foldcp/fast_runtime_and_vram.png | Bin 0 -> 148957 bytes cookbook/foldcp/fold-cp.md | 182 ++++++ cookbook/foldcp/single-gpu.md | 180 ++++++ cookbook/foldcp/will_fail.py | 56 ++ cookbook/tutorials/binder_design.py | 20 +- .../esmfold2_local_applesilicon.ipynb | 5 +- esm/__init__.py | 2 +- esm/models/esmfold2/config.py | 4 + esm/models/esmfold2/distributed/__init__.py | 50 ++ esm/models/esmfold2/distributed/comm.py | 384 ++++++++++++ .../distributed/confidence_wrapper.py | 191 ++++++ .../esmfold2/distributed/distogram_wrapper.py | 93 +++ esm/models/esmfold2/distributed/esmc_tp.py | 129 ++++ esm/models/esmfold2/distributed/manager.py | 586 ++++++++++++++++++ .../esmfold2/distributed/model/__init__.py | 2 + .../distributed/model/layers/__init__.py | 2 + .../distributed/model/layers/atom_to_token.py | 192 ++++++ .../model/layers/attention_pair_bias.py | 118 ++++ .../model/layers/confidence_zbase.py | 145 +++++ .../model/layers/diffusion_conditioning.py | 127 ++++ .../model/layers/diffusion_transformer.py | 184 ++++++ .../distributed/model/layers/layernorm.py | 234 +++++++ .../distributed/model/layers/linear.py | 226 +++++++ .../distributed/model/layers/msa_encoder.py | 198 ++++++ .../model/layers/outer_product_mean.py | 142 +++++ .../model/layers/pair_averaging.py | 265 ++++++++ .../distributed/model/layers/pair_init.py | 383 ++++++++++++ .../distributed/model/layers/pairformer.py | 169 +++++ .../model/layers/row_attention_pooling.py | 116 ++++ .../model/layers/single_to_pair.py | 190 ++++++ .../model/layers/triangular_mult.py | 446 +++++++++++++ .../esmfold2/distributed/msa_wrapper.py | 354 +++++++++++ esm/models/esmfold2/distributed/recycle.py | 358 +++++++++++ .../esmfold2/distributed/structure_wrapper.py | 270 ++++++++ esm/models/esmfold2/distributed/utils.py | 542 ++++++++++++++++ esm/models/esmfold2/experimental.py | 12 +- esm/models/esmfold2/hf_adapter.py | 8 + esm/models/esmfold2/layers.py | 18 +- esm/models/esmfold2/model.py | 326 ++++++++-- esm/models/esmfold2/processor.py | 19 + esm/models/hub.py | 3 +- esm/sdk/api.py | 2 +- esm/sdk/base_forge_client.py | 146 ++++- esm/sdk/base_forge_client_test.py | 210 +++++++ esm/sdk/forge.py | 25 +- esm/tokenization/sasa_tokenizer.py | 29 +- esm/utils/compression.py | 22 + esm/utils/decoding.py | 13 +- esm/utils/msa/msa.py | 30 +- esm/utils/structure/input_builder.py | 7 +- esm/widgets/components/results_visualizer.py | 2 +- esm/widgets/views/esm3_prompt_preview.py | 11 +- pixi.lock | 6 + pyproject.toml | 13 +- tests/models/esmfold2_api_test.py | 97 +++ tests/models/esmfold2_inputs_test.py | 3 + tests/oss_pytests/requirements.txt | 2 +- tests/oss_pytests/test_oss_client.py | 10 +- tests/reference/esmfold2_tiny_cpu_fp32.pkl.gz | Bin 37946 -> 37937 bytes tests/utils/compression_test.py | 57 ++ tests/utils/msa_test.py | 13 + tests/utils/track_round_trip_test.py | 60 ++ 69 files changed, 7954 insertions(+), 136 deletions(-) create mode 100644 cookbook/foldcp/README.md create mode 100644 cookbook/foldcp/cuequivariance_cp.py create mode 100644 cookbook/foldcp/esmfold2-cp.py create mode 100644 cookbook/foldcp/esmfold2-cueq.py create mode 100644 cookbook/foldcp/esmfold2-fused.py create mode 100644 cookbook/foldcp/esmfold2-none.py create mode 100644 cookbook/foldcp/fast_runtime_and_vram.png create mode 100644 cookbook/foldcp/fold-cp.md create mode 100644 cookbook/foldcp/single-gpu.md create mode 100644 cookbook/foldcp/will_fail.py create mode 100644 esm/models/esmfold2/distributed/__init__.py create mode 100644 esm/models/esmfold2/distributed/comm.py create mode 100644 esm/models/esmfold2/distributed/confidence_wrapper.py create mode 100644 esm/models/esmfold2/distributed/distogram_wrapper.py create mode 100644 esm/models/esmfold2/distributed/esmc_tp.py create mode 100644 esm/models/esmfold2/distributed/manager.py create mode 100644 esm/models/esmfold2/distributed/model/__init__.py create mode 100644 esm/models/esmfold2/distributed/model/layers/__init__.py create mode 100644 esm/models/esmfold2/distributed/model/layers/atom_to_token.py create mode 100644 esm/models/esmfold2/distributed/model/layers/attention_pair_bias.py create mode 100644 esm/models/esmfold2/distributed/model/layers/confidence_zbase.py create mode 100644 esm/models/esmfold2/distributed/model/layers/diffusion_conditioning.py create mode 100644 esm/models/esmfold2/distributed/model/layers/diffusion_transformer.py create mode 100644 esm/models/esmfold2/distributed/model/layers/layernorm.py create mode 100644 esm/models/esmfold2/distributed/model/layers/linear.py create mode 100644 esm/models/esmfold2/distributed/model/layers/msa_encoder.py create mode 100644 esm/models/esmfold2/distributed/model/layers/outer_product_mean.py create mode 100644 esm/models/esmfold2/distributed/model/layers/pair_averaging.py create mode 100644 esm/models/esmfold2/distributed/model/layers/pair_init.py create mode 100644 esm/models/esmfold2/distributed/model/layers/pairformer.py create mode 100644 esm/models/esmfold2/distributed/model/layers/row_attention_pooling.py create mode 100644 esm/models/esmfold2/distributed/model/layers/single_to_pair.py create mode 100644 esm/models/esmfold2/distributed/model/layers/triangular_mult.py create mode 100644 esm/models/esmfold2/distributed/msa_wrapper.py create mode 100644 esm/models/esmfold2/distributed/recycle.py create mode 100644 esm/models/esmfold2/distributed/structure_wrapper.py create mode 100644 esm/models/esmfold2/distributed/utils.py create mode 100644 esm/sdk/base_forge_client_test.py create mode 100644 esm/utils/compression.py create mode 100644 tests/utils/compression_test.py create mode 100644 tests/utils/track_round_trip_test.py diff --git a/.github/scripts/airtable_issue_sync.py b/.github/scripts/airtable_issue_sync.py index 08205f06..43693d36 100644 --- a/.github/scripts/airtable_issue_sync.py +++ b/.github/scripts/airtable_issue_sync.py @@ -31,6 +31,7 @@ F_DATE = "Submission Date" F_PRODUCT = "Tool or Product" F_SOURCE = "Issue Source" +F_TITLE = "Issue Title" F_DESCRIPTION = "Issue Description" F_GITHUB_ID = "Github Username" F_ORIGINAL_Q = "Original Q Location" @@ -342,7 +343,8 @@ def build_row(schema, existing, issue, repo_full_name, action): row.own(F_GITHUB_ID, login) row.fill(F_DATE, issue.get("created_at")) - row.fill(F_DESCRIPTION, truncate(f"{title}\n\n{body}", url)) + row.fill(F_TITLE, title) + row.fill(F_DESCRIPTION, truncate(body, url)) row.fill(F_DETAILS, details_block(issue, repo_full_name)) row.fill(F_TYPE, request_types(issue)) row.fill(F_PRODUCT, products(issue)) diff --git a/cookbook/foldcp/README.md b/cookbook/foldcp/README.md new file mode 100644 index 00000000..b08ec4dc --- /dev/null +++ b/cookbook/foldcp/README.md @@ -0,0 +1,66 @@ +# foldcp — ESMFold2 folding examples + +This cookbook shows how to run Biohub's ESMFold2 structure-prediction model with +[NVIDIA BioNeMo libraries and methods](https://github.com/NVIDIA-BioNeMo). It +covers single-GPU accelerated kernel backends for faster inference and context +parallelism (CP) for inputs whose `L x L` pair representation does not fit on one +GPU. + +Tested on 1x and 4x H100 80GB GPUs. + +| Script | Input | GPUs | Result | +| --- | --- | --- | --- | +| `esmfold2-none.py` | 7ysz (1 protein chain ×2) | 1 | Reference single-GPU path | +| `esmfold2-cueq.py` | 7ysz (1 protein chain ×2) | 1 | cuEquivariance backend | +| `esmfold2-fused.py` | 7ysz (1 protein chain ×2) | 1 | Triton fused backend | +| `esmfold2-cp.py` | 7ysz (1 protein chain ×2) | 4 | Same fold API with CP setup | +| `cuequivariance_cp.py` | 5xgo (1 protein chain ×12, CL ligand) | 4 | Larger input via CP | +| `will_fail.py` | 5xgo (same as CP) | 1 | **OOMs** — too large for 1xH100 | + +## Environment setup + +Start from the NVIDIA PyTorch container, then install the dependencies: + +```bash +docker run --gpus all -it --rm nvcr.io/nvidia/pytorch:26.03-py3 bash -l + +# Main ESM package +pip install "git+https://github.com/Biohub/esm.git@main" +# fused dependency +pip install "esm[fused] @ git+https://github.com/Biohub/esm.git@main" +# cuEquivariance dependency (cueq12 also exists) +pip install "esm[cueq13] @ git+https://github.com/Biohub/esm.git@main" +# fold-cp dependency for larger inputs +pip install "esm[fold-cp] @ git+https://github.com/Biohub/esm.git@main" +``` + +## Single-GPU accelerated backends + +[single-gpu.md](single-gpu.md) compares the three single-GPU backend choices: +`None` for the pure PyTorch reference path, `"cuequivariance"` for NVIDIA +cuEquivariance triangle-multiplication kernels, and `"fused"` for Triton kernels +that fuse several ESMFold2 hot-path operations. These backends keep the same model +weights and outputs while trading dependencies, warm-up behavior, and speed. + +```bash +python esmfold2-none.py +python esmfold2-cueq.py +python esmfold2-fused.py +``` + +## Fold-CP for larger inputs + +[fold-cp.md](fold-cp.md) documents how `wrap_model_with_cp(model, dm, ...)` +spreads one ESMFold2 fold across a square grid of GPUs using the [Fold-CP methodology](https://github.com/NVIDIA-BioNeMo/boltz-cp). +CP shards the large `L x L` pair activations so longer proteins and complexes fit in memory; it is a +capability feature rather than a general speedup. + +Launch CP examples with `torchrun` on a perfect-square number of GPUs. + +```bash +torchrun --nproc-per-node=4 esmfold2-cp.py + +# Larger 5xgo example. +PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True \ + torchrun --nproc-per-node=4 cuequivariance_cp.py +``` diff --git a/cookbook/foldcp/cuequivariance_cp.py b/cookbook/foldcp/cuequivariance_cp.py new file mode 100644 index 00000000..3bada7d7 --- /dev/null +++ b/cookbook/foldcp/cuequivariance_cp.py @@ -0,0 +1,95 @@ +import gc +import math +import os +from collections import OrderedDict +from time import time + +import torch + +from esm.models.esmfold2 import ( + ESMFold2InputBuilder, + EsmFold2Model, + LigandInput, + MolecularComplexResult, + ProteinInput, + StructurePredictionInput, +) +from esm.models.esmfold2.distributed import DistributedManager, wrap_model_with_cp + +spi = StructurePredictionInput( # 5xgo + sequences=[ + ProteinInput( + id=["A1", "B1", "C1", "D1", "E1", "F1", "G1", "H1", "I1", "J1", "K1", "L1"], + sequence=( + "MAHHHHHHVDDDDKMSTAKLVKSKATNLLYTRNDVSDSEKKATVELLNRQVIQFIDLSLITKQAHWNMRG" + "ANFIAVHEMLDGFRTALIDHLDTMAERAVQLGGVALGTTQVINSKTPLKSYPLDIHNVQDHLKELADRYA" + "IVANDVRKAIGEAKDDDTADILTAASRDLDKFLWFIECNLDLIQKMGLQNYLQAQIREEG" + ), + ), + LigandInput(id=["M1", "N1"], ccd=["CL"]), + ] +) + +# torchrun entrypoint: torchrun --nproc_per_node=4 cuequivariance_cp.py +# torchrun sets LOCAL_RANK / WORLD_SIZE / RANK / MASTER_ADDR / MASTER_PORT. +# world_size must be a perfect square (the CP grid is n×n). +local_rank = int(os.environ["LOCAL_RANK"]) +world_size = int(os.environ["WORLD_SIZE"]) + +os.environ["RANK"] = str(local_rank) +torch.cuda.set_device(local_rank) +torch.cuda.reset_peak_memory_stats() + +n = math.isqrt(world_size) +DistributedManager.initialize( + grid_group_sizes=OrderedDict([("dp", 1), ("cp", (n, n))]), + device_type="cuda", + backend="nccl", +) +dm = DistributedManager() + +model = ( + EsmFold2Model.from_pretrained("biohub/ESMFold2", esmc_precision="bf16") + .cuda() + .eval() +) +# Shard BOTH the folding trunk and the MSA encoder across the n×n CP grid. +# (Trunk-only sharding leaves the MSA encoder running the full L×L pair on +# every rank, which OOMs for large complexes.) +# +# Defaults applied here (override via args): bf16=True (bf16 distributed trunk) +# and offload_esmc=True (move the ~12 GB ESM-C LM to CPU after its one-shot use +# so the trunk reuses that memory). Together: ~2.14x faster, ~18.5 GB lower +# peak vs fp32, quality-neutral. comm="gather" (bit-exact) | "ring" (n>=3). +wrap_model_with_cp(model, dm, comm="ring") +# Fused kernels accelerate the modules *outside* the CP region (notably the +# diffusion sampler / structure head). The CP-wrapped trunk and MSA encoder +# no-op this call, so it never touches the distributed ring math. +model.set_kernel_backend("cuequivariance") + +start = time() +with torch.inference_mode(): + result = ESMFold2InputBuilder().fold( + model, spi, num_loops=10, num_sampling_steps=50, num_diffusion_samples=1, seed=0 + ) +end = time() +peak_mib = torch.cuda.max_memory_allocated() / (1024**2) + +# num_diffusion_samples=1 returns a single result with confidence heads populated. +assert isinstance(result, MolecularComplexResult) +assert result.plddt is not None and result.ptm is not None and result.iptm is not None + +if local_rank == 0: + print( + f"pLDDT mean: {float(result.plddt.mean()):.3f}, " + f"pTM: {float(result.ptm):.3f}, ipTM: {float(result.iptm):.3f}" + ) + print(f"Elapsed: {end - start} sec") + print(f"Max VRAM: {peak_mib} MB") + with open("esmfold2_cp_output.cif", "w") as f: + f.write(result.complex.to_mmcif()) + +DistributedManager.cleanup() +DistributedManager._state.clear() +gc.collect() +torch.cuda.empty_cache() diff --git a/cookbook/foldcp/esmfold2-cp.py b/cookbook/foldcp/esmfold2-cp.py new file mode 100644 index 00000000..23f9a673 --- /dev/null +++ b/cookbook/foldcp/esmfold2-cp.py @@ -0,0 +1,74 @@ +import math +import os +from collections import OrderedDict +from time import time + +import torch + +from esm.models.esmfold2 import ( + ESMFold2InputBuilder, + EsmFold2Model, + MolecularComplexResult, + ProteinInput, + StructurePredictionInput, +) +from esm.models.esmfold2.distributed import DistributedManager, wrap_model_with_cp + +spi = StructurePredictionInput( # 7ysz + sequences=[ + ProteinInput( + id=["A1", "B1"], + sequence=( + "MDNFDNYEQVASIKVIGIGGAGNNAVNRMIEAGVQGVEFIVANTDAQIISVSKSKNKIVLGKETSKGLGA" + "GANPDVGRQAAIESAEEIKDALKGADMVFVAAGMGGGTGTGAAPIIAKLAREQGALTVGIITTPFSFEGR" + "ARNSYAIQGTEELRKHVDSLIIISNDRLLEVIGGVPLKDSFKEADNILRQGVQTITDLIAVPSLINLDFA" + "DIKTVMKNKGNALFGIGIGSGKDKAIEAANKAIISPLLEASIRGARDAIINVTGGNTLTLNDANDAVDIV" + "KQAIGGEVNIIFGTAVNEHLDDEMIVTVIATGFDGSHHHHHH" + ), + ) + ] +) + +local_rank = int(os.environ["LOCAL_RANK"]) +world_size = int(os.environ["WORLD_SIZE"]) +n = math.isqrt(world_size) +assert n * n == world_size, "CP requires a square number of GPUs: 1, 4, 9, 16, ..." + +torch.cuda.set_device(local_rank) +torch.cuda.reset_peak_memory_stats() + +DistributedManager.initialize( + grid_group_sizes=OrderedDict([("dp", 1), ("cp", (n, n))]), + device_type="cuda", + backend="nccl", +) +dm = DistributedManager() + +model = ( + EsmFold2Model.from_pretrained("biohub/ESMFold2", esmc_precision="bf16") + .cuda() + .eval() +) +wrap_model_with_cp(model, dm, comm="ring") + +start = time() +with torch.inference_mode(): + result = ESMFold2InputBuilder().fold( + model, spi, num_loops=10, num_sampling_steps=50, num_diffusion_samples=1, seed=0 + ) +end = time() +peak_mib = torch.cuda.max_memory_allocated() / (1024**2) + +# num_diffusion_samples=1 returns a single result with confidence heads populated. +assert isinstance(result, MolecularComplexResult) +assert result.plddt is not None and result.ptm is not None and result.iptm is not None + +if local_rank == 0: + print( + f"pLDDT mean: {float(result.plddt.mean()):.3f}, " + f"pTM: {float(result.ptm):.3f}, ipTM: {float(result.iptm):.3f}" + ) + print(f"Elapsed: {end - start:.2f} sec") + print(f"Max VRAM: {peak_mib:.1f} MB") + +DistributedManager.cleanup() diff --git a/cookbook/foldcp/esmfold2-cueq.py b/cookbook/foldcp/esmfold2-cueq.py new file mode 100644 index 00000000..20a89dc0 --- /dev/null +++ b/cookbook/foldcp/esmfold2-cueq.py @@ -0,0 +1,54 @@ +from time import time + +import torch + +from esm.models.esmfold2 import ( + ESMFold2InputBuilder, + EsmFold2Model, + MolecularComplexResult, + ProteinInput, + StructurePredictionInput, +) + +spi = StructurePredictionInput( # 7ysz + sequences=[ + ProteinInput( + id=["A1", "B1"], + sequence=( + "MDNFDNYEQVASIKVIGIGGAGNNAVNRMIEAGVQGVEFIVANTDAQIISVSKSKNKIVLGKETSKGLGA" + "GANPDVGRQAAIESAEEIKDALKGADMVFVAAGMGGGTGTGAAPIIAKLAREQGALTVGIITTPFSFEGR" + "ARNSYAIQGTEELRKHVDSLIIISNDRLLEVIGGVPLKDSFKEADNILRQGVQTITDLIAVPSLINLDFA" + "DIKTVMKNKGNALFGIGIGSGKDKAIEAANKAIISPLLEASIRGARDAIINVTGGNTLTLNDANDAVDIV" + "KQAIGGEVNIIFGTAVNEHLDDEMIVTVIATGFDGSHHHHHH" + ), + ) + ] +) + +torch.cuda.reset_peak_memory_stats() + +model = ( + EsmFold2Model.from_pretrained("biohub/ESMFold2", esmc_precision="bf16") + .cuda() + .eval() +) +model.set_kernel_backend("cuequivariance") + +start = time() +with torch.inference_mode(): + result = ESMFold2InputBuilder().fold( + model, spi, num_loops=10, num_sampling_steps=50, num_diffusion_samples=1, seed=0 + ) +end = time() +peak_mib = torch.cuda.max_memory_allocated() / (1024**2) + +# num_diffusion_samples=1 returns a single result with confidence heads populated. +assert isinstance(result, MolecularComplexResult) +assert result.plddt is not None and result.ptm is not None and result.iptm is not None + +print( + f"pLDDT mean: {float(result.plddt.mean()):.3f}, " + f"pTM: {float(result.ptm):.3f}, ipTM: {float(result.iptm):.3f}" +) +print(f"Elapsed: {end - start:.2f} sec") +print(f"Max VRAM: {peak_mib:.1f} MB") diff --git a/cookbook/foldcp/esmfold2-fused.py b/cookbook/foldcp/esmfold2-fused.py new file mode 100644 index 00000000..f5a9f1ea --- /dev/null +++ b/cookbook/foldcp/esmfold2-fused.py @@ -0,0 +1,54 @@ +from time import time + +import torch + +from esm.models.esmfold2 import ( + ESMFold2InputBuilder, + EsmFold2Model, + MolecularComplexResult, + ProteinInput, + StructurePredictionInput, +) + +spi = StructurePredictionInput( # 7ysz + sequences=[ + ProteinInput( + id=["A1", "B1"], + sequence=( + "MDNFDNYEQVASIKVIGIGGAGNNAVNRMIEAGVQGVEFIVANTDAQIISVSKSKNKIVLGKETSKGLGA" + "GANPDVGRQAAIESAEEIKDALKGADMVFVAAGMGGGTGTGAAPIIAKLAREQGALTVGIITTPFSFEGR" + "ARNSYAIQGTEELRKHVDSLIIISNDRLLEVIGGVPLKDSFKEADNILRQGVQTITDLIAVPSLINLDFA" + "DIKTVMKNKGNALFGIGIGSGKDKAIEAANKAIISPLLEASIRGARDAIINVTGGNTLTLNDANDAVDIV" + "KQAIGGEVNIIFGTAVNEHLDDEMIVTVIATGFDGSHHHHHH" + ), + ) + ] +) + +torch.cuda.reset_peak_memory_stats() + +model = ( + EsmFold2Model.from_pretrained("biohub/ESMFold2", esmc_precision="bf16") + .cuda() + .eval() +) +model.set_kernel_backend("fused") + +start = time() +with torch.inference_mode(): + result = ESMFold2InputBuilder().fold( + model, spi, num_loops=10, num_sampling_steps=50, num_diffusion_samples=1, seed=0 + ) +end = time() +peak_mib = torch.cuda.max_memory_allocated() / (1024**2) + +# num_diffusion_samples=1 returns a single result with confidence heads populated. +assert isinstance(result, MolecularComplexResult) +assert result.plddt is not None and result.ptm is not None and result.iptm is not None + +print( + f"pLDDT mean: {float(result.plddt.mean()):.3f}, " + f"pTM: {float(result.ptm):.3f}, ipTM: {float(result.iptm):.3f}" +) +print(f"Elapsed: {end - start:.2f} sec") +print(f"Max VRAM: {peak_mib:.1f} MB") diff --git a/cookbook/foldcp/esmfold2-none.py b/cookbook/foldcp/esmfold2-none.py new file mode 100644 index 00000000..35cd56fc --- /dev/null +++ b/cookbook/foldcp/esmfold2-none.py @@ -0,0 +1,54 @@ +from time import time + +import torch + +from esm.models.esmfold2 import ( + ESMFold2InputBuilder, + EsmFold2Model, + MolecularComplexResult, + ProteinInput, + StructurePredictionInput, +) + +spi = StructurePredictionInput( # 7ysz + sequences=[ + ProteinInput( + id=["A1", "B1"], + sequence=( + "MDNFDNYEQVASIKVIGIGGAGNNAVNRMIEAGVQGVEFIVANTDAQIISVSKSKNKIVLGKETSKGLGA" + "GANPDVGRQAAIESAEEIKDALKGADMVFVAAGMGGGTGTGAAPIIAKLAREQGALTVGIITTPFSFEGR" + "ARNSYAIQGTEELRKHVDSLIIISNDRLLEVIGGVPLKDSFKEADNILRQGVQTITDLIAVPSLINLDFA" + "DIKTVMKNKGNALFGIGIGSGKDKAIEAANKAIISPLLEASIRGARDAIINVTGGNTLTLNDANDAVDIV" + "KQAIGGEVNIIFGTAVNEHLDDEMIVTVIATGFDGSHHHHHH" + ), + ) + ] +) + +torch.cuda.reset_peak_memory_stats() + +model = ( + EsmFold2Model.from_pretrained("biohub/ESMFold2", esmc_precision="bf16") + .cuda() + .eval() +) +model.set_kernel_backend(None) + +start = time() +with torch.inference_mode(): + result = ESMFold2InputBuilder().fold( + model, spi, num_loops=10, num_sampling_steps=50, num_diffusion_samples=1, seed=0 + ) +end = time() +peak_mib = torch.cuda.max_memory_allocated() / (1024**2) + +# num_diffusion_samples=1 returns a single result with confidence heads populated. +assert isinstance(result, MolecularComplexResult) +assert result.plddt is not None and result.ptm is not None and result.iptm is not None + +print( + f"pLDDT mean: {float(result.plddt.mean()):.3f}, " + f"pTM: {float(result.ptm):.3f}, ipTM: {float(result.iptm):.3f}" +) +print(f"Elapsed: {end - start:.2f} sec") +print(f"Max VRAM: {peak_mib:.1f} MB") diff --git a/cookbook/foldcp/fast_runtime_and_vram.png b/cookbook/foldcp/fast_runtime_and_vram.png new file mode 100644 index 0000000000000000000000000000000000000000..e8eda16f9e77f967a1d3ceeebd0567d750933d3c GIT binary patch literal 148957 zcmeFZi942S+ctiSB26liu`(+qLS~iBB15J~$V?(*#tNZOGG;15rbL7?RHl*)nI(!4 zDrBC&{aUT}`MuBf`wzbDySLVQo~7G;U)Oma=W*=&e(c8;d{#|i8#N;}iA36_qS^qjREES%j;oXkmQOq{RSIXK%{o9=ftce-lraQWB~ z(IdzB_FFkSU%4vI&u{;qHym+rvgEHD*3-g=Y`&tXj}K#gOZ-E2x!FaUL`EVhoj$4U zp78UFi!*I!|E5Vxb|%iSO`B<$LpI)ce3zEjAZNvo?ad>_c+ak{1kiT_Pckn(Za33?df}plQbtz|JO_V9m5X^ z65D^hQp)_c{rInUoj!e$Tzuz$zepkvcmMZK$o}7({_k6YA%>Br}0K;U|L zxs*-k)6mObUViH|_N`f*9=R7A8*3sKBJ-@3)##OL`>5eWe`TI_;LE^#e3Rvvx0w6n zLp8^t<78xHdc~JnmS#q8_7vGxb#&~ht*w3b{CO3M# z#OQ&Uz!2qFACs)CY*$y;L(Mehh5q%mTicjK_8&Z`+|M3cc#loo(L2d8i7=GiYS`3GDpEk4!-zyldxlONEeogqL%XlE8u+2mFse!v6 zhMpA^oa!zx|MvTLShCO3X5ZCWdQ&qq;pmLf=|eFeGzzUWN=iyDwms1(xi*{C^M^K1 z{kOR570sW9NAAC;p`m%$y0+0sr|*2dPxtI{m?FLW(sn;=QB1Kzzv;r{H&L&JXZ61x zJ$gj7ZClNkS8h{SWinC-E$@xv4t)<5@39%M4E|_IrP{n1TT!mFSZvpwVl?^q)349P z0$vMO7M7N>dRh1E34HeKXzwex9k!iM175w7cwyIlqrm*bu+!tk`>$TTa+jUxExlT5 zFr}=d^d>_+d2+OsTheoGtLOahn(E-)`L4eVoPYO7a#jf!fS4ff`8kRccOk&aL zOmi}5C37;q=OjGIaKCnKpdzRfW{%aFE7t|^%a<_h>DBblsjl@ z?mBhq6wR((H(t3-n6{>$d1CZR#Pw&_b*$+n-aVuew+YtE{S~2D<}t(Ql*8v=e7&{L z=bbZiu==qasp0m|oh(GR7>qkaTdVFCW%bD=2 z?pGjnotTkv<0yjq(OvQ%5P zR1MaJPkntu&cVq^Mq*=Uzn74ZGds0=*RE%t^Ox^1i)otmMMl#8>@C$AFKEp+C^~3R zX!Yn^A2r>xCl=#VJx;=~cS^5p7BV??|PFE20dG%!3od|`eIHT5Z+)n^X<(z*R+ zC>SXTudgZB>ZzMZampCQa<3?{bmHQSM6fQ+{mRy2c=P6ssLx8tLQiX}mi^jH zf1STSMVZI+Cf%ooL`0BCs7}{!-JaJ}nwt1k>ezq%`tpi?b$MPvRrO9R*ZucrQ{}gbN=Q`o^|4}S78a)1iS0zz zA}6J!rsn&sUbE{i*ofUU5O&XATG>QF@%7Kphk>BE-P~%gtE$L#^DYF0hMt*A_T$T|<4-S?Q^)Zwck<J%}HF<;CNnTU}>qxZ^|qMH8nLFij(~&G$_t`>KeCZ zBb)Tp-^(B?H9s|yU>0|#Ix+Ud+K8#y?N<+{TBWVId0KweDS6-UA3v;AlO!2FL?0%< z+jFn$!v~e|2`SRg?&k`2)k4C;a)yS?q>CesiFzeh4%`V1Ejn^2bYS-4ct>umjBiD) zH?8IKcMY@th4Xd$d@oi9?KDknX>8m+SQ}b}TP4y})N+~o)Q08xi8`c+$)#Ca1dEMS zfU|sZ|JSbrh)@j(a+n zdrPh#vuq?O{_^nr(R^SlgW&0*@ICuyZ+%=}o0nl0y+UeAmg%(V5%Ki&EVS+1PWn1N zej5Q~+pCbz78fPlLH#^}Q-w$o?lV7j`m8NlMzszy?A?12N3*NK`&F##uO8t9ahHR~ zWz$pzj~;!Ev&?baQrc_b?AOi@90>gz;j@u9Zlf3|IaV)pKeyx@kFk|?%=+{D%d4t} z20krEJw3gPqo2|u(z3&t#cm+!+86~6%rC?k`|ONyprWRZxD(MS7cXL89oWSp;c8`= zEXBzwx8g;A*nrml^NWvdj!Mt7b@K+0plFzy#@n-RVcpCIs)HFslE_ai{uvU$VUzS- zW5#ZMh*lfhzI}UFxo7dE;i%d1PIf^-!4KxUhbT*@kok8LIPIPet-}c+t7ek1_iR+h zGESGycv5#rl>Hf7FFASnkZpcNww?07dWwp|f~9Ty%EXrDCw8DDhI+hm9%YNOm_{uS zV_-CYjsp6otxem@dul*VN?sNyG13Kz+7O9a9g$1V^gv5f^O;%Q_S|`ItSxrxKt+d; z_bZXIvNDdmqFnOD<>f&fKawVs!=|l!{gR|SLq5|nb-I|?+1eF3if})E{P>~o`r5#|DW{?OZ-`7DjTHY5^Sw+=gIK87i4txX z?d=nXVz+MFHuZtOGAIwJ@(Ol=FGeMD-}jCQ2{xB5Kef}R4V2F7Zdd$e6t6{l==0}d z2VIHwKNa4~)xw!GQdxt56MJQRB*!{(w@^}20(Csp&eXi(=2o||IK6Y%u83cCi2j<1 z%JncLP$KK>*|X<~VX5Gz?aWUKbuPNv^b}?;7&@|~l3biKRaZsLvgTu-wJvn)wd?;hUwb`*Z{{bwrL-kSLhKB<(HQ(vNa)eCN?%9yEv5}s>B=- z@1LQeAfw#Q?3aU#EQp_|{JkjK!-NWi{FD0hwP_wV1P38)#Aq%zO>?SL7F7)9)Y zaA&WZn~%2I`cI(ds3%L)8Rr31w1$3dY;5Gwe5{y0cP7tqur{L8dOUZ8UP(!*>xDg& zLGk5K)YYo$YKpM1upe!iyVsV+^R~{bC@P+4Ns+6mT*P53l(x9fb*6e1uqK8j<56^$ zCuR6PnZTGBdhFvS6%`ddef>wsL$RWcY&UM+6pkMK{+%N_Zf^bO)L`A-V-{O&+MbYM zg^9NzRmKWgYo=Gqjy-uP@>uDWrO?IKBmsLQhlA_kPYf*1GKtMDt9|hx`?PHGQd00G zeKZ#b2gPlAe)(*j+_UMb4A`f)wY8htvvup<-DBrWtmE0SGbrfE37fvMQql?1e$`}Y zsSiV;dj(GMuhg7H>zjZl=Sh6v74J4ncqb#D;}(%8d3C1j7yz{ z1bdrM`Wr?@N8{2K5PbnHsR~;H0|QOtvRFk#MBWa6jBk&20SXlGMZ~rbm%B|Qef{|} z;oHTs6g@@7 z#5Mz;Xr8}dYMOG?`N!GZdh-1!X!-?}FP6z@MC^MWPcLSPH?x$VepwL&x97YgWgG#lUKgQJ5*>aR}@;e$_E7pYn+c3*&)Y&p{#jK{GQJ`emiS5 z3E*E{QAueh_;gB2ihmf3MAoX$+S>92W4|u0PQvXsmCIOLi+k9)EbX1Y!Bft~AyY0@ zUsEpKI&UV$X^V<1F-t2WBcmktpizm2;((Z#K95%VosuR5v-YsoilohSx00gb-S5c~ zZcl5yQRn@UHE%EH7*}5C)xn_@@J4!I{IH8+)NOer*>?{v1rc$8M8n6&=l|}VV$`yn zoLq2nBET=zjr~+I>nm4Z7;#gaTv|+$_I^3}qlNtu4u*{kfu!hn?b;<2%|cJly>*|o z;h@XAh6Yu=eA9rQ0@`)A1PM1aQE;n^wzk>Y(M3fr>3z2f$3;lQug4vfW$IpC{8&G$ zlz(~a2@sb6wdIVMipKTdUnJgq{CMV9e+A0fsne(Z>;Ke-u^bv^l5~HjBOr-N+FSSb zEn8JV(W76U@oN*M79SbkR9ACUU42npOuw?S(#HADtGQ zR?5Ci@LlS+4e1m*6W*}WPwSts?49wujN)YRRV}Y4uUPX8OEsUpc%k#EQk82rk}^+q zU7h9UQq2y>MtU~3Pa)5rKVMiu>K1kT%`7P?>GWN1b!oP0tUa6T@!Sg?mjsZr5UJtw z#g}ti3k|wCIXO2X*H!iVt{+bC?(Tl`VhEBPehi)7v+1$ZqW4k5j#aRr1}HZ4Jnj8x&bg5<8emP)84TUe;!&OKPR{M z!`B&>%_AnpK-?DsgS^x*jO)yUho&_lLAP&LrK!YBj_|jR4ne3U%xDJZu#>$mH zTL@MQ;&kdrOx4Ga+mPR(KvAqMj_mXI_dh{Hfq37vjZx^VMAq;?&26Ggfm)DA^78Um z934+a9XxjtzyLynVTBiiqobqy%7VZHA?pqM_9YvxN&@1JGL7@r6C^~gf0mY>o?bie z0t=RabXS~OYE^M^rZHKDU+o+?F-L0obGFrYs`g_i==m>HBl(dYHxA7HD((;Y46brJ zQ!4`t<`_5UGnVDhU$HP0V@$)s5{l!Io$%BLHFdy}`?$sPg_+Ua&=qc>R>6F_YVm8hIn(I z9^M8@FK}(f3fbWzDih;i*|ph&P)x3aNOXPvoC;EU<;s=Uz^;zVbM}c%5=V|~mzI_u z7-yBb_QOzKP06#V^-^8iWa$9`ft|oVwSa#nR#rjjXW~;JWdwzUyoI}yGxvgLfk`tW1m>1y`Hq++3dT%dfaDaN`Z9ZeFSFF3wUkkvZdI2t0g@ZA1*8|UX`@P0jyeTGLh$+ZwdEanGnj_^^4#GtRw-(7 zay8Z@?jQM6NkKtQXI;a8y3hTLd))afEiLV+S?$(0{rxg3Ls^X72dHkGO;e^L^E>-k zapwVXaRcj0oVVM{z`-iW$(;(W4aqXbpUz)->%SS_mcY>jWx*d6`?klOJ9qpcdSvpc zf4;}U%F1dgEg>QC4KPjF14Ww?i?!LQ)G(v(;{)ud@*lD0wyJaV-2w$>*yXq7p7VkY z)8DEnA0f~N90}GgXngkU-HVqlagZ|EZ;3ec@r4~xn*gAGi>%L?x{G$S=Yy_ph_CUP zA(?A4iV-rY7uLk2Hb~`;IDdX&AJCa+tn!DGi;EhIaDV3Q$0yunFH+whdmcWp%n|cp zqk@9MH-un>Bmhc;p1qx&Kf^H#DiQ@M_P3!AF%c`@zI}_B$b-BWSm!(cJMnCpyQ6?Z z-i7kWzC$Qy^Ar7BpbqSuZqGWWqNuF=X{>xd5-DYeR=v5qYKN9!_$;mY($kEUxz7%! zi-@<*1*UUT)9W}v7P0D`JIj1mrfzXmzO0meaGjhoWD$#aq%64jPY$ViTy3*`nm`xqG+sX_72=_d`0w4*TIeShgK zs6&+e+lp>))h-q=ibB*R!DxFOe#=fhZ?|o-}XuO{*6W<{9>$!IEWutF6I}u** z-@ktg4#3%;YTcTi$)`0n@$~mJ7ggr`F52;CRE@?@i$lT(8VQRWsSR5L<7 zy<6&Y5|+F(K6+RaQDZ7LJUsjsct7foUDvadyVf5F+iHg$mGPX*`TFAr&v2rQuTT15 z%Fmje(C!0X=bP0P8kli^ME1UDY00gWt(VV6Pz;-waS$3}1<0}g3RQvIh|Hv` zs|#I<1`4cxNGX<+VBt`m9KXIMoB93ae1Vibi!cx;A=$OomHMoTz^yIX4K-H5}K>!AsvWl6_7&ISG;;Lcl>~FnbZ5JdDg@uxeDXI(Obt ziW65{^73WYN8=nl0q8Dtf7K>YqPh?e!}?MUkao3jn&;flOedie4+z#j^P`2)ti2cs zc4e|^$G|w7T*#&65!v%oQ&S6nKJZV?_xo18Q(U^*_U;|kmOXs01GdnA9T}lKN%QsB zuh3-QHGTg44StUv9Z*Y_&Yckx5E80JOgJK7@QpM$NVxe;Kw{USzEy!KNA&mfFrI&5 zcQSyQj=29V^hdrzZ{^W1c>GBKfPgQok5SV7N}z2q9q1vHQ8_^JD_CMeR$ZR^^+79@ z3);Hi+kdpSmTjnZGe+uIg&io_{*eE?7ADCu)Dll251bng4Gw+-&R5ep)~G^1(N|uD z?XT_(WjZeZ@quvrZ{Y8nNIoHXt>OM|PEI%Cu54<*0*c(*n*>06eJZ1C!sXlwBCKw6ys23n(-+H1sNcq(~R* zBYA&(F3!_T{0;syplSjrQHuaO*H}vwB~WHoaI(FkSN5YY2-tS;G0XTE+9x0@#U8T^ z6IF>#X)qi#uJGdZNYu5MQC3r{y|Hmibte+(kc%+CGUyXZfaB7PbsB@h)~#EQn!Kfe zhGdFwCHZaKvga$5C+($uGMC=bIO@be!B_0)Rhs~xF=csa)1GD3Q)um)6w%R`AjZB} znUId!x`5(bvwi@y;w|95P;?CLuHR>Q8`7v&j}A&hg0wdyLZ}ugi-wu`4q`!B`uO6x z%j*@bH9-B7(A_pDu6}%O|5>ChTUR}T8@jPsUD(FE*K#GU&G6uerWZ)=*s+87oY}E< zS(L0je0qKwsq$}ndnGN^{e@cHwn*>XIkdGVDd45F;F!w;B(6KGQU-%-IESYoF$T!q z+HQU4Sn53%B;;M5si5En1yE$eOTq`l72_DXAN?8x!H*CG8e<j7LAZa$?yi_+Pani=xUg;+ym8jJBLkz!ePX%-w$5$~iy3vg)$?VPd@81vDu}_)`QWQO-kk$U-W4PP7U!P-4;!ulN z{H_evZYwf}aB0x+IrseWC=VxFiY0ZK)yFRykT8}0K;i4RnLcjYq1xcNFlmzZYoCk{ zt%8Z(`SX?~Tq$fd;bCsn7W)6x7U-7^??Wnq?9k6UUb%9%v|}DITP+DTuAF+RRZDn- z)T+liIC+FOP!9Ly&xGcebf3MMj0&rCE?$O@PxaZeXEf8OO=o%4p?TK)!trX|i)K#9 z*&>^BWQvK``FMB~dBagtT3R#|RaM*8IE3|H*u{O9Iy=9patx}1$@h2n7^KlY$a0dg zS9nOdapPvhRxZZ`GE811e_q%LN}=$*>WYdq2?+_+D_MQdOG-rFG&ZU=OyhKI{V?AA z=w_?~+Ex1E^e5l-Qq2H;WgkAq#&#+}?ljkZ^96k^=Ia+8o!S_h>{eqNf$CAevS?57 za{u>+iB~ycC+4T7cB-B^QzJOs5SJ?0j}uuVd2`dYPgy^~;DeWm*40QC&$1(%Hx~Q> zGFmjh@pfZlW3T03#mZ64xBL2<_htV?bD%BM z@PVr4W*T4(1#o^MKkqN%oHumzR_3H1pVGGE^rh>sgKHh?@#So1dORT|?Q|af#Hlqjt~|9Ay_1QF z$$6|z$t!W~!95fW6Qog#ZE$OuRiUkR3~oZ9ys?JCVb2Cd?sPmGOR z0!m|$-p(xgusxryay?Z(^erlMOZpk6$V0l@QBihJp93|!I^59mt9s@gM5^`W{`IzD z0TGdC!@P5=>rm0|tyFfEIHuMY^p$(wLYpLLXl&f~%KYyy?U(sAA5$IF$)Vam`~2z0 ziMtaO%W8uo3NfiMVCNlqdV#E|OV6JvZT)`f{K zBiVTsyOO6X)cRiY_67%0Uq?kVtG&a}t#bh}`pj+O0+{9iVwRv0M4Js5@Zgy@?nCu- z1pN2$@wwvc+%_soPk;kBD|4=_|mz_`YeRLTK{4jBOumq9H*sc|7g-$#ZI8qqyr&p0yJ>=d$;# ztxPxZ#bm2NqvAaNt&on9F$AAsg6@ZM)Ym&t9*DHHx6?t9sY2r1iH;sV%1&|F%hP6fEYFN;sRgELpcUA5xwxo^E*YIMeX}7Qfy%7- zAcE>m10+}ppW(TCZE!Q`?c28?rMc~)`@oXDO{F-^ax$!HuPOKCWjq!RR5*Q_keso{ zRj3G@9(@(whphb?pN@WR6|(+BfnaI3p`Muu{l?(3DgVU_d8FH#%s#YsZ$Lpr^)D?I zBiK68UqTaRa(Y@8dx67s3&QCJMSH%+Tx%~QBcmGe0C7NyTEI6l#XK<*0>kLEf&wCR z&KnzGz#@@bb^LCRVp>>wW7o@rnva9NyEr=+swIjRm?AhCrweV`sj(qC5Benev(IHm z>&&(Q!{2Ud^^N|7W)1~nX5-zS{O{;VpUb-H&|M&B*q4H`%A=E`$Hm_CAfcr5(Z_XY zq$+JW`n#c2Q`}v;(2=R7FzB-=cX_o!iyJ*iu+7je$6TnhUC&Hp0~hHg#tDcBJ-*HT z4hrP$CSfB=Ubkmti9lDm=y+RoK4k&3=T84p>AMdFzZR5fCoAiAWR@R@>Tm$r^BI3C zfjrTA-Gx)q_0r`~ZTPNg`hKtKCmlV#y-dx)K~I_yX>{Yfn&|RKqxM3)GP1h_sXJbi9MZ|>dl3;ENId7V zXboQ2Wwr{)hcWX?tE1(QMVy3g_6-@HIS>Ye^R=_ImFz9= z!cq~u?FV`XCoWxjtSOr#vNSu+9d@lBD@-5_hI`5J?ACOaqGuJRma`XDDIdGUJ$b^D z%NW!OolNUhxm8Pwcuc0+Qs5}mxX^7svMiM1fx|68y3+K()zKH)#?8$QvM}&!qN4S) zL7)CTmgF=c2;K@%861&T2pp{t#(Wahp3?(rF^BX%A*;K;$t>>^;r78ZuEQ^>#WSvd zXlCNo9jY6Cz#l=!s?e9%gUfz*<;yHC=>7WS%&|LGvaK$6ohXTw<2m2{M1vgNhgww9 zmaKD!@;hg6v>?1}iR3rFFkZlS_;5>+t*%{v1q_#UYoj0Z|WJwP(ttowk-3 z+n;J?^O;=g&^D_B2M$=ZXVI;!u9A@e)~e%qa4k0UUhi=@Qt;@fHZuCH#N$HvX zAu#KZ6ccNW^2t_F zmE8Y)-%NVl(y|kBfy*AYe^Sytmi`Yn_#}M$%`btj4^TNFKQT*t?FC3_Iw~_UO3Q3W zj$Gs950cf9XUwmJ)@nrK5Ko`i5|EDL%7P;)vvuoO_S6o)OJ@cOnf5&38UCQ!;X^o0N_^L){n~ibK|m7^%Sc{*iy*G4*S^OlcV%_Sh0man3f;ZH`}cRF z6TorYv@ll{tq~NRZ76YX5iv)BO|f2z2Y;$3NuI=JPhrh1CzQ-n;ZNECD!EG`1CeAh z`Sne=j*tBDd7i7ZhboYh1NKvi%g`!BU{hNxb250)$QXH+6w&iFS>(@o8Vba)QLif; z8}ALuyW1B5Q6Fh#{L?yESnETKp>%$@vJU_=|Jz1IBU853rLmA5W@cvZdDLr)T3teg zbMvJ|Mfajy?y|h`?pHUT;PmPl-;ZZ=KD1?OacjB6{WR>6Wr9|G_YTMG?Ci((LCb)F zAj_IY)gHy<{@PkDiBlWymNrEIhSt|!-S1*QRk#3-76?T%_%m%r3+vH~Cj!mC>vm6- zH{4lqP3C^()1OxCz3fWpRxo&ylmG0>CpY$*3h43v$)s7}9@fd+{4r&?ZZWie4IxMRX$nwv?z_ zbTkBqg~@|t5(J5qnaZJ-B$?^R@_~!m@F(T zkkmFJg9YPovO(Af?NWd;StAKRR*mxMyhW0p4!zt%Iyp4P$uhpI3gN8h;&k&zN=3>q z%$Cc0oJ{SyzX)7)?T$Zsc^6HkcAM1&+A`Xcd(uc672sRXs{8S|;r-CjM zP7p_wGIG%5_YHhB9!9R^F#+GC)m>LAw7w4xvNL!II`m0;O@BPjIR899|DJRGH}Axi z+k`7MGL_c%=%qrDJK|6VVU#J>j3e-C0rPpCw%qf&UOq55<<2QFd5}mT+^vhTAbk<_ z$g!P5zf;e!%Atp_FdXktt@rWcN5`LC0t8JczIZimK-{*c_rxprDbwgMOACuY7<&eg z8R0OX@M|0WL~L2)asD&9eQ7H6@0YB#(p5F$)booAM2??8Z#ZW5tKb3t^5SkK$6UE{ z$-F>k_2U1~hkuSp7V_o56f}DZwEW=aB8+CHtm^7b%wls89;#<^Qd3g2!%h)oFyjoR zA6md~K}NC_dYb+FRn%a^yAbb?djwW-*wf@4{g4US(EU>F&w5zg@ zNvn)*V-eT(s(>MZ^cp{{8T6WM*Y$7}iO6XlmY}7hkD#GJ^rwIw$R4jXOFI9!`}mD; zo@T<_+142yD8My97Kw-Jii(Qlh(>y6mi#evu|t|#<+(=h4>*G@F7t2SC!G_ikpF2B z(o@!9{v$`!%o$1#<4prD9|JQNtG?_G0qb9<$(6)s%EX;hx5L#M5J6DJZGT2%Um>K9tgdP^K5rt5QN zQa{cL#Mg{><_W7YF|)FULCFcuYc)Ow63Wr}h)`Q>?}(*A^Sk2ey2p42i#RP(>$N+I z5@>s!g49*h*%Owe4jBx5rn)nvbMhz6zJ2_>#!(P23D=)j?&!KY$aeD+4R-VmuAxiK zBl4L1^}~}l3uqks5c5Yu*=ynR3V`bliKhwH$;?QFm}N;~{lJfeYoj}VzrWNu@e1ZI z;P@M@(8Ery1M zaz^nO5nVSB@f+~5K2q_t`8f{LZC2{K3sxQz!&)K&=Y^(=16yl0z#8b z=k(OvT*nwC|6KzQ!bErlN0Q`s@7_-INpC{yw>a3?8~q}km84`YUH<+1rL73lpC=#e zw7)i(u93i{;`sdxBU(aJ9v5N<%AsvX#9Jd>>eQ!pY0W|T8({RxQjHfm^J^~Ql9?HM zUnKa6S<;hd=9K7(I9?Mc?UnW_=}1n5j;a)J4_We|8IdI;%fR}ZXeAPU_LZM6xa`U=W@9x@#+m5wm+T`g&s-`i6bNXJ} z^J$#QMy`QyL;P{|UpxMwc{}s7dpFE(yIL!7D)~ZdiJrKjq2bqo z0XBUt6=OhEj#lG!l;K!$m#nVAV;?~{n~`W8mrk>@vqx#wT4%eAcUZxf=GV5*h3Ipl zTO0Z3S6?|3`KGPSW0XnzsrJHIP;xo)`gJnGOw|5T+CvV3S`)vA{)pl0630OrNHFiG z*H%59$2+#6RJNDA=tR5dY=#;O8b#t&!uQeZm?Cz?_9%Kr14s}$^}5+Qho?%P%B)?j zC-U?NbO+_+N9ebwpjl4R%-8^H9z94ohli=DDWONBt#eeR)zmxyT_Dk16ZGu~B_>AH z>1|%+S_t|N8x(VcL}0|jwSBmcikw5 zW)Y&Wre-7lPuxN7L*Jdm)>h|@i9@cOoCnp5XuM+|>02$Dl5z_kJ%uh6Q8|i@7VueF z*vh0eI5_ACb0v}D^ZFf-72XGVpnAb@PL5bR`&mg$Qj!^+JxapC#AI@iLD+^CjG1!( z>~cTYD6`ZxZSy|#+Xzo767;TC`Eb_6(R(i$?tK^{I^S?f5>iq%_H{r*Jbqhv-Uc57SeS5!%bV*lK^^jO?NeGYsjXkV{&v{hvT3frkIk=`$`@ z!1@!1>I0$8$mOSDohJHzvGMWjSQ&T_UL!AXL`|SibMbd?slIf0nFT6CiPvI*sP}Aw z*wtt7ya0&5hT{+JWY|#>9_xqmuxgx-omJBABW6-Z-y!lO}-<>6*=dcJg_!JsRm9e5L{Y{9gH38%aA|JKyIy? zACe*N0|`$>nfuPon>WMAMuk1#8BNQr$Jrum9(Ndpsevs|qigi->sLS2n=o=428ILgBrQCl+t zY^AxBWBwTHsQjFVaBjgYPH*WWi4Z2}B6@Zyn8zT-Z=9Wl2sbi>H=JlmEdj0VyZ&Q} z+^@9*z3{X@+gT-ffPRcJoCr$@(2@pBH}yvwi8{UrvGlofBiuXmA7F?z3|y3ylB)4a zN=nifw;)`KHf=m_pydsEzO-)TDl*66Q)}pl$*af%gEQ>fPtG2w8hn3A&;)t{JbV$( z=hD@alSZ3DUe4?5hB#V@`<{ctC;$mKJG2;y`Yi$JDmoEHpO&OhuirMr9c!-@F|VgB zGPhc?dy=Y0xE~AqCgA2~oKFp>1v4w5p=e* zal#P4Q5-W1_-fW!UbcB28;90*3dv|A$bD|*Rk;w_oiLkmNJ4|Ga(ez}u^st^`xLsp zMHSL3Gp*_dt1LxPa4*jNIv+X(<;(!*@4i}HsrT}W$6M*?W9%d-<$5kGyR3TJOzR)f zeVPzEf~@)$7s>l>=+D-Eg+qr9z4`q4{9r!OD@8jhdo>>I3PrdD+mSaBNu{=^*U~V6;oM+##}DbOVHk!cWjYkZm@v zt*sIADLSkn07y7^^i(@`C;($XMbv-=f>6T=tHrgM9|zP<8lGifU@*Z60LB#+x!fB> z*tStpcP$d{$IM%_Q~22l5B`S~Z|DV%KAHGV{}V`poII24EnBK9HZEexO^0MJVgR8=tb(*+4xf|yg{a`NH)UA5Xl}j{l@cz9 zO<1v7$ly@)FI?=9<=1`6jP@V-zJ2@PE2T&O_%uv8KyvV4QJ{80m)?Bj$PueAFNJ%* zd^x9qgfFs&^SVK;1CXJA_Y+KM`zwX*i;i!r-<@PKy1VU3g&jg1p^xn1}<+xTf!Jg zIKpDo@G^mp(C)y|V?ObQ)jP@JNAn{P>?&kTVwUGCjEks%=%9flvmAvbk{Igat9$*L zL==XYBSz0KEl89fXyDI^ixqIk&6uB>`Sn>>Fg2MAT2B>hH7Fp|FgWf(N>|X<){gF+ zJfc`Ru{K}1PH2%BkvR?_XEPR^V&ldG7~#U`2KtD!mNsbqB$j9&7)@5C5$$9L4gdG0 zVIl~llw;}5fBTb5+kzBtB3cNuEUp*l!5xOVk|Wnne*6C2A3d85ic-etG^}nqt*!X~ z8?AN`vFqCZ)a?s^;f79kwbLH|Gzkdp;qQy5DFmY>gjb;xq1tICbXvRlAMNpf-hc_t z9PsY1fSx2joRL&iv1?(FE(F=VDNjHk{?99N7?^@ES;1lI2wk|h%tNae#zILcdHK@; zWCU6J|6V*z#GWIa=R`{Tthfs~^cn8J+^Y#O2=~8U_4gY-e$)UZB8IglXJ*>mL{Do| zy@U09hK1ta1-q95qHp*4#Wp098Wc1@S8AX&>}CoYrikh%I=041qK+Ggkx@wdK(_>H zCX8p;d155W>6_}mt06-SO#Xch@gj3s^;bv{l9fn5TqXWUf517s_PZ&_uvm47{bRkp zh^hrl%4z>r|GN?a_peuwzvD-v0x4`0&fq3Mb^qYt%|yzC!H1aAMRsY;N&t|3h->08 zdMTu;s(SXNvo!&4xz!Sy#iL_m_k!5bKz}1LC-JMGw|AJ1Z-*dM|C4vu-)kH-($tK2 zJhgJ0zv4A~CEo@IZzAYS(6NA-ghV1bcBq@0`i+?ItgfBLHk;zBVehU113wF!A?Y=4 zLI5D&Q1sB>Wv}NK-C(rEBXi_cWu;6{u|qhpM0W3Gd;3rAq~Ypkg8LE z6;5-ocUn?2N~wSh>3z^Nl75y{^nj}id;7naGLfRFT{%NVy%)x@}k!7q>P0|Da zD19-!!b%bR>?ogz`x5}s1SOqwe;IQ7_U_Ibn7ev%>A|(XQLD+%%e(pH$&(~M1S2t6 z*tB!#40I(2FTgs+r@z2P}xCHA+jlk7mRfv_es4x-s_wL;rxC+WxRHxC4$zkP;y>v|d|DGw!AE|2% zn{cd@o8}8U@pd4i7QIFkvz-ukfWgqBx4N3e=-K4EFnKcD>EEnfzy^UFXt^2?H!50` zTys1FH$lW$ONF=R`xo$%O_drR(e-4mS#q+ov)hx(_0N*0+`z*<4Bl;#>Tj)Y(GJXV zDzrb*nKoaBRID7$m+9j3_g!ys>uv)%7qA!YSA{%uv5N$_z{+!x_vdPbQyD)c{}TwgS(h2MPE+M(dNM)DbUL^4}{2Giwhw! zko@K9Qhy1K7NpksI^WCblGW<<>xBvW8zXyLXt!KPx%vEYoWg@uGO} zLf3rBBK zjqir734f!76n3JMCvV0}{g8qd!-IN_Tjk&WMxSCbT778KmcBwI@UNo|_c}awix-I%%he;p0=4gDr3{?4hA~(`7HMhT0k?Jox%#CCKj&tw> zI0Esi|M2tYPXfY|Ca0z}J_kMR62*``9EGY}vd{%ck2{hvj+ySXj=CK@JC23KM1i^3 z-wY@%QjBm$a^KBX~tMZ!Zu=)|~tQB;{b}#i*-0NW|<2uyVGbF-$MS zNcfZsq56L=FDt8lxd_4xPu`(!Rwky~SRHc3+$v8G5A)9dRHeF2&cY~+PE?ca$u3BMnqZ3F0Lc#$Nh%61p)JC^AB_JbCxL1Ei(MW5 z{#O5Q+-mr|A<`Vs1Ewib(ZVG8ZQjjw5tl-!Vt_z_xSmJQ`3X=G7so(6oMpEI<-fli zW(yNKI;j3AYQ)qP5>o&kwt`j~p&+hI_^wxhZ#xb*Fo4)(@l}C1V4mqUHme2fnc%TF z`oz=_0*7caVUUt_)21?E%q9}A?0J=2m!Bw&?l&jUU-CROP zXg`+6t9&{#Q$mu^l8}<)NSj!qeWh=E0H@Fm50XGPNdUtZPz+vUf)|8@fB-&Aqujk< zI|__Ve-HYiDzx_Wm>koYNHk0bU|5gIAR`UVCDpiSpY+zom{BXIPnpW%^P z0RbCbTAo2)LsSY$LPT3T`ozs!onBk?NqYFUGj~&LM{QP_@S|CQ! zDrAX~N45 zKtB`$DR;^Pc=;?;AaubCrV@gj5|P>9%?-WlH5c>U2^;=!dAZqjtgPPM8#x`xy_b=(VEY+-W5A|PqQm?@uc8uKcJp)EUb}|(`En>Nm)f7xd+;po z1{iWUMPtL=^r8f^SlSi^p~%pb-KqKChj%IUsL%S|2*?jeVyi#Dko>>H-bvfbRPcq7 zC$G}9jXr#ar#bg zFs_mGPdgkN{#KF3Pq%ZHW#iw~MQ95A=dVT@FJCmXy|_-ZpG&*HC%9QqDKA$8n`5|cPMGc|P* zsR^P31%_}44F>c0ZvfhgjR28Gz2vD;b*e(0UwV&-B#dGKCrO;s}irx&1d~|#6d3uP;-I|zWd43f58OJ}!8olv4Sf3pKd{=_! zVK
zSF>f#x_4551S?N_>pOx8;COp-qyHVvAlbBZR94%^dW2iAQ1VWM<}5g0Tkb z2nAYDf#9J;$CY>-4-66$V=q`_@T#+@uqq`UkhXjWRquKsj%B@ z7$cVr+<>#Fc{6G!#=FAPTTq+)|Btj>iM;(xsW`%}Q^ZtI^zHV1oTEU1mLW?^p zBy<|mJ?7W8;WHrw?M9iZLIaZ6Z*ubX7+1*SL_I`>MBAEZq<^oy#Ril?&`Z+MD_>6( zVnCy+>(k{hN$(}?Q7*-6P-6e#?J_t5hAwV9Ccpa&<#w$5Wf5?l$j>=4hs;*vW-W16 zQr`NrHwRM(gB;=!PCLM<-zk8^f1Z4WCK+k}{rAoXi4H0lB_)aI zn?WSOaLdh-a@-s-cmY?>KW2YuJysn#ahhV-mnx`1nBNyQF0zR#$wlkb>hp`k#8^5K zC&sB5(T;_y_4A2(Nd7AMT%SGo}#+&QRk# zC?(y!OG69`lKk*Mkc)8A+L#>zr-Y%On3o~MuBT6*G7Nvgvp-0$F$X||Eam{(qmkDl zbJ_aI>$nwL`9jF-N5;8X0c-E;`9_=mXK@0F#kv0a^$ACtXJ~AlI1)21mXgD8{4z5X zA}*b^V`hNn5c(-e#5A(zA|(kA<{`oy_dxOktj^-&KK!MlL&uyIEjiT$(VZu3c1#yu z7=1}#S}$*JG7=cM#^-nz64cxb*Ox>GJOyD-@`IGfb7o{@Xea=-uni=1)i**U3Kf_e z@7zHm9x{gUx~v9bmiFoGPIQq&%^@3@A38}2yF)hcHf@w8nl$WsZ8LK1AJ&fKsQk5t zrDMp08j#u$^Tbo2&&WUkMIxTc{G%c;z{GK8i@Cy%Oi@HL(gHwF20gJPmZF6^fU6X$^*Fm6mn!XpL!;B-@d z0ju;^c<6~yMna>5REp)8aMfA({f5 zY#^bc|<>D|CA=H3F)_{Z&>Iq?2A{|9iKvkn8J-hUN zSN<(`d17s#@>oj_6Xx46EnO(m1Jf{rXMa5ZkLqOpg83v3@nAD}HRZ5X{D=)9-^KlC+7L z*-bzIqCE5KR5Cx2C(!`Zh8U>v37ldb6j>eP-Te7!HWxU39M}F+Y2P>qBp7AT;u? zFrv8QVHbgofR;6UHms9#bDK!0t!8jiiq}z+ z07ZzWZ*un?)hr{sE3#^cYHVT}aV*&bH^HfK@^y=|R&o6QVeCD?xqSP-@e3g%B}7AH zBqJ#_jgrcinFg|w6onKLDutxTh)`s2A}K2w8AT}35GC0%BJ=+`_5I%e`*?oOb3BjZ zKJKIX7T4#x&g(qi@7H>Nq#Ib#gN-!ywEl7%`v3FhG14IGtlYm4@j5)uKhqt!Esik>b!T>>~FX;Nv~B2pHi zg{!Z8<#+(O50l3}Po1HJLf#=EbdBf%qRl3P7s7&qgAqUDfkp(5kwsv&3Bv$9^GFD) z=9^e2ewiC3B#v8HJaW8b)0hwj$v+8lp(r90Z+ZT0%n>ITOrLFzyCm%f3B&hX;I!W(fN{ygi;ZOyho6B=EO zpXT0Yh**nX*EkeL#HHV0i2EX;Bch;Sa>?$*iBQ3=lj?w~A^ssG(U%gB+bFxtkVOY8 zN=RHCvR!cR_NGQpp-Za*iCNx)8^E*8E;2&C+0oH)FO2W}i?2L={|UN0{>8K~yQ1z0 z?a9HZ;p8w+4fB8gS=1Y)n`n>ff660GF|}%H)Oz7P*4g8zIH_>rPhFYBgTwbOpWoSe z_nYjHz6v~BNuDlp&T)@&)DHJ^ieQvE!s!juPMvAqg1v*v_4BV?AwpvP{7HDT7Q&s?t`yu5Af}O z3T>i66|G0iiiiTj9whJpmQo}Q;rcdKt;p?O&{+6nVmbL^acWF3v?O9jx>*lOJ$yJfxPP`ow?m2(+op*BJV2RB_;}~sr zoc}7okRSeWhHd)RgvSNPCi4SG%ah3&oDE|36u4V6f;v%XU~#;DMKOTowg%Nph-P4S zG>RPJ4Tdb0RH$u*ejnj6T;FO~%8mJbEBJ4RBYIbNyNR$Hc`}eN7im*@cJgm?TKz_KEYJk6xZ21!$*rgiZ zUEQ19lrNt0o;~TU0a*=Q%ihO44j8w7{clUa9Nm>4_HF8UUr_E;MW2O#Esy-Z;EA)$ z7X%sspzR^L2Q_)JVpI-d|B8H!aZ z>uiS-!X`H9odcQ9O+iE@;zA17N3nbLH9p(1|HsgUgxnq>K3O|I{T zS|2k4n-HsQEp&N2SxCYMtHc&xdGCTo<`-UQibP}@gv#Rh1wZiuR3@-Mgm%O{{Y(S} zEtw>z86Y(|pV4IgT;m9;lY{^PS1H9|R%nBD75pebn~H6km9%-jEK4HcZ_Q*edwAEv zIu15AUmle8eT<`8aQ!;KN$i0;nckKbxHKM6$}N5rtKquhH>+NzMo3&W|6aYSgb$+^ zmzpm(nBOn%Pa-H+y23zK8KB8qVSwlt(3p%A4vDMK)Rf`^`4lddB71Rc<%tB=wQ@X= z>`|W3_0X?d*LYqR$|9n@77`Jmqj35p=0xU>0l?R|C$-0hu*3^{B6>*k1Urhr$k2>b z;=OtI<0i7bAoHkL%>XTiZVj5;RfHv`q`>$$XBZz9)rR(`!todQXuLw_16mQ^ipmf+ zjlDSJ+r|g*RB|d_zAW{ajeK`W|JTyTbViQWOFDzTjd2=(c`4iAs}c{L zDN&u{(E^Yn0UP{bwG{(cCnWF?AyL=R0h*y3DsUG3ic*ZLg-?A5m`TvuK57_2US)W6 zK_{%OZvZ2}9G=u};0z&lM~KLhB%x6xUHsnGGe@phk#JcQn>Wb*iOvUI?al3{t`L;O zmx(?g?r36dGsDl)&Ii3xV>) zr$#fqO3j7rTI3qRK~F-}kIzYCZQZ@*_5gU{drlnC8MpM{k|LdDK#Y5M}{#a&~E zgb7Rl(Q2O>eGX6>a=10??wr8mAi^_^^)GREj-NUeVYU+0p3!sXjpz-@=>pBHj~+Yv z%dQ<&9t^RHYE5mDlwWYeziTYgZ`}Glf%P`nAGvUW_YGroSe!w2`jslP+vCf%FOwJn zm1;l?zdqWR&`^Zk!?}=JClXdmdKB~>B$yf!Vn8r+NAgc16az}^Ekrj@0=EHuK)YSX#l;pw)PBUs06FQzvdUU0Hi@GNI1(Tpts2Q^%Soj$w{W6U_pKX zm}-kV8@8YwKi=wmHF-9#6Y`PPbVhrjwumVsFB|oDL0m9)b!SSQGZt2TESp`?pYV8` zQ)iB!qjAObx<(?Y>g3c%mjX>o?)zy#$DD}T!DT4{>vIrO77&OUZo@uTUvZ5#bv2C+ zpF(2z5jw#PIz!=cZf+fh9Nom&nH%R%3W%M&cqTUO|AnvE$>hIS`Sv6EW#P+26IQ-r z9kxzY)MzJllF6lgwu-l&;6CVTLC;^oeDIC;re4IS!w z|5WK=p?+VXB7*|MpQevWE?=p!Uda8$zx< zlOh2{+t;h6o-}5EeE-dL{|U|9e+Ets{W|)ma8p*2s`{h!4YhqhggMEz9 z>&6eu)#XFW%04J*>vSpp3j@!5s}}v*&f6%{9lAT}%arepe}CZ8$z`)N7MuL{<=l}_ zoqsoQX#3$^Jt9e$AVFC8*1?M9*XeF}1SfvjJ783n|0mj`i3{V3zPR_U$Q zc*A`&>sl-MF;0WIihp408a^#eT&r4BaWsoj*!%y!)=TYwp00K}6!hcA&+9|B9+`BbAx-`jU8DuPzI zFfhSgo-s@G5jkY>rD6>n#d(E8ybL&COV5HLhgDrnG3R22cMf@93tpr;HMYJTzu(%yB zb96tjM@%SQ$gHe_=_22(Nzj+XBYaLt4ST-LJxO@-^uwsRmUr0DITU3J>*{`|fago> zN#8?Dw~22*zw7ThW#>9lYrm;#{6dSc&A8vYmpoycUHy`d|0`ST(o9UGc|aR)lYW9` ztJOFGMi;&UqoY{2`)bag35w&A7QbFGc+>3k@wCJw<`w^W6yH49o{FZk#$J+sI7fj9 z{AaPnS=X0EoArhy2g!gnlW;a=Wt3)=-)Atq5MzNmHxfFj@^y$W&ZcrjvR?k`Eardz zrMLFK$FHbikV1Bca8cQ} z=snh`1mQy2=5InY+E0ul35tYPy3l$590aj;9!QsM?jebpT|53k_Z6kWFK@M zo00xY8;(( z0}D5Vez()5FjcM zCJk22Ee0nvbXhMK&7}$r*d}JSdd|&Xq$jph{L$%^Ta>+xdu~^%@fgNnb&#D5$}2IL-(HLu@u{b~#HVY608EX=oi4 zPIE2s{rxR(meRMY>6d);2E_*hMRH#L zhDNO9f>tv~^ww`8Y| zzqmqAN|@rQUU4My@<2#k2224O@@rto;CQhb-L>bB0iaUDOp{wEU>^chg)sw~V`U%+ z9RmYV->Q}_UcBspH;gEzsqQ5CbC768LR(AUEy?hR?Er66r*lDu@XWy1KXUlfz(ZNS#!-_V5UKoV$*sBs$` zQLIYwu9b-0A|d6IlaomjRaI37>ZD<;i5~#chXU*twL8f$J%Z>Gaw5?@`*uM)BWNp< zs03k^(PjLf*0h4lh%!X=Uc>N%oW6Zie<70->fm4Y3BBa*LMkp+WlrL)FG`@6OaI-P zYjgK$JyLe{e<0D5|rJ+A`sD-sw3ak@oy@z_o zFvi0{8L_?xr?+XkA(BRH;6Y@h1lErc#Sbi<1)!&%TOrg;zFkoY?ni9gNa$1*=70nl zcI!ZmB{BOtha(d$x=VRB(-o=DYE#39(}$9La*Pdo^~YMVYm`r%1Ffa@5onbdNO&e{ z5fFlP>(}qkkp~n8$v-=_7&728pmh#HTJK{lJdyn z0RN^7M#^zo`ga`M<;7Imb-0*6@}1^BWtSo9;~(5gH_ESDs<*c8 zX&<*{>wa|wBs3#-HyVm0%1Le87Iqp85Y$Br3;`DutI=99|1DWnvyCA!77&$D6z(#Q zr4;jl>>DEl2pIiy@CA`W65prZ9PDMhq)tX_d-nS)cEl0!?(TK)Oa)&DlVUezxU_!zao{kzWMx zDFy&1umFxgoP?dqPKYtrm_PNm?n%J9JvE6wg zdhB}3;{TOdl>#lK|6~03?-iM*psAH%2a)!F-Q3Ixk#3LPFZWkI4<0<&et&=4WpviB z=#SB4%&*fF|HyN4y9TO%o#;@^e}PLCYVyCqLGf|~ECj1e5w9CWGIviyOIU?nn2HOy z2Hwx+4TGl3s=oR*vwsN38u@Ks+DipW_65nPTc}I?GNNU%U_lA-G>b-_1>urn3WsFf zry=w#XZVf%F?HjA1j+oZ&WNaszm!6>Mr8uyHysg`H~vD;Yb&^VWWCW+kO~A~QzyJU zIS&Zgf!`j70-n+pfBGl=Wd)SiS(qS1*jB=lVrx1t`M;9AB)p8!!Nl?e?+}S8qoH;= z_m=|3FbE}Co990IunT4(%!2B{`UUqy-{F}5fz zM7wnKVJC%E;ov33X4Q{_X#An0NL91}H#oW6oSV2i<4PqkVUsALy zdg)s}so!NrLQIYAN{*j@#1@`+!F`_`&d^_vto{ysYTTuzDew7R;dGlUMB{j>R_{~B z(+84Wc!^`c@Qc+xEQ5IH@R~p!;1#SbW7uFTAD+{|6cFC(cRq%d|fW-_J!5K7Lo=5(&>2ii;c zKqpeV9>61Kv+E`3|rJj)unr zat>`cB55c?LqoAd_eHuT`8FW~s9OXf>du+PMnkMw)I(5PYL84CEzi4ngEsN4kBW*q z|3y(AHg+slg7D!3@NDfL0%%4=`{E;g`V@0Cf zq0blhC-q@`(M4=> z@}BbAzf~I5j)qpwZ0de_XF|<^cXfkw2k;YOxAbg2-_&^dLn4k)5y*&NP!wX6f4Q7a zb5irqb&v^(NnRtihI5w=NNQC=d)vsrXd(n6AZLn#;qmx)m7b|M6eE+!HvziN2xbN3 z3mu;6;U$WP$Z9;MzHwXVp@W|=3vR=gu_i+t@_@*gDUya(A3dP8(+M$kMiuU8!gh@U9 z#J)WMZv~MQQ=YKs5+sxI1S@U%Ay7yYW<-oyQM~mlG5yfJyE$$y^u{&pi)iqU?}xcN zH=qTIH=OV!zj57SS~3U{?P4s>q?64_?7)L%*P3#HtS0Q%Btn4M5w3s=o0cMWOXnp| z{hj(huwyh0BsW!{yEc5upYJ02a$W$pi<(+%=6iE=s5JFL1*E!BzcA$y{rV61cM0J%fWh8HVZhpIMtmr#j8wm#r zpE<}>YtRcEt+-ChBHp|0`lqd5__AJoNq!0iFOiK$!eEDcPmE50 z%efIwG-BDr=0e=@k60$4KktaCINjz8S0I_T&XuUOCGZ} zX@@C7utK(_dEAxJ6CRJ7^?dTKaa%Jxf1}>8N{NKxn)J95SDC|5ESm#OLBEPKy&QH@ z6dgM-u!|UuVQk!oFzgR3YwvtHsg{WznmB+cGFj35z%6|27$P642zGAT-DCkW^6+-Y z0n3RI=EEBg90j(AQk}#W{sLLq|1p0lDJ@{m6S9Ksd^zZzBje3zU3LNoUti3I)I!Af zY~kXgo)#3478K(RkJ;Lx!GQr6T`*k4wN9-B$EKg6Q4o)}2^jaPd_-WVuGLdN|5jnW z*>f`xpDKq>bbEKJsIkjf3kl4sX)T&tMJu%5_Ko^s=Ph4_w$6&o${(sr<5}^1)Aa>5 zGgjvG?v_2~<^l+nT*80;+6Da3cRxV3%OPl-p2B$0D{X}~wl)wLl!4#O(fq~u_?ZK# zBz`}%#~8S_Z#pj!HLD1CDi$1jP)!ju*+d5egh3@iOz#2*GX>Cu|8U|Ns5?*MK0T?5 zT8$meJ1z9?N|d`K-I|cjow)mCvL)U*p)bKC0g~j3Kk-MTZwLI0j1&Z#Bv*>@j;lpC ztlr@P=@1-I3b&WHk!*8}zu96e32Mc{_grZVyXT#e9j^~Y?>zr}-JuWxf$_1oflkU8cI^P>NM%N0f2l!9olOh!X3?oT1 z>_z{YYMkWdXQdyj8#kp%ZLV#-sRJ2NwsrklaHHQK=_Sl1V3Xr^c5D=+Rf+cw{n~$5 zj8t6EtdzIFti2z`xCFPWeR|HDGVd4$T-{qa|Dmxau-Ya+tQc9v0^1&9%hymO zlN!2#eO7aLMM%N`T1GUrRnTuS!Qf01(2&~7v-LC&RM;Xbg&(G)G%|;jnTR6{GmiO! z>!X{zWPvHX%rT>*o?k^8WhQ)gXQvRLedJc1v!w@&m!Ugzx>>x_*gHzL0Us3Gs3nt7f|m`KDwP21OxY3qX}<2rL@E7P=i| zqxvj#JMgEyz#c$R5TqY~=_S{1llEpQm4O2NdutgAM!965D@ICPM}NPB;ha~q+ND(l z=$KfF7Z$zDLXk}A9 zCcc+e^D-d|gEK9k2=1b?1)qyW{LhAbL@T)T0#M{(!p`VK1VKdd;Ap!GRp zyKQ9MvY$`b?sR{e?=0J}Pvu2i?cL!mO0H-6`#-JxEPkOyPd|=%bweIJo!1dgL{X>S zr}?6jw8=9?9b2K+2!mMz##mo;bAzgLHNi*iF$aFO|YHQbNT- z!Ag~4`$lZ+uVcR3c2##d5DWe8Q{5cr`!eHi9RY{#1;*d<{*zkqLXUZqHSTwsYQ?d8 z!}^0eXm>CK!XYw^jM8oC*6V*tAhTa58ld_^PDhbO1qqF!c0>I3T^8-c7IH}EsF?D$ z`f@Hbw53zhIwwv)TjhE0n&q!NFIJu}g|TZvm4^h(vtCnQ?bM;bel{%=0Dsnh0RHpS zz4!We?(lH#-B^l4NErHZczfs?iW{R(puRO9`u4wZk*TB~e;qC@@{)`kyMn&nZ%IE9 zY92Cm_r@(Ib8(2m82)8^lB*E+Saz}|XB~!za(fU_0GQt%8JnB@wv^=XQc}nxLEhGJ zaIKwArx6^tthQSF|E4$LwQT$^y-9WN2=f*GlTq8CX+`G9qB}d?<5O}Nmag@O8wO1} z@WmxTC1jTe@I=CGN!Ta0b_@&D5j$`f+Z>s#4JgnH#-EE#fVY{$Sw_7cIz6h3m}liY zq&U8jo&Di_K)hGAGJ>Zps2o;m2~T{l<{&B`AaiGJqU5mCW?9v)z<{T(SjLc8sBAs( zU=(+bj0vSmkyi*O)w1f}$awVU*-Tqqom^xo?RvYh5xB)>7-r)KpeMbzZPWRvdjPji z^fHv9#BX?Z-uB$BN`Hua%y2cGORnZ7i*uzT?FABQ+B>d~!40*kx!^q&jlKzoGq&Mm z$2yGf!;QzIm*o(ie9sVudY-?vv;Y+oTg+x!O2yWBL-=9qYr2BlQOIsU_=?3xDso%Qe!ZE%i6 zeATNBf|1*F)R6&I?XVK3)Lzu%Ir`TjnZWV?Fei3TToN>AB<=0lg`oiCrNIRt$j?uN zdES5)DUuNfyQh?_!QkiMd-r_MjtC+L3)6HU1Z4c~Q1Fx_YZ8Yh96zYt*QEh|Yq49j zkjy=0(NlySpwWVtB6E`<{!?p&v6j#o60H1_U!-ldJf@H7sbmA}AjG<59kqEAVEaHZzCmxAi;r|MO49Pzu zkvuS90_KhdNJCZ-=B5)8!0z)-t_?-9&%!7q88tsfwp)v?)i=qOmGb)XI06howWL;xvr;8+2Ak;ovd z^u1XAjsmA^ZY(qv3mS1N&C^O-wjK)#P?&$V|6K6nZB=id4+-^bOy$1n%f7nT$H7>Q z`RwuS%eW;q?emvqou8W7DZ2LJC5?zgwOysJgc1cMq|EP{cbxyi637%7nZ3;N313A|�qVGX@oQP9E5~q3O1|-knJcUCS+5!AXG7M;A*r# ze4Lf#iN}cKqhUgUA=vcHWgs0m!)^*LY9#9e;RW!i8RgqW)vQ*uK{g5@T%SIF4$O;L zaKHbuL;)g#{n@GX^&6edkx{1I=epWCl$BRji}SW7f3=wX^Do*U`+U}9ozTubL0fAc zEsyOCl5*(Ys56CKp5X<;qn*WvrOt+ zFRL7rn}DjlK?c`U%g@XQ59dtv%{eQL?>Gw-HhiP)XB`;h)cH~dArNyC_ArZ5Pa51( z1}t^R$T~1569Qz?&`hFYmQI_&psAI}wqH#>gy)zCfxb1DpaKHU>U=j-{0J4Rj<1i$rswuOX8zfmRWI4mwg2 z!;i6@;_8oBkOp`giukWk{N0nWS&n%Lz8Hp1EP9}0h^!ys1n4?_OXlZRqj5NZe=TIy z-6Y7z3!W@`CPaAeQEFmO`{^@L7*ja3H zGBDjiuV_tr>H6hCx>>(F9aWD!uGy!PntPtXZJADn+dSW}_A@qT!+_m4ggrG99^7M2 zin{IG`GKWp@tog#uFcPTPQRKo>9*Oj?o2kHz!JQt8@2XuA&U>KsBIAzx`H^TQm#y9C&sD=hI^mw26Db@G!J`O3n^o_XsOBkFywgJ+k`5)Y_ z$2_BKzVLWu3{6QNz(90->HL10tI~!rn%9+&;g`i%(lQk_trrn2b3wV+wVY2HDcH+Y zE@StUmJSX(X1hi0{I?xeZ8v@{<#97hBM~(}p1pp}ajQEHPWuJYc|+fq&oq+pN3C-& zZY(|H|BosO_PDVxy9!x-1LJPZ)Ew4+yDTsW5|x$~l_tLy+Tw^56iG?fzAy4GkYa$u zC1@O0S705K zyGiZnqr<{lqZ+{?$~W-WRhs-YyzKfYLCHsVqJB)jsrvlzc~EXbfad1d-#)Cr9~S!j zD>RY~3#0hr`qp&TX78@v^sj=WD@AXq!q@5>8y1c{7{29U9TwO_vCUJ;MPUi~5%ip? z3+shWd+G7|yYfLY2hgS<{xSc8S^>>1bc1}5ytU$lx1+EV)F*{{?09M1#@y>4#ctd$FW(N=y$ zTSw=_zgnLyB0DypXWkCN#p*gKP!}Fy!K4h&#FtA!aks|KUiYFo@PMPo*hlf~jI~Wf zfqa7vSLsTD98IhXA`Vz5yeCMZsM<+CJ2f^jX@mH`DzR`!7tvvpXBphZYv_Vg9^F1k zzO|3aQp2c_0EJD~_5r{1)Otl%2`m>Dq4iiozx+{rPkY*eSW$!i;D7L`M$wUJ{l2Sg zTswb?iq(ky^DF}g6+SUa)`cZ-r8CH^Mv1)ey*IS{DPHVtzdK22VEPA_ndPelMuDQ| zPZwWw+#6w7&ymz-qWFvR;QtgG`Q*^e-z2w_JtK%K8>9(>$HLf9RbN2g{RXZW-D6JQAD5jC3|XQPvR^J03IfCUR)0!v3ws^lw<@ zO7e_se?&>UMZZ|IeKfjg8@o=Adh`tO~(*!~8%9q9}b8UFsvsBdg#ra3$@0s<-K z@BjEbu7ADc{I5M#oz@SIuIl>+Zj~#EU1z20McOst>)(&P_B1o`d?WsZ?pB)T;XU-W z1|K%%U-r^Yx_!|0)Mw6ir%488Z>3O;%(Ajl+G0U>mgWtNtN+Rj^>civuXRh!(_J4$ zT76wujY%~Zc|!aJw>?;#cYD{avbuwrAG$?Dn2qi`bcw|}#OTu8SWlCjZC*6Lr8MVD z>g`m0+V!96ceNB`IU2cHq#t}Nl~w9DD1JQ2c7tBZp-1`~%&d&^HeKJq7It>w^Tc*` zNq&xw`V|}c-069=a}@tV7}eXp)~~A&{=XxMcnNV~MnEk|W>$ zDnk#;`KnjUmAt&x#>m@O)_6@lo1^;2pORA3(ej9JUa7P65BI72+2-JJYfOX(l=~~k zC=A@PIh*GgBFYsavi0vimUJ?3!mEj;BH}#V!>X2lzjf=ys3rSP{Jz;5HuX%+f1q=N zg+Q#Br1X*xiiXEFvr*UWDE|Vv3y1FrF^x*Tc{%%GTB+#JXOqdeR*^07u_KiXe^szt zS|PLZ>iLP?qGEF%JlS>Zl4Pcf{Jf$^h-#xGNV&z;&XvQa+h5Mnh_fI-qS%#l%i|Fnsw*Or9 z>79I6ay3GxiZgV*nC!ITFJImt%G|QHf;sQW`HeA|wJw*HGlhOSy>W3fk&U3Uf3xv< z%c~knY12*F8*H9-?>;U9b3hG?`Qa{;FQyrC#*a7e_s;wzAZs^ z%xUa9Q=Hl4ecz-Y?C+&eXn7GT2IizB5bd1I^!T3w`tw7pMnTq)?h);R8Spw}8jzti zsNjfzec8&Dho9JY&I5 zK?#X0K7BGk38`ESEiDJYeiL$ql}CyK|5gb>-JS4oQMpaKcCAJ^e?ut-7L#>@P-1o3 zreikytotlxZh2RcVMCWSVh-+3*6##~8}GUl-)MRCbk6(AM`uXdY zoC}^=4O9ICm$_qWq7^y1e6*{}GsDW~e_Vf(e5-9mylJ@D=8YUfbDbiWgcGbnXOsoR zHV89pmpPo)!$ZB?M*EDxYGP*(Q|S}0&O@$Bt+z5?M2jd^X|#vG3d*@QRVK;OVWc^o zpP^ExQJyHGz*W3lnnV4vIoFGoKb1rDaalE|)5gY}Aj;x`wBU-6YMBK*4U5^>DskaD zC70bH+l#9+$(JJuXlPEYwiqHAjDeNMvo@?k{1+kIq0#}CWe%6dBocLR@>YHPxEiJ^ z1yH~5@{lD%k^%u3Q~8) z-9I3JjbzxOpI;|0&kKBwfM+_jXgRDwJOAJF(}SGZ=*Q%~NESf*HR$&nuR9^&@Vy1AnF+tGjn^*iOQ)b?@t zN;mkXQ0P42Vwnz!z8@Iq1uo4TSBuvlj`j)yb0BYy#2|nRtAus^77*6ZbG;)W6Wl5W3QZ8rJZ{Njz!6!=Ap1ZMR^>vcqN{qTiuCidZ0;8h?uzJ5i!nrTN5t^gI(Kow% zZr>*E1QMo?#YqSQVv3`_0xCeaNI@ECveP=FUSXOQ0GhXuY7(Xzf{MCf6=U|+&AdzDH~5LrNd<@7 zqIPlu0+F@~XAB%`C@ z-Mb0`{2yxJUg8IuDwZrh#23TcOVBa1!B1iBRX<=+sz5d#&4LFDRO4|ovvq5}K{bRv zk{cjEC8%w))5j(yUSfjM(6527fm7P5#@F)M*W{>iSc)^CzT?_X=L>l-1c6Ys;$NIK& zf7m;9%O4!obCtOxxCJnLnFZaL-4eLKjs%Bnpw;9~;PVf`lx zAX9PUjg9DmpS%7jg(;Ica;CclG&UxW7c^8sU^$Et1iNDnBFIC>D1q(O;WvZGz-PQO zYF3M;RK2ddzduc(XVakA6k2m)`Sn#>q+hLsZN`tGvJoQhozn zISLMj#)+6&)D2$iPd6oJaUP+7b0f+=SFV5g^_kMhE zX+$-~Vnc}9rLngM7SgH485uSaU3l-uC94YO49cMQBPJy3dt@+~rqWvl@NOAz)MLdu zEJHIplV^8R5XQcY%7Vwq(#ndcLZv@*~o~(B5#`lZ6ssO-NYa!ZRRTLM8&<= z#r7uEpO3m(X?P2}qJtSKGrwHWXVh-Mp3Su_$@*esz8~T@l9Q7w+GF+$ws17hK%{qg@eU7>A~lCllNI#TE@72 z0V7t1v#RvwQWih~{>qFr93wup%5_!BMBCLBx~ReN>ErQUI&1eNKdI9>WNWuF1is5;3nPr{qvA0e@uN-BH^x0H1A7PY>~lgQqi* z#ym$Rtyz)w#sTV63pKTVp_*)5j<;v!!rRl<(=dMaT9t42;z)Nbp{rxcUrUX4*Id@o z`$@gLOv7s8?#5LU7++I{dStOb3ukXs|4y_0&pY$GVrp(KaS{;IA8b5D5%v7GFBj<` z2p|&bXVD=c-ViF2pux$`o16UB^pV2{Im|fckyw#t5+MP}2H)k$bvch8n;{w*AnN>X(>AkXIU+*}z9u?RWEiX95(U(SmI zjjJ}BFFN(Uu9c-5!_$!ax{idHqt+{HX_*KqvBt|%iu@B6erw+%P-xVvKc!Xg-~W!U zsx>}EMtdJnQo1s1GJxKn?9kdWEiu6<7@zsDZD zkn-bNmtAd4V)I?mmDTA#)n$Ff?y2dnla$vQzUsx93vZ{78|+83Y-W`de(21PqKKWim)LD8faD`Aa4~;(%GAUcx20rI zKvTjJeEiLbvmtR)Ky&FU-TKn=$$On0NW=NKLWRjqDKQ2B#**?rk zHXk0%Dd#UmLnbHu3%>>DdYI@C=`_-PzhZ%x< zQ%v{t!-SbpF=!d=x`DlYq32ju{~)uMk`C<4T^EC z&%h+8nei6WeE|i?ASG%Qs##EOP%_*RN-p2DEq`R5Uvy{!c=DqU7APv- z<&R$8J+j5(J8xpv(!*-CZEdR*XNKwU@}9-#CecJhJcM+B<|^DyGt6`#O@Qw=VG$7v zaN|m6px}ik+r7)bptj);93o*KYiFsi2AOp@?99L0KWrJ2JirW)CB1yKJLuQU4#xTi zsz)QAt@iS?_EkE=Z2f@4Kq7>ByUa?JTU>p!VYG$;XaJOH&1t2k`%kOfS!k->F2F-7 zkFo#{d2f++%a-#kE>=L6l+O2TxC3|FSOp{4aa5v8XV9U>A92;>=$c&{k}GuV z%StAf_Muw;zzdN%lhLA_%;f%%pdy^w=P*ze=?qIzMHuouMpT2`rOB)h=LnIv5C}>gg3{oYBcq1V`DL|=eil>MH=DQ4Zr&CU*S7Xl!%4j8N_PnLkgvrCc*F8^54*nC)rKKU% zp(__RiiV5cb3G*{pj%$@lHncSwO<>nxPM8c_&rsnwP9tUW0?%NG%lQ)8|dxj{fXVL z8EGItdwSMMOLMfgwpOgYKXK0ZbF4?&KQj^NN#2nle1GVwl%bw%fIB^h$;1nxz$wMN z2-!ko!tHVzjeut)#|1puj?GZMMDDu4hE3vCO^w)y+r3@;2OVo}>L!;bKDg|M??X|v zbaX88lK7FJfW6e>ufXR=A_S?){>Syhq^pA^4=EYMe$i0qG8*>g*V_YpW*ueTgl2?A ziH5@Y*&$wxu>)edb6p>KcT!W8Im+tgP?g{7n>d@HG^a`r2kZe8@kJm&BIaRYOH(vBGq}G_^w{lT7A@ zV~(AJo!77xJC1Kk*x>nBtRXA$B<9W)J(_J{-?M9d(t+QS{l z%gie>6n)bBmR@LozeyyIXo&YF!x8A6zG8LK+2jHe0*>YdO~26uONROusJ?O5hqzS9pl*lmQP_u8^WD~9N(|% z&&VPd4a9&{_fVPxtipM_>nqYhq4}bA@bO~^X1aeA3ET>U$*XAX3nWM90tsLtf`Vb^Ys0WUtbrD ze5av!jZ5Djo@snrTu?AQIP@~qqoRbjZtQi|@7H-*GP%K`A%#vM&rTk1-96c8aH_rJ zj7tXsrop&S&BPhy}5 z4TUT(N#{%{%@DVs)mVf847e}K!I^=Rhn50X$u&i@8w>OuUnGfUhx|nBY#j!s`#itk z7#bQ5o6gvn$z1Ozl1u_n1@psIv#UNMiW7h%;~mHG8SGcrstm-qP#awJ_h-cpVg}gA z+k}jTZ%>^#I^CO^srKnnh7?eW{YQ?hYHVz5uH{}127tLbe=Kv4Y-vWGgbvWdT$JMK zm82NDdtzEdZfzo6xx81^`Shi9#=PVx-z>VhG)C22vx?&R&GOKeUpqmqY$%y;#neX5 zQ_q-<)B1~-2fSl8nc2Sx%Pm%|4|PV_m!pt2_I|_;RA%rQ2Y07SctnIbcJ7N4pD+CS z`Bo6X%2)K{#MBxX8cN_QGI;9Mn>W`pG9d3Lcnh#RS_LBV%$~~D1Jmk(@@}O9hZ<03pG3u`=eQ2irH?z=QU>y1rSv3le2g)<^|FHa;`s5IXP& zK<~{@4_?C!h-{^wt&c@eB5$^;2{yMjU&bEDIAZ(q-FBdM=E0vZi{z-M@JdGDy+4%{A4PdZnA#aH#E&fQ+X(= z^in2ire&Kxq94l9f7Ce2*`R5q+Lx3Ks<#hzKv9z-@`Mw?ZmS|QE zrc9XLLo7>bNxiVUXoy-5keVPgtco1)A%GwK1{ghcC zfw;Gd%!>`Xtc`FL9YqgRa+_u^&7B6nq8T3)*L6#{$8D6`G)#TYeV=mX;Q2SnD$~Pj zj=15F>aPFuYo_4rOsl?&{KaVgche(FlLsTYO`{Vhhcxuf=H?z;w3{m`5IVyrp!(-> zenYLV)~(Q~!9H=>_O@e_Q`Pe#1&?$@U%AiK34iiUQpU9J7q}I-V3Z&%0!elO)JoJF z*ayDhegl3x>(%X!NjOBCO%b>>3E6U~hEOqIBk7LRS-9|#AeJ!mMPmmcKggg<=uTpA zDkFXQI*0?3RtvFc2f%XjLa42=Mi|)B^L;B6=4Twq`2+I+8K^>Hc@Rm7Sy;wJaUWCf zZMI;DR&e)2UqWVCV2~AWT6%g+YAOr7r({|yuH_}!>_|I-`zw}}`w13d67dE|3X;{b z(ud!>u96I4=+3BD1pL3(C-ch^^Z`{VfQ;+%%V;ckva zs;=kS>ke}Bolg-tx;9o!cs__%5FY@O3JVc(Cjg$CUl zHQX#BG?G>BZx9k4%G@6t>+S6gb^+OXna zjE?=u&uPSagy@W8f_% zJuM?S+EP|-UbPE)H<4Wo_wVf^QG?D$|qx=X$6f1!-H%>B@9-qtyJVZHJ2N)Mu}$S<^y;3TuT7ExT_Z<+C5Qhk>P$QM99H_Q zts7mu8DiCXKHa9j-+e}*Ir7I2Dz1X2Z>L}0;XR(J=MQ%VS6)B5npvjamZ!zjT3dT- z{iw<-mC}ax8_crpfyG%{a)U$zPjti^FHq42n$9bRhYO8Tht_MaAZ)QfP{rsqnf`I^ zux*z!_wpKr|Evx6Nn&OyVxE*_w+#*0-ChCKP^j=~QTAZv&X^Nvw*Gnc8Q!qW%KH1_L}(c+X>@y|8lu50F*eU`A>+Bs6w&bZ)3N*sGP4~)rJrfl(^Ivv*f z^F+I5f_~iV4}VkRQ%*5|oc}xSB*p)A_ue}Zf6RZU2IzXJ+G%ZBFOVWIxE5Y&vG@HS z>5Q#k-A!ChZfUcg9{c^Z;>`zS*LqJx6pHAHH9G%$mBK5lcmK4?{)eE|2?@Lt>U$<^ zEpy(B^!~5Mtt7PhV^ah=M2DZ>FlyRHf8B*<=d%&ZMu+eejk@7*udkEGW4;tPTi4kr zG6MQdu#6k{_d?ay8q4~LCT>P_7l*HKrwZxp& z+=C?BnQUWl#vliaaXTQ-nEEgwCUp1L!otE10)?u=pA#`730;J_a)Yy~FPa6y#=>^Y z(TnK##ful0f>$IwR*c8|T;}inM1Yj!!19}G$nnoYep0NMMR&--av|%*dDWj+zY4yd z=TA8Jh@`lOp9G@68Rof-kwurmZjsROB*a$h14pL^)=U)jOil!eV4+Y z?;J_`Vi+`H4goreB*!fy@<}99c0iJc#zx%1AG6pDKg-zz7$T`^m#^k;S{#Nr_ z6y3hU_TRP2)N{CXc;{-NW>=Xfdq&&dS%S2nw@jd!uA*t}F?WCETKYjPTI0x|))?<4 zmqGP^QH#dw?wiF{pMDtj6^J>3`}D=Qo;eTr0l9pLAxRzwAwJ~?5Oas!6Y=E8M#W4p zC`%(TU=aWpML{P@m@0@JNjh+jB@?+Z+jnq%ay)_D1UwAH6yP5rX;4@32WT%Me@FQM zG5iZ4%t$xsX}LU4HZgtQYv1p?XV^10+ll|PiKQHz&h-A~a^*w${kFT0>wRn9^~}?< z&->*C=uwGlN4~#xs1JJZ;#)AI?qgl=soZS|G1^56;rqA;L93<>sg zjTV4}ptPr>;2$BgpI5jZLr?e?vjM46>-7a<)%FEFo!`N369P|ZEUKb=($;LGq9J{} zpdcN*XJj68)}=oVn0grZs#`q>CCB=Q5Pob&z&o-xAS(7kvNaiS0H&2B9m$NWk)0UQ zi{8r-nDEMtEoyoPpG{c1y~3W8{@X1aPu;%0C4OBBt**J})w|^p0W?xG+`H=!Vo^&R z4Y{?W`v*V&`rS$Ssc+dvk9F^PQS?38KilG{$OJw0x+vLWe`@*afB#X8Bd>d$FwX?>D^$Z41@ocEJ;X4jcf z%Ya6fJaS#W(qfyaIVCT$rml}gQF z(BLnlz!irZHZwz0RKnxO@@RLcD8#pO+qN$pyVZnq(T7cqb}>PsC04XhU!2fE{iu-) zl*AhGBc*WV-~c7AR2b}to>h>;@eULYhoPmV0HlqK0)whQ(cR?~8@dAVFD+H*>gpn> zH*qGvrn@WP+BL(uvh3Is$B$ot9FHRJSIo`ra)XWal}5Di%a<>+#uE|}>Td=gtqS!^ zS-(|KD{KGg8~>ChwwTxig#d+1Rfm4_t6T~t$B6gX=(A~s4?&0IF4~#Se>r5j=1?vB zWih2~&0)^yrYI`Vs$#|wfUpqG9UA~CVNmb_gGf@)HzF%W7|*0y6$whr7-;yIM3HJp>3)!dj}h0Z8<)wRM&H zPFCZRZ{HlCXko|=S?x}>npOYSwTghF6$+IVFG4hFJaHrgg+TLMgKf&_m^>(E$N3qO zn5!`{$Vr==G}_+ex|WCM1r!@dr9d_P9If%EuV2mF&-BTmZSTXHrjTHSp`oG5=g%+H zZ(!FCsym`34SOJ8{YcfcUipVNqJ$aj8$-NPwj*g}dy?a3#jv7y&gGe8c|oP)5M%G z?;xp{!RwI9B{Wl42eC7WApxz5IGTY0Lk~}cu5w=<%>z9guKV+GF%Cd0(wvm`VJahvuUCK=u!qfn$JDGKh>2xc7 zaOv#maGH}B%vmt%RH-#9tp42;s!2mfOWO;248~!w*$a#7Hfj>=u0CQo*#of77KM(yl z;dSK6FMlf&v!gpVNiLW1u@N}Trrh+qd&;|{^=y=gp>_J5jGA#PDRRoYxJdga`ntHC zJxemoKD{gjsfgs=l8hFDvLTuHC6>hm!2jGxEp2VX9?9(8*NWdH#m9>&-cUGT8?RV( zMQ9_y-?Y#bpMCB{DQpHTkPhQpmbNxUXQ6bl-cAVtxr2G{NYj_;Nn+m0m0&kKHHt7S zRG81|>T+II^uQVi!89ugW`6X{svh`K`pknWKNru_PeJq{E>M0FiIoyYvUqq;1O1P$6b{8C-v7f!uH-# zx2nq))vQ~+MSV^Cao%9rHG;aEat!bY&?%=q3Eg%4Wr*tCLndM|$V*(iME)TP-)5Dj zQe88B?*{|>bahR78Y`ZfkL?h-qTjNk>Q#k<&w1QSdoiNYRyukNLd#~%KOpeHpn(cW z=}U{|I%sAj==#{PXJx{nS;MszKBaDZQV)Iqz{JY>>KSNTDvd!Au&*GpE1Xl8$uuO= z`RrNZK!iFWT-C(3)O)7jmpeh@eaoIk_LO;>WN;5L22n$HcJKauRmjdTdPNhh#<5RB zV;QI^5JeH@mQAvKTK#&d2|wYZnhJ%b>}Ri=zYff52%4kW6;?&~Jm*3cJs+ALI>^0C z;pf`t*PaJ?jXx=qDtc`Mf{yVEdM^@&*UB;pEk1{*If|}UnOxj9hb+U*=VGm!w#*dy zU(hZZj4*6jBXY$sF?#ycmd*Qau^};T>-M`PzS4v(n)C4aG?E-!RF;S+t+^&EiAgsW zbbuAUauV4?W^<5G`@_;^!lOIvh4#*-*QGILT3j)MT%4Q-zxCr8`(}T?d>pzm+c(1V(T}q%G60E?m9DhtFfTRB^;jG zS;bl1)J&z~fdUNHz9PENdoHa=Zv;h$323Tl)o0(RW-Wix^3* z0qAce7PFOjlKgVVr%AYZGwV$3mkU!*VRUo(x7SCGE>B&_ZDC(OSUKxAZOUqvBHu<9BV!r!iHeTC zWsD+vcf))@4!s;zntsU3YbVMWasm+7Z{Gk9jD$#dM^?hnzXKppip+3UDun4)5 z?%ck8S*MN4%#S`0Gt?4$nOU{mLM+?o zk7E&QtkV{C)DrjHMh;R`^z>HYfHnSH`0Say3D>&<3CRGI7|RtD6cjyw(3Y2%SCw~T zg!aw~OHJ1!Pdk=toeOpZ=aY2`Fu$44Rdzx<>sojh5>4QdDnO=2yD8z~pEXt0j06F8 zPsxwM35-!`#`r%cL491@eXyGd3jTb}V`*t=?7~TCAwU3LTV~W2eZz-znl=b>juNL% z7U{iP3ch|xLB!V9w^5?{5gpf=A0C;A{Wqx@5X;HD#gxk2f-+tTUv2CF)pX`Ii-cJgxfrT;>)K3{LFJ zG}}Q_wgwzNbcZzl)FuxH2LN2*9;T>ZVFuNp5+G_=0h=!+DJd3dR&d(Rwz+|lbq(eR z9#3%*k%En4i6QjW%;$F->K$ikDseT1HIO_tHDlTtY+Q-sCr{okPg1CVx)myCLyOAu z7}5*YY_c_*?mDTY_rqm;u9H+MIPmjhxlx>D<>e`{3wIT7ZF@1#H(ZByINBiF|HU;| z?gr1P!_=nlVlj5`N}O2pwn;|=julycoqx2?X20f`)C7-->jmI=4EyOQaK>Z9Dim|l zIeuJ_B*v91AA251T|)u@${v z${O@NiejtK>T}ZG!Y%dXlC2p$otOs;2O zkNSeiPTVitB<^4_?Mn83=2d{_m2+mWVDjMgr_gFB#O~aPX7cu-arTg>A1)_w5B2pQ zm^x9v$4C@Hn8d`*Z4Gu*7Y|a+9pt*5ZVj$JQ_7Z!t)J5L#!vRJJb3>)dgJOhi~ZI% zA`l}!1EYXg?xOd{U@yuRAqB$>+}6--kF->1kzr*{BCwI8?HtWA;*)xUg8Q5E4{x7p zJAnuph>uO8YUr0Oqw)6=oiPXUh>)!pfWP42c=+2$l9Km9C3O_%N0mSUilSd0sa+LN z@gj&QJksL}lXXHv#L_0!ESoe`+@F}PyT4-Vm+m*WM6}pA&5!zTZYo$lrGDeey@wwR zhBmpK*r$H|qD-^)v(8TBu60#n)Ixf9*a_~ouvmwswniR4nJK%7u}$`j_Lgv9?|AM= z)VqS2)QiT(6*#oWyTjff9NQ0Z#F)z!=qyp+)Rz9_en@ibBd5jXVY*B|utjNvliKrH|u zBa{70mzDwVXTma{>IeA~$=j5x4`_Y5zmxZ`uXdc<{U{5E-T0%sG(0ji1g=z#^`rlz zw5iRmh`KxR@a%V^tP1|$K{4$I842^&V|r3;|F!rZfr53Hz`ZArQ7og^r=F9?j7siM z?X{qgArYkJtplSSHl(wkF5wqef{9IBHyl1b;UJJcAP@=#<=Ta?Fzo;-U?7?0ly2}P zuz#yj)PYxgD31(SdDAOuN-@?DMtI)qsv>X)69I@F>-v z1@*kO2Wh%H8}nO^KCjSU?8|kv*nL5`mCNl&;Fos^K=85z8vl*1!x_bU#^|r#I|8V1 z!~frnM|bi*O(j5}doUxGehq$?(VIR4=?Ucu4OPckp6M*(*I6MptKMj9MNm(lXU@9+ z@|WJ0%KOQf541e^Y;)7gORVMo+rVFgj>6~?qmSWWxNw1@yj-+FIUgZ_L#lPAYYc(iZ=9<(% z>uR-I>I_q9EWv%(PNvl2$(>5X+QZAJQ>}~2;N>*~fE1b^z~8iliP!M04h!SOKfiJI zEK2W#WHr+)y5#{8cBE~~is2hJ-X67amA-XUA|%GW~8BNbd?1Fy%mdF z=0{boT@`P;zjySt|BAsoH`-hl2X^qdAIy3ubN5oplSNvkq;!+(g3;{%#CI_)u_X5j zKYOx8`a$7$^ANp}j}Pg&+hwmo93jHS#mKvcNyUpWXoVh8Za=zT)v`^2#=Xz)+~USo zab|HscgQ-a2P8yIgcQ4YWl6sR-}K0;|4xT@mfj5(6casy8OSRD3wI?p^K~e26FOVj z7aBtH)?!LpsT^5l!pS6eYY17c1UvMVwHQM=~0rQ4We^gFhHqC9{pDa!uyqb}u z^nIg_UW}Zd$lc$q?aky#&1$^C?>tkW1-t!zdg=nhfmazp{4aM^g6pfNC6@Ll<-p?Z z58RI6iy3)!WkRpM^iC^$ykFn&5B*DBz~A`rKIIR-^W9d}(Qf7IWZ%TTMXcX{ zGKVPBx*R`1>vAh{po0Q_aHbK)72c<>C_`r~0K4qpownlAl z*mBDGMV-8_+|=U^%b0VA&VPJDuks-6)-FYf%`7QNf0%Wzr7%vNpZawSnp*3mh@%(F zBX<329T7Ume5DGQG5L66VGn5q8Z4F$O7h6EJ$lYxzVxQ=(SE~{8TmIOTDsKJKD3_` z+A^(RiiT^}V43rP|p- zV_jCim<8fQ@dBa}6X<61vU5EilO*gN2Zq=!u0g7=Co8w39 zCk8W=gAO^qeQ)~Cii7zqGrs3ZyZ>NQ;)JNsOx3*o#y@B-rmSLYS54U@7DlnTsg?WP*1hAm-;RKMlRFxZ#HX% zQcXwD8^s;>9rCKoDaS`|$OzP!?HY7FkqsSOJ72l_o%c!PqyPG^?$%o1rODGXdMx+s zCA3$QcPuF8ZGRx_LDR?~9xIiwL(;cAJ<%$iD(&5p=8lWQ(y^{AN{e2M1D))ubz*G_ zx-{|3S2teUurGr8hJ5H>F!$x18ZyZU(dF9mZSLOJOCEQ=D{!odKDt-u5ObG%DZ~E? z=raHNh6NUXj7A)xx)rg+=fS3MIO4nJB1QwMj?oG1XWeF5mEUZZFXmmoddrZ?$3~SV z@S)yF$92ZtJb!X0zlb8+@W;_oEAjsPGa`jja9Sa~QwmoD4a3 zrlbCPux3475`E)>K%0(3K=F2(o#rKvg~2!PzU4XLH@2lU!Bp;9^l9zX7h!YRY|XVP zv>1+HF2_s8{Un({U7aXw{@Parz^j$m+kt}m9F{>|US2L5ODAWvZYzzz_w1#9oNRo9 z5&vV&5Qhzk@ z?JI+ol2x1mYmCGd+tg+B^v6L7=|IdqW)qBxl3Mc04d|jMmYCvGC;&L0PPHALg@rTG zPYey)H6X(kubegb_zCUwc|5gjL1dqX)U~~r0S(8V-8bANTgZ@GP~t`m)wkcYnxUK{ zXBz2--J;M0$scoozmmghJM|~(roX?~waR{?TW;i8+wEA693jwf{m}^Rjuv*y;H6AX zPGXo#2TducNrK2$$7DR@%9XvypP&dxNU(HwcN2FoP-GzBkR@u%4I4JNl!=hHmwwr} zZoI)vF595=!x`LcNfW3?tAm(##Ww9mel5TbUkdb zuBXG4Nsex{X{#G+ycuD#>(5=y4}ha=2nmP+6-{sZ>-_wDsZ-4Y#NZSQSOa}WYKGzL zI%=RXB`yPuBm)b^j-Mc+@|ga%w*1d8`vpuFZz}uk+HfE-E4lFRjuwR+YGqTXoMg|x zKj-~qU>i%|8SZ-#;by&WPiEC0d5ZbbEp6h%)O*@~QcoIqr03E^a&OZ#xqMio;na4T zGi{!RxUmK_9?^SYoQ_{lrKF@p%x1>RX83=BB{3}0;ppJ7)v}U8@LfnIYXF}SsDjXF zXU2PK(4eudTNepI<^9AS2J)n$!jsLHwXna8&qz=-Tl!wJ8WbZAJRAQX8M`i6&4 zsg^){DT?|Z#x-WOY+4DzqN{Hjyr6&cq%W}j6U(LkjN9uV=F0LB)!voq0~`(56#=DZ zM~izfX{~I3UqS5oXFq5H2@ zR<2l~4rXkK!Ltpxr?ljx$OT#m_}WboQo5WG7k)PQ{%b%m84pH;QQVfX&29lt zl0Mh}O`f?cAtuVFa95GjPUwStWm}gX`R+C?oayc0Ej($t;^_lE+)^8_ahkbWu`J0r z!i~i8Joy=x87km}Fy_s7Oa*M#541xs+sH*@>8^;@xaT3V1j^}xg%Q7>jw2t#U<;% zd=MhWb;Uk)Be;VY{}MwrsB9S(`5~$Tnn(iS$jlewDYR5kQO_1_m!RXz4FAJM5X|RH zi4O($gUxQw`_|wx5t|0Kn9KXd7+NKqdR>nqKE?xBqbJ}+4E?DE-heV_9Mf)hHU!-Br?>Gx_L z7_?qQBfFIPjQ0u>iUt^d10iq;LVZxE`s4Mk_U&}vKDq zDgH3yHdaw3zezBzDwy&;vWoj(6LZLZylAW1hRY9>H^2Snn(i{yt|-u|n!BH&eXB9{@+b~RQKc~MjoFFfrkd^+fEtQ7?4BZbK`jBExF^b(^qVT6?bb1 z=gsO6-N>52YwW9%S4*~jcz5cO^~W2#O|qR8RM?F-U77z1?3$J`@%=mNFuVZ*;oFdK z_}RcVmssE8q9TbIAF?i?Vd_!2+mqxSP3{&}+w2$HlLZtdjkBP9)fa_0^wOOWIX+*@ zm)rhwYXn4O=&tGRSGP1z)jRjdrFu9y_Zaa%krsTN=FBn)K@MsIxbm&Q{A&;JUPUD( zXH3Q^l^}jIDh^^4M4>1slrp|vs@9A?{R1oq5CMtTLxV^Ap6^^&7&@iS$5hf*d_$Bq z>PD?5mAl_^ko2a?!vfx)LI1!++&3`MThhLH6pS{nz%5*L|BcMp~xMQ7hWgyqwsnU*zrp&u!O|$s;UDn(P?%GA$TuJYR ztB1t9gmcR=0#q_?M;;4_p5~Z+0f=NdiaeUDGTwh%UW59-KwW&fQt>0yQC36X;51i@g3wozd?ghCetU&zv_e(WFUf=;t z3=hyLyiGsoal`-m&;_{`FE-(qw`(}F^jBcX(vW#QQj#t}dD9EhQP!5p3Cc>2zTKn4$CBS5YNyqdK(W)?dEv?;R2^ynTi-e1W4G zgNbXLaL|!(@A)9n8h-Z5y|LGBPWPQKl!^M%QcJ&+GBh%RISRiBe>>u5;HP+tdg&x! zeI+i#;_+=`+rK`pWhagTQ}@1PJ^NGkm&;$<)m;zKWXi0j^4>*#|83Q;UOAPiV4R!u zi={u67kox+(r)LrR@L!ue$#UM7G{C62?k?-^ZCFrj}6Fs$Th>UFO`4GhNTL%Wm`hy zrCwW~c)j^26*AZNHSXFdb{%(CJnc&aJ@IXzdL3NHrp3d{u=9e5N$he`ly0xS<(8_Wq~+?DIh;1Q@CM zzUO(nHT}(U)DgBTQKg_n-5jgD{O`j}-zO@^>EUwZKW%-cF||}deOX#QcGb}wq{v9A7WWTG$-a?Z!F>C-_m-!;@$Gw^bU5c9IN23 zktp1D-K$dJ1C`ZgPifnG*Be$7+$q30K=os1-P&C5g5Wesle3(3tb~NraWf#*TXgEZM)8#LQ_d0F@)MXegt({ zxu)dW!ZQHfTr)iR3H1pp$y6|m(^Zdny3?&>2ThzRg61;=X=P{y zc8cUt%*==AHmwGATo2-dOZCdV9WSg@%i?#- zc})FK#h4BQ=p0_z(-&hvOnuhg&V{_MXTZEI9URQReTNkd;oV}%K#XE``Z1I=fv6d| z=1M|CwW4rajur!Vqo$CYEYRtKuU%uuCnHQ%$Vd2q$`Q2T0$os3gyz0|&*9qcJbtv% zKpAuT7pTLC69XVhOK9TYE>wfx>VzxD<@Zri2e zF%AFV6RJY1#iT^?$_R}a4mqiMn~-Zh>r!uRZy$!gu9*A@jcg%4_>u{U)ATp9HERbJ z5QX=D%Pus0w(c~s0t2g-Vu{ZT3jK{=QYI!QIE9FyW)BuQ2|I&@LPgyN_}>D#yyiFO z9`?jPCIcmiZ0FC zESKCh{!VjjO=f6IpX;S8!~C7CIn8e<1(!JTy_HnvrS2?H1c5>>0oMm=C?3&_Df<24 zh4mK44bOFmDd_edS6X|G>ogJS)zv+I4qk%cN5v*<2M1=P@4>KH@GmVo5N))R&i~Gg zo-25+dVt**dZSqYna*Dt)=tetknjAVdBE?c()y?in*DqpPLxgqAUsh8tX)Z6m<}xM z>OLzfG6#9|R>dSV&bNbEm<@C4Q_`#0#oLZlQ%Q8Ar>CD6nq4Vge%~}pG_^9lAgnpt zeIx1VcX1al+esnVIs{n(DBFd#h&~FTnc;A+vcfVY!Bp@p&+nG;K|4$+VGY?#(ZUSy zqoAkA+P2W*CJ~=30R)YVmPF-5pjk$iTh1a<6BHsU3Q3>3b$`O|5jpRD=dmB|#wC-F zbs{!~O7W=xsjtKY3}ZrlXJ#0R{zUvfcKZyu?lDvxUbv9q{G-t|ZQG+

N_PlTuN= zE!+aVD+>>nN;KP~DSFCwbl!U~JH1d$H{9dC!09_O6|+_z_QBsQLn8#($ zYhVTu6Z^$2=_kTo=$2w&HFb40z;F}Rx`6l$W@fSJOM|czDFh=&@%+M?Cr_SanLXG* zZ0GTWArPM2G%0^>ynI$D8zl0oh?JS{utj_R=8b-dMg0FpGF~b+Bx3yLrsE~3*zL{I zSJ~eGNL3m%pYL!eE!6n?pe0#d0|T}B8%sDiaRtxV(aP*m0)lLtnz`I^e4~H!s)Ljl z+o(BLpb}G?IX*_T0}!YM{kpVY`DF*E6#=Fm+n%LS+VWR7x93qibj}e@%CP!}@RjI3 zf|=J|0HZe@HtEl$z(XAVfcvj1V!U^vk4v?CaG>lAj0BmO!q{paV_CxC2S#mw47j*q zcEmA1i?c#oiIbEqF;ruI2aM8tODu8gnPpvZP&5E34zrMVx2#WEef{pO`ubHhZH9Tr zDq(~AM(Sx_MqwrPk<$+Xg3n+?xF={s;PL0L7~tWzdU))>*v-z<8&)$0>AVUN+n#Tm zT3^-CL1;!E2-JF$sl`r`@1>>Lz_lWsAJ_!Wu!kq`EX->Hle@!mm$S25EU3tU?mg}1 z5_@!+KM2*I3cH;5acA6fK6*PKxi}Q4vYMK-wl?MY!cx;u_w8|oM#&jl%h2vAFF*)& zs*$1;qt}};sIB8{#CJiw%S^37XML9Hz)AA>?+fR8@PP?M|PZKl{+&cDohuWRJ zOjBbkK1x-*_1VP1G2tpALf_G%o9FAoy!DaTV*70ujuigLH#O=^MUKB8rN!?q5=_=F zx)*Tt0?>a|8{e7XcTAu)4z=H1NAf;l8I@;I!Jpzg9rx=d>~W?XeSh~YF0cJCbYMTn zVr3oQolL2H+)Ku>4{I}+`V8E6{!-yHnftNIHWkGj?+Qin*5GZtj5K^>v8L~|q60Cg zd0thb9DTe2Pyg8>z^P9kg5uCcB)_p z5)Li;`WJ9aL%@XTJv=TMzqzTT%UIwmVfc!&PHZ8mN1VEtT7x{98woyKS@UPgcz^Y@@;c7 z5!hl5LzEcU(2(FWK%+yUoIH7Qxc|qN`eX|im;1lm|J-`-S|e&39$hV}r8WPMOxE@3 z`oM;+!aY(tKFVo#WVG~ocle-OOc0Fo5&Zk7rHKh`15gtx3g+`K2%(YdW+_5a5ne=s zm65F$EuuZ~YGXB?Ux+L`V214)nV440-aj0Jp1(h&nfY?oV0m=;)|3!du}JCL}3Y&|&udaX;e7 zp0+M2ebq8|v!!sD3tj=_{477~@bVDWMdh`f9+^6}n%7`vq*}H!>CPRSfOnmsN<&$| zfVv30)nl+zg#O^VkQf>>LG6fT0-Kv$K7;q(@a?T>wVE3Dm)Y3Lw+YAdJM~gu(4u-5 zAk{lp`Lm!2Ce@E^wVcW`zt*;?w~>O^OST}O++|=N#UIzm`NiMoAgnU$R=K6uDn7SF zcw0tGyV^_V7-76|t+4^yW4M1O;$4Q;n32>zL?=Mh+1TAQdIck1gI1O&&W z(fY{avBb7evHXNmfBVBPH0%F_nJdS6&ayHae77QWLNyKlALLB~ zd{@w{K!g8WbzDL9b)DL}YfoIgR|!%ofBFRoy{(E*mfHHmq|(1WYVeSstCIW24HYdcq#xsP(0P(jQ>?iH2B24`jE3e}p4 zpFa(oRMrt`d~P@H^$5FnF4J|Lj~ow2$#jDvoKvoJcBt9JdkOCm-l9b|naGYc9;?vF zPWdX`;t$^TSzgcj0h|L#xH{l9A zI)6FgSYLAE+a9~+u78Vws}E@{XD+E6%rRY8!gg;!?8#&Iu8#pz+uQ$AUfeB04{ifV zcdt^{#9rUt!qyE#uFHIak6cu5DpfX_U6d{&4*7Iv-CR#H5kZce z@xtDy_;^OBm?-{myg+Mk)RGyd2tL!DHWZfw!w8Q!+ItgP|6@pmqPP^KpL2E=tx{NO z9ImiWQ8ax7No{FRf6&^hsSNI$5Hl|6yjf#niEbZUJTjoc+?aGR7?DhV>tu(tcdw6+ zPrzW{e-uax_FfChiHSV5iWu2tVq9c|UumNb={(ED*3+#c+4pn<+( ztJsCxKKL>|~^yU zL>Ve0!;TTbC0O_%h-w)qeMBEpQ?m@(T*&NX!BhMhB-hL%XBP;@-SO{zRX7MWFt(Ml zY2<}t3K`;(W)}QNDhlKZs`zG^UQ64ziIWPSV7zb#VFi8-K;T(z?Q+7xg};qI4CQb) zWzR${p&k;~{iYv#I#$Nl7qJdJnfOdigc0v+s~0E*(o)cOLORy%^K(C`{V|FN2@emv zd-v`yMdGMQ$dpj$F({RB$t++bPAu*AhxzhBy)`uoJ#=hiTrm|&ALF-=!Z3;0c;M!* z`xfj&Ur%*Cl}m@sIDd(&X=};tL`2xWTE_zbRg;J;KZCs7p5S>2mShiKwK3G%@y!|_`Oo$N>7w12kSsfi8g}b zzLCUrNJ=w6GXP$i`Gu{85&^|6OyFN0inrgRVI@&@}ELi_Ohdaj|MF zI5>FtgNP$bY;tlzx#NiwlP&VlA%y*Yf;b)RNJN{yw^}M)*m*2^CeMB85mD2Ms_c%W zGAjk_AqLPy&Ta_Jtt*_GsWaHw6tG8v;0J7l zZAUCWurrVfZT`&_MI49+#wX2)91`Nyll@cUJ?mhE=9e#pWS*@^VrocMDh)RvZAMhJ zhYO#k?kf8T>zSENQI9@&`)v`{9G<`Jb$e)h)@#r-)A#?rB>J4!eIqy1y7St#YwwD+ z4~>;=6&WSmj~EjW>;DYSXlAjTcD!_O1?LZ1!IsrI4wj2+i12iLC;O>Zjbt6}4KV3v zUyKiV*`T&GfCj?&Ly%XH1wa^SwLdKj6xISAMq;|o5m2mF^z6kY;P>HS43`}fs85a?14>|Emo)Q#_->zTmgRlsE2g2QzRV{y-Va8tHG4R&Uea`r155hQ<> z%~t+3B;@n!>m4}L6ij@hI6(&sF>mJ}rT2dm?*u6skK&A`_(Lz9fy#Oy8ispfTrd)> zoG;LOAlvrhGe0KrEt2$JJ7@1%eWaGlp-sOCguDuNAs*gLmUcq8b)w^AF2g3bZ}&Hv zwCP@L z1M6D^0I!s=dhW>@P=0f#r4zI;kg`%ws|-5cR6sr}Uh0 zW`efSI>s2o&!tz+>F_<+X}Z|vb@OY&EB3sH%nklj%5_N>>HFAy#heONLk2oGJ(D@6GQadIE(~ngY7T1G#oa(TqLGi=2ZmnwZ=U3mx{D znpSno7)D!=9(`&r&5>wf+GWc~`W-nX>EKrdzpuD!2!;h)Mb1=K|hM;Jpr`v}x;#4b5*q*|Zw%C2Ib`N})pe z@OTM$vj>QkKaUjFLUAS)5zNa$`iqWcdm6ihHp#v?wM+)tCjsZ7M<+5J90kvl9VoyD z;OgLIi7_aUwPKyW;8mT9%j$`5j|dNQIc37Td0g|g*Y{&Z5vt2x@T`cHigM3^WyZkmU~DPuG%E0sOCwyfP;n)`*+lOKosL=R3fWkVw_^wy(Qtbi^#}+*@t}hfoGh zDBJ&RBuC!(7(MgRGsfw4mM>njbgaE+n6>sR_py(MN1B_Xxt!Q+ccQhoIdOvEfy51; z;;j3~98tU|ih|MZsz>mK68~oli;DD&C2YK`e3ixb@^Vqjo&(^nbE`!Qe|uc4ku{a~ zmR(L~_SjTCY+9r4g?pL}L-Gw&=W=UnYxGLVXDCb{Gzvq_z8+rp`uh5snwlgT4sy_D z1R4^{jDmuy@jEaxFSLETfshx;&BIyxic4X9+k7W2PSaK@rR}p8Y&4z}9j< z@O?W?Vp>+8j6*i#{LCsRyK>XFGW6Xa8I1ZHhI_{T1iNi-2h9vb{IeDTf4dU3EqjD` zh6pzjIMs5kXtJ$v-lm}gAoEm!T3|>L0|_F-gY!J$#A4Xf%5sie34;02n43OHqT-X( z-T!&9Yu&T&&r}^x^he>aI6d%Aamz?Mp=c!}9A2CGiYT@w$9pOos)g{D2ag#^pP%yH z)Y^5pGp*=ikho{a39j=jJ9ezM(5GUar@Ln_q%FL^kMG-+y(XLtOY~D~)}8HR;;3ep z0L_`{>1q^PfMc5Byhm0AOwS}HJ3s>KhnsX24u?X&#q+#MUe`BbMgVB=h%m$QAFTG~ z2% zY$X(Mg@`u&Ym}u~`trZrs1VK2%C9wdNIN3**a%}0R^v5`m!x(~H_4<;YYloi-IKl{ z8s%PJckq*orcMC2Z7B3=gPR1tJ~s917fQW$2_S9lRLIE9Pck%S(Ip=*YVfRl`gRt9 z3^7#3oD)Mn&$w;NU`)1wSG5;DB1ZB4E)DmjMcW48C=O89b``~$G- z^iXE)cc40KL#F7_zgJE(t=qMBc+K;&fYR{LoaEVA4!r~LQ>@eECsn#`?S7b0yO z*KzJliXM-iA0;+=vk*ws#MoNWqC6a9TB6Xzn0x5MOx68p;x`TfbU0i`K0WU7`N^_t zwwDGfyh0sn-4)xow zc3eGOZ=I3fs*{f!`WO$X9omQT3hZq7+HJMt3$MP1 z?Bm#Y&K)J}T+-m4Y`wbc;UPz3tAF==SaSIR3sd`dBH$o~roX%$#-`IGwkyA!5Ts>m z3%pAM=bWJ5@_B(%7 zX7TwO`=X!?`ENn{Jn$FKqcfIqAEbvwbEg}81Rd`erXPm2G~0Zh(tGp735+=R3&? zb4HTq`3XJXGthPMm97(w_<*;`}gMXI#oA26|zAo{ZkC zw@8Z6cNtDg#$-MSRf!+E85so?0g@mjFvE1iHE+m(tT9DWC7 zA321}ohb-ND@X%Ela+VMoQ6d87TLW+y3!h%lguPM2@RZ(xHvOTZcLce0O-MYKN2$^ zD*&vZ;ebU9Tc@0^?@I)feRuD{#w4etY-XBqMuJ(qx%B9nxE3SHe=NRp=W6h<-XMp3 z4G&KSqLn)CTQfnncNs`IIU9fyB7a(Rt2$t8qQ*uo__al3Z8sg3-`o#HzL3C&;%U>b zDK=wPI>y=tv=bjR1(bH@ol9W(J0yyVj=PY3=T4v3OSiP6HF+zxO}?l6r5Ng)wbUv# zFCuQ62rtm!Re^F$0?Gv=VdQq>#tq_BQnG9*@XR){LAKYq{0nq&Mpdx2>UHWpmbUHT zC%rX8Yih4&xggwW1*I4M%;J;ZiQzNKYg!5+ywy6BL@&y678Z&AKzt`ZyjCZLj}l+L z015lF{Yg1xO)|z(aoLpxGybzR$ICjmlB~89Evp{C{+ZiU!1#Yul=p8t%QRD^qOmOT zo!>?ZyjuolL){M1!n={yg<`iF>#rH^#*eL*+K!l`L)HSDN2`2;*s@H{zPOwvw&Tyl z4mKI~cQYJ%mTDQS(i?Kj`}+DQl&KM}atmwBaa)TKUGwWjMVCm|wdm+NaUN`w4W?cT z{L&9ZrhI+*M7~Tq3L-R)@wNeS^=OVAzqn>v5*jxZj-#RL!77EaO|)6@?nyls9xq3g zzl@OdAE|iXt}x^}2H`jnYY=}jKg4F?kfcx`Jdr)q%?sJ821t}A4lHp>Cm9Kwww-=! zUjt{(6Vwa^{zp{vT}K<8wz^&w)gQP9oCNK-ukUZwbz5?hQw-u}TJa1$hJHO1(ghP}?8wVH44^av6gECEgvRr8C zmbvA#E9J?Mp8xNVUTphbj_%TEgrio3i*sXqyFO!d(yEKS?AgWf1s2);B@LA~vkpfa5AR(>IT0npaps^`suH z0U}K!Y_q|t#p0TilB&6UV@o9)&%A@0P`x^7*)FGp{rs`i^~NZV^oix-5)#^q-7PI_ zXh~~OVc)ZFm;6^Z{UlOjxtL6`dp_RdUS-D|Pe>T1*Xc%QMz6&4&7p><`QEQqnzXh@ z1tPz;-4(@u+^=gA`bWZ!Gb>8xzD$79I;uB)i!`S?mJR*9h{{RDK%H&FZ#jnTf@hYy zZQirl;aeFiiv&*{yN={eFO|2?2yi-i-D}9W*yJCr2ri4HPzBW@dgUKK5F@Mc7D+O< zi;g<^VvXqgR0D1b5KmC!V_eU-2q=o0XI-&V+;>|x21Y<1OMDO}rO$7FV42`9IYUBw zimTXCANn|rzpbq2;Mf{kN;Iv9A5~x?#|X((JVTu`ba0>6TCJP2?P^`&PeogE9e(ms zm~*M5ZfowLP*EsaYP*fb9by&eVrX#0F0}sM_}JDLcS`ls3Bkh$z@BMES zO{gbDo9q>7@#0bnsQ4+jcfUPle2kj8;qxou2YvB_30`MZY3k|meG1Cs0#kx3Wo7+n z@eOYWltllj++?76I`@r!(_@GA#`{6BoM2(!BDS{kgs_b--k1L4!!Cl9xXvw)*%tfw-~?oLi(EXymYeqF!05K*hAoQ~H&-ln^T!9v+S=4tx@m#<6XtlX)Z4Mo`$J z%w@-}JptB0Ljkaj(x%Hw`+?2UbrR*Uq;4E29RIUR)8qTIB1}x^cdWQ7h2P9-Oov>{ z=kslbY1`fx@eiBw5I@`tijS#y*DU)y$HPm3Y(THnPV#OF0-HiNGHq+hXUXqW+?ewm zwQBbhH+xFue@iJU3IN}u{7d^Oq)z=MpMbmpVFDW)sk_s!P~0KV3aZnwyqc8-xT~`k zi25m_o5Sv1_~h22jj=Yj_et*J?T}}z)nOeubAF{PkD#{jg&?X!3+f4vSoS=a2a%>M)1}UOuz58kYZCZz3E_<=cEH=iDkiR>^~`NIjAY1Z|+Hz zuS|L3BdNO~PUM8JW#6mN5^eimqj~rN+`jR8NiR!&9u_9EFhZ8_dDT zw27DZW!ZMsJ$p!^5GDf%V5)?*GeGGWQ91G@=!-hRfVJr3%uC}MbGs)_!5VaA;7E-2 zsVT%Z1XC!c$V-8Nsi;###`X|1a_AHbyIj!4$6Q)EBv;~)%nSdXW`87(%~*oLHg49X zV#nKa-zRQt-HtRQeU7xcMzcr1<2p8}f0V4O}E=iaKKsqhYwJEvQDFJ93zt3mjraNm4b4^z=x(w7%HJvz75;`gHXhl+KF zQ!fQ&M=+Q%aA#qISEw2ymGp8CN)T2ZoQz}EzG#(zi zmzGFt*-h^fe&-{Qka$)RGkU^d#3bk#ru=*PH)X8Ge0h{NF!tHJw_CsPGbwr;FNB2O*Z^6wmM8xNS}o0!(9sePkz_Zoy6d-xJtlgoe0+V= z^fui?#Lv(O*YAMrzkoMWE8v$xWWgjV`|-o3SCI?QT|R3{bm+S~>y{U7a}#f_*t?&& zeNYx9mZUt*)E<6YEFDC&h2^X8anS0Nu={FcVaRD|n6lp~uUeulWYZ<}Z7AJP;m#ue z<@;cYABk_n&`HC{Xf-7jzEd9kSGI43-u3C+mjvmJe?LJGT84H9hMT6OWjKJtcNSd3 zv}yN@;!EKJJ;Xq2#)X! zG+W{GuDoN_T^z#@Qg?KnlNpY0k0D!H?8=$Hw%oR5|8*-czACKM!6zWu)BPfCnO(fo z*VfBU^V5zL6@a%E_K6@OQ&Ill!LUUDz;0pI*6rG({^+j-ioEY*76xMjXbU4)RlJ^> zj{l%*{(^?@RaWhn$AuExW{&+#3}L);T|vbbXCvmOz^(;Fhj3_F?HLaCr$KgLM8tup8u&0%Q!vr5+a`c> zM!w|F4o;2JyIAcY%G-0q9%v71Oh$vOW&6BJmr!9h!i*jaJC+%m|MbyLk-)jDzx0(L zJO#}zu_uN@PcVo*ynKAljmj^+fF|Xad(p3v+nqWn>6-ef;=tzN82R;c?Za`De^QFmgYh2;D7wr}nyL zqSx7dS!I|ZZQ?K`j6$mkIT#|U*@n&DeGLt zS^W!TshCE&cEAVks`$rB6JrHp=cvw^AL6Y};=Sa5laiMFLGj$yb63fdV!AS)sd~%c z8Q4#_9H1;C*_-!p&^zDo3D`Dwr_vK*qeZJRG-CC{p;QK=Q$l$dh{4&GaQI;j~WmV z8I!xzIE?N#QG{|1{!{Y&Ks$#}k;V^iW5fJvZ!d#Syxs(BG*vxi-ydhJkCYV5cZ9o# zDiGJ)vplJTac#s|*rp{uT@T?g2>L7YTh?q~Z%m9(_xGz+`Jrg=*@gIJjpePpq*c7A zYRVkC;pQHX;8)AGI1wtDF#|;$=sdKX0gINA+oHU^bXx7t7wo}ui?QB+>g30KsdtxY z;NHq-2xV@xAftojSmpzAP1ra%9EIBwHUJ@@w;|lGPq3dEvo@k=$598Gq-*Nbbv$Cg zvr+XeDEocB7635U5F&;lYnpnzI*(9q*EDK3M9x=@5-gU%c}g z`ceH8&uU!ipt2+Y*aDwT$;QSew?XaLG2(Ic0pulFm*0$dP8>GN0E_Z$Z<-WP)=h^1#f@%9(TyEyZ}} z(-u-H$C`l4&cohfndAs{3b6EEINu+JJQ1pE%kru!I#6Dej~_o`V!}t%*Ca&@)?7f8 zDC6-<^Ypk??*56z!OM2-i@(d;w`WqvMugd-J{I_6Y+f71M^$~1=9O7x@on4pF9%Ow zs>>N_z7(`zyMwxy+Q%6{L~FAdD@ITeWYf#s{;L8+_tY#_8qh!hY)4b$rtmnxoJi(! zf6THzV2Gd4QrMzb$%kR=tLWwH{k4%!(eR=1mJ?g3DNqT~0~xD8hV5N6v6Yod1a8FW zQe=9zt*QvV_#l}Ff;wJE{gIS~9dP5EcRzKW>bv6S;uP(6z@lN1S-~uuxiVKb5ApxW zq@R}v6ldMkRod=pD2`j`Vd21^0l5c;X6AyYo~j;6WoZ6+GUtfy3GLLt>*8OR3ABHY z6Hj~4n82lZ9k%_Hf$8b_Ch2p}ue<`Um&i)MS^JN};l__0UENZ-RsG*m1EjcZxcoW% z2}fWIu>zBGLrHhWTh3W7#|j|=CLkpxh4)TAVpfmxxOwk>IZK^yVY>nG50?dyFm!VU z1pJFBfep_4!?rW`Yu78UeIZy_yIJu?y9`~-tS*UnPr6PWVc%cFcQd6~DCoeR4TmNU zDa6)K*%+8mx%TXISS?IX#)3@_ftV9V;E)r9!yv|(y;v9U(%bkPXC4k=HpyLxOvhM@ ztY_@1M %ppf6ow`^S@T={X6yz>nm+aCgGE2MBq$5P4o=nj63?KT+2JyqKPw>z@- z37u&kg%?p^VB&QbJY(qAbE|kFT(Fez5tHXWwDAXsJ(da}k2ayCFD5b@; zK%Qkp@iWbuT~C!l6s_y0$jugUc_(f*O;aEScIaA&Knt#Ol^87|`#=r!9i4?)M4mIl zP#fGBnzw&6s(^VA9`nhXemzo1DJ!mj&J%m~7z9Z*UGXU%RuV^IX!oa02;FbeP=On5 z)e!`=T+T#z?=s@@JL1Uf+k67})aq88B9j-?wllLEF3FvbQ{U^Sdb=Wu)p#8j&yMDw z$A0D(HC)8elJf47N%a?)+Yo34!cf4-)YN+O6SnAANKPYw!w^168JS*`3zs@dih}%? zLEFiHO&`*X2(7X$+w>&E1q+Ap?L@|S`eNAy_n^B%xw(=rELqvV55Gl!%4o_rDAvFV zGqgHQ`GWhCCLxDRRQco%Ok7SNZ%YbgRBgm^0St{#q0^K=FqhN~7^Dy|n<$bI6HhSU z5|A6nG!UYgooZ9AAYS7>{=S=sC!QtV_Wvs4o3jVWGY5g`#O@NT33$Q#fGbW0!x-F^ zRmtEST9`W*=#z*h;QvBadh5T((n~+@( zh}GkUX>>(HsGdEx5D@)<@&`a|olq-+uX&7$_>p{DWC3_f`3s0Hoxzr^CQ zBbWbq<&A#v>$k-P0{jCEezY2TPZ*KK2K&)uX4|A&G)M zt3q4fjdtI@JnEuF5aA|3vUlgkxLBc}NszsP6(WhSUE+BQap5U%CHFI|loV=Fm8M=O zpcXcE+ROG-yMF5L?lFd6suAHii@m={IqTK?voS}>&f(_k|UMp^RAnZgCW&V{87oDzUS%^ zQ@8f{r@Es$#Uo7uUCk$RFB;Lh7Bh(M<{ryTVoB`yD%&|eLASqwLD6Q%_)I}hcArks zDNQ-Q6Su2l@@&2JCsHS*;&s%A<{768RB#Q#2O8S~6u}NR3eO+gn2~|c(Nz2%9FKj!Z=lTD3O9#+ zRX~P<0T`0pKyU@hp)<9@@MGS&hPJhr=+OItx zH|{33n8<2xXNzh}y%ppyBvX@mxz+Wzm&Sv}mz+C-0}s2$S)9|&*nV2Rtlsmg zJy|*_-a?Erd)3pO^oo>BGsX(JP2bhMWz8tm6r|bM;xGPrOiD$5E#DJ8Bjlx~$NZ0o zn3$Mx#4LjNASMm#l>#NxOE=biV>=QH_C^n8N8P_qk^(7|eBIlCU(GLUgIg9{y=az0 z|2p0OX146Xz9*?g6bh&&sOp26I)zdT&(Z3(~6=GCScS!_iM=Q=sP zCTD7&eCXxt?w|ENg?CMw2%FsLG&Ap)SnL{pP?8{it}2DXZUI+{x03kcwJinh5NZ=w<;-(cwWr>VJ(lWov(1JN^yv z6N9%J31ou9(2lpQ_)J|)mf94l6zZFle5`p+6B|o86+gP?ziC7C{C;id^*nMDsfKU4 zs}EM?c--oKrDmN^?aQ)8XKBK@^?v^~iwnGoGYPi)8t-sPQBw#=awf?UFNkAb`a#U} zT^D|RCVU3854g*Kp^s75K`*h3WJ?n*Gg8iN?B4^r|0kZnx@Pfxj&?6-qR8cYDf)Y0 zBgdBuPmr=FDse(zx4UdEF~}l?(%K_rVkMZ;Utp@KiE$@^h%=egCmKptbn_B zzVuSU{>N8Ow?++?7zy2be^QV}hK|xO=KqVc-Sc-|AKnIU@3uYDZ#7U~C|{SBpjiO^ zgnOFQ4=t@2&^(b z7kNalv4vtEl5t{Am-5M+zzcYTL@)*UpdmDlkP`!RNd(9sq@5MhMr5XUqaVMTf9<7^ zc_V*2U4hkISggW9qN~fXtNU`UUY1;A9fM>R$&W^bv}WRPS6voB(tHglGWT?RhT~q#Ch6T?ox@V31pt z?KZLLU;5LKV}9ZOJ8N@A5$4_%t}Tq z$U(xU_F={|%GU^bCt-nLImv(_K5GFTkHN*gj}l4{gL$bbc@_RTUI(~BCSJLEaB1Dv z6TLoPlaId_-6IBb1x-20Kl`lgU+CoRSnK+fRkUeo)5&f4K`UKWR`gVI`y_>LYjHT= z0PKNHI@aRh(W6g1x{WRg?Xn%lg@PUj?Iw{ov8RK3N7({FUgy;OOs9uV!L`i@(2k{^&E+7FcuCjr7^w&d3Ulq?3;902IXQeBwTw4Zl%c7Y8n- z)q0)o=lVE_T_4#`tDX6$(i~2TR7urwo(|SXx3u&b{Z)%eh!9U(D+5>j&KAacuM-Xw z{%H@|hkk`C84v6gs~h-(pXlMU%a=)AZvEJH#2u_qY6{AI_%&tkra;cN3L&R8lQHuV zAt5(VA<2jEku91694+oWtRQL~ub6}_EXr8!FNUllujKgg&u@pN$=Rb-{w%`Mfojr< zMnF^3{_mfw=L3rQn&&=u{lLcIt<_IIzG9oS+~Jgk#zz8SsxJp0^9sM?X=AH%KYQ<# z|Kmk$?1h&X?}Od}lK_p>ll(Z}w>1nth>9|iJ&r4DKzWOllvJs&{F@g`YXTe?yI_-Jg4WCmq}{LcPfloau>^7 z6f%u7_s-_1ZU+Wg`XtJ?{PzipeTdGOB)!#W%r7bDKFzjmc+u;-t-BT4Yid`G$mv!8 z#}{!JYNSFT3Q?V;>#Qg!1%!P=f^R8}r7jMcw(!`bqX_BCEk%4^1@$35&>q=Brv+T zxJ*M&i$A30iSqNC{M57Ukg}%5-njIz)vlk5P>=X^C^o4Xob%YCciw;F?OT-kvJk;j zTaD1b*u}oDmxqQ?7Hj-A@@4EsT8^=&<7ACsn#R$*2`4V@{)PfBBIJPMLYh|U4KTbg z6;d3D+9Yw!L{<)mUKRHb-1NQ(p~55Wt9o5g@$9|F#|L?3`^FAmv$VWJa=lhX){DFItksA`yU3(y(JH0nJ756? zAats4&QT?VkYJ&>S@jFaWrQM!*7G$?F>)~35Gwpc$9(Vo3+~_4e|H7meon=6MuF{H z$+0cGO{*@O8_Z~I`5vK@-Lzrosq$C>6~i5d>YQz_pG6Oz&P+GDP;D<{T6>z`PuX4i zv;1!DEG{xbf;I=c#H2YLUmD-f!YzjhNy#|IsFEA>8g^$#D3HofFy1w!cLMPAdq-EG z)yS4l0zA)=UtV5*rNP6t1l2)>4mcn=aBmChC4nsc;?3sbn4jjA&@CT+qL~W~7y6YC zX{F%ee*Z95q5e|`n`wO=ETg9H9Z7lxPISVVbVD;UUjP_f4fmz(&5ul#7{0zmtO{ac zV%7n=Lm}NW*vvwc^J{8~hN6|CL4_!;3LT_e?B)JDH}^8% zYjkWZKG)j?rDf-~<&@Tb6?2yH@gPEl4)6-Rv%BZoy9)^|Wk)>jH{E@y`_fekS~pMB zN)YiLFn*C=P~JdX@qiwK_Z80y2L>LC*L9$BL0eDpL_po<5WZ6J zDG^X9$EJ%3Rxz~MJC3GVB|Bau%u_MFfagTA;NAwg?DL6xqr>awZs{jFKyVXn{c-+( z7WJ8Q6ljBalbjX2KS*sBjm0%5>vBDMq|Fc9~pq3oVL z%*&}G$dDrmI3vV6E!2@SMIsdn{V4rXlhWxpcnNl(@S5 zx#f9T1>9G*h$sFhtfyaT0=acQD8x1&rSiuNv~o~ANsvSL_wUiYzyi9LCxTcBCHAxh z|5n?S9FL{Nf{ql3q9(suFK$(ct7H&wxD#t=bO{)lGY_3AZQ7T!E6W(e=1MyqZdFTs zoH-CFX4HO_UqhwLQr_*8>>a;5>%MkU&$L%1Vse9 zBfV}cy?mJ%s#Z7>3etlynkwSNi@x~-;4t+#St;YOeDtGT*m)ERw1ise5x+VPkmd&M z4gen#SKW>39j*t2xDKf(32;I|hf92N)Cu=kVg(Y>QrW-Sx~y#uk?t+>_gdAKR$LS1 zH(ihGWgMx|uK*$gqp1BPg^X22Xa_xG3Y-whcL>^jzyelFi*AzA-)G+K=1oi-Fg+<8 z!Bx7g`TLn1g*$Worm9L~bzk4DTyjr3_bq2bKx=jR^F?{p3>}9*Yf?UI$Eo}9O`8hc zTHLq(-o!nHQs~JG%nK_cy9Eq>7vWc$s%D#s2yaMiFBX*iyU%g#HR%R#X&* z4LdwDGp}a7XISZ2`pyzH`qKly>nGq6ML11%@4mCX@(Vt}Y<|!#b(N|0{ap3At>VnB z6RXB~wK6^r(-gW3o_MUlGPifVVld;YUhis!_M+r{E%xW>Em$p_ZlJ@xab;u&erOJy za)fe$B4F}oG5YNzXox5DiLfVmSW8c7%{FkfFlRmV&Wm$w3XD1uCT5F@oNT2As5y`W_47_{ndfq-SeN$I<|$t$bmLQj$BJUArR3 zQurKKVnx@+fhXdD$sV^+8EEoJ_()afBd%B)G)bH0)JFEFmi-<$EPfn8ifS!{n$8!BTcg+m#`E^!*nKXvR z>bI=bmwWfrN$>SzIx=eFd%K^e9VZrI$I3Az@gi^2kRQ*`Ki(fUbNWZ!9pRe5_53Zpk;bEicPnDa#~t(t22w$KL`jcDSx|LWQZV%)rzDJ@ zF20MRf@-!%4=oV7pN;U)3%F0eo2P3!>7znsyzN1t<$8#I-glyYsO{|-oij==IMRVB zJdQ3q2}{ID`XKm4y}9@9X+E#!+_cF|_PDzGyK~?e35!Zp=o^gnTP5u8daY{pw0^Nl zh&Q(9#X8ASg$#~JirfNC&fE4DvBMv3My?f|e%rX?!JvgqR)53^B`uS;Ex7oJLkA|u zfa0p)s6niyV9i95_s|`_`y|e{-K|I+>lVt3_3)b;ywF9?buCFF1uruP!tqQVyiWU* zB`#7_*z5QjSo|xF^?Vu}nhEc$ccc(n_y1P+m&DbF!?n5oo;>D!RmP`IeUx3wh>37R zHkSOy$8AkF{e*TmU*??887tPv6Fp-%e-uMsrRg=tvnj{lv0ALAdDuDT8M~O3Jl@q$ z!}yCkAVrBx35G)Rz1Adqbd7#1*bP7PKp@4ya~ObNk@hv%!Bac+BrT4eBWT4xpq}p@ zwI+Kmk=mg_+x40*i_p0fP%HUt(Tw$5aF}5bpGg}P)f^sU37dNhRT@l15FE6{{8Jjg zLkuIdo3q1D;z9d8ld&#Br(vg>T3t}J)Xq9?!2A=EPx?FZLOybj0E2WGS$yGZVYu7- z>(KM#s7J=!ejQQ>;Iwd6^!}X=oMg?J7&wyf+v#V6;l)b=C$?-(I<>S{ zN=C*QBf+TsS2Y8zX{&KF>!)h1@c&9$lG#s9t2` z1kFP2^{z3?{E-VDRGh9S@kVNj{A}6B_HhGXRwzA_syqI@>&<|ZW@Plr+4mbQj``+! zWzmN;h2(tyE}Sr+`T8c)^wN{OrM-}f5g!6<+j}yr$=Wq~v#6*@G)WHjjNhg@xw3I+>h-b!t5E%@9QEuk=gH>iplAiE9Y0R-2;8FE zE3mEh(SyHUG)eg*n+GY!Zm~JXRP?xYZaNquSbu&Mg9CK9xMM>`^*& zL8+a(C*F_TlXu!3ooOWE+t-^f#QudNRLAKen|>Gz`GZW`gMOUQ-85lQi0Xq}NSHj! z-`{I0iUjOgLP(WpAaLtTS0{>RcDCj7>bXCEs3^i(DHM8o-y6Xo6?|#68<5*2=eVo+ z)^j_hq*{Iy7h&`TcPK8ms6vipyl`Lpj9mn`X^yyoh`^uYP3FtMhu&r~CkxpB55?H8 zn6eq@Z>)NGTm7=oK_cpeH~2nK7`qYcg?4&9tQH6y3oMQ( zX9-|?aDJQy$KUBLr~TL3pu>B*Adbkhpk6mu*S+7fIlIuXVrx>G%%b9v@KzUZ4~#O1 zV+enS{+ae~4{D1{ZaKAbg|RIOXNtl&1g_h`$yt-zdzM;;o|-dX$8ygatNS}nJf;?a zI^}}ytv`c{HsP(q%G^tV&|p`)C_)EziHXJf#fCV0iO3uy3QJ>sXG)$hp$p zjA7?bp5}ohmV`WDYJCW^e`28X?CQ4-cs$^EL8hKIa+ z_Gq%7P*<;P%^l8r1@H;~Qlpg_!ulonTGhB@ZQUb&ETv)W^xDTqT;mT(k9JEL{1uef zeX3(A%Q!S{JSVrpS>SYNz~v{84cmX&>l$M*7;8Q_EV^GbVk-x;=AzL*5k*4T&D+Zt zP7%bd$@uTs=+Vhd`iyB|!iCzlk$afg)&sd?_RM&<7 zTMk2*zpPAS!M*5>s69B~`NMj&0_No;;@uzTV-4JZQ*^$$%dIz4h+CCU+VX!2kJ?8^ zwLLT4yqX~-r~xY*SS%X-RnCH)RdeR`5lN!Id3Ab z;C$lmjr_qt_O}WfVB&S*&_I7aj9B)JX?SXgSeo7#lm1K)#a)MKDG%Dc`ZgMs zo;mTbdbND2$JB;+!2{J%#>Q4Zhs6`Jw1oB~{b8I=Yf*#2kP_|9f8&8lYSG|)DfW~y ztGVkg{dZ^wcaX(Ku73yDqp;>*P?PIQ_3K&Aj2+4Jty2vHx%8qb5!n5i6@gt}~ z?k^*G+()Y6%Tk;sY2^44^)m_LuGifYLx)GM2<`&n9>X&jk_TSVMfIwEuGSYOR) z&s|D-<%PYy>moS?o9m(@-$ehm(0EfE0415wB{doL9q20fG)>>dVf zrNML6{xGpU%DwBBM&2HYig%;ui(~DyRAqDWcdxTPo$*bJitZ=GrHC{^-r9iU5{>#VUHwsw^x_17ACVG5fk?-SOVW%Bys2#% z+(CN3;&hr-0~2{_Vfoy9U-zv|dloLpph@+O`I(|)M~eD3Es?Xw9*c#=qj*ImjhAcD zvNc55IV`NoyxH{$`}2-o4SOm|?w9>1Lp^`SQxXO%i%%amXpHxKB-yc^p z*-|JEEq}cvXC1o`lZo;XCBC)5T$bS07q!=PHRcXf$(;+y8ba%Mcy@CQg0fCULq{GE zR|zI=_qOdKl?)sG)|WlD#v%{T(-34OS*BsEtEV@R><>lqlzKm1fR$)P1$Q_H(P(q8 z%3qUT`_QG-20Ky;JGwP$)8$Wg!P2y?;wuXyho|BXa~L#Il@}+<7OhrJlx7Zf73@}O zU6q;pf$4~`R;gz3pC$j*{?wv%C10sM4i3@Kt{1v|{$%=*VY{bZa<&8NFu{>A)_)Gp zJ}!fUb;jTe-2r`zP+d_U5loXoYUlEEZ^GQq;R+M`P~SQ8#F?l2E|Z`6Q@QxkI0XCI zpL#Go;Iv&3XUg5EWcDB_IXdllzq)21%Ia75v$>_ufOS30ICZi-d0o*d4Jx6_Ct2LN z-EFEH&#D&Dm|VvVfoI=D8UJDyzZFjl%MFi_3B+_`^ytHd8} z%5)@lQ8BjBw|)xaKXb_6FRklxx`bTXkY(hL>-^zq{zPB&r~ z6BII!*`J%fY_c1J@&Zf-a|EsIALvMk(bN5)tChHmrzl#O-!CUK&7q*@w=YnQJ-qyD zu*HY0lQQH)8K%SJ3`j2F!(wDpBv%b}2?ax5J^56E*`&1A$^LD7Dyxx}wnNizbpQzL z*eEGQtuG$N(9Q@x`iw^y`>!dl~(uYn@WK)%lMdG zSA}%$H3LT1r<+rz6R`gLJdzK}9h{d+WJ&UN=E&PSTl#VA(j{V0qoh=Ox_?J^ zmgLjnwepV2dhpbg-6P=-a=D4sq(J~1CatMiRzaJqHO3}oT8o3D;v z{-{bGE%2j=C+k#Yw8E-sJ$kJroj7$`6Ya)b?+!?8DJ^_eOu%O`3| zTIYwLeD3HV`VT@^i{5`VI^C9NvMKVP(#|WuIS6-}d?nkKKR!Ai^X{)yI>8#>(@DRy zI^X^M!b7d0w)gs4<;1)k3y~BtIDRO467_0;?@*E_AR%O^IW;t9kH4StQU1rthLdp? z3iXv5{(T|y{#NRYC@!wij>?nG+ zc{tL|;rF3TO*2`E{h~)2(hl@9DEWK`Tu593PH3SVh zq^xO!?`)R4NVBjs+@^J7k8zd!qm$h*ULa^1nzX-vlK{g5F29M(8_p|dCAThrN5@qY zZ1JGUH0`?_E%u|hRE?mg?@YOVorm=jSzzG6Br5oRN{{B_ju$P@=yaLlDX?-zdRz@dQ5ipabK^ml8R&!M)JF>ZrU2!B&08orz0UQv|G zHo1fIPsj}FspKmtTRy6!!F@^aB(wR7`E?vT+neWPW=IwN4AV9vQ1OXs06^QCu3uNC zP(XE|BfshEOTWx8``+d@gI_2qQb=^WA0)(`kOf%~)tBGaP`y#|vMo`@gAt zM`q;fticna><-a4_%V@N$Yp;vMdV>QewPmoMe(H43m|77O*se|g!=)BE+}dY5cl~i z+*=f=`}lERuOuvCz)VKYa?qwH2bT)T{cvxMlOvt7n$yC0LyF|Hu-$7lsalu!JUaR2 z?H7(d7SqxIFnW^|g%}c$y+lThxZKG%Q3MUUF9;-|!>!rXV?D>UVl*~&zT`~%dKPm* zp->n1@+>b;fzufU>=ise zDo;P+NQ4(n91In16j}xYrj~=(nEL&_v^#a5-QCVlo_x6UE^CkNUjffOWh?j=CdT^} zmsKm)zl+?Xp)_1B7Zp-ZJ|Za ))qWZG~{rTD#IUZfT|;a6WQ&8BLg^gtNLR4w1M zY)(CmhM@nis@F}C*R+SKg;zPmcfLtUN*b5!&ebRods-QAU8E5u)pF^&Q~DcFNFo3O zhywcLfAE_lB5s|0q>ALvuVBN5%kVBg>5F$vc4)z!FKXW{oN+>XKGfH$f;YlXSz`pI{?(1?vSmhl)=A-<90xacPAidhO;pU9T?dAY``nc0B6@%>hqJq)7AI>9v$C%c)sGrKj`uym-*Q}1lYow zU`KIdp3g}CD04`Y9_7{ZhyH z4))eQIzCy}kx$apr7cu+hmG6~gRggX+|EAXn)yjK+wvUG1L>=TW-zq*-z?hc{iN8t zv??VCdJ3&|%lw!Ep-$>>HLVD2Y%wrYi0d8UTM#9y!7FNFGK178&lC`z)ciovpk=?w4W4-n)A^m_4H!Q3{?<)^WH(GyN@j zWBdpP>}l!gq9gBiT;iLcQ{L#ZG2>_9h8BUAN$+jE!aa7sH;8}+Nth47x|PtOlh_gX zX8e=_azj)Q$=>AMCeIWuNe$;dIAPf{cgi~O#^U=0q;C3T;#P!S1#u06B*+uAc_aKk z1bQIRnILoTIQM)#T1to>jG_7aKcI4Qp(-f4-oolssiOY1xPqlyG7lak?Hk z7Lr`Rovk4Re%|`khYc$UEh({Q%N+N*DfxevCt$ay1EwFAVK?RDP_LH*UznI`IQSZe z@`L?XD?nEC@9XBCk-6#No1g*^V#8hmh!l$2T{!O?K6=AD+W4}qYkNX5 z*2AK{f}Qh}th7!>WF|f1ERE>%yMo$ss>ju;Ov-8`e`~nx?e=rtCi?gl`)^ec6GCxe zXX7Pdm_lq4Lik>xq4|_{`euH&4czE*d=e>IVTQ8vD*w^57BI5Fvyb`=abL))OIrg)Ye9IjOz+!AS<@;bGLq7>sQFzq9#p!a}8zS zk%@nJJgAHl{U3H)OAKUbNN8~y;UXB)D7r~5g6TW&j}0$EB1bMO)7tf}fBGZIUCn&& zcqrehn-VeA0mcsFyjxF?N`~?DBzn>qS4JmDHC{}Vb@H3Mf28Ts&+yilHZEcNckV=~ zV~?=F*@I?7{13f|)&=`3P1ybT(J7 zf}4F_(BSNuxbuI6_yktatXPbcBi+r{$WDUo}mQ1aG zSrFef=wX@Ef1`JH*owk(ve~9O(ScuOxPHp}iBaj4Yx&~<+53KzU%vR|IgU?yX^f1u z>Ek+RCWsY4{G)*TP}lJ}aAE(!;-uTx?eo36Ap*^VGimg;#Vk_!T2 zKMseEappKkT75g0@!)Inn3G=Z_9gcS_2b@go=>s`F$D20R$N7o(E(lm&e}TGpn$bg zP37-4+B|tkZD4G4bL3`r-DX9zVX(?k6jD<_tm$J?z+W16Q*CX0?-jqonNH=6g6u*? zC$8U{o?c^R5}>oMfTnnNZ^vM=1lC$N>-TM>GV{GrR=u4b7YTXI7|W8J;}t7bfR9*Y z3@OT1r}@3B@|KGEn`w>tBObD7htHpE44Hw@C_(ISlg$Y;&d z_Zhw`<#yPb&b7BFIg@I;Q8_wj@S^ZOyYwdDaeTgTS=({;>52=*c86fZ^Z_C)T*k&w zkSMFMM)rsEty+RJf=0gi=^*w!)aWGs9pB2i7OH1^qEiY5QulOB+944&n+FORz)it$9aF6TUb8Y07qZ>f3f;aeRpCdVDkFE5>@trCVK z&j*)@3l!Pn9WBv}^lA2Y1yNusA5(j?%p-PqGMWQNz3t33-SCDTYI?+gJ@o8542lGC ziu(Z62ThL}G>lgeUIajlMy0&8_cP95jI&?&)r1;aTJEk^1MZ^md+^gyNzQfr&25{` zlphtZgElHaT!lLLJ!Mx0+X!7%x!^Z1)%w1g@Doj9m!mw~<*u*z*~n>YYx6p2!#;zQ zzS|-|$BzvnVNWAl?*Or|gsD^|#JYPx<_p_(ff#CkiF*Qgi;FAf=N_y)r9X!G*|Tx4 zYn#`_yXv5to7UrwqdC5~qeWGlUV2{W1Q1r69n&Y}Yo2l-O3V9kgQ=UsjM9-BU8SYsrrbL>~g=^XgPx+dz+;IV`=Un^FA ze8INRpPTA8d2U6Fu0Tte@WW1lhXLt3?oF!M=6lKem7P4s&L#8D59M_nN1 z5iZL0AFWM}9TX97K_FC8T53HtaEzUU zgE(tpsZ)ab zYZU(o@b9FROOR8@y!z(3=-#elHlBdc2hpCv>5w?r;^_bZuzUk$kflUcc zi;s!BE2=G2m2eO>MTwje6`-R0s(GQO>1WHuTT%UI&LPirZBIz^tob+ogzMJkF6`&} z8s*E1S!YKqemY0Kk-vFjYL8(zgA?-&&81)kr4q-yfT9Gch=L2jkNcxxL^!m>Urp`d zx~oO@<`?m`&%BhF?kA>~*S(@|iNd z(Ql3NUfowor1)5$=?aqkvhV4Nw$GH>^Y7fZ2rGbbU?tl3P-Emmro@==?pxRebQI(a zx{vG;w`h6i*oKr;>IQsb@#Ww8RF0gHJDM-rWz)*2!3{z)*M0>2z4a8842N0fOVZ~f zl>hX94^3ycy~1b|V(jgF`D2sLCwDgU*55nek(Pk4a?gbzMZ+0^){WPD5o;M@rzw_s%IhuSv9Fv1v0&k>NQ>$&l#_IN;quj zROy|dROMYNDk>CDRJ~=t*f%}+oW}&FmVVcLpbj&$D-IhrJQ7W4$-1@Kd4CGHgu>59 z=NW!*(j zPA*ZtudV}BMxmcL8rW7FBnO5D`47dO*I`(}v(X~onfg~DkJ#|q+sUuqm&XqMGTVLL zuVhP5ey-)s$qiS6PQLGW{tk9l!`ZFLTJt$GV84i+G0bw=478qHGo|60GV8&Cmna4Nz8DE0l6RpixzL%`}9r2sEMkQ+7!Ac1}csQY8DgAac?QZ>- zl>oEGVX+6}Nm`;A4i9I8d9;C<89hvvcbqR2HRdRZaddG>f)FXEpuiU(6Bynf@G1Di zLFrQ%oNU;s>kZq+3a}J6Y}i1&a0x#GZlEtm)PR8}a`8QS-0ksXbX7jKZADFs!eao^ ziPWARPXfj3Y~&00haHsG4SOB6z9ZVx$sA>SZ4JvO|B9%4O)HbW-yyRjyny2PHT1ka zKCR6Un(_s-+Y>9;oViBp|ANO07ku?4GgxB zc#v!D(pSDcXF_RF($Rpa z-9O;)+aPzz$gt+OcJ2Mh*D(>c8bFFTn(pm7TV%tJu!YI~LMV{TWUx^WR}a59k^3`7 z^3W60PbY^ulM^xf)(I$FePx)&#Qos82AZKs+k02zEclo<#OQlo_4@X#%31PP`MVAK z3{%9yKm}knJ&e_gdY#Bj2|)@MP8ZCENU#O*6-5`b-Z>5;!9*0KBp5>YOoksK77itS z*aD|a!VW`ANW`)r?9y!9bzUA$i_fz5z>FhkzzJ2v9p zOx*GC;PV69z|^)<>V}I{D5?LzhnG&xAChCFE0W8NLsws_L$GpBN-1yPcY0#!Fr&v_*gy ztDz+$4W*>i<*`S_&|R*eqobw}3jB!^4MDk}k0g=b=Hp`~oHwF|6)U(7x}P86#IZAd zZH=Cr_c*&Xhsk@n-A1EJ^f9`-T7H;Q@#Ib@9)5t?jEBR)4SG6$h0T(g6MHY3jvEK^ zd`MJaAge8eZ+mCfo88X-e$&;7+YYU~z2V@L`efGB)ab-pqFW!QZ@gYD~Kz};LcH6;Z-O3dT+|{%$f{H@y zD=o_w&WsEae@#-XLQ>Jy+k2DP$U>i1$6k4w^uvg~FlhTxsL zsR!Z~bY^B|D`;sAu@iM(x)xapE@5)Ij+K>$!VZOFnC02D0iZS2_~H}_yxL3&rALSd zy+M$SW>3cZ4_T+BMzVyY`c{>!WV7HoMqk?!F1YrC3$nI6eCB=P-!FLs)XcP&IVJP% z(jJwGL~`jsGrN+3;WgZMyN8CN_FSLrQ|T@&xD0&w-~zg8f{_pwi*SkCjiN>@RLW=lj;C#P`%% zb@RQe&QzW8i6&r7z+>BF`=wHg15Qpw!K#mHTMr(X62~(LjlK7leI4G&p78roI;)w{dbT|Y z<@%j|;;bU&ZsOw+Y;Jyb;e0BR>upvZSUXJ!*k7NCoq2Z7t0;2-0~t&4Yi*KXoG+(c z+g0Z0usi(TthImXzF~ z3SRX;mH}s`NOWOHSeLn4$2%&|fWWiiK{ z+d)B3OBWy99t90%M|S8(ahqj^et$ZP*wJWf% zMsa#g9N2Mfwzk%Ni}!X^%ShQ(tmRqs3=!aGkm6u7B}_sv1EGTB{-;xpk+h~JdII9T z;o#sv*oG+1d=*~(1Ux)_TE0?k-ue7|$*H6a{+D+4A=s>&zMIUO7jk7XUGuZIeK9>y zWY6zpQ|il_AV^qHDDihUGlk!ZYPrJMmRo|UV@9xd#-?8J{de#Zk^I7Kd>XE_yp>{ay|p1Qk10WdWAN<6Q0Q0@``aWWVL z9_*u=c?9Uj^K0z^L?(mK7$Ur&Tf^lCaP;lw2he=pliukzeEruZwSrATY#4INF&2H@ zz?R%6**Y(^ax>$uRi-u1_|^jc2iC9H9tNTr?inoaKn1wq*Q?cu_z)%3X6&oOlqdj zZi$Gi67A#!)QEn((i))CkS-UTE> z1f_ygfcO1z%NM~AF2(&mhd8mv`o9#Zgcor)56s&mxdT^;SEOHyy{2gn|J13Goiu| zN2pT+RgDm+mc6c3TScWcEHW&@v!cGZ3aD!doQ;7&-(i1jvD)ywxHrF^7w^i;&#!l! z#}5!xoo`-^|EmWC$M5PPiOUR;?A%!`&qNj zP)gA_oJH}RRBe=>4gXfo)}==}r|wfQTom5#r0h_t?CwQ7xUtLN?W-Nt-_BeK9Ik(o z^lbas?2{YX`22YiHwzht6}&e{&I#d}L4^kGETJ6`N+`c-m^{jHQL8TbXXbC%B&JkZ9)ICPpRif*kNeTs1k<6)%zQI9yN{U7T zHKIJcb0x~|z8#fDq9})2VF8AM4&C582r1gRuD%|#md$+OftJ!`hW!Ti(R@@M$0$bI zDxH0Ks9tbBRpPB>Dib|8E!`+m2Mh9h`~&&XzNS|r0o**bvbQH@V#JsW9YF$!x=$Op7|w|6Cu`dT#F!p2314aRqvSbKjpKsOQ4T% z>}zaqYkRhsU~g~l6MlM5P6?yhzvpKBIuW-~W$fTEQ&P4L`^o59FHkJ?V|9+#9%>ml zjtK7+z_D3OQ)n-rA(c-P%@_%+d<}s;e>RJ`+W!zy;y-fvb^(Bwg=>F5fEruoz3ofs z*WC-4lvSZ|)kKD2P-mp&-z#%+?_u_SLhjFsviV5{%u2}24&V_lOmj}a#|)kfuv`j) zR<-Lb^G~;<`lwm~w?H$>az5$jc`oO`Wz1)~gg>xS4)7l=XLNy*UXHN3y2dy>I;4L?^-{>YhU zY2|EAxHxYE)RVZ@=7)-|2Od-8-Riw`y>G4OZl(e=h~H6=(svO z^Z!$PU<~m`_-dp-HqLSUU!}d2a+L76L~GuNz>^_S8-%`A$GMjpw(0uZY**j)tx_tQ znwPVBdn)68#EP~v)T*&A>1y7mx}S8pmv4sX;4^2Q=(F-45qT39FZ2z~E^5zq`#l9| zKZqE+u&4eh8CXm3wlC}ekgisBGPBJCIYA=Vg6|73uo_}$D zXPt$5tTY=4^b}8!{*A!3AR;_szjur2&b?Xf~lb_#F+ z`cvC>wI8S5GAo&rN`H6#wwAhvLi zMV?hX&-&K&zZiQ9s4Tnf?e_r$L`o#25fqS+E@=f3q`O->73mg91*8O&R6r0AknWIB zNdf6b1f(0J&b)Ph|8e#?-#5Z-pp;r zzex}b7BY+p{({?5G+3rlZN7{boMm3A&zA528h8+nX*eP!Dgygou2vx>bcTUh`&gVl z5w-{<7Y~`XLq`rMS5Je3F+qj!==6^(5)KDAQS_HnKhHozf_P#ISCGs+#FZSdOSleU zz#(yQ#K0Cnq!U1mxjhd`zh4%~i~<5=up-)_!a_sAv^fCE9tB0kS-3aDKc@#yC#bK5 zeE;xK@>#!hI;Szh!NrvVDij(jGcyx9f4v9k;NHG*m9x>LON8X+;Mq|P{)fa81e7Qn zhh@KdNmTUu(6IxS?cPgf;_v#c*{!C~$y%*0W$xU1wV-ci`8@fiol(WhI8{^D2kCk6 z-V}pMA4!>oJ?X-xS9>*#{gEaVQV)Zj_2`ewK!3l9$xB};BwCsx|04_(alQA${68Xn z6S(=_E!27k?bV-2Q^N&gOqxlRQy3QO!B{mew3_yONUa_uII)v!4d;{cEm=LvvmFv45Hx z>2GDWYekmfR8))S5@led(1EoH{LJGab7-TMfH4}pmnbfKPsAyRf z1(-xZ8n!c(Zfau_j_6UKm)H!89Z+!zj|5NI;3R8;IS_6Z(zVa(1(lnQ`*xbjib_7- zo5h;-LAAACoXyGk$2<+lRJ;zb5{HS_ZHSAV>}inwztWi|_dx9)7)&`(s(Y;3I^Q zTc=?J`~waILQ6`jPz&j>0tn^aSsi$GXtr-M5$zCQUM%0%Vmkcj0=cy@Mzv)WLJ$tX z(|xmO3~i|rhl>YeE_UJTtNZ5@=h*7Xiuc~?87K~oG?AC;b=@6iHAFzI(!uvQDVK^Y z0~1-)nT(6PS4X%Zw2TB)vllO3YyySN*6JuPREZg&WrKSU8iI@WJN?t#wYt_>%bM_@ zz_^hXR23m$J4e702s8~07{HH^Yj3c>ou^mMiWnpTc!m#veQacz#mAFq6zC8kKRq&c z20z`&i89R9G}u?G$)j?0wt`NqhfmVOWsa7$&8+%9+a~=tF=*84e(%-6*qmfmqbPM~e+e|-RxE`42 zXbGybR^a!R3rf(|GV3nq*E6^1`eveAYmsaCp#l%|A-Tn0rzFoo(otFf;vAk1!bJ%7 zy9m@1kObktoCt1kzw88Oag>jPRA6=Jd}Qy;iYd-Wr~ zb)No+!)XSO-hoU8smDn+%F1b|?G zlSaup#WpU(=6R7&`P^l3ZZ7snBk=iiTxiZ;CL;?06T;{3G2Tg6V$Md~Z&QokQGTYE zF0=3KI+Xe*QpwjEd9aDI_np$tWE>?w#&O3+ zokLd<^weLbB-0@TeJD`|Zd9OMs&0VTF2q;+{=Jo72O_)@b6=9VK`SbM+4kM|4LmJ-wRH`L%Nv@l(R7%VF+RHwcgM7$KE)I(t46{yswD5uY94S?#9m(*0DIy+; z&pU&7Xh^5Xww`O@GYc2bP0G@xaNudeH^y#Vm^I=bm@cS~3=hWv&rB$cfJ^|5kqarM zfffjy>GV3txKl4%r4JXH1Z!z}YFGEvoDw>lrs}XwUq;-Ov~L9se0+!oloX|`?Q@5c z^1hK63;v-5ucB05Ut#+VQaY$&0pk7Tu&xmJxDWt>C=yTq>;+|KI~L9%N!=he`dK&+ z!nanWL!@WDv?AdT?6K)wH6WC-_jV!y?Dohr*!F?T&H*F59&3$(SHtt}`a1Rs7xT>b zuf;2a2l7m{DN=0AoW>I4RtKqHj{6hXE){lD!xC9IgKGj?P;svnTA2C8XFsUIf_}M* z|0Gi8x~Mj50@z8Bs0P~~bA~an)-{}!E{NTce0E6h3F1u&>Xi?N@0$XPKs`m#XGdew z#EBs%9{yXJy|Ij1AEV;#)BVUMFxUm$zO|TP79OFSqNq^?4lFpkks1eX*)7e@ch+S+ zhcfP96d2W0()s1qZ-2>8-a|o@cFdZq(b_u#6iiS&d$uew(!yl5scUG%uM5)Qmap%1 zzB6C@1iIV$xLDLG<%Nk;?E>vmm&(W*ZHbhOg^PF)KDEaC00f#qBrXCTf z)O-N770MvI1`^3~=9)XP^~{;Dn8?qYJ$&?b)O&=PQ@PAqFCtG5Q5TCm__zz<+E9sL zp>BYX5Jp2#S+@WZoeQ+E>|h!M^$T$iQmA^Q3>;hnQ5{7zXhL_mZxh9czMWU+I&H$w zv@~~omOOb|E__?g46nFQ;w!AV#AW;>n5;2Jz2!B^`nOMRi%DZ z#H%{hsIXk20niXZw|OB7(a5$n1s{K}az3wCp<0-`kR4mv>)M~KDA_k{RyfunIiI=x zGt-O{#JtlVPeP`h&e9eNb^vb4Rl8s>q;s3ZPv~tW$lJ>&h_+kD#weksMA7?(&juZ^ zQOM37giRRo<_#4@x<^9?N_3taVhdk%0JLG$NCE+zl|b2&wzE*S>iwTS(&7pr%JOxw zo#u4w4&A4_XvJKr z6b9ap@!`mAtG{|s*8U=1hWwye$VY33_pcZ2e%3kCXaW%^F7l+3xXpoS6ao_)^nl`M zZ>7#p9UUE3qCGa%2L(UNI)sT-O@+NyxCsnAZwv|#vIYU*?kn759&|||j!znfA;p3~ zwA#Q3MdUdNrcE_@q(g*aSU=_Qr?W(IO@9*YDEQDQ)Hy$>s^6L|4!uj<#PMPn45|Ex zew}ohayf;Mzu5i4OG?h6Nk2^X)t_Ib+q7Un_sYm1A}|yi;7o{O>}+WZ8*7-h>(^~n zI7j7;83g_{W_ofPNB_YGD4lw)E&BbE)`+i{RcMPpohz8R!o=FOtc!hV%K5UMxXV(`i8DJL^r z=DQ>H%t_f*43_z`A7m(6W# z1b;|Rqq;>(OPjWJ*u5}J^N%$k>a}|jX6~a#2KOeD5+%0)RXO-2_m8ctN`%Jb(l}jf}*Pf54H6C^R;#B8@NdIZdQnIX(<% z?xuOzDdiMHjlD`kP&ZnvK@QC)beUh<3vbOBXLM0HDX*V`vC=v+FP`x>*dE1Zq0sR3tmnWL#oCe6U}@;1YT`RM45)5L@FcJaI|9MLx( zEh;uTht04z%l1_}c|nTiCDcvbPdCm^c9fBv*ePUp%Mzq*RA@ms767Sty;(gIm39Pf58zk0h?xTq@P961W@ZfSQ8c(Hl{}3kF4Mbb z(2#k1x42K_@+>!?`D;1&Rh!3lkzK7N0g`@-THhq2WjxT&I45fwVOZID5@wu4$8@YH zWAH)9+bCPZ^mnNG_gXKfzY|o|Ofc60Z~*Sl3hCkqst#U2?UnP|(0D#YAdq4(hLrc8 zij7C4HZ=yyH$LY-&XLS_g4smd1C=M2k=LuqdaXWA^Af4dAYCBmZ@=bIR%y(KaM3yh zukH6*cPE5!2cl-E9VDUd`e@u31s(Uf!Mt4y9TTSEhIgg4(9}V}zPM~njjzlu$w?wQSBI+v zOD6rtN7WeZmfQ>bdn4~P-#vjXUs+Zbg(?*J0v*M$sgCE*H0;IL{F(=AW5mA2dmQ5l z#@<;qpXn_z>;8hiRF1{fS6%rr>;51mzre&ufTp(gox<;Fu0|k0>~i40%5QHAbKzW% z@TjPEkdT(ZaH5jd9B_`150U{g7~QDeRFSI?%RN$TiUC(AJUI9Y&7t2Ca@*zCPkl#V zo5cdf1(+2${KQg>WV;`+vt#8e2SmVn+UnEf-mtIpIS}fSfc}|!{}9uTE(d*KLYqmwVF-2;&1z1aT5O9~!LLn7VS_J`*YmwF}_ z-MJSANhZA91jg*iwDbj(_Jm&=gN~T?KDcyeGY+~!t#V~?C z%9Yg-3gl{#7oksenMieg*#HCXvg@P_q6dAor0!`>#gmL}-WVw;221grb z(vxq&u1%BvZF-T8lAn9wZSYbqR5gd)U&qseQH?)OZyrHlD&^n)dkI_)i-7Ch|NDM- z^!OHRr_M*SE(PbxC-wuyvHT%rKZi7S3HsFroXi?3)V){VEiaE-FMI5qJRY)q}B7#n2E0jhrzI>6bo(B?Y4DHAxagtu* z)?Xuu97-@aW*4i*KjBp@(2Yl{;T%5Vw9mD!IiC~9qM?0ek@(7nzc=9$V?*vSrkmS1 zk}CvCu@?_rDVf$9u)3-AMW`rlmgFjnkvlKDeajz&SnnqG*Nc}jRyZYBoLtXJN7BZ< z0e_`Z_^UcN6VvZ>NgFR;daMY{`G`nOS#0r# z(!dhwkm{hC?}G#F3^LbC#Rs^OsW*ovwa|ZMLs8;&ONfaaTn(*z?}y9VsSBw~vUkW{ zoSTTgz$zWf^HY)N9-a?yvil9>+ z$E?kfZL7w8uxijH%1dk;sYmW|b2gyRu<~uBFI93s{A}WV)r=Ehe#J&1{B3CQAvh}} z21Kvic68kd3-urG(u*Ycdo`q28?Qu#U6uPw#6*7NqaJLAj}hfL*8^$ z^EpLS(ogcgkdvw*6Q3WOY~)nroy16j%+f8#XN^tY5OIxC@D=j!IT0{V`x4;~(ldN% z1ELjH^^9}N482hFJ zicFnp+Rf>#Jiv#w>gCx61oo6)Wr>DeX}z(EbC9}#Izw&y{8k1qoZz~5vGW5c7{Qc3 zf8x33f7w{|Cdtbi;hiPO`njff9TAQ*suPFCTF;qW2aC$~44?D%)e@~S3Tp}ZG|{Z# zw^!bB&7~80#2!ljC16AM`8zl%Md2_7KHiq?@_w1%kRzb{bw+B%@pk$=ZnQk-LTo|1 z6*__y=;D(&8jeQA65`yXG>L~e<pUX8+^ljgETJzCH95haH_hPt zl2{Nbi+Uygzt~o*k#J-|en!2DD3@eLi$o#KfLo56e5?23v$kNIy+0Luvp*Vuf8t68qf2|=KPpYUirD_OYzQ^@rkalgo{RoWX}x5%7gSH#?+Byj zLeKpq&2kDGq4-Z4HDjs~aYeEwiI%T}2?qorch@h%pP7YyL#|(MuGq5rinD74=xuFo zi&A)OsT7pIL63L3CQsdO{D|cdx$6^>r^aFa%tDDz+Q03yL5L?D(y8SO6HJl$FCG0ES7E~jB?Re$8is%pZj8pQ%lTeLEAW@XxbeYxK^UWkd4Avmnx z$bcm0Gv#_Tcxk1DS6WW3yZ)&A5SrUs2UyVJn-baU+Xq27a8TlF*K__TQsJpVfx5i}~jjx8XyakNmx0)tB|(PqEPhlvK?i3*i=;lU0ess6e3s9o>?)8A) zsv#yo=GG969tQ0=ig>b~Xx^apw$IA(Eu$OH^S@Et#N#U2k_PucKz=7k-U7k0Q2KW zH~8!1>Qm|=Fhwj1&)_&w%pBV+ONX7=hm2cjqtmC&o`tBhq zlwPk;sB7RkaOv%Cn-cLZZ~vgD-L_rPsahg^4+IB5d(VnH*Lx5fzXOvOX=z=ZRGaAa6`!o?^J2=B{k7gOH;^7MD)++lPBpaykPpX8PB$c1(c?O{L9tO z96u@iL_OqllPM^g>`csck}1B$M?~Psp`RsRN;%T>Uj3}v)FFZPU~*z;)n-oJXKjc7 zqiFS8{-Fvp{gc5z%#b~D)sGAM2^H`XuFucM&2w*UZ6O{t2IHp92kcS+JdAK(bpf3kS0FRbR%IVIuG`$ zCZwW;CL_Ai-~G}a#0OdRTfuRl1{QZ(zG=>Yj)K4zC9Bt%rrK22f{Ji-B%XmPP$KnQ zF8|EDlWt3OA^IV*qe#b#$mQ9PmoG4+wP#;McH1SjN`M&TeoPc6o>JE!68s z))R&5EY?Su=E_}?5i$mOU9fUSZ&}ToherP$Z?qZ4G`g-p2n|9NEOA*>vY&1+To;1B z;YJ*D$=4B2dPuC{E4&EI8^AE2u5H~0Y-@m^Wj{I6`$7+4`qaj7`gnDgTS0=~F#pVO zV59GIUML=N6$pEb90;TC7{e__7hScj^nJ#^>monrjSe*ZV_U&qQmlYuN*c#BJCldr zdUs*kXUM2O_C;Zhe`vZ|f=MtHPm0lh zgP17hWcO|0+2IyoX?%R@y5!(#$P(!Hh)ddk0^ajov=@(EW4Vdwf6+sJl391_rKM_0 zmEUa)vNuYBi1tEUfD+oqnNziTPCf%mR5wXdO76jms9_UjA~JTUf5sPiW6*zHn-0!Z z?!uQU{=tVg?8LN^e)i!EyjC;^Cli$TdH{NnacBs$+G4rut1$jyN#i~8M#zTM zq1i8Ewu+IhFtVaB4$vxk;N!I;vRyA**1rv$QogeGE5nS3#Xjb+395c1PefszXB)m_ z=O6bOqrPF(@_e>5YJ`<&q+#1x^wvGgmcC)0-H3Z)SP7N?((Cr&` z>^W~)4pj719M(v=B9HwtSWd>UWxqXn_bHN?%UZ&u&^xXZK5|R^dCm+X$toGkww=E^ zY^*Z%whl&VH-|N3gllhIsHc;{km6$*mtH3qmIuQH7S_gour{zgxIDLi`Y1^u4^tsE zrfn z$wWJlM^^e|#n!IxYCQ?ZAhrA(^St{~;MI*FsBfDzZ|dm{7`#woRKc-1D&Eurc5A1Y z*jG>C@?<(tsHhu7n8$m;oa)8Gu{*Skb>TQnyuq>v@~6 z!uFeI0ZXXDlKz}*?xG)}e4Nt;NR1<1{T?!Ez~T7Q;{`+y15^W;r3i{0T%nH|<_Sh_;O#2!~}4(TZ(8E`QnyRvs}(Hn&SkfOh9n>JR`Qqr09 z<7a=tjySoI&D^w?n9Fk9BFm=rtCYlnU4X4c)%4B!wzlcOMkW-1PPJDq?w-qx6@#N`sV6=F?cE`aL0$;gNXD@TIqNWdBk4qA+D<_vhiNH}(VP5@w=U zJo0R+f0z~8i#6?Gdv3p+l7;-AVdjSR1ZM4LUYO4>=?s|SKX@GyxFF|*&9ik@m%ry%>n3%o^hTb#Q%ZAvA zoeNtsv9XpiDy*YA%>BBQ_z1*O!Qsj84SFv@m{}wDQA-DV{OqUaPH1A*8S?*ktNc^F z!dL(9=f#lrP<6s3lk23$&KbPo1(o)a%c-D28d^Qj@b^F5Pa}jl`oy;$%65$6OFgJ$6 zJ*BYZiVP!Hej9sP(S*_8H}6P_nQ{p#J_s0T0XfQZAYeiC19a;%Kz)TkECI&B69uMu z2!&~d5WUL~P@&4p!_xw4YK{RmWSZ0RSZoy&P;{(=GBp~3nP`buS)V}c)91L>*LnCe z`)_)>A~N3EEc{3qp)A(SrJy38ACC6Bm)h_8@By-~$9qFXiI`!tY;I(p$v6JVwY6e_ zg{BLNf{%eshL79?NFHPyKb{E+ghQpJ1iORlboU8tN_qT4Ko;t95~D*hRbIkwbn_hM z*o|Ofr~y-?AEcdgZ1BUJ;iJ2|Dy0P+`aYED$^y}qfFw#-LJ3_1POVvYvmPTxcFM(m} z1|rGjV6n;r#FUts6M&AGr>Tww{!^ig0gQff%C3r(zNg2*xD>D5UqcZTC#7jqXl_22 zMC)%2;di4&7Y3;7dp@?fLPGQGt6v<^34@MZL7k^3yLBlFf0dF1xl+7EXIRmDk8o#T zWD0ZVzK@0?KnXxZX^`^(m>&j^L;=kP zH-^sL39&zh#RY#Dd2KFt{zrwix|^tecAv<^L1gV$bvQrQp>nEq8f-X)X!={r)9Em0 zL<%1N-NPWHVc0M7sYg=6WAhp+17xyDLGoZvSPAr8aKjKlV4~pkhJ3?kz}iCCYj9Y7 zAqBH>*uyJZ3`79P#VzlBtu2G_Bd(CDvurg@5e3Mc)*uf3A@uf%&X}q0uf=hQ3gzXs zb)r9t@0|KT9P^(dx;w8{`7gT|(D=>B&2HNT7sNnKU(?~X8KAJ?Z>ma0UPX9QUH+A| zQ-o4P`>y)HEY+|4zdD-(#m4&py~|YDYzNtFzXW^@kVpSNeCj9^+>S#aX%cC%s&bbP zZP2O>82MM56sVb5%ipIUCYAbH0{j)LIq_XKGq4;bJ7d(*je4YRN~=6J6Jke4a7pA}}8 zz`>SzKQVApDsbs?iP0HGlSQm(a_*PfqgAGMYPz7GWe+pgXIBGD@H?Q6{`)DfsP}W3 z?c5VpmL>g7JfmduF^VX3~MW_B&@^XU!;J+$kVNcXrWB>H`yLqtO}ZdF_@& z;g3v|iAv!eL!5isSw9Px}0J{Hxu0(n3be}aVg_eA!ETq z^4CFQCZ`lKH+2-P&E&TP5Da+C-f9MG{hgb#`g{`T+xhL#S%>sA&LZTs<_{46BV@{` zewUdN*?Are`j)frO{v$%%-gjj)A1#jKLn>gkEkmi;d*gsb>Xj#zcylR6Rygc8eeJ# zNRT?_W1$$u(;#g+9DZhfmy`Y?@%RT*mvvU$Rg@cCeL3ojoylb?cy9e#AcnnWY{7USb0!^22MEO@>9Afa~(#*CxPb&@_C_Ta^p z--yu+*O2UP_D{u9IJGqRQtEat>=0sK;)ji?4vRk|8%NuQbzhA*(D!$`*ck(?ys(qLl9U*3u(EB^uAQchbS{nql6-13dZmwdwxLBiE zGi>jEetvp;AXJ>FcELsriJ(}I0{sx=nz(ZDcpXo1g5Kx<9Hl(Up-llJOR)1W+`K89 zi;abagq*+}`!o1Q`e1q`+MovuYcj4t>2u9_t61guFnbWp=`0>-;AQ2ZO=AX3Gf3L? zieUEHqW8I7N_2SdekG112)PyN7$m6_S?LpYj&p>T1eW}(N4A;xgwf0^jYED#R(1h< zYw#T!+%JFP%GA&^k&N6%g;jkJ(|SiW(?H4f>cC~KoOzgQ)alcZw+qS{h8~y>WEqh4 z@OOpArYrIFX&34xtR3cuJXY#JI~tdS(*gTs!+{t0^S^^;3~311*w`Raalf-8E70DX z134F5#SlF^u3yCe|I?{%2k9r7phBnU-kJRz3YkV2pq1X-+G>V`f@h%LL@FTU?*OY! zqSN^;*gJj}AE*hz@SW_h>d!OKlL%||2;IX&#=C2R0Oqbps7JLdC%nMsyCvVNs9dyj zwK;WZ;92AFN;!5S!~1(uFvt5WcyyiVjseTc%Kncn)ova*c~c*qlN8%shPJtP*B*98 zBy&c*lc^3Av>{Yfo8t@98LtNQ5F7Irw0{oIV4sZOAEjD~jrX)1sO@z8wO@;tK?=CuDHK|I?744;=@Ksz#{1U@GQ%57sR)dIs8nhq-Y1Jitk{}s0O-`eM3X9Xaw!jFoDLYNp0 zr~CSa=>rfLg?L`zHxG=CCgd=x3j-hg!WkxPvdb@lv+jJJT!lSut?fsjumQwy|MzH% z^<^D^BAVpOXcN-4l2!Ts>VsS`V7)*wd-*pa-gW3GpHN1aG!Z16jh9bcZ{5w+EG=!~ zJf)J%>ze*5rtu6=6OE0S98N>DrH|0XM+ruRuljuyOt~l)2q6*lOt)q13IZK4x}13F zKnz4;jfG20>UX@0QAkO&%w;a0u_3cYN$N=!LI%Ba9pHOEJRSbp)^@LlE}hfB(2$oF zY$SG(P#OVWA0?aqeF}u zoSE=jprIf>G7+INu*Sjx_KkS1G2fLnUUJ3OzHA}?y?elb6U8#tv5RBtLyczryOEia*eorx{fIinRjahUj($utz$)sctI(Ph+ zFV<#=X_aI2dDEEK8ZMO>!P=U&I!`aTZXG(5q?Xx=Kng!i$>zpmo_mkzSOk3B`$l@$ zOaImwC>r@{g!!etVY zXArhC(dbVH*_42@_qwU)6WwV)KIS1i(bAQ)X|{rlyp?Cu3zne9vUsMQm^u;+ucc6<1Jj;tAceY9*hfWI32KAm~J zec}fto4FPi5c2QOe~PpK_`G!1_=u#xGU&1>qU{EAGx;IA_~qK+n<;9-ix)3bE>+iq zr5%x~!l}WO0u3yU_=0UBt8{G^hp1?~!~~YeG`T26rxfGyJTYh@2zO&s`lg*ZT>I{g zbRgX(rx*tnKtxw~>XfVdAqO%m;6K|O^!FlT-oP--?1RdAg_LVqp((REYwTpU;&OUn zzglZeCdJG?mCE}>(dw6eyeo8-=S^_wjY)Y`z(3WX=G|q_O$<(do-ELO_50dqG-5}} z1VO9cN_1nmD1C?T914KR9S6tHl)btD$Xd1mfD2cNDO} zI4$6l>m`Zd^XGCA%`XOcoke%6hLRu-5Y%38AsxVAHWEr_D)?%lh=U7Km{BOOP4hVx z)K4hdudWCV1Ab@JsCVLI@3Ar{gLRD{JNKb$>V!bwwEYZhFkcyWrvB+bO+g2`gd>OE zQ;2nzjAgA>-a+L{;B!K3@=$B(yX^Vu#fd=Z5aRG>-wh9TAbWc+_&-A#FTs3YDFnf# zCdV&=%f7Cij{jxu<~q)1hsjJc%NqCntFT!#b>;M#*FUD;MsE-~qJe39S^EbFg@@@* z^VhL|yX-w&s-=xD!^2mneGh0*VACZ4vD~-qs;sQ67ad?W8#UVLmT?E0?5p9g%apeb zLDwRih$&syx+S?4bau5czezSbo+XAh%pZ%I9Dibeh?bWFZC8h2(2euNAS;moi>?)H z=;gl=??~u3ntxN8-E}1(JI%&OW@@!a2bTv?er`!J&L6gVkFVH9;*i6yIR}3z{b$SUl z`<`;VQa=6hY*}@s$LDC?4JfnHk&x`W<{7Om&{iK zpgLC&zfCcmW-##Ae2#Q016;nwSKD0OfPr!kUvL5SV#s zt&CoYm4bu{m>7w+{(o@_1FoOq258q{+rnIKF7-HZU5imUt_ucH>7g5AZ~q|Flub_b z*KEo<^p#L!ikNtKxMGyc?@NiS5ba2R7GI`#O|!L+)(Y>x_pAQR;X{5I`~8m28|5nL zBi|{IoSAF}U6>WV;|<_#w2e;4MeR_%(0t-sZYX*D6zVIE1| zI$(#A#Au6ZS=3{9iD<0SDH@Okcc9Y%_jqUbUcavcf^YgwO?BBfc0vTL0<5N@|J&1T z;^2@v?+Wa3m-=(KD921_9fwHnw67At_z2HYRkuO%-jC_reA#eGdB&A8mbOvTjLE$3a(_dQ$n9caCMfl*`#%227iB^}9 z!2)xk37B0`5D^hX7hhh-$_(-`Hly z%XtntNC~)1AG28v=@7GYh#~bbhq$RK1xEp%c>*oqk=N zz~D!#=6KB5ksFQd(G3GoE4hDwDmv)Pm9)0QVUTDGP{0ZGaG!tM3Pdw_N>DZeB z<9vUPA{B~^)5ySl8g^e)UxkA{Bnr%sP>FgtA3ai>hoUFP2j>7pfi!;=>}dik8-_`R zr1Fu+9u(D{vdn~*gH1XQW{2<3bmFL_tWooNG#J&zg)A z`3}|#KMDm`|I*5C$NBkNbS{&e!zIj4(egeD@40=%e%X1+>=x`yz#B(NMcNmgnVTC1 z52X|^Whh$zlSj0EhqLp3jm;(|Amj}&|8)rW%>30#fPHNzFF?M4YxRr}5H9cDy^A{h z8K+!4zQ$pWLg;7<9UXc`Mo=+`f4Rjvzmw2oeAaQ`Apek8g3w~H>EVrL?8VO?+zrg> z*Jg}&OuzV=3>Nv$iDTmu@_05syvYk2T1;H2$h()`4F<{! z@{TeXT~#@RYx)(!xF>Aw5VI+BH5 zS()PIIeyfx*F2vyTV&(_3R3mT$66FuvI2OBe^}tr(b26p96O=_vE{LxooMg}GM1kL zFAt0bfiAG>D%Ys4hU>%xViNUW*_>a|-Q1_C`H2R(s7Zw$;!5}OL;>1+Ha z7a1ej_ zzeP${E|vAGYr#_auEj7Usc~6wmx<)B@1VcRVmCdBp4V;gua+;c@_>@<(2isMNeMO& zocA_$W&0}l9~=JIlCkTcnyMGmIw5ft@(kt6PknuS09ltW?+vGx=S(1W5c5p=AlxP!}#|T@&bA(L6Srv7u=tpD-_RI1}Wv_d^?$38|s9USU~TH^4t00KspN z3Ty^RlRE5V5GUceRz9qBDglsO*;{Np`iIt|9PcYs#q4ik);^47^P>>X+M; z=li5+XhK+H`~g4C^0?{#kC(|mDMfaKJ7U+STpRDc9<}}Tqr7Nax5bKs8v~1JFQs+O zYAUKyRj?TPdQ~vI;Xhoj-S`%L4q|!e6lt2D!@ai@#I&sDyYfaRB}n-m`)0Gi(>Ptt>`M5{gjGu14J>V+3o-A1_!{*uk?KcO??j8peR<$)yFuO`<9@$R zzF4_9l!rK`-4c2?z>-UcyXPlhoPaSLTFSs!6iD&i9|ebHIK5N|^??T*w^krs2YcA- z-CejmF(6|YC_!f5KIiz*mJaXw(@HW%ZvT}#?<>{|pB`;2)EiXj#LL{=(mr(KN!m#r zk7TN{7H*wfjiHKj|F!;emXWaZmfah9_7;h!i=m>ux`krA*O9lIMFp)siab&Vj0|mH zgiQt(EnHw8K70u2KS~A$)RhyDaRG85`b!7~CzhasuG<^Z_c`4gzQM?7D)0sbXaRD-V$?$;?N87KYwwU+uaf#ckoJ!vmcYTiKrSlnjj zbe|UBNcAH8faqxDa`VwOH^=%ttwgF`@KGiMQXL2x7T@Ao>WlKv$(P9c z=nQh5i7id@9oQKyR>Nt;$M{FF7|X5pbR<=&585r{{I-KEkwoO=U!(}dTH(Jxa|&Qd z|DHOaQoH?9e{Rt2xVVS*TTU9=x2GFUp=*sb?LKdslZIz@w_MX+@!^xb3BLbIyyep= z9N(W$KPTTdV~Pokq5Y;HP5EwcPy+?8-Dgs5gXWj?Qr5#CX|(bafS}c z2NH1F8vAVBD>oP0Y9(H1+kw{4PM;KEf>ZYX+;h%P5H|?HeaI~`GQT-ocD{@aHegGq zjO_)9cW9OdlW*K^JyExPr}dg}dKwEyqPksg?%GT2RGe2 z&^wDYJ#!>}GLQ(bK8*^RA@V&_epoI}w)M!rJg}QiIiul!{-bT(VsowgW#cseP!6HN z8T~WYg5$a@x#Cd=(b72zvUT01!KtqI6ukGlvJ6XW8jEi-MC!xkIvHYY!RFo3+1be= z0Di%bjb~n9mu!XC_Xxgj^;*UF2ZJ6l7MAd<#y&&=;QcNEn>y@AXb_MyA1lvc3QGk} z>X55O9)UpAyF&cV;i2^Fh^K0}>f(k{e&KC-r?by%T$F|{5?V02GT1Tuc;aU+KJ#mZdJ|Dfp|hY|gfHE;3X(DFa`TLqzruh9!Z^tu(3^&dsGCSf4LyyVgc<-uKU&t45yR0NxWgk@M8uYEvuEAE)_F)oXHnt}WkH zk=aw3@`F8O^=O=*>@vapR1nAFFzjksb+ucD;_`=GdEFgLpI^9G_3^%aANuJEQ6Rt# z66nBIP}{oZCW2(R&Ln*oH)#rB?VtdZ73_D{xw&JYA(AFIKV{0@y9&ZJ)+xR*l-CCJ`CCH z8trv9s;%jjht;Pc2`RoY!u#%OSZ~x_QpX&r$lg8Iynmy|eYHb7Wr1=TpU0~IX>1_H~{>XQ-cCx$vsaAGca=MmBfm$jF` zQ0xKNoxsxxB#SpuH07V3%bM@sZ=p9j@~u_Zw+;~8nC^>4(HSy1#CP4>oHs1ajv$rS z+jm}dr`4)tBYc?fa7*n({-mrlLTn_^FXq(4@5@fvrNIMn_}!JiJB7evQhPsT^ix#5 zz)%PE&bwk?MPM&1o5atbQmbK&5{k&`=+7B>CIwz{`b6%@wdNe0?Tt%=mQ77ALFwi? zyIj3D7#Y;Q!jQBx8i1|qSHK^X%WVW1a(C}Oy|ly-sq@$iP2I|=$+5_qmvW{%|M)VE z)`7Q?Jk6I;+|r`pzr~CBOuu`xU~W%8vSgD^u5%W8hNtvXCsuv6o~z(>H1A)I+qBJ^ zK1TOdJn>$&;c#G3i$1#0lACBSp4E)Yg}FJR8(m!%9(3l4sB48w{Qw^4W+_}g$x}wm z#5p`_uK2)`$A_1Q!2U^Ou=M9c-M;k#czCZG_e0O(#8QI?(Iw*OwA~xW25biPSu@~XFPnPX zYhx$O#JK$+@2JJ?UF8;Ak~>`zF4TbLGqDsi({<}Ru}l{3>y8Z*`;0CXz0Gce4KLM? z6&AiZe#C^R9c2$FuXO9Z_)#Eadr?zRjc=5&qBZRH_d- z81@poEf*VZQ^_fDKCx~5`9eT}R;^OrU7n_R+;s1l>zt@qKh?aUn_Az@l}guYIezrp z;lmYXndp-5L%XZnzxxqQi6`_)Xd zwd0r~KR}~ygLblg&Hki?|JMTURmC8`xZh8-p7Gs+4PxOqamTY|kzLVH&mvv!y|S5F zubgX2a+vH7xHWZMj}}Fj?0j|0b1~KZ!>TsewER}*=hf$vpvkz<(bm@H_(e_n_Ztd- zrfMI;W`>p*mf&F9l(KOw*;P_2P=cCZR z-@C{nd-nDdgP+1#3}=OJ^T!FpAN#c*#x*Af7Sn{g6r{7fVC-W7IWkbz`uO@n!AB4i z6QjAmx3j}0A@HV~KT2OukDID*diwm?W8tCL4QvAH4FB^pX1<)Ts^J!Yg00^h zu|0OC_I$SvY~BiP3ut*B&*rbtyGGg-bOv)lrT}hUoW^$dQC-8j>#rL<%~6w#Q>?|L z(dm0apWlZ5kQ0Ze@p$gvKBia6nIA?cu({nhVnVH$1H^0?sq@z=kwNvuSTVkwaU;8Ri?}Gwm5_xptbJB3e;sR0x zIl>Y<4BGN}wMpT>b__-XuKO2`FSvLnb|&}vqB}et8#Ih%VBjBV^p^m^kvKAQFyY#q z4W;98oT#FlRd_Q=fd#``62?Fnzpve`-D=YE*QB+lgr^a>|rFU}x+aza8CM>Dv>v zO2fY&i4#8idc>}(YM||chn)aC96AaL3J4u}3e_|^a=1eqCB(d;2>MYA9S}p1U*fdj zVQ;@GXUWR27mn-zMFPdV^?iA#{k-E`HqQ5(MXNMr_wCB2V{vKa} z09z@s6|eaz-lfhNuE1s>HOh2*%KXc__!75-r}hoD0daNRg8d)_ThfTs88uU)nj>={57kl zroP-v17dG{3_Zd=wKWE{ji0}MJ)RA|x(!nz6cVhrUNs*z@8txyFB>3U!y~qWaU=h} z;irc{6CO!DY?!k5i1I``z z-_T6U(`3xF71Sx6%{ z{-{YuE2%$N;>U};XCuehTD9e44S`dEFIGgO@@tJ1Yc1D)bIB9M%xi0Cv_g7`Zlxm) z3R2mGfJ(dhsBzkai_2|QXUcPt23lzC(3ydXDL}q4fld2WuOyv_TfrqehR7zUI6Y4Z z7k{|JQ}fvP({Om~;dztoQU0Lo<`37GiN7gDjyZ%jY&Jhk%1<%Ax-vC+$dPV;dg1xx z+!ewd-1*t6P>Rtm!6yQM>}>4|)-mifs?wTnBW66rB76I9Hn_q)ejTo3ZA%XYFacwrBzLLI~0V=ur9 z^kQTp+9Y-`@fi{hy<6s?#gYJlx^&OmS`6JM+kh*vc~DdpSX5P&1RWV%RNU8A0Od}` zHimg9sp+8|bdN3#{<7eM4X8fc_M?L>WpHHaY)5Ug-tP&=?I3we$s#s8v0&sdN+&(1 z&(4Tgs}6XS+VRN0@fc?dKX31EG}HdS6QcZWR@b*s|Bc7CwZ-?@ge&p3Rb*V7%t9p6 z0!f`}+VPCIPE^T#=*anx?qD9yu+90%S8hr{ z6g zV3dtKW-);jLf8~UC@u>jceabzS7LF4sR6r zbn*|eUt;nhaOf%cK>{DkgXN6TZ|q1tVT+(!c=KlybGRL;wgvK)DalCtV8hMQ@~O9^ zqrmL(elsavN7O6cs}SOJhKVgu)&BZk^FnDx>dJNrLx-KqA4~D+{o9ReXZX*%UH>n_ z-a0JmFX|ozQBV<3l&-I+q;$6dB2o$p(j_3>9ixJDsB|kK(%sz+3P>|U4?VyD1I!F_ z&*1ys`~2?xpRY0hZq=QJjN;4)&i6l=aPaNZP0}9=((=N`u*KI@OFMr|b& zAM090pjgu$=TC$tZW$@Up5_yRpvdfn&&pbAp8cdpSCd%}{GBhc&`)p@Mp|hp(uIYL z8-dw};Cm7Pp5FNC82Gk=pbRJ^w`pj4cxvX_0hv-J?Po}8=A=Jtm}Pb}bw4idc#5H^Xbs;h^KOD=c`LY0!4)Xl{0-Kn_S+C`2GEfXfKcDhD1Ay1kX|reoeI6>OpaW zP0D?c$3J^<1z#OH6@cm))N1b5QuicdKML@GD}!Nrr_@2DvxP#hg+x#1FILy}Be(1) zEtxH08LJaY!P_U&i!T22M>cUZTTy#Q2dhhJG%l*!BCdA3^{5g6c|SL+c_5O2hTA$? zBL(R_k;$@^WJf3!4d=}l2MaYl##o%b+2f*x{E`3akB{kKU#dS^YDquUP}7LpC!H~B zy@zGF%liE|J`z-Zw?;HCE~dFPWBY5kGx79nVR+7<0}J(7X@a^iP1{-t(`RHrip{Ri znj8P7!rryQl)3D-im8W~I+VJs`bnyr)t2@4EXc=h*Q*-)ChP~EeV$PL1U4S)Q3${_ zusKp*=56kSS}B^_p?}IEm%HbGTEk6y0NCzVT=q*`abLRjO;M^OEOlgS%g_>pkL}vx zpi%Jq)KiZxKg|lB=s(3D%>TJR=aw?CUYMaHM*}>(K3Q2u8nsl)FA3-;kPYO{=RB%(<6Ccl$ zk1eR3V5ww<(=j`g4Tb7@_L2=x=h+|R4(nfSks~Xx)M7&T$>2K(&GvFt0BDVFg7Dk@ zt>mgv6QqyOAH&k`Zu-#>LJ95VTEN#cu3mq}1{NT9yQEWi_r#a!vqgcX50b#aupwXU znHf>CpM6Dp)RodRzvt&!Yt3wO;#lW(h*0H*f!L{1K+QcLk_>j=;l?y1M_K&q*^$}x zq28!_b)v_*-j(aGg2CAXZ@~xhz3F}Q$;b%qrR%r{`3L`<6d^nPk)Wo{H=0!*RK5`I zcRQ{iMQr;#_I-rxC27V^TYG?to=Btun)(eLf_Wk>mqd2RYB9GCI|rUsqvhZ6%3TS6 z<8#`=Qf*u%!RIn5ApIrUJ}m30Cy`EnBmZr1Soc|;>_U^#q6h*k+hfGmfBzM#jm88G zSyzurWzmf<(^#YWJKxi+) z9^|74M29P@AGcSs2q9-pcKO@iXcrg(5nKGY<5 zFqw1U*9L+uKP8ABnOAXtm8ASoaxg=?SURSby#7a3o00(u0R1**_08BI;y9}wRI%>K za9m_`hQ&F{X1}K5Ez1%g>oitm~{b7#yeA^^Wo{@)`D z=zO+dgVYCA`IH-&dfbR!Jij%+iAaY}H{A87F2u3iX;?#KAJIkWP~=9FIN&6e)agZ2 z=DQED1E&uch2nE~L1vz4M62!E^ohH{KxyDZyj$!Kd_cpZP*FFSF&6hooR#%W7aun_ zdU~l3?=WwC99)TIoVo7eZ~?U#%JO5J#L>L`^|)u-t9b*bsD4O{?#m-Gx&*hn2=p0L zF~lKcc7$gbo{Zn8zNY_vo-bl)e0@B4^9P6 z6`P-*e46)au-Rr{pdP3yn{Z{>gA67POPMIvT{-Z^Rp z!ap_HIXih|@;m`Rwpx*y52yC(%(}gfi<8ko@>22d5iUfRgCh8o+fw(iU&{elHyjnlCwLV{76Ca2tC`8(bWFeKLbzrtAw+et_Uob>?P;72g{{j?SOBZu-8A)-|ECApA zYo&ir{gXv(7vI4W(6dw9v3r6!aWrL|*Aqd$w0JYs(pFY;y5`WjRKJ}Sljw5SUeL9v zR4e{^YRn#7B)m&01JR+6rV}y$UltmA(=UJN9TFKch}^&aUGih z1qBf(aBr}a$SSG4j{qACM1{y;fDeM;fbVw9LLMMtFiHc{+$yW?V6m=S(}m32k!GBq zi(pdM;ctfQe+`OV_BJ8~A=49%KNZcU$^vo-OcFTB|8cVCbvLIym2$D$t^yz)JtT(r z7$4(bKj7ra@M)RpU3)!A*WS!_FFI;_n2bouSlD{PE8trbLvst!F$RE~v`pw!3ZEN| z^Jtm!LEUaeJ-MZ(NQ6mslEG3Nv~6w{VF=bE`j4^eRvrE--TzmL=D)V6+V7m>@vXAR zP~M)~n8dPK0H{xqsfCzupq2|OQ*Me8-2z224Tv*UmmZp>jCE| z8>(@Ur0B22#X9xCx8BxwRB`xqv>r=)IZ>z>GS}O{ECLK5ZJQLv=n`X(rgg31buyXm zC%C?we$>ByakBp>!2Pw4lnoT926h?72;*)yx}Lz@+acsp#U0!D@J63Lo@&aB@xC_@ zBV=G|_>9OSWA^iGaL>X{9)jN4*;y0RQ2(7q=Pz5%Ux&+FktMX3v1^vdn4D}HPVJO< zV_!7>|70xnM~z?=Mo<1-sA?K6VW*Rd`}djqJ*>rO7xrJ3snE~;5vs5WcQIa0xQQct zH_;d<+Gh71)&ef-n~XZ_8%oRN|L6`mGfQ5<7EJiy&Kp`6NZ^1nyn3;nZMWT}Kx{u4 zr_k%;tE%k;a(u6&e4!00DqR#Hj#SK=jA`!;Jbk%Ye@6~*3r1_cFH6|V7Nx;Z=flW? zboM6k`3Lp@cmlt8R~o5?EjAokxsz14G*6{wX9Sb{@|%E#0yvfSypFxX66tE6a>9K% z(2I@5DFqy1d(6?Z+-mh_{!c)#1x-n#f7P(Fo`%cmWY2!~YJy}H+Ka(!Cm*Ws>RnK; zzPUi%16@m5d!+Mf$WJPr$SLN>EXUqxiG;H=9_2Xr)b}pFb~u85&Ygd<%dn?al&H!h z6C8kEr{0GgtNV3&i7iLQar>bGl6!j!^ZUr-{AF+}+{tcJ*Szaw_;+7cto_Q!CnKSn z^kd`2$LV@Uj294?r|sK^>S@?JxmDsrMQ|!q>C861;R7%W??L;3LpwZZp|hj)%_YWQ zrt`6OJUAPdKgdy%J545RaSBS|GRIke2UFN@DnXa`vn13@u;_oUZ>oue5okH~?MU2^L{# zn~DCxU$oYItL!(MWE` zH#`^C6Rden$1?7I6DPKSd(SXF0oEKm>CcNx=_}iyMTO8#!e!YLp9E#wN4fmahGLex zgjK$;2JD09l>r!EYR3%B5x=yNn)W=x_5Jq$VrJ%kON^r|)^Re!1HdL=X*siRBhqoj zVQ~Di_E;SiKio~J_!zrwn(!86*FPh-`g+rgWWFS&e(OBl8=HUW*o*JJnHjqHYP)2C zh>NiG4OXx5*IoTqZsKo(N5y$J9d8l)iQ&(b*hP%V4&@Tnt3wr#z1dap)In6Mu)s)J zG$N!6GE%COHo zgxz#R<7JRviSq%9Qb1ZG6M)^6m5}gyP`5X0R(f8MzYIDc zOOyZo_EX;1v(t+I$!mVLxsgqstu+lV=S^@jf}yYOlR3?L;WLY<3sg1x+KUms)iw3W zQYWJ*s8I+`B5tO8d`%8TSGPUJKl@`~;g!{r5f~D^A zb&0IeT*@*GN>6C8GcVnlo;{T~wOrdtRn&F*tLN?Ze&Qqkn$#~4uSwMePnykjfd>i; zC|RA(9lmND_!b5=BnF@(O#&*?8K+ca9<@5XEa&)e((pFok@zx~;-d6{FbdAHTVOoiOn~1fFNaQ_N}Gw{|9R9KS}AEBH0;P3#4Kk;y_ysPb~hrSSUkX;=hDSB z<8t?*@8~|Zh|7TYgMdS=xm)RBrmO$-5vkk*oV=;KX%$xG+3gq~wKb59fCSB@AHQ91 z<%M3{>Z-kAD=PM|y z_Q^~%;5m=Ke>o-R z%%Aw4E5j;Bst4m{bq-EG%6wu?XK+SvAa4kim^K|o%Y*Tj-fUnFJ=nCZ09iWCZ;}?{ z`?w@npe$YOCS=&Od859@dhLITwG13U0SVvOn!@%OY0=by>2qAFS}AdVNwks3;fzzi z5yTyxu&C4YU6XlyeY;vk&yYeK(srK7uB%4cxbl)w?3OqFHUd;qb=ZX}jTL6%G7S zkUi2hyhh>I5ZXdHW?=e^Xiic%$Y&Vwsibd!Xu3~H*?`o=As~V){-xMyM+ii!dP*?e z^Hzhe5z+wpoIBs!2-y#sWba&a|3qvE2LY-<>fR~X!18mCFIM>3pl#Xyx<|^XyFr-p z_9y1Ajj}()&SmZEYt1X#Ge+8BC9h%ZXAeVCZpiQw#+3LBb30%1pFSlWDfXSn!6sE? zEMkt~KL6dj*bR$tRkYUY8})o$IbPbTC8q=%eCHVJ>40xR2igQVqM-7+$6@&iP?X6QPA5tS*vH~+u$Wl(`|JiPRek0DLnWQ|Wr$o9 z_SXYWmMEV@WhtZ0q-^|$->jR<6g)=+^I&S7AaVOP!`3~^S*F(x|C)CbuP#t0`fxMO zY||qTKDyv2h@pf%I)k!*{DvZl?-A)ytSlvlONgE6Uuq;$TgwIhbGP!S&W5x=Ot>|Z z@TRjfdRgfOAYh2mFXRd2IxSB#&3Tj`?(n0ndsUnxkLqqclO{SJn)i_X8gmoEpN>AS zrvPaDxc@9&jgvw2mUL_!{=ULizw?o#@T&+7@Or>o7T?y~=;*~4&RG zPr84_jEqnKWw#@D7uiE2kPvVC8u3!ycVx}UddmUPwk|@LkSw_i-gAN zo#^g>$te5&yE}<#GuWo4M35g3BW`Up$JpV91dxrwBX#rdKYitUGIt-9ro3|BZfLIG z00LFWs~XS4f~u)@Ct~)0AM(w+YJTjboo~IVdffh>CSBzA)R-0Xx@J#OAEy$G90uE* zLSo)OyPb->d83X3prMQwipS&ttGEMCOJ|k(@2TXeZDo4rD<(BH)*AH<Ro6yWU>Q_*wMZg)yhn_(u^uJ%O_uR2 z{{d;%srRY#o06~e`9e9;gbnb{heOBB0kHL#Q%2LtQ9UVex_;5}Bd5C{GU|LHNE=R8*r$yY)N)jj{Ws{TqmY+ZrO$iOXvTHe4kKq?q{?B@4= zO&dw++r#5SG1yE^Zs#7E<}aZyNaXAKRbHA8@NPLtf7U+al=yx`U?>e&P$7`9L zJENYS;)`fK4Wz!QKlB+h1J%G0(TvT6*HhD)j${gxHr(SR{EDyNNN6Vgi} zAdsIbN$mFK(}HH=#_3`1r+47KXM7` z)uIQEFzE~Rh`Bgyrl-BWdSla^@PPZIMxqPR9ZM!E{?-u7h3$sNG~?OKy*xwaAOYSU zf^CgMxie`>C?HHMd^>+R!Kb}sTrqv>IL&K_;ki}O<>tPh^yf#T^D3Qd*wuqa?#`pm znhW^aY6M?f5yNFRT#?9Y2%BBR)FcMPkBynef(Ai6P@PBf`jK1^utKegFsqrZ0|ZJy zv;3*)15|`~t|Bmx1!GMO>5&sLu(l@OVHaNK&upw7(G5;b_U3{XQ<5^)?;MZ*HHo3)sy>(1wS8H$}` zVf|sX%h6I(Uzs^bQb5@w{4K$dL|+u3SAF92^UPpvcRY4kl|Q0W$6a z0x<&g+Md9NtPfE2!?=r$ftm}npSD&YZ^05yRuzso`)l+hoc1DgBo$fwBdyq^0mmXv9X=ue9c$!h6`^&3;X>1oB;eA zr@jXXcw+fPp-u;$e+|Uwp`lJZ&l(R$m2GF&Y8ei}O0Be!usn<=V-)`a1j#PL67ZLx zbqxBXr^pfOcq>Yf-|Qfm=;n^6bq}KJr>K??Luj$XEJzPODwPBGpRXHJc4tF(eN^Cp z1t=>o7DC6Cn%VGxpPRhU(b{1H_6^17nc;|Y{NI%TrQN}lM~cqg$lZj*Hi8&px0R2i z4WN6J)J4`mQMZTmdt+O3<#J7%2 zwIogE<-giSU8YWVBrFCKUl~fvv4R;Tz|{#qmhT^pkzUAR&~Z~!ORF0UBqzsPaf2pt z%YYXUD)#~syCW96%_}O$BX%B#ZBj` z#7z3>K^F%1!Dc%(*mx?-cw8e&LvCtY6kpV}Ah7d!GUH;gNi>7mc3)Do-r=utt5*Ca zWac6k^UBygw2JHqp2IzCY^*|xPhW~SNY~~;XRAQ4arsdBgfpK6DcbzqZz#vGmxtG^ zKi4L8#sc~~`X&;FKlp!i>UrMQhG(!JE~DSX2`)QT?__ z1GNA!B_#L`Sm-xs`M>x36A|C|4#-DfNX7vc1w(=WD88eI1_p*-V`7L}BgLR0WyzUc zs$%nt(*^Zh=W!q1>A1jis<(R}BBp2$gD!g-7M!3Kb0i zA0*HT@hG}%S0Xxu0qpT}2Aq-q^YgYmncq@h@HEuKmv#adJXSq_B|6;s21~PHiiSDn zp!t=w#7~kT%POC2MX| z+T2?2s2uQiUbHHb2IY0-H+>KvZBH5-^QMd!MhGDO*RQ`*_#h|xfproQtw>AujJT4y z6(m+`jqY%(e@8tF|A~M^q#w_-Gz3JW{lLJ>xtGrNqs#=8LPA4LfZ$IJfv=P}b9)D4 z)hElnYqkvCYi{gxcF?%@ycd4GKYSa&QAV(<4e_n4daVz1R)rz)tW;LkbZrfb)f^aQQk5-@dxH%IGxhvuG(xGiL96WBjAc#7u))oDIh{`Kn@J^GoVqV{QXe+d{a zagB^jW=$NJ^03Otq>YYh`1(a72422+@mJZ4$!Xyc6?!2oK?ZZgt8Oc(m(%CkPD|$(04O|5|qfSX?5;OODCdxa? zHlqqWN3ONNG_YlVeNWtSmOGU5*}Qy^n#h}LSFSksx^5$^0B$RoKgNO2;6SnZ_ZVUh z&iB`6kF`5&bbxclsng4sFC+cH=N*V9y`I{#!bk^yYxY`Xc%S+=H_e`cd}qbPZlA?L zu1znOD>PX8cTc!j$xg?fJrDSZMX(N3&EtPvbA1dw7&9vazvYxAcZ_L^8OU(8f?x`J z=;o~|Ndy7!)k=I9yKw1vhTKb%EKfnSA?^F4_4z*_!{%SG6g*hAAd<5`RJ&|%0tg+kk{rT;=TD9mUz zp;C>Wj+%c(|D!`;m933HxG1yi-=jFOfN++SVbo=i2{_Y1=`VG4b^W&y2%}Z;9{T0C z%i}BS=F8FU3$27_Cz>Lzj{WKw;xO^E)Zc*|bObN?HLc%d?)^2U=Q97{_yo;}Q)SPS=<=m%iB^~$eLlqb z^ZWRd@JB3}$!MWcP10v5>!&qZZ9OvS&zoK+UfaTvgtk1I;N|Ee#PMIXBHbAYno|C+ zKFm9Gh?s8nu3OP^8H9;(W^?Kr(L#5|kL52hBQ`xxPA~^ylTTSbE=H%Hh`ZPaLYhlF z{YKc{@BZ9j-AW2YvX`&@PQL9u5=`U&T)&|9U{bUp(x9$kDzz+$hfZ(mTiIuTTn&`G zVw%T!0@iP0jSDlWJD;)QFu7p3(6vMI`*U)shPRr;8-}L!rxx3-VMedDI%}kw%>l{r zzB&6>qxk5O1im67y)nLW7Q8B8ct|SarJBZL5HP42_4mWu{|uC~fEin+(X&|z)^dWw zW!>Tv;vy0g%sU`;@_DGmppLDL8`0iz3nkm3Z%dgxWSc@vDIYOuKKo|Yf{c)#e2dsJ zKIAr*Y%Y?M-yA*O-kCc_+uhMi)t;7JB^!k4jooI_x$e=L{)` zmZR42->6X`bO4_YxeS;YDW7{@WbR*CEB@wTAK706{n-#$nyP=^=iKg8_c^)T=S>xE zycl1WJC+hPG>op&8k0}f)N0R2CsFpvJ5#)6qKTwTUTOI|EfO>%U7>%Bvke@ja;r1G zL6$%MWjTbN-e(*oBwEoE4`St3?6iRTObXGH6fyqRqANL4>oVVP;uU{NslCpw{$z2Hv ziNK&BW*1=e60!`%fhtNGuOLh_DqJqM-EnW2-|&7qQhgV6l)fg0{0F|zhc8s{-R9}ISG}q%MQgcIBxBO0XFT$jn6iKb z8heuOKqjG$a_9j?*yYn?@kH(R;)qNBv))&;nzNLiRxc`_EV`G@{>OjmO!bpHQEQ9D zo$SZQyw5RIjrp7<(>qi>&m7*+sPcwo%@nIqa+WwT$`di^hy3Y236uka7KUCBVhj&KBo902*U{PU;yAJ3*-KG_Lqsa*&HzHOt$!XFY(^b>1uU8fR4=1>ew@(kKI&5tV2gJby zOFOR?LL%bV5F?)E%pD%6h48uTrZKtL+?#^{~8i05$E=Ks04ONpu-_glS}gK|p{tm$$vIJ*40E zGgcpJkUB_(&3q4I{cROtRKxg?FP&n8a=4ITc~9WY9ETwP&ns}((1q@3t(b2HgIkd8 zg$?EWbctqZ&a6gHS$y8tw5j5&N(Jljro1FCxSC2kNIe^h?YWi5H+p+9*N3qS>2BE} zQqjiOSq0On`H+*GV$53Kp}_ZOw)uBHyC=FW6RzI$p&3F{=uOk;yEw8mepc0I7Cufor?*da< z_5Jn7#DP?o)2Q7KsCC(6%0cP#6>8leOSKkwA`)RrDW3Fn7 z6o|+leI~4V;+mS}xl1m}jd2y5D&k-9zZx!4=cF;&wwV;gidom5i_yDhTrrEk%?Jp8 z5)3;)nw2xsZ^_{Myu%^lL`EesMyIDh|6U>GAV$EKihL5+g}j&Fuv;Z-=Xbi#V>zc` zp0T(Sbemx<@m&KTqJ1|1{N^}jbHk`y*MgsxR?7Y1LZU=7(7kh^i}dTteTmPe%FGO| zvjGc;d-pr{6OMqe^wYe(2OZG@fTMWjn8Bp&yq|omtW_(33s>yvAwhzC0aTio{7#$? zbRuLit4Fab+W{1OfjqFns%uMk**-y)sJ!6R5^tFnb;8B zhP8K9I!P6NVESayQiu(ehsJX3G?%?=*!6sWcfZvn)ZZbCgw4DUh9p1L(!%;y-typz z8+J2mr@Tr&FD;qw)^G6=sIcAE%vE!v>*(nTZa}y{toicgORF+p)5@qNfO7_uMMyvV zcp6MeB9nlwA=78!{zWHhQ08hda)utr4S zNk@in{vR9{J`rjVy57hA%H$dSN06n7?>bp^IXQ{GV|)5a%fNEwOB2btH`Gb`%$z@M zWgAZA(B3u_HNEw8*Tu*`vQ@p0YM{o5(nYB#4M+rn4xV6v_XCFmU-g{|F$ca4g^j80 z9_XbkzlyMHuQ;!8{ykRi^XPoy`}FD&sZ3vC3h&NG{~Hh-R!m7rX|+U1kn0$x^5TVa z@;;g4n5tM^#Gfh~ZD!8}!k_9H!(e1Kt?>JsKvVz*)C$~ZXGa33i&Fn4v%b$jCH;lh zAm6Zj5#UNI1trb+f$7qRzn_9?T)?FLGEmU!0JH^EupOplcnr>_q{+Qst^ug&@%ppR zu0MJ7&%?07z)T>p2s^QI?Z3&IuR3hS!fR(SF{(_iYp?;LKP*0pjKD zS57XzKnd{2gmAS4bS*e4bbI;AULQ?3$1O{ZUe{mRLah<}21p$t)X=xmmWG87Hk3Y? zc_3?`OYAhHvr}vT2)(SEX_yUF3iXf%4lVLxQ^F+iP2|DVuMe-;QCJ!h5g{(al5w{6 zLyhR~g;%`|ucZa0&CyObnCz2+`=7I2?1_T80;UW_(dx9b@y_xyOSI|+)$KG8fC7d# zQoLL`QVOL@%Nre;ml)c;c7~VZ4Qqb0{Zfrvjz9>e14dM1Nj~h(1RJj-2f=+1-1-H z{3f52Gn8W3_O~lWv{IBE9r=N~p&1a`0~3z`pj8ueSyRcC9ZD8iMvlnA2ZcroPf5Ty z0?9a?m;Etv_U6!^M7(9n!|yx76Hw06Bin&GgL?x+rNv>88eP|*YerAxCpD#lpYxOh zS@u5a_CqzD?Q!Ncn(o0{LZGFdpX`BWae)g%ZVaOOpG-?^^G*t27alsbI6&08+#QOZ zT0@`KslM;h=s>%qS$_Ahr|Yb5lDMBk^F`;St{oxds5mUC3hk2G0|-_3bEU#Ifr_SX zwW4ZuU_iV2UeO0j|FhszJ#xXVpW>f<4U{RMZ1=Lfs{x{vM>EQ#w@o=STt?%KK^OZu zlaCPN;3T!XQt#J-cxY|PH*B7t&6m-jKaZqg#N}XH9PJtY5}h~X%23{3Pov98I}(cyQ=8@U80 z(Z?;;@Z~Fkj@UrovEkc!DkaYP1QB}lyx%M1Y z#@CNZ?*0Gk38E#`fO8IBGvUy5L~R7Yihy&SIAj6Pb2<_0JE%Dqik(>-IlWt7&sYI> zP6vWmYjER?)zwc}sB`hO+-3Mfm9>8dA~7I$*`+wxuRz}na03RQB)H%IzQ@@eJVDyX z9lJEnyv;NSzQ>K5qisT={|;Vi;K{tsU=%F4@BKoZQ*Oq~S3gTDGrv&*PR9 za)$M~BG>jE#e4Qs%a}l; z=}yrIgx-{o5N0f1Nky$1kp20~F$K`;s-#kM>(*b(<{ec>FKqBsgt50y~_I(O+_<^@T&zn4^n+kY_vP-C@mWs0l>s(J@7ZjtG(K$72CO}uf zdxZpj8QEW|J%@fO%F6YfrK5)hvR$r%&I48tcYs5PWRAckA|kK#AHkmBXAa-Vt&IUH zFfV5|mWT6wnVpARXwXE#GkW6K9Z9^aK#H?%a~INLW4q_)xO>}Jx@&rewaT{pjLtXi zBu+C0(86nmA>sBW^ZyHT^o9!Q*-D?yTWq^03TNK~Oj4;IDD7rK7@T(C2U;sEAy$Is zcL~GqD0S`aHBi_Z6+@MklHxewuB>dVA?+HZcvqqGNjqR!bK*${NU5=mZb-;09fim9 zwq>_gY?cZ*iL53wP4i9&({%ZL`U$$5nok+e%Q&aOYN_8bM$|LF}pBk zpddH5bwlj>0ZH!u`4_W|a{x$K`yF(DP1+@}NJ&DYp(Cofu-8!P`G?79^t^hA!_|@! zpEMvnmz}~8MpJmsnHRQ~Kf$Ya<4Cl$yo!P37061s1%3#mP9Lav;5bx2O(+6*%;`30 zpw`%E(jIlg^wDkk*5UBBW;WSmT6p@>Y4`NrqC#<(ON<;aqDaNXf)p9ySGjr*3fpcJ zP6iV9za}N44dwvNe*o~-Zy^2xJ>p=Dx@;d%09~V?06J4+e%kzb(IPfr$O11P;JwGi zb=}CwC^Iv&cDsBaQ4Z3KjsYkgAyhY39^72`C zw)sOFAy4I4KQSrSj2L9}_f95^9)%ihU_n_TW_fSpG)G_Vy9`qYlTRKAv&i!NH;4$i#sevITy)!88`vUNlX!VB~9wH=%?JCt>6Hl&(;FHbD*>=(UNui_xo+Kz?++V7_0*Z+9_az~5y4XzRqM9o8s{5dpFCt(zlN zjDJe!r+HFqM6h+IFcB~WS-E3QlYDA=WGJiCoKlGywMKjS(ABO)RL}VtS^}yY)VaX%cK1*})V%Ke7MO*V9G_iz33@tEHnKC8koxSyNT|^# zio!~W=htc7r|KNVcPRGUnm(Q%R^P>VkKBYB#tx3897`G0tOHyU9G<0e*;j|>HCe5c zlnaeGyMJOQ*JNU-T7KXFP9OVYqA_Z27~3`ITQweA&&pZh@o+ zZcS)pX>9w5CR2~)y!XK*%AGeT(nvgP&&;r(DQAtLG$RH=+zOmCJ~Qza9wDzJa}|?N z*b3b&v1XG=ieNjd*&P9@89aoP^WW$7g|N`Kc#$Bb z9z7~f_YjeC@i^!#XJ5ZP9qfqPn?Wzq-@V|_sErOEUXse8UChXA8cDP#_B{!hC3e^||29+5>CHfYiSW|uWU^Z51b zDoIJd*ADhazZvX=`g62kJQbXYH?Blw9-o%+T-y`)BbmK4)An)h8~sR)T&}6og6Q4z zH^lx=7|L4BQ^N!DJ|1xU%FE_s{$El^MY7MSs$(6tD*kagme zg2dJI^tVo_$Lb0hzx;nU1}u9{+eE=do9>zgFFRGH%24&p!k<1O?Qt3zP(RB3g{ro& zeIPOER5A8@{YT(Q(D<2z-j!-Dxu_ezPJb?H5mNb8yeRJ{T?g7Nd>Tmrx=9PKrAr>R zN`2^SVneQvDsooe5dsTcyyY~XcYakh?1Hq%4oGVi*4otI3h82*C3hOXE~zFNL~>Bl z#zY6p-BFr%GR%33Yuzrm>r-P>h`UqkNbl*@&@I22l*6g)<+_nI<#hh^@KFm1?WB1s z-j(I+*#gKi@AL9T)z*r$-n%#53(Nm5LHg(3!532j9c z=iZHP@{YDvMtsI@G4>oMYx}pli`y4|rc7Dg8)6qF#kiu%(CnCXPl5=)hRhH4PIHG( zHtA1}#p4}tM?b=r** z$SH!E$2*n<(Le9@r0(BR_v8#(n}Ph;cMqjjoxS#V3RM_oqQX2H>e5itxjxJnaZRqx zqvrKPcDKMk47@-wIk(X45;~q)6+Gb#GeVh^=>ez34L4G;boZ=(E4`FtnJ%#^VCZ-B z<{@=Yix6mVbMU3J+`4(Q6&U%@iaY*C9sYUJ&?ge~!BY?LB-pvTK#A3B(ahM-ZT@$7 zkRDwOT@3;JM8((0P0QfPm0&WeiWS8p+Z{>-tn3Om1vg%+Y^d3VyWC&|MT zPdT+x0TKbckq`lJAGK`#K=|gE;s5I-*{8(=zH`{j_&x6e*R{7W| z@E!f!T3~u8AN^cbuP$Nb@X5r^&`%h7Y6 z3Un_imGwm`sy%RIUBTLR@5O#&y+Ffo|5sme&R$mro}c^)>zHM1K`2Hx6ob zgHkAMCY^S#^BPtqGU?CQHGEO0=pz*FdDc^OBlVnQ3^K1}w=X?4(#i8ws_YQ>QHlPj zL5IV0PWj`Y z3>>&bj%6s-m6S|?SiCAN8LxvtPkxVD9r!MXUDkThpK3`TAFJa9-uuJy&vOV_v3Env z_zRzG7CS&Ou7Na@H!%4&Mo7!7RdS)bs_nZ=?V~OYPuggNlvJ*uZ&`UgXl!gPM&Oc#Qbof7SkJ{!QFaGH`MSX{u39mP+f%G*xFfa_3R{wwV3IcCK zZ9pQGT@650(*ub#Pk6~Y(VP{*^R!hwC-|7>(o~F2XG3_*bUm!BYcAGB@=ZhG84iq~ z9p5(P-5QHee|1+FiDawzJ@{>B!xx}yUFKRhh&|osHe5ps{imVqe7kM+G6TKbWsV(% zfAQUyAM>nx`-0^H4}xoG9weyC)*?O?S6=^fblvJhkQ}iT^`l+cL?6;^y(?vAtzxv~ zSfT;LfQX%_W}pb0ZGID7aqa@2BoWhQWXFaSn>-eTxXn_ zw#wKuJC)t=$(@`nuLjNnX=h@2SMpX%X0Oy_slQ`M5WuS$&VTi0)y__+z}ZSLw-7to#J@Va#~DpyFgxbUy%PBC>p>C z=&-z&@l`XKsI``Nd7U;BA;Es3Xtz25y~`GRiCz*2!gRSgl^)&g_I^0KEsgfiJxV^0 zT?hZlY&qbL8|4?o1^l*ch*p;gVb#_tz?@I05+1#I%MEAe#tp z&uW($8NYu0I?^vIi`$~-jX!Mvxspn)c+PBmLvMk^kyTj2GE*vwOfw|>p7!lW%U%FJ?3=a zeX%s8prqz^gSZv|0`^S0V{3m{kKP)K>!WylB?!yi+X0f^^T*qWt04>%^AQTN5I4_x*3f7Z#_4ZR5eDH?)H=!Z@871o6jc)+O zXQ|Xr3H(Q$He9^J$C8|&Hvq%Nz!ZSOp6+`U7@w{dki#IyhBHzv{%VVscRWuQR~{(O zxM5Iq2jqO*CD{=YK;Hd+dY3*y7>N97y}Y^Y!Tl6|tIJF(c5JjO~rC^?Q3G zt(_B2I%2}PIRIGhVZRT|LVtEH~r$xrjMg^$s2y0aZbN_ ze=O)JP(SuQdgbN96UXsxNq^`n`ZJ;p$rRG8))QLCPink{GN18`Ao~e zDgob-$5%q2<}C+pj?ZgP8k-tJZ-iuP%ky=2N8~_k?66+tcsBJ{HcKO|Tlfdn-bZtA zRK1?%>z(ZUi_NM7=27w@6u+>Y{b8V2W;~`{Tox~5+)i`z=c;@5A_o`uQWOmFpMcb# zye8ms2$ecup#F%L&B- zZF~a)==qJy5!>S5z=uyiM1$;kX&CC^Y>AMXPmE*y0hn$f!gO0G$zw5` z+W?`h$hHGibUXtn=p5hZ3xg`!Ek@ry4A2aS7%t3ra}XaKhJ=2}Z{Du>JJo6lB*^ADM zJUu^JC``d99z)$Ub3S266C?7ulxacNDT;`2sv*f5uxY z?_%Hg%ou@CkAKm#{d-$3jwiKb^mPJyyc4pL460*6uQG%tHN?lbLFOFpiJn zQ_*1EJUcD0Hk?=d>!-$=ix<2g;&2v=>7Uhh?l#}WQ(f1McWAgl@2bGrMC!qVobKj0 z%da--cs4$G5njC8*Xaz+LG}R;{GnM%c-_+C^*x$KyIVnMe>)0D@kr1qEI`k3Ni z24*H{#_b8e%YA_9k@Izsp5Zmo2zOujIJ5Efg)}7ehYJ$;oPz#V@;7D~ka0WARu_Rc zT`nsttBMD-U;Im$5uYzU&no475KIxjYYJT>dTTSa(C8KdTf`2{-dZwJ1n*h&AJf_# zfloq1x9M=efaOOB5_q|-+!>k^j#0;*j{m>yC%^vu`LC-Qghit&JolIA(Y!k4mp=eQ zrBP7MNcQ5V%7fRY)WwHxE1%iMq&tS_D0`H7duT=a`BalFf)%FwTOk5_e2H^=Z5LCjIDJo0yzZ9N zZ)Qh~KOoKSB}r%Bc4A8QInMl*bEn#uR1QWOPYF-#qu}+r4P-A#bMk6`)cfT2F`ICM z8Jy6%+zkh@IU>5ykms6WZkw;65S$-i*?P>o(fljzTBHs2^#_6d(v=T=?@Twx^O*Zz z|Ic~=vv9D}+r(qF+k5bvAirsaIsrTfY&IMuR{~Bi6+rLx z3Sds1pnIzG@2RsFFBIULO+{#zq$*b$IlCG4q^dLQ(7lvXXCT9={RzR_%y06xebaw# z0N3Z|>z6)8&!mxQ`(J;2t12VwInP@+V=9V={saZ{o8$iQBoAlcW+j7%afMx3eE+ZJ z-a8)4KKvV3-RUmc3RxAJqJeB$WRHww71=8p+1k1h+1bioN%p3aL|tWX%F5n*J?~TZ z{QiE=pU?Ap&Q~cH*L9wsbsX>GeXOx{5snk${%N|E0cj>pr{l5`xBMjg-&Bs?1)gK* zXdjUlQLZ;}`nGcU$w2O%3FRJ#41^mqrS}Q=e!E!w(V5-!N95`*1Ws`@nb_<_Xj*Bt zMslO7<*7WeLA_RiXlCkJqX%9I0mZheC!U3O9tmJN`aJIN`a0^Fq+0V`>YNNx)>nU0 zTSUSClsibuKa$RyN^YU!MHf_U>_*t1I}@&R?hE2F-iz|NM9t{;<>kKK-sfy>c`Pg} zT5ngL@Sx|dt}%o|hm-ceO1qL5_vcrp<&-To>&}RD8=m~h z_|rN@t%fQ=lK;(}TP;l4@+L=ZjEa*hbz3_|SDF_mQI16V?f)RtZ@1VIv8sSkIcq8B z)90@_Z(dbPb0sN{9LSHyx;jaVewcmwL+b%wb*}C+2j3&dA8()wqv*9edRSOfgy&+} zIYq5<%MhW?hAO9dx7vG1JlwXIq5JxaE{n)9vJSJ=0Qh?1oT)R4?;hSHuvYx`moR$5 zX|QI!ikb31FHeripS$tSxk_0rbNTH9$6Sl+4zC!#hQAg$%jRH~eL?SbS$NM;zOr3pBTgq}@ zvR&3tkv}P`K~dI(5l?RWnx%EAv_MJrWz56m=awgjh z9u14TQ`uf{dT0;eLz^E8{{D6Kt(8Z!eyf+Bep}^GN3!(yhR>UhvM1#loP{sv?DqbT zD6w_8h$SPFa+_(&5x=a+J#5(#sSL;61cV{W#k5ne1WzSzDR#_YIwamlm&&(TQ$A5-}|seRSnue?jkU3^+=-<3_gTMQWOAPWd4DV=z*6Ge5YDx!YSdyT!i&Kcfn zd+=~6YWS)8^kKla(RBf`dvf7NhI@SbtD0tZa%iGgO7ux6rkD(d(V)epZF zOC3D;!)BgUw?{Igd{k)WbpMTeLIhdgu+>2J;@GKxb7!gFu1tHLpZBKio!=xQ)D-#A zO|?hf;%n^LIr1>9Qu&McT zztM8fCn7_74T%M)&Ubtglrg|wtoIFyWh=ehbT>!h z71^{lxUSsuZA)+J-jqVR#s1v&V(B03J2@0KAX-t>LUjJkfEbBh_^Q>j*6LQi&qq-= zzW!dyACw-^*4GE!&d-1EJ6ZgCcgL}FzXUVBM(E@taU z>#&Up>hjn3iKC9u5o`((kU1ZD=XzQ7fUElte=RZci$A(zR!4u$7V!k0Yb8x?znzfy zG-ubfj<|obV7cZ?LC%s8Ns>urk5d#1(Vr=oYIrNvSV&H6?&tcWeDl#u=Oayp`|jI` zG!l1L2ukt58H&TMqW(xxxn+=z$+z?(iv9FdCT4FFD9EsvE2Esa&U7LO9-}|6AMAu6 z`Dl>_WD(|93TM=u>EJnu6S%vRZNu-cc-taMU(n9oJmI>v<$5XiAMpS3K{1c^%H|g( z-(qMk3qJA}-3X%l0r!C9PmHI;f!bcka+x&0LXiRUcLTd76_ucJ7eZ+@G3w z8Kv)o^t@oT{&{hpOS}ffypfx(J4U1#dqcFp)a7GKeo$n=O}-}geyiw?gq5W06RuBQ ziS7B6&%dQdH1!I-`T$*in~a$uJJMNi`rYIScRzhhlUmnrR9w^OL`r(WdZ~38n~ko> zQ(<~R=CZbnj{avE_iZDv-@e@;TW`+>P=6AaeK_^o<&JH-k9yi1?>)J~>viXP-J^+T zHjuUhvIiKIwT~W1XYoZ10tA>>Qv47r{&M6?iFSIep&2R<2dq00hBN8jeJ*A5%(nU= zi+?7O_b72PI4^P^?;f(C`dG6ol%i+vS@yz_-HP)2g(nP}7=kY)$hDUsW9yCNn~Eb} z_NVjsPW`YFJFpbrfEtM1fjtgvBow{5wo&Pn<Um{vHtJGjG2heZc0J;{S1(aW?axaZEk@uf z)=NLPvhHKk;B%$$!gBfjva(hJ`4*!;1sup)hUJG|XX?fs2-X{xi8C)=H#KU&ci`PbSXW z<+$Yhd$U|Fj7@5UP50T%^?rsM)cwJE?}pr?*SYei{di|YvRa}pzsaDEb%gOd z(>dJq$~}RDCExb;Qa|uT;t&I8BI}S=@@KxsyJT3#i1qUsJ^mQLQ>@Y64;0>TS zQ`gRJhKJ`Q=l(iKbPvhrUXK;M2$#8VQRbouO>5exwM=L@Ocy1KXa54WMydk?HV)uRVBb!B_YL~0NyD_cxulZV=v&8-_ zc~5@7oBK6WL^JoYrq7i{-sejPGQzrc*6%!+cf;%F7j8+qqJ_BUOx)2v8j3bmHx{zY zzhF{4M(Tohtj6YF%&*g9jpK~d*0bGurYv2OVWUMzTW^0xm@RF=?~@e#ubO_KdSkeZ z%0E&%x|Ar48T|Oou8(bfd8>2YFYV+-hx@KiJ^pI&yXoi=pR?vJb2)`wJSW$0;|L5x zX~XSQ0{<~nDg8j*#?xfJ*0L+-`=GQ9(MfPyg%rJVxlJn4?Hwq}NA zg^gkb$55lo)2VF3!QW>s!w+Ed$gl9#o~7c5nhE=73U($OhZMNnb^XMHR=)v??v0y_ zg*QE*ewbm+?&g}6bmaM_>emZmeMFq?5iL_Z7Bv`@Ocm zvsYQ|bD0)Oi8ITZf5!AsdKi9-3^2%U8Ad*Q#hE^Tj*t;WDvr-KK_|lXVb(O)ZJF8@ z4c+;6>D}*>TcdKdEQ1%h4p(t_oGg||-Cn*V#Au0}eU` z1sfx~nb92|=88-SLZQ?-T;pY7l`{6VCPdDJCH8l2uXhCLPxPjwAN>Qg`>X=(+50?7E^7(OYdz;w5)W+p2DkfPI3rzfg&oqsi8?C$- zl9Orn%HA;>6jt3hB^vqu+FO3Q4hy4@uTE&B;o7ftXlFuD+qT~OZZC5rEea+q$o+cl zIoq^#_8V$?(?_!An|@~rdDp1>bblG+IXA)3`#xmxHphpXHP^QX!BvF>v}?c3^Mr43 z`h6?st|4<)f%l?bN6bH&Jn9Z=7@MAHJr;ODSb+-u)hi}PU)3zNmEFB3*YS0qduPv? z_shY%2k6`?;$4dVIo}uGFDu?Nh@3%7S~~NB(gM55#{4@RXt!WB(i(I;wZNyvyg2MG zS=)hwT(6rVdGf1w_1Enw+g^TxHT+ACx_4V3l>^V>(23)MVy{QXn7u?IdxM%sbT;Z7 zC6A(g@XaGTJ4kOG_x&Z)-xgHIXEQqouRPAq-sr1xlsxtHNk8o~e%I3i<&2KU#wR~H zD7llr(ydWJ`627z*y-tn5@G8pot!@q(xxZ$*`T+@V@JdeunWxZvmfhs=%N0y=xf=N zq7S~+^#3KfiESQ#w|&Rwkh5M0Jq~DXbqgxGbmM>P z<83;c&omw}JtXp|T!nQ78A~(|XdkE?)zs8;EI45sQbi-(`z}o(*-uW7qtuAQl2@|g zyBPW1p0TKP zx1IVXMl0mVgSHn{;esK^Y>#;V{-UTCqcn<4!)pPVW2BwUuXiWJoxUKm*FWTK za@riD3vWlPKk_NDQKf*VrAGMa?({Zysq-H*nAq>`KL5B-)&J52+x;}{-yUqF+)GJL zL6h96^tkl;#tT7FpKW*eT9D)~nKx+Xbj^Pp_s%g+TeQ8pU6`!gX4pDkcTt>TYFqLV z$3?}e8TJUArAhCDwvl)ClV8yI+(~4Z{&gjw8lL%eg!W6w%kMno`^QF#y~x@xnP^0N zIZ4zlC*mE^U7ZfSS+=0?LG-3{(~BnDuc`Q{vp+wQv+17Wq3D!Z-d>#5FkI9kmcBlQ ze1k6e27J}q@O)LPb5(y}cpno8oJLjW6#@f*-eL#qcR*H(Y;K(VOqi4uO zC1cj-?4b<0&hxO;z~WfIA{`rzN4-u=bN6$PLvcT5zL7pVCWUWh9NO)X{jKG{+lZ6r zSmfFLXkNC23L#|ZrB*>CtsujmB*x@)@%;Jq=%KXFOK9Opd%JY}Gre?1vGKgy1q0gI zg-f|RE~p-O{N`mBUm=6oxn}K?e!_w4lpm~fU&luCdS4`Exp9)$=F0Cn?xmeq#}~&e zdu9~{RQ5HH>56S>qd7^(K#CFDQ=;Miou1u~pRwLI*wkd>mTkg`^$%B4kM|AsNMgEI z4!J-S{?55^S0{tW-J>NvykGNs8@(+!>DJ1l{CT>gM{pSKI<79y_LT2FER+1k>zhIM zV>+|WZ(>ZW{QM!sD+}&BnMG^mABRu2C26EJN~X1(_;N}A!HWU|(^7eXEXF-a>IGSh zN44p;UR^5vVg6qG_YV?h8fNK_-dACFOloYpZcA4f=H3)bwp6$r=GG39W6oibe>#D^ z*L}7sYN{)au#%rzxySO zU+1*a>No4r_O>3ur(uC{yA2)|P-^x(GS-UXXxD5x@_3w@B&G%xx{RVjjh4biHN+vHa&oyI;UiJ6x-J{yO_YFqT zB_u|qT#loyIsNQa>(2(}S2rqe5T&FUKmGjVz5@IiJN+DgPSQ#PWu2sDD35(-+OB!1 zpfaLxeCG6b^TZ?P7wvi7TDkV5dr`$C9ed2Q+*}#ovg=A4wz*;cZ*C z@ae5GZ=8aJz`d=kro?-cnSKk-(|)~2`{eVJGzF(<98b~|j`ImTXsZ1AD7&;p{t0*R zNq_37LZ~j(?QxpB0hXRltLA(8cOBs?We9=DzH+an<>yk~M6PGSEA`*c(n_Ovf$Zwx z-G7~D;$LsnE@&6|%dJPBUYwe$BRq!Ame(p1gywrbj0%L^|M$8);k<=Iv;*$4o^rkP z`RB`$l9KDjW@m**Ea{DNXE9-@$+OzU2(R|PuIK~AP=9&2!gZS{C@AiAsf{c@(HTx5 z`VIYg-98OR38*uovT+3M{>VqhDA!Lwdef1$)*EZLMHPO6yH%7OV|6X zjqE3OCr#&hT@;ca%jpYuK;Oq#HT3^5Si~Lo9rMMr@19&3Og&@pb-x?>m3H2zupLS( zr};;zApK+gS7iT;6t6mO*N2nuUsv@ltar)ONLxs1(SjR#oq84&2rxPfoeC7v$hXmL zFR=S<(*_Q^~7Uxhb>Oox`I+j_*y0#x~Ny zgSFhRCV7CJf3<}vT!&&Vlqe+e0}@vsACC51o#=^yOo!YbZpU*bzwmGV{r54e;D0?K z8CAeF+X>ec8B?@8Q(mVLErhyR9NT%a!?L8hKQAOG-o*W`@;Fi{XO}V8Ts0GQ;7BC$3tT zHOC9_-^)=x&z_NqV85ISlB$~-Y%k2Oboe!C(S}JYYSAC%+4f4#k7Jwy&;QIJdv|5t zL9u64<%cFKD=YexlI=i)IGv8(&mp^MXf7jf9tm#zFDBPI#ia^_7hS;IiP+pY8c;wP zm$jU5lxow0{SoG!ys7YDN;rE;(Of{PZy>u2?wf zE_?Y2LBYbM9Bc{3sz0CYhNP@)9BSKy%S6e4`kxYhH=isdyv1zY9RnlwdXmU$#Z`XClz*!7Z>KK@O=Ip^o3 ziDU35eWSFipHyZBn$qK3(+vxjmX?}{R2ti6+ShNM`T)ljt7eXL70pz!tU9Xv%n?Z^4US7rI{j5wax zaa-h?DZUbX&?)w*JI3%o-d*e45s~>2DT!+r=RRmBsEG~F*1CQT4Gm4KU-$#P34fjM zr-A?4x9>lESpN*Q;>xrWvN~`!)Z71bNd0HSQZfqA++rgYp=goO;t9=yoF!YNU;qE_ z*W3LufdNb|fcZ)@X-y`M3;l6qsM2uSac(T{vh=U*F~CdEt7})Zd_NlZUU*Eo?mE2O z=q~*>hE&N>ECn=5M&x|8x3^mdNBtUDt?Z#+Jw}Ov{4h%`Ug3i-dWm*P6zn;7%NZR! zplPnaoj=r+9)vc%WRU1RD~qGEx3Yc>T|w&~x%X6Nw7qbCASpEmX}KROE4$Ll87=#3 zpf+$&#BSYsz3DWP{c(9Oe7p`8C0yqGuP443&5h4@MK!80yUAW}AD{bPUfa>EtQQs8 zI=}Fbvh3eqQgifIdf;V2xTgMGmOuab{rZ&E`d^Rp_t$BH!T9g>&x8K?`uhLNuiSo1 zv@t@1BRhynTHf0%XvIoHNe+Z^w#}+$v)5Ud6-V~`q0`9p8JnJFJ$B5kl0DyJezK2g zmImhO(roc+3?#zCM~`+=QC&n4v+V+7WT@~ZIAW1y*7hkSyIlfOZG3k2@WF%kQO3=6 zw%uX7d)e)`!fF>@qc=NRK7T-x5mGBYG&BT8M()o;3n`-H7r?fxfq+wZxPM?^IaVTf zbp;i#o-*`s^@uPsEz^m~oyLv4oW5~^S6p}x<2NPlN`RKPk zRbj=IRwH{#I_{z8I{jx-v`(KsO}vK|nZha8GT-91+@{U9>Q(i}5oPfP07b|11*tau z@m4yURu@sPzELfw@PjW$Nl)F2gwL`sth8qQt5|3SG8nRGrphlC`f~Ttec_ox%E)=2Pe|KZF!kemx-sS=?(wcv-UrIVH_? z(#MZHBN3?wc=|#ujh04Tx-HOJx&^4ym|ikMfBE&T?{B*nq~5)IH;fFdR2zDy@aYyj z;m{%9ivHqK6bwupP9>!~^M(sY-E~3$avjEyu>y$XS$N5PFPHNnMCPQ-EH6k6xGaD^9h;&5I?R9c7Q;&#ido@au{-fx^S@rTseX6PD+!xu-}(vl-KMqA~gOt?VP3L+pH139$JTt;&HhQJ8elrZktalIqQLMO8ejlnyJ zr@Je7dSbODe{$?dqT|yp^ZgHhTmSgsrF%Q}>U#4G(EPUX;w=#os&y9USnhVMf2rwL zsg%1#bM~Cb-g9CcJJs**KY4%0I{!;7mv7zrZR_=`>^o)zDGs#cx{O&#pDm2uaL3x(vRTMa-NStCTpR%v@{ojPR^pHXL{E13s=U8ChBQQ%E~~Fvk3drL*TyNkN1D* ztt`zS;^L}Xn(OjHw=LdS7+|s3ZK6H^T9bK93HqJKjvYIVEqU5UYPdNwWq5YaN`n4% z3zIH%y{^JRDQHLE@9&x0jrLtdQNbI-51KQLLs0lLq^B}41P$7o*fa|o340&>S5qs> z$fc}H3Is& zyPu;N9CHP9K(L1grPnghTeJO0@$#^_StWYI84JqdIRt0q<>aovDq1jNu24+XZ0oYz z%D`7^AC82Oxy+}fohYL~Jme|F1~LXR2{|@$nl{%q&(tNUIGBhDZg9=M#;5lB`n}BM zLWfx!H5tJTEe|*E_F$wmNSP41!`WA6<)42<(>Ym(__gbVs_6&df|W}<$ROb^weWYhE~i1=0;0e^1Sml zGpWc8)IS}C$Zk1DVYR0liG?&eW72psyJvoW{?M^wpDv$1aV=xa{sRe#VK=6ceL|ky zgUYq_6WQ(dl{!o)Vib!OWuiU+$Fs(C!`k0oUaDU~rAU2&tQX;>%EMbb9sF72i_y7iMq%QG#Si?^H~I6=L)2FwmNkggTEtU9MI^AwkwA(Zi9N`_Z* zGMjP!D2iNU_tc)@ftHZ2XX5x0uYY~;Ta;_1gQ@uU2z&Baig~iT!Q=r`Q&z_LA3SI+ zHJ#*YI)4;IfNJ=e6>7i!$maQ`cFgfb=WUude|CyFXUPgykYgmO@BmiIvweST%mZ+e?;BD2JciV+?GABWsoKs3t_Gd{^P5)q;8x^7VIk^AR@l-Yo zw)TazB#4_q=SN#c%!6vr%tgn=4MHAyvv8TWx9ZYPQ>tTa7GPfPVPHr>Wa2{Rw&@He zY?$6JVa9EvE!nBot8?RB3ou8lrhb0gg@aT|Bwf}o6CsT!;=;*3?HfH_`e?~Scx|f5 zrzzRG_3ehsU*YM!Hd}S>0y54XiJ%ku(o@iCS6Dr@d*8l9h!5K#W`2G($ic$Sw8o}H zrKIG=@R^;{l-qK1bBVigq8Fd_P@_C_t539kgj0PC!!)JTVVTg^M$ROcK}gwpI13QwqzQ7I}h2AOEi+dBqo-G zzM_e$>9PCHnkB1fIVm+YH8B!{Bc6(4FWWN~E`zTk*&WXLYIFmeBa_FAcBx~LM4Y!d z-!xb}AiPO}i4yGgq-oQb*HGQJ1NFI&@Vd(?|&qKVL@zO0({* zq8%K^Vf{p7S*nwkgz6}T*kbj?2#Ivg=B_<+%L%Y!%jV7UmoGm_R8C2Rlp6pTX()ZL ziB+6g)Z;6bdg#qIOaZf)XD!ShtHG4)0g!;x4uS);jl}UrTU2Q~JG)O`z9@@|K89pX zkO|{UP)XI~vzv%+;j86*y##RR6Af)S&>t6}3^HOG6<6~znsP1Uurs67%yKWmRE zCL%JQ?&Fd`(y33h%TgB;VbW~bvPHcatpHR#C30o8GR{ezLIyW%zSzX2) zq44yTY_m2N@sP9T${j^cO6uxE)tk7sGO}to%e=&%X2hO;@JoDLc$}pojRTo*`0t^< z=n*9AyzLtpsNc37Dyu0AMTF{?YvN=bhW;LH6i#ISBWlfOEOoCm)KE@kveDC?{1png zHY|MYMPJb7BBy*GU*AkL5fxk-duNX_x3y7X+XJm3Kq^S22BPMWj>qHeOs~M&yiy#5 z^z#b}>L2%LgytC}77!!Iu;>znqLjer*PI~<=w!Z#y;o+zJq^60n5KOxCOSG^I)uA^ z0xHP~IBEcb3H`V;Y=*CWju6&Do$QFZ$HC;T*qBB=pZtBI0RZE1Fc@3pWyy4(&iZ!} z8y>=Q{{;xoZvDkt-=!ftHk*siS=)>5Z9{jn@89c}Az&g*!#JW~JbX`!@L3qElU03M zX6C#&OdtVC=&(f8SHf78;$#@SOre=TSSUC;7JwqDOH?{8IM?|g7@70LmPJZxygn&> zq1#thy&lWn^Z9ItqP>0IK2C#3X!t;TrvcjMjv6wF*xL&el^iL;^@nA{hu(w+lke?w z`kHO76*x93_u~A;?Aq&GA`g6KId9yoRCn=${N=Xq}JJZEjfMmCc8 zC3VBgwZ$;X>+2bE-a{nmwQ=A?V+EF=E=H1?LD*Rx zlBO27seN%_Q|fSn{+M02w^(xJ{;hP}QsrJu*@PlP^zhz76#4e~@>ry=a>Km4J$W3q zZxU*K`$PjI1a>UH#Jbrr@vw9CiK7OLQmUq)xWct-!41%EoeNVnvaDf(HB0w4F^)BR zLS+~ir2fV|k-qF5@n92eGY+8V61~7j(!U>npdr<~tt0{KE(Pp~E=Q z5TRp95fGKR>e^AsAGE1)|R%DO0oe6me=GE(gNav1TH34)P+$4|biDp8tY(1q>gvkqYVsnaEf4#q44v|Af~b4MXbuFB-dsF`n)O{k z6ag0n9|WEH(A9;9&oF7K&eKv<>`qV7u`@7;uf&UEpF??T@LhVUAfgo7@~m0Klax~?nrv_;e4;VK2{+z7X0HTv z)o+`vBS9)BpAiQyJ($({jRc8kjk2kL8^W$}&i9)td=Fdqh26QP?@Fi*`+G3YEt(@6yHX+s>JKq*qIF^j$P=#&a+G?_K-+@URn~8xZ_=Ee+B82uaZdWdZekP>J(*1T`-NMct-E#3W~9}S zg;x9!&P}TIV@g`>6yM_Stp?o zGM5jynER+WZ><_05@HN;-h%~@#Rz0kZ(0PPQ`3LJyz4dwff?_J2=ic0Lv5BIbVm-s ziO%K*R?7@OeDr8VswpM8{+G~G8hKVnr(I0?UWVUe#>=5XPR;ru<6Y%yEgi3KT(}}9 z*ZC&Yo_xE@TVX&b3Tk#MhtItHw>t0Uj(?y_aXN{1+Q11oPp*aNRGfQ4*m}S(_xocy z$H$yT`*Ieh`gKePQLJRBfvn77|;hhFT-(=g*j&p6DO_yz>%>5;;RC(22vV*fMK3uJZ> z5Xv=ESU(n%(61r2wThFyA?Cqg3z;iG96f(Zch-KGXxbio{?QgupM)!MoQtoxtil-nJky)EK-+YGfgax7KfnTTeH3(N=up`g|d*)j3m^bOdg)LBG2c* z#@R66BqVV~?2jdS%8#T#rqfopAboewH--N-1l{R%ci;UebC(ABEDfPXJRLF1{Xkx- zZ*Kn4p#%KMJz6m7DSr9#Nw`{Lf!kuSWA*Uxda!?;XVVR94o!k434#g?85_H`d-rY~ zLJDF-^-lVnD^29I_H%S0>l6{JQ~{8k2N_j`Uh3CZrs9Ny_^bz>fBh=A8>Sv{;D~*X z`V~@;BwEYRw7uqYcOt`k%pnkZAqBX=h;jh{aee#t?GPtt?Zr!%f`F#J2#mkqtp{qs zc~(q%gP4s?7A?O`vSC|ZCg?Zr8#ivmIxjEuLff8%Tf4A>m^}-??S0||O>yYVnaP5M z$Z#@D3D^OqiOC*UN(RC>z=>m;NEduWqvJvL&y`c9bYyhhKi`w>_#G$jpYKH=fd2Vz z{~w#J{`roK<cu70Ci=@$?X3tQKJ_8BoN z>c81kh8bin8v?1yM z%mY&(>P)_4h*-Mcev>V}6A&7j%s-NIA=9W1ZqSFY@?yQ2#RjlBG#6=kZbf3x)It%8 zD}d?r4Gc~e{Ha0jN0?XV66Nc~lz`R?5jU1p81U*J8Q=p&Ln&2KYg zAXuNKBbFrVvN&@Sqqc|G-^0X|ifsS`Bmq=q9<(K=?b!9tL1)CEkS)|mkBSDkS zvwghnvk6CuMq@;|Pkj2W*b z=w$N_O>ok0peshL+k~7JF8fSOOiV#c`p>Cj%}UCEx3|v+Ee&nP7fLecQ`m3ok1Hpi z^zmRhV?SjOQHixzt4Y-o9_;9AC&(G^9iqk^DnLO>iV7P-`q!+HBCiW%kQrTSUmD4c|?D{SXiY z^XeSn5x}>`E4B=>ad42};iD7p?FrOkPeY_~&NDT2^;Wy5b@Mb*V4b;)>#YVI5K^JM z2uTtIu(m)i8OCQ_g^3Xm>L}x3r$2q>j5Lsb;Y<^JUg=i-3SKQeo&8)2)y|W8fGGrZ z*=N&k-Jo&k`0+~9wzqH3*JPV_?3zV!)wXWSP!$bpfA5>*TlaOR6N2{9V$WWS$8ESwvFMiT6N6|{9h!hGm9MuB+5{b{*Va}dM~VbMb$+6^yHhQ;1U%l*c8AhuHNgF^t3r;sFNBOIT7#Ft$EbRn>;_ zT28?zFvMz0NJ-TqOjf-rj2dOi+^w`sC zW=<{V){E)S6j=I50q=-5?}&=E^(Rw*JrQ0C?ZhZyNnXf81nT5x9%X0e2HZd8BL$n0 z7r;`DuJ26TaizqYkmNn}KoB*x0Bn~}L%yBbw4KpP)%k_~SBK_9vif@iKM+Rhg23+P z?;wc|ojuzyY@xN-9tc1fv}MmRcwMl;4+#r*=+5Vl78wr@vPuTn4s}6{#v(^q)W;Aa zG9kRAD42U z&Pg0N0$6LdY@_3@x_nwi-QtnBq~v5Hf!pd;TaZtGN7d$(mpkT2wUU7TUd!0rje}=5 z(0}?txiBuM(<6MI{XMxyoHsC#?YL!@*MF!G;WuVpt`8qx)z7EiyZ0Cz z5BkEg0^<+O@`t&z%c1*|EV@33^Et#WqsZ2$>$TJkph;Q8xr}P%z%@14fUAO+`iO5| zYk^%lYU;*;gyk}8y$YdlB}T$Oxd#?GS8z0FA0H?dA+apJ*Um1JEsuEmEO8 z@(|(2l#2UcIErJxDB9ZQ2rW-tQA$uK)5(WjB!$&X&-6S~UX%yPv1i}F@t!oCr#5L*yfl)yZ9?-Jr1r`=Y)*Kz(0*tfYMeZ)Fw1`^C2 z;JZ&W7B6e+LBV_aKsJ78o}z_;B=;3ZF;Dt$ZFf3S-~$2CDGhs75yjyoLy)9z3~lKX z4Qr!A-{lL_d%vmDr-un%g$0j`HWoJsJdyhvvzRNe0;5O5r#Z`1Z9r%l9dNcgXks6zNZri z{)2-LA|}5tt-IHi^BYF|iDJHq0lJKhIIU#2ZmeOEP{cbpX!I_X-vL?OV1ox*sEJtV zwzk6_uB9f`O#f$$%;lL|6*;s+^To z22@29qEPar!i_&ZJ5W#CN9+xv%DdMvS{7JoF7s=h|Ez^`<`WGvJkY)k&X)v`bRc;x zbLrJF-+Qb$KG05iULTQG9V&LWR;L$sZq3uGrXBQb_*SMMCl?9lq@R7ErB^x}AA>Q0 znhKDhaFZuQN3&pS$wF7;VFPk$zkjl;*q=F<@qw;&t{Abo9sd-{FveG(MXW0c8!d(v z_Patm`;ERT8q3Y13O3Ez=BeOjkXay$Vi<(N$wZkgpacno5;QE$qX2x^9V2+Bg@iAT z>n_cW6ABhABK>Ng%Sv2B))%G{)bgkv-euiBH%VvGcY8^u5%}$<$N04l7@7(4QKF22 zy1Ke{%W?#X&b}%mAyHj&Z-bo+_Twr(kU#lLi`w+mUs9j?Xi8Eq zT6a5!5@@@Av?8**9+DE{d%XHaOvds5gHw+?aL&Fs0t_=oR##;69;|rwBb>4d1dv+Z_oBH&IV|(7~)d zLQ7*8B8I~@xWM|a)UEy$%xtiSntBlUpEW#;k&=FqLs(b~UYUAxk1(${LhPYhyCJrw znzEwJ*Yjw~_AOo`a4`5hKd{JHa`iEbz0}lkftvQ>9EDRpaN%S?Gkp%@S7CCVdjXWn zi7ttmW^K)R7khF;i#1Bt?B~}TAjo*XdPiu$gzNm#wQGcoObzsOL-EQqM`|Uaj;9e? zb=gNP65hgK>yCM^WlJChMaZu4yw-^HISxgTiPK=$kDoa4nJ=9W-BW6p2@fd&edOi6 zwRLm|_l(jj^`<_)FY6598jf-PQcXlqB`?Cpm~4IpqzY;6YrP0ynq1ic3n8D)_}j}z zEQauwvU`~W`XJC}^imsR|BsPk3PJYJ**m~*q%77GcM8HC3j~9 zbv4wQ9LqC7^G)Ub^OagFKi(um^wiTf({Y=OHJ5`sa}+9Oww0)che%@I$i@WCLIOjw ziVF$*jddejfIxuO1GPtC(kMZyPYvG5CCHgD!(~j$PxPIPVakcXNiuJn-NOS&6OGQT zgUVO0#v`tncu%4I^czO=+Ia7kxvuTv)!4m#Z)6-Dmsoi(6r;ysqGEy%3n5N{>ewdx zcsjaX!ZfSj!~)WOdZEP*;Q_49uRC*CI)zu~Wm5x_TA-|_9qhwV4_~l;vJjR&tLLg? zt3^3Oy_44BCyOP50(jr?2O;k}2B(o}1NC8aFe;x>d!ONgT)(BD7i=#|(xL2i9WE|_t? zEXMrHX;9|kBR5WPaVfzxb^GHFdANK?#!}RY*iCSBJ!9$DT3rYXxBam>f@wWHT$aFE zX_Y$AgU_PDLtf@nYw3${Y1avXXF1Sza~_26Wgmog^o?IgheF;8Os3C7WaJp3cZ|6G zd{3aikI#?vW$dtcn6>!=Gl0SKt;JtDw8yeifjpwXx+?h$x+-82MP4%t2YNt+%sgI# zUx{*mqeb&}vXTIRnwVosiG+7$pO9k?Dipo~gJTS{z#hgdj7uT$YQzl3Vz=a{2;;n- zP|uNzlZ~h!&hB^xCg>;HS}KBSLuiGXGy!Q4VGrTHjY(+8f)aL%96pJV$Ay^LM3vNV zs58Bd-++R0)JMJq!eZ(l$sbjQNkKT?%F4@+^YL9nd~!4P2U-HZtg4a{U)^T5Vhit6 z#_b-7H72Ak!|)0+J(`Rc0p{W{yY-wAzCB zCJyAa+}klL<6bUD1O(L44`$NA@e85nq{H}>2ujf7l(WVU($4h^Og}>xZ2|r`Dq_|b z(!x9SSx(AaDdS_OANKX} zA-uU@E2UvK*P`fn0>aOSo6^Mqs_Vc6)xvCx=rH=(QCv)T>+xzoeBnwkDG4TwAlLAu z`-GhfVCh-|$QVJmF)m`%c?3eiDeFfsWx@kYsBu_n@qqC@(E#;>*i7~JH&RR?8o+3x zyMo=?1wKqp?f1V=kF=>FY$pj+>q}?}!Zt%NR|6Xoa8|Y@2u>Ts5q35#fM%I+2qF0g2OCYNWB2ZAp{Aw|`T5hpt6ULY8*m!V zOLLQqUT43QM+$o~3K}8ea-d;lurV#Fyzkgx*V_ZI(>RUmtBi{YV-6W_C&n{Fz*`(s zPX3wZh57Ie4IOH5#(o`LGpZW6NpMeeWi8~v{tQBuDto6Dm>{5yv3oUJFgaw@Zfm;P zfzZ<%N@Cu!wyq<~@_+U$(q&~b!XAT8PS@4Z)~*FbaYFUYMX1K9+!eG1o<<|!pII$t zR{+bAeg6WJMFbZRK@fzyHGt&l6p9InCPz1n0Yb3G@cBj9jN! zv|A>4HKgs-;(T$UTDnsFEyGm@q{4iK)sz93$5z_Y7ZEj4yGKR+1(u1Nc%BINt5)QzdQfm)s_ekws363tZs@a>wBeKCSD9 zwM|TVl_!B;BcKOjbdVwOlE7JLyYdZ{UDfN`-srj?k_}HLs2db#PJw@j3QtK$4%{1w zEglHuO=Q=|vvnMSvO970wMcc7c_Xo`Hs_s-I6so8A7gdN?@F@1r_uT3u!D@u&mZe= zgwqPqPjn>zY05JF$vAy3wF<$N*99RA<1bjrOQ;)zw^=;rik`R^#E@eP?rArtOM~p@8;KZ3v z8|G>6YJ_%dufPj_zK|V;kPCdUZ(+8@%+1YBFzzyW6(4^Y*isp(xQHB=Z-2MunkfDc zrM~|s!i|#)T<=z1Mt|Q=4!;w3xTX1#l;ii2L^gC^GUwmEkE4Ih8}VCfm*3ene*3R) s@BY8y;`r-<$o_BoRsLVTsrZ76(Rcf$_RjH9;xWZ9$y|JO{`&p@1?SOhO8@`> literal 0 HcmV?d00001 diff --git a/cookbook/foldcp/fold-cp.md b/cookbook/foldcp/fold-cp.md new file mode 100644 index 00000000..fb3bcf5a --- /dev/null +++ b/cookbook/foldcp/fold-cp.md @@ -0,0 +1,182 @@ +# Fold larger inputs by running ESMFold2 across several GPUs + +`wrap_model_with_cp(model, dm, …)` takes a normal `ESMFold2Model` and rewires it, to spread one fold across several GPUs using the [Fold-CP methodology](https://github.com/NVIDIA-BioNeMo/boltz-cp) so you can fold longer +proteins than fit on a single GPU. Wrapping results in the same object with a few internal pieces swapped for versions that split their work across the GPUs. + +## Table of contents + +- [Quickstart](#quickstart) +- [Requirements](#requirements) +- [API: wrap_model_with_cp](#api-wrap_model_with_cp) +- [Behavior and differences from single-GPU implementation](#behavior-and-differences-from-single-gpu-implementation) +- [Larger example](#larger-example) +- [How context parallelism works](#how-context-parallelism-works) + +## Quickstart + +Install the fold-cp dependency + +```bash +pip install "esm[fold-cp] @ git+https://github.com/Biohub/esm.git@main" +``` + +Use [esmfold2-cp.py](esmfold2-cp.py) as the CP version of the single-GPU +[esmfold2-none.py](esmfold2-none.py) example. Launch it with `torchrun`, using a +perfect-square number of GPUs: + +```bash +# Launch on 4 GPUs +torchrun --nproc-per-node=4 esmfold2-cp.py +``` + +Compared with `esmfold2-none.py`, the `-cp.py` script changes setup, not the input or the `fold()` call: + +- Assigns each process to its own CUDA device using `LOCAL_RANK` and `WORLD_SIZE` (set by `torchrun`) +- Initializes `DistributedManager` with `("cp", (n, n))`, where n is the `sqrt(WORLD_SIZE)` +- Wraps the normal `ESMFold2Model` with CP using `wrap_model_with_cp(model, dm, comm="ring")` +- It keeps `num_diffusion_samples=1`, which is required by the distributed diffusion path. + +The wrapped model is still an `ESMFold2Model`; `fold()` and the result object are +unchanged. + +## Requirements + +- Python environment: + - This ESM package + - transformer-engine 2 (installed by `esm[fold-cp]` dependency) +- Use a perfect-square number of GPUs: 1, 4, 9, 16, ... +- Keep `num_diffusion_samples=1` in `ESMFold2InputBuilder().fold(...)`. The CP + diffusion path currently expects one diffusion sample. +- If you enable `tp_esmc=True`, ESM-C's MLP hidden size must divide across the CP + ranks. The current ESM-C `ffn_hidden=6912` works for 4, 9, and 16 GPUs. + +> Use a fast GPU interconnect for good throughput. NVLink within a node and + InfiniBand between nodes are the intended setup; PCIe or Ethernet work and still save + memory, but communication may dominate runtime. + +## API: wrap_model_with_cp + +```python +replaced = wrap_model_with_cp( + model, + dm, + comm="gather", + bf16=True, + offload_esmc=True, + wrap_structure=True, + tp_esmc=False, +) +``` + +`wrap_model_with_cp` mutates the existing `ESMFold2Model` in place and returns a +list of module paths that were replaced or augmented. The model type, `fold()` API, +and output object stay the same. + +| Argument | Default | What it controls | +|---|---:|---| +| `model` | required | The `ESMFold2Model` to rewire. Load it normally with `from_pretrained(...)` first. | +| `dm` | required | The initialized `DistributedManager`; its CP mesh must be square. | +| `comm` | `"gather"` | MSA pair-weighted-averaging communication. Use `"gather"` for exact all-gather behavior, or `"ring"` for a cheaper online-softmax ring path on larger grids. | +| `bf16` | `True` | Runs the distributed trunk/MSA path in bf16. This is the practical default for lower memory and faster inference; use `False` only for tight fp32 parity checks. | +| `offload_esmc` | `True` | Moves the ESM-C language model back to CPU after its one-shot use, freeing GPU memory for the trunk and diffusion stages. | +| `wrap_structure` | `True` | Wraps the diffusion structure head so the conditioned pair representation can stay sharded through structure sampling. | +| `tp_esmc` | `False` | Tensor-parallelizes ESM-C's MLP across the CP ranks. Useful when ESM-C's replicated weights are the remaining memory floor. | + +If you enable `tp_esmc=True`, ESM-C's MLP hidden size must divide across the CP ranks. +The current ESM-C `ffn_hidden=6912` works for `4`, `9`, and `16` GPUs. + +Common choices: + +```python +# Recommended large-input path. +wrap_model_with_cp(model, dm, comm="ring") + +# Exact communication path; useful for parity checks. +wrap_model_with_cp(model, dm, comm="gather") + +# Also shard ESM-C's MLP when ESM-C memory is the bottleneck. +wrap_model_with_cp(model, dm, comm="ring", tp_esmc=True) +``` + +After wrapping, keep `num_diffusion_samples=1` in the `fold()` call. + +## Behavior and differences from single-GPU implementation + +The API and outputs are identical (`from_pretrained`, `fold()`, output keys, `type(model)`), and almost every step matches the single-GPU result. +Two differences are expected, both small: + +- **LM dropout** picks a different (still valid) random pattern than the single-GPU + run, because the sharded table can't cheaply reproduce the exact full-table draw. + This is the main reason the pTM/ipTM scores can wiggle slightly. +- **Rounding** differs at about 1e-3, because numbers are summed across GPUs in a + different order. + +Single-GPU kernel backends such as `"fused"` and `"cuequivariance"` are ignored by the CP-wrapped stages; those stages use the distributed CP implementations instead. + +## Larger example + +If you have a large 12-chain complex like [5xgo](https://www.rcsb.org/structure/5XGO), attempting to fold it on a single H100 80GB will fail with OOM + +```bash +python will_fail.py # expected to OOM +``` + +Context parallelism sharding will split the memory and allow this protein to be folded on 4x H100 SXM GPUs. +Launch with torchrun on a perfect-square number of GPUs: + +```bash +PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True torchrun --nproc-per-node=4 cuequivariance_cp.py +``` + +## How context parallelism works + +A protein is a chain of building blocks called residues (`L` = how many). +To predict its shape, ESMFold2 keeps a big matrix with one cell for every pair of residues. +This is the pair representation (or just `z`), and it's what +dominates memory, because `L` residues means `L x L` cells: + +- 500 residues → 250,000 cells +- 2,000 residues → 4,000,000 cells (16x bigger) + +This means that the memory requirement scales quadratically with the input size. + +Context parallelism (CP) shards tensors (vectors and matrices) across GPUs. +More GPUs means more memory for each tensor, allowing for larger inputs. + +CP arranges the GPUs into a square grid so the GPU count must be a perfect square (e.g. 1, 4, 9, 16, …). +Each GPU process is a **rank**, and the total count is the **world size**. +The `L x L` matrix is split into blocks owned by each rank with a [PyTorch DTensor](https://docs.pytorch.org/docs/2.12/distributed.tensor.html). + +The model weights are kept replicated, where each GPU has a copy. +Only the big activations get sharded in this method so per-GPU memory drops as GPUs are added to the inference pool. + +![memory usage](fast_runtime_and_vram.png) + +### Stage-by-stage: what runs where + +The base model runs every stage at full length `L` on every GPU, with the pair table +full-size. The table below walks that same pipeline; **a ✅ means +`wrap_model_with_cp` splits that stage across the grid** (everything else stays +replicated). + +| Stage (base ESMFold2Model.forward) | Split across GPUs? | What wrapping does | +|---|---|---| +| `inputs_embedder` (atom features) | — (replicated) | unchanged; its cost grows only **linearly** with size (it looks at a small window of atoms at a time), so it stays a small, fixed floor | +| ESM-C 6B language model → embeddings | ✅ MLP only, if `tp_esmc=True` | splits ESM-C's big MLP across GPUs; the rest stays replicated; ESM-C is moved to CPU right after it's used, to free room | +| `language_model` → `lm_z` (`LxL`) | ✅ | `_cp_language_model` builds this `LxL` table already sharded (the full thing is never assembled on any GPU) | +| pair init (`z_init`, rel_pos, token_bonds, initial `z`, pair mask) | ✅ | `_cp_pair_init` builds each `LxL` table one block per GPU — no full copy anywhere. | +| recycle loop (`_run_one_loop`) | ✅ | `_cp_recycle_engine` keeps the pair table sharded the whole time — it never gathers the full table between passes | +| `parcae_readout` + `parcae_coda` | ✅ | runs on each GPU's block plus a shared trunk | +| `distogram_head(z + zᵀ)` | ✅ | works on the sharded table and gathers only the small result | +| `structure_head.sample` (diffusion → 3D coords) | ✅ (if `wrap_structure`) | runs the diffusion with the pair table kept sharded | +| `confidence_head` (pLDDT/PAE/PDE/pTM/ipTM scores) | ✅ | every `LxL` input (pair, rel_pos, token_bonds, distance bins, mask) stays sharded — nothing full-size is rebuilt; only the small final scores are gathered | + +### Summary: what shrinks, what doesn't + +- **Grows with the `LxL` pair table → now sharded:** pair init, recycle loop, + parcae, distogram, structure, confidence. Each is built one block per GPU and never + assembled full, so **adding GPUs raises the longest protein you can fold**. +- **Grows only with `L` (atom-level work):** `inputs_embedder` — a slow-growing, + replicated floor. +- **Roughly fixed:** ESM-C's weights (split by `tp_esmc`); ESM-C's own activations are + the remaining floor. diff --git a/cookbook/foldcp/single-gpu.md b/cookbook/foldcp/single-gpu.md new file mode 100644 index 00000000..49d74105 --- /dev/null +++ b/cookbook/foldcp/single-gpu.md @@ -0,0 +1,180 @@ +# Single-GPU ESMFold2 — kernel backends + +The reference implementation in pure PyTorch is accurate but not the fastest way to run ESMFold2. +Most of the single-GPU runtime sits in a few repeated pair-representation operations, and there are alternate **kernel backends** that keep the same weights and outputs while swapping in faster implementations of those hot paths. +The result is the same ESMFold2 model, but with different speed, warm-up, and dependency trade-offs. + +ESMFold2 runs the expensive Pairformer/attention math through a selectable +**kernel backend**, chosen with: + +```python +model = ESMFold2Model.from_pretrained("biohub/ESMFold2").cuda().eval() +model.set_kernel_backend(None | "fused" | "cuequivariance") +``` + +This propagates out to every module that runs Pairformer-style blocks (`folding_trunk`, +`lm_encoder`, `parcae_coda`, `confidence_head`, `structure_head`). It only swaps the +*implementation* of a few hot ops. Weights are untouched and the three +backends are numerically equivalent to bf16 rounding (same pLDDT/pTM). + +> This page covers the single-GPU backends. For multi-GPU context parallelism see +> `fold-cp.md` (`wrap_model_with_cp`), which is an orthogonal choice. + +## Table of contents + +- [The three backends](#the-three-backends) +- [None (default experience)](#none-default-experience) +- [cuEquivariance](#cuequivariance) +- [Fused](#fused) +- [Performance summary](#performance-summary) +- [Decision guide](#decision-guide) + +## The three backends + +| | None | fused | cuequivariance | +|---|---|---|---| +| Implementation | PyTorch | Triton kernels | [cuEquivariance](https://github.com/nvidia/cuequivariance) | +| Accelerated Operations | none (reference) | tri-mul + LN+SwiGLU + dropout-residual + pair-bias | tri-mul | +| Extra dependency | none | `triton>=3` | `cuequivariance-torch` (CUDA-matched build) | +| Training / autograd | yes | inference-only | yes | +| First-call compilation | none | yes — Triton JIT + autotune | no — precompiled kernels | + +## None (default experience) + +### Dependencies + +None beyond this ESM package + +### Usage + +```python +model = ESMFold2Model.from_pretrained("biohub/ESMFold2").cuda().eval() +model.set_kernel_backend(None) # optional +``` + +### Performance + +Fold a 644-residue protein using the default backend with [esmfold2-none.py](esmfold2-none.py) on a single H100 SXM + +```shell +python esmfold2-none.py + +/usr/local/lib/python3.12/dist-packages/torch/jit/_script.py:1487: DeprecationWarning: `torch.jit.script` is deprecated. Please switch to `torch.compile` or `torch.export`. + warnings.warn( +🚨 No checkpoint found for ESMCForSequenceClassification.forward. Please add a `checkpoint` arg to `auto_docstring` or add one in ESMCConfig's docstring +🚨 No checkpoint found for ESMCForTokenClassification.forward. Please add a `checkpoint` arg to `auto_docstring` or add one in ESMCConfig's docstring +Loading checkpoint shards: 100%|█████████████████████████████████| 6/6 [00:00<00:00, 151.51it/s] +Loading CCD dictionary from /root/.cache/huggingface/hub/models--biohub--ESMFold2/snapshots/1ebf0e3481a5184eb6171d40615c79e384b48796/ccd.pkl +pLDDT mean: 0.751, pTM: 0.458, ipTM: 0.096 +Elapsed: 58.02 sec +Max VRAM: 19682.0 MB +``` + +## cuEquivariance + +[cuEquivariance](https://github.com/nvidia/cuequivariance) is NVIDIA’s precompiled CUDA kernel backend for ESMFold2’s triangle-multiplication hot path, giving a large single-GPU speedup without Triton JIT or accuracy changes. + +### Dependencies + +* `cuequivariance-torch` +* Depending on your CUDA version + * `cuequivariance-ops-torch-cu13` + * `cuequivariance-ops-torch-cu12` + +Install the CUDA-matched extra: + +```bash +pip install "esm[cueq12]" # CUDA 12 build +``` +or +```bash +pip install "esm[cueq13]" # CUDA 13 build +``` + +You can figure out which one you need with `nvidia-smi` + +```shell +$ nvidia-smi + +Thu Jul 16 13:03:23 2026 ++---------------------------------------------------------------------------------------+ +| NVIDIA-SMI 535.216.03 Driver Version: 535.216.03 CUDA Version: 13.2 | +|-----------------------------------------+----------------------+----------------------+ +``` + +In this case, the CUDA Version is 13.2, so you would need to install `esm[cueq13]`. + +### Usage + +```python +model = ESMFold2Model.from_pretrained("biohub/ESMFold2").cuda().eval() +model.set_kernel_backend("cuequivariance") +``` + +### Performance + +Fold a 644-residue protein using the cuEquivariance backend with [esmfold2-cueq.py](esmfold2-cueq.py) on a single H100 SXM + +```shell +python esmfold2-cueq.py + +pLDDT mean: 0.751, pTM: 0.459, ipTM: 0.096 +Elapsed: 19.90 sec +Max VRAM: 19680.1 MB +``` + +## Fused + +The "fused" backend uses Triton, a Python-based language for writing custom GPU kernels, to fuse several ESMFold2 hot-path operations into fewer CUDA launches for the fastest single-GPU inference while preserving the same model weights and outputs. + +### Dependencies + +The "fused" backend needs Triton 3. It's GPU-only (Triton JIT-compiles to PTX) and **inference-only** (falls back to reference under autograd). + +```bash +pip install "esm[fused]" # triton>=3,<4 +``` + +> If Triton is not importable, `set_kernel_backend("fused")` silently runs the reference path. Check that Triton can be imported if `"fused"` is unexpectedly slow. + +### Usage + +```python +model = ESMFold2Model.from_pretrained("biohub/ESMFold2").cuda().eval() +model.set_kernel_backend("fused") +``` + +### Performance + +Fold a 644-residue protein using the "fused" backend with [esmfold2-fused.py](esmfold2-fused.py) on a single H100 SXM + +```shell +python esmfold2-fused.py + +pLDDT mean: 0.750, pTM: 0.458, ipTM: 0.096 +Elapsed: 16.56 sec +Max VRAM: 19795.1 MB +``` + +## Performance summary + +These results fold the same 644-residue `7ysz` input used in the per-backend +examples above. + +| backend | elapsed | max VRAM | pLDDT | pTM | ipTM | speedup vs `None` | +|---|---:|---:|---:|---:|---:|---:| +| `None` | 58.02 s | 19682.0 MB | 0.751 | 0.458 | 0.096 | 1.0× | +| "cuequivariance" | 19.90 s | 19680.1 MB | 0.751 | 0.459 | 0.096 | 2.9× | +| "fused" | 16.56 s | 19795.1 MB | 0.750 | 0.458 | 0.096 | 3.5× | + +## Decision guide + +If you want: + +- **Fastest throughput →** `"fused"`. + - Fastest steady state, and the incremental per-new-length compile is only ~0.5 s +- **Huge length variety →** `"cuequivariance"` + — precompiled, steady state a bit slower than fused. +- **Bit-exact reference for debugging →** `None`. + +> Tip: For any benchmarking, include a warm-up fold at startup to pay the ~10s global cost diff --git a/cookbook/foldcp/will_fail.py b/cookbook/foldcp/will_fail.py new file mode 100644 index 00000000..dd8b2d56 --- /dev/null +++ b/cookbook/foldcp/will_fail.py @@ -0,0 +1,56 @@ +from time import time + +import torch + +from esm.models.esmfold2 import ( + ESMFold2InputBuilder, + EsmFold2Model, + LigandInput, + MolecularComplexResult, + ProteinInput, + StructurePredictionInput, +) + +spi = StructurePredictionInput( # 5xgo + sequences=[ + ProteinInput( + id=["A1", "B1", "C1", "D1", "E1", "F1", "G1", "H1", "I1", "J1", "K1", "L1"], + sequence=( + "MAHHHHHHVDDDDKMSTAKLVKSKATNLLYTRNDVSDSEKKATVELLNRQVIQFIDLSLITKQAHWNMRG" + "ANFIAVHEMLDGFRTALIDHLDTMAERAVQLGGVALGTTQVINSKTPLKSYPLDIHNVQDHLKELADRYA" + "IVANDVRKAIGEAKDDDTADILTAASRDLDKFLWFIECNLDLIQKMGLQNYLQAQIREEG" + ), + ), + LigandInput(id=["M1", "N1"], ccd=["CL"]), + ] +) + +torch.cuda.reset_peak_memory_stats() + +model = ( + EsmFold2Model.from_pretrained("biohub/ESMFold2", esmc_precision="bf16") + .cuda() + .eval() +) +model.set_kernel_backend("cuequivariance") + +start = time() +with torch.inference_mode(): + result = ESMFold2InputBuilder().fold( + model, spi, num_loops=10, num_sampling_steps=50, num_diffusion_samples=1, seed=0 + ) +end = time() +peak_mib = torch.cuda.max_memory_allocated() / (1024**2) + +# num_diffusion_samples=1 returns a single result with confidence heads populated. +assert isinstance(result, MolecularComplexResult) +assert result.plddt is not None and result.ptm is not None and result.iptm is not None + +print( + f"pLDDT mean: {float(result.plddt.mean()):.3f}, " + f"pTM: {float(result.ptm):.3f}, ipTM: {float(result.iptm):.3f}" +) +print(f"Elapsed: {end - start} sec") +print(f"Max VRAM: {peak_mib} MB") +with open("esmfold2_output.cif", "w") as f: + f.write(result.complex.to_mmcif()) diff --git a/cookbook/tutorials/binder_design.py b/cookbook/tutorials/binder_design.py index f2242669..7909cd6d 100644 --- a/cookbook/tutorials/binder_design.py +++ b/cookbook/tutorials/binder_design.py @@ -2,7 +2,7 @@ # requires-python = "<=3.13" # dependencies = [ # "abnumber", -# "esm@git+https://github.com/Biohub/esm.git@main", +# "esm", # "modal", # ] # /// @@ -648,6 +648,9 @@ def prepare_esmfold2_tensors( del max_tokens, max_seqs, pad_to_max_seqs, use_vectorized_msa_assembly _ensure_ccd_loaded() features, _ = prepare_esmfold2_input(input, seed=seed) + # Distogram conditioning is unimplemented, so forward() rejects these. + features.pop("disto_cond", None) + features.pop("disto_cond_mask", None) if max_atoms is not None: for key, dim in _ATOM_FEATURE_DIMS.items(): if key in features: @@ -705,7 +708,6 @@ def fold_and_get_distogram( num_diffusion_samples=1, num_sampling_steps=num_sampling_steps, num_loops=num_loops, - calculate_confidence=calculate_confidence, seed=seed, ) @@ -1233,9 +1235,9 @@ def _load_hf_model( esmc_id = model.config.esmc_id if esmc_id not in _ESMC_CACHE: model.load_esmc(esmc_id) - assert model._esmc is not None - _ESMC_CACHE[esmc_id] = model._esmc - model._esmc = _ESMC_CACHE[esmc_id] + assert model.esmc is not None + _ESMC_CACHE[esmc_id] = model.esmc + model.esmc = _ESMC_CACHE[esmc_id] model.configure_lm_dropout(lm_dropout, force_lm_dropout_during_inference=True) kernel_backend = None if TRITON_KERNELS_AVAILABLE: @@ -1309,10 +1311,10 @@ def load(self, use_scaling_critics: bool): f"Cannot reuse ESMC trunk from {reusable_esmc_model.config.esmc_id!r} " f"with LM head from {self.lm_name!r}." ) - assert reusable_esmc_model._esmc is not None + assert reusable_esmc_model.esmc is not None del self.esmc_model.esmc torch.cuda.empty_cache() - self.esmc_model.esmc = reusable_esmc_model._esmc + self.esmc_model.esmc = reusable_esmc_model.esmc self.esmc_model = self.esmc_model.cuda().eval().requires_grad_(False) def design( @@ -1386,9 +1388,7 @@ def get_base_image(): "NVTE_FRAMEWORK": "pytorch", }, ) - .pip_install( - "abnumber", "esm@git+https://github.com/Biohub/esm.git@main", "modal" - ) + .pip_install("abnumber", "esm", "modal") .env( { "HF_HOME": "/models", diff --git a/cookbook/tutorials/esmfold2_local_applesilicon.ipynb b/cookbook/tutorials/esmfold2_local_applesilicon.ipynb index 3e46acb0..a3669c4e 100644 --- a/cookbook/tutorials/esmfold2_local_applesilicon.ipynb +++ b/cookbook/tutorials/esmfold2_local_applesilicon.ipynb @@ -44,8 +44,7 @@ "# 1. the featurizer, decoder, and viewer. Brings torch, rdkit, biotite,\n", "# py3Dmol and ipywidgets. The CUDA-only extras (xformers, flash-attn,\n", "# TransformerEngine, cuequivariance) are gated to Linux and are skipped here.\n", - "uv pip install --python .venv/bin/python \\\n", - " \"esm\"\n", + "uv pip install --python .venv/bin/python esm\n", "\n", "# 2. the pure-MLX ESMFold2 + ESMC. --no-deps on purpose: its packaging wants\n", "# transformers>=5.7 (hence huggingface-hub>=1.0), which collides with the\n", @@ -74,7 +73,7 @@ "metadata": {}, "outputs": [], "source": [ - "!uv pip install -q \"esm\"\n", + "!uv pip install -q esm\n", "!uv pip install -q \"mlx-lm @ git+https://github.com/faustomilletari/mlx-lm.git@main\"" ] }, diff --git a/esm/__init__.py b/esm/__init__.py index 903a158a..a5cfdf59 100644 --- a/esm/__init__.py +++ b/esm/__init__.py @@ -1 +1 @@ -__version__ = "3.4.0" +__version__ = "3.4.1" diff --git a/esm/models/esmfold2/config.py b/esm/models/esmfold2/config.py index 7fd85718..ffe39908 100644 --- a/esm/models/esmfold2/config.py +++ b/esm/models/esmfold2/config.py @@ -248,6 +248,10 @@ class EsmFold2AtomEncoderConfig: hidden_size: int = 128 output_dim: int = 768 + # Pre-halving token width; wins over ``output_dim`` when the checkpoint + # states it, which is how a config can serve this and the transformers port + # at once. Absent, ``output_dim`` reads exactly as before. + token_hidden_size: int | None = None num_hidden_layers: int = 3 num_attention_heads: int = 4 expansion_ratio: int = 2 diff --git a/esm/models/esmfold2/distributed/__init__.py b/esm/models/esmfold2/distributed/__init__.py new file mode 100644 index 00000000..da377c0b --- /dev/null +++ b/esm/models/esmfold2/distributed/__init__.py @@ -0,0 +1,50 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: MIT +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. + +"""2D context-parallel distributed extensions for ESMFold2.""" + +from esm.models.esmfold2.distributed.manager import DistributedManager +from esm.models.esmfold2.distributed.model.layers.msa_encoder import ( + MSAEncoderDistributed, +) +from esm.models.esmfold2.distributed.model.layers.pairformer import ( + FoldingTrunkDistributed, +) +from esm.models.esmfold2.distributed.msa_wrapper import ( + MSAEncoderCPWrapper, + wrap_model_with_cp, + wrap_model_with_cp_msa_encoder, +) +from esm.models.esmfold2.distributed.utils import ( + TrunkCPWrapper, + wrap_model_with_cp_trunks, +) + +__all__ = [ + "DistributedManager", + "FoldingTrunkDistributed", + "MSAEncoderCPWrapper", + "MSAEncoderDistributed", + "TrunkCPWrapper", + "wrap_model_with_cp", + "wrap_model_with_cp_msa_encoder", + "wrap_model_with_cp_trunks", +] diff --git a/esm/models/esmfold2/distributed/comm.py b/esm/models/esmfold2/distributed/comm.py new file mode 100644 index 00000000..9ec4efc5 --- /dev/null +++ b/esm/models/esmfold2/distributed/comm.py @@ -0,0 +1,384 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: MIT +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. + +"""Communication primitives for 2D context-parallel distributed operations.""" + +from typing import Optional + +import torch +import torch.distributed as dist + +from esm.models.esmfold2.distributed.utils import ( + LayoutMap, + get_group_rank_from_axial_shift, +) + + +class One2OneComm: + """Point-to-point communication with parity-based deadlock avoidance.""" + + def __init__( + self, + group: dist.ProcessGroup, + rank_send_to: int, + rank_recv_from: int, + parity: Optional[bool] = None, + ): + self.group = group + self.rank = dist.get_rank(self.group) + self.world_size = dist.get_world_size(self.group) + + if rank_send_to >= self.world_size: + raise ValueError(f"rank_send_to >= world_size {self.world_size}") + if rank_recv_from >= self.world_size: + raise ValueError(f"rank_recv_from >= world_size {self.world_size}") + + is_self_send = rank_send_to == self.rank + is_self_recv = rank_recv_from == self.rank + if is_self_send != is_self_recv: + raise ValueError( + "Asymmetric send/recv not supported: " + f"is_self_send={is_self_send}, is_self_recv={is_self_recv}" + ) + self.is_self_comm = is_self_send + self._rank_in_group_send_to = rank_send_to + self._rank_in_group_recv_from = rank_recv_from + self.parity = parity + + if not self.is_self_comm: + self.rank_send_to = dist.get_global_rank(self.group, rank_send_to) + self.rank_recv_from = dist.get_global_rank(self.group, rank_recv_from) + if self.parity is None: + self.parity = bool(self.rank % 2) + self._queue_send_recv = [] + self._work_to_finish = None + + def __deepcopy__(self, memo): + return One2OneComm( + self.group, + self._rank_in_group_send_to, + self._rank_in_group_recv_from, + self.parity, + ) + + def _prep_batch_isend_irecv( + self, to_send: torch.Tensor, to_recv: Optional[torch.Tensor] = None + ) -> torch.Tensor: + if self.is_self_comm: + if to_recv is None: + return to_send.detach().clone() + to_recv.copy_(to_send) + return to_recv + + ans = torch.empty_like(to_send) if to_recv is None else to_recv + if self.parity: + send_op = dist.P2POp( + dist.isend, to_send, self.rank_send_to, group=self.group + ) + recv_op = dist.P2POp(dist.irecv, ans, self.rank_recv_from, group=self.group) + self._queue_send_recv.append(send_op) + self._queue_send_recv.append(recv_op) + else: + recv_op = dist.P2POp(dist.irecv, ans, self.rank_recv_from, group=self.group) + send_op = dist.P2POp( + dist.isend, to_send, self.rank_send_to, group=self.group + ) + self._queue_send_recv.append(recv_op) + self._queue_send_recv.append(send_op) + return ans + + def _dispatch(self): + if self.is_self_comm: + return + if self._work_to_finish is not None: + raise RuntimeError("Unfinished communication in queue; cannot dispatch new") + self._work_to_finish = dist.batch_isend_irecv(self._queue_send_recv) + + def wait_until_finished(self): + if self.is_self_comm: + return + if self._work_to_finish is None: + raise RuntimeError("Cannot wait without unfinished communication") + for work in self._work_to_finish: + work.wait() + self._work_to_finish = None + self._queue_send_recv = [] + + def enqueue_to_dispatch( + self, to_send: torch.Tensor, to_recv: Optional[torch.Tensor] = None + ) -> torch.Tensor: + recv = self._prep_batch_isend_irecv(to_send, to_recv) + if self.is_self_comm: + return recv + self._dispatch() + return recv + + +class TransposeComm(One2OneComm): + """Transposes data between (i,j) and (j,i) on a square grid.""" + + def __init__(self, process_group: dist.ProcessGroup, group_layout: LayoutMap): + if group_layout.shape is None: + raise ValueError("group_layout must have a shape") + self.world_size = dist.get_world_size(process_group) + if self.world_size != group_layout.numel: + raise ValueError("Inconsistent world_size and group_layout.numel") + if len(group_layout.shape) != 2: + raise ValueError(f"{self.__class__} only supports 2D group layout") + if group_layout.shape[0] != group_layout.shape[1]: + raise ValueError(f"group_layout.shape {group_layout.shape} is not square") + + self.group_layout = group_layout + self.global_rank = dist.get_rank() + self.group_rank = dist.get_rank(process_group) + self.rank_coords: tuple[int, ...] = self.group_layout.unravel(self.group_rank) + + transpose_group_rank = self.group_layout(self.rank_coords[::-1]) + self.transpose_rank = dist.get_global_rank(process_group, transpose_group_rank) + self.parity_transpose = self.rank_coords[0] < self.rank_coords[1] + super().__init__( + process_group, + transpose_group_rank, + transpose_group_rank, + parity=self.parity_transpose, + ) + + def __deepcopy__(self, memo): + return TransposeComm(self.group, self.group_layout) + + +def ternary_parity(my_rank: int, send_rank: int, recv_rank: int) -> bool: + """Parity to avoid deadlocks: True if my_rank < min(send, recv).""" + return my_rank < min(send_rank, recv_rank) + + +class Ring2DComm: + """Ring communication on a 2D grid for TriangleMult and similar operations. + + Sets up: + - Transpose communication (send (i,j) to (j,i)) + - Row-wise ring (left shift per row) + - Column-wise ring (up shift per column) + - Initial offset shifts (row i shifts left by i; col j shifts up by j) + """ + + def __init__( + self, + group_2d: dist.ProcessGroup, + group_col: dist.ProcessGroup, + group_layout: LayoutMap, + ): + self.group_2d = group_2d + self.group_col = group_col + self.group_layout = group_layout + ranks_group_2d = set(dist.get_process_group_ranks(self.group_2d)) + ranks_group_col = set(dist.get_process_group_ranks(self.group_col)) + + if not ranks_group_col.issubset(ranks_group_2d): + raise ValueError("group_col ranks are not a subset of group_2d ranks") + + self.size_2d = dist.get_world_size(self.group_2d) + if self.size_2d != self.group_layout.numel: + raise ValueError( + f"group_2d size {self.size_2d} != group_layout.numel {self.group_layout.numel}" + ) + if self.group_layout.shape[0] != self.group_layout.shape[1]: + raise ValueError( + f"group_layout.shape {self.group_layout.shape} is not square" + ) + + self.rank_2d = dist.get_rank(self.group_2d) + self.coord_2d = self.group_layout.unravel(self.rank_2d) + + self.comm_2d_trans = TransposeComm(self.group_2d, self.group_layout) + + # Initial row shift: row i shifts left by i + self.send_rank_row_init = get_group_rank_from_axial_shift( + self.coord_2d, 1, -self.coord_2d[0], self.group_layout + ) + self.recv_rank_row_init = get_group_rank_from_axial_shift( + self.coord_2d, 1, self.coord_2d[0], self.group_layout + ) + self.comm_row_init = One2OneComm( + self.group_2d, + self.send_rank_row_init, + self.recv_rank_row_init, + parity=ternary_parity( + self.rank_2d, self.send_rank_row_init, self.recv_rank_row_init + ), + ) + + # Subsequent row shifts: left by 1 + self.send_rank_row = get_group_rank_from_axial_shift( + self.coord_2d, 1, -1, self.group_layout + ) + self.recv_rank_row = get_group_rank_from_axial_shift( + self.coord_2d, 1, 1, self.group_layout + ) + self.comm_row = One2OneComm( + self.group_2d, + self.send_rank_row, + self.recv_rank_row, + parity=ternary_parity(self.rank_2d, self.send_rank_row, self.recv_rank_row), + ) + + # Initial col shift: col j shifts up by j + self.send_rank_col_init = get_group_rank_from_axial_shift( + self.coord_2d, 0, -self.coord_2d[1], self.group_layout + ) + self.recv_rank_col_init = get_group_rank_from_axial_shift( + self.coord_2d, 0, self.coord_2d[1], self.group_layout + ) + self.comm_col_init = One2OneComm( + self.group_2d, + self.send_rank_col_init, + self.recv_rank_col_init, + parity=ternary_parity( + self.rank_2d, self.send_rank_col_init, self.recv_rank_col_init + ), + ) + + # Subsequent col shifts: up by 1 + self.send_rank_col = get_group_rank_from_axial_shift( + self.coord_2d, 0, -1, self.group_layout + ) + self.recv_rank_col = get_group_rank_from_axial_shift( + self.coord_2d, 0, 1, self.group_layout + ) + self.comm_col = One2OneComm( + self.group_2d, + self.send_rank_col, + self.recv_rank_col, + parity=ternary_parity(self.rank_2d, self.send_rank_col, self.recv_rank_col), + ) + + # Fused transpose + initial shift for backward pass + coords_t = self.coord_2d[::-1] + self.send_rank_transpose_row_init = get_group_rank_from_axial_shift( + coords_t, 1, -coords_t[0], self.group_layout + ) + recv_rank_transpose_row_init = get_group_rank_from_axial_shift( + self.coord_2d, 1, self.coord_2d[0], self.group_layout + ) + self.recv_rank_transpose_row_init = self.group_layout( + self.group_layout.unravel(recv_rank_transpose_row_init)[::-1] + ) + self.comm_transpose_row_init = One2OneComm( + self.group_2d, + self.send_rank_transpose_row_init, + self.recv_rank_transpose_row_init, + parity=ternary_parity( + self.rank_2d, + self.send_rank_transpose_row_init, + self.recv_rank_transpose_row_init, + ), + ) + + self.send_rank_transpose_col_init = get_group_rank_from_axial_shift( + coords_t, 0, -coords_t[1], self.group_layout + ) + recv_rank_transpose_col_init = get_group_rank_from_axial_shift( + self.coord_2d, 0, self.coord_2d[1], self.group_layout + ) + self.recv_rank_transpose_col_init = self.group_layout( + self.group_layout.unravel(recv_rank_transpose_col_init)[::-1] + ) + self.comm_transpose_col_init = One2OneComm( + self.group_2d, + self.send_rank_transpose_col_init, + self.recv_rank_transpose_col_init, + parity=ternary_parity( + self.rank_2d, + self.send_rank_transpose_col_init, + self.recv_rank_transpose_col_init, + ), + ) + + +class AttentionPairBiasComm: + """Communication setup for ring attention with pair bias. + + Manages transpose comms for K/V/mask and ring shift comms for K/V/Z. + """ + + def __init__( + self, + process_group: dist.ProcessGroup, + group_layout: LayoutMap, + cp_axis_0_group: dist.ProcessGroup, + cp_axis_1_group: dist.ProcessGroup, + ): + self.process_group = process_group + self.cp_axis_0_group = cp_axis_0_group + self.cp_axis_1_group = cp_axis_1_group + + if group_layout.shape is None: + raise ValueError("group_layout must have a shape") + self.world_size = dist.get_world_size(self.process_group) + if self.world_size != group_layout.numel: + raise ValueError("Inconsistent world_size and group_layout.numel") + if len(group_layout.shape) != 2: + raise ValueError(f"{self.__class__} only supports 2D group layout") + if group_layout.shape[0] != group_layout.shape[1]: + raise ValueError(f"group_layout.shape {group_layout.shape} is not square") + + self.group_layout = group_layout + self.global_rank = dist.get_rank() + self.group_rank = dist.get_rank(self.process_group) + self.rank_coords: tuple[int, ...] = self.group_layout.unravel(self.group_rank) + + self.comm_transpose_k = TransposeComm(self.process_group, self.group_layout) + self.comm_transpose_v = TransposeComm(self.process_group, self.group_layout) + self.comm_transpose_mask = TransposeComm(self.process_group, self.group_layout) + + self.send_rank_kvz = get_group_rank_from_axial_shift( + self.rank_coords, 1, 1, self.group_layout + ) + self.recv_rank_kvz = get_group_rank_from_axial_shift( + self.rank_coords, 1, -1, self.group_layout + ) + self.parity = self.rank_coords[1] % 2 == 1 + self.comm_k = One2OneComm( + self.process_group, + self.send_rank_kvz, + self.recv_rank_kvz, + parity=self.parity, + ) + self.comm_v = One2OneComm( + self.process_group, + self.send_rank_kvz, + self.recv_rank_kvz, + parity=self.parity, + ) + self.comm_z = One2OneComm( + self.process_group, + self.send_rank_kvz, + self.recv_rank_kvz, + parity=self.parity, + ) + + def __deepcopy__(self, memo): + return AttentionPairBiasComm( + self.process_group, + self.group_layout, + self.cp_axis_0_group, + self.cp_axis_1_group, + ) diff --git a/esm/models/esmfold2/distributed/confidence_wrapper.py b/esm/models/esmfold2/distributed/confidence_wrapper.py new file mode 100644 index 00000000..1f7a84db --- /dev/null +++ b/esm/models/esmfold2/distributed/confidence_wrapper.py @@ -0,0 +1,191 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: MIT +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. + +"""Distributed confidence head for ESMFold2 (#6, inference-only). + +Keeps the pair sharded ``(Shard(0), Shard(1), Shard(2))`` through the expensive +front half — z_base build, the nested (already CP-wrapped) FoldingTrunk, row +pooling, and the PAE/PDE heads — so the full ``L×L×d_pair`` pair (and the trunk +activations) is never resident on any rank. Only small tensors are gathered: + + * ``single`` (B, L, d_single) after row pooling, and + * ``pae_logits`` / ``pde_logits`` (B, L, L, bins) for the output contract, + +after which the **serial** ``ConfidenceHead._finish`` runs the atom-space +pLDDT/resolved heads and the pTM/ipTM/pair_chains reductions on the gathered +tensors — one source of truth for that fiddly code, replicated and cheap (it is +per-token / single-channel-L², not the d_pair pair). + +The wrapper consumes the sharded pair DTensor directly from the recycle engine's +CP tail (no ``full_tensor()`` re-gather of ``z``), which is what removes the +confidence-phase peak. +""" + +from math import lcm + +import torch +import torch.nn as nn +import torch.nn.functional as F +from torch.distributed.tensor import DTensor + +from esm.models.esmfold2.distributed.model.layers.confidence_zbase import ( + ConfidenceZBaseDistributed, +) +from esm.models.esmfold2.distributed.model.layers.layernorm import ( + LayerNormParamsReplicated, +) +from esm.models.esmfold2.distributed.model.layers.linear import LinearParamsReplicated +from esm.models.esmfold2.distributed.model.layers.pair_init import ( + build_sharded_distogram_bins, + build_sharded_pair_mask, +) +from esm.models.esmfold2.distributed.model.layers.row_attention_pooling import ( + RowAttentionPoolingDistributed, +) +from esm.models.esmfold2.layers import gather_rep_atom_coords + + +class ConfidenceHeadCPWrapper(nn.Module): + """Distributed wrapper around the serial ``ConfidenceHead``. + + The nested ``folding_trunk`` is already replaced by a ``TrunkCPWrapper`` via + ``wrap_model_with_cp_trunks`` (so ``forward_sharded`` is available). This + wrapper holds the distributed z_base / row-pool / pae-pde-head pieces and + delegates the back-half to ``head._finish``. + """ + + def __init__(self, head, dist_manager) -> None: + super().__init__() + self.head = head # serial ConfidenceHead (nested trunk already CP-wrapped) + self.device_mesh = dist_manager.device_mesh_subgroups + self.shard_factor = lcm(self.device_mesh.size(1), self.device_mesh.size(2)) + + self.zbase = ConfidenceZBaseDistributed(head, dist_manager) + self.row_pool = RowAttentionPoolingDistributed( + head.row_attention_pooling, dist_manager + ) + self.pae_ln = LayerNormParamsReplicated(head.pae_ln, self.device_mesh) + self.pae_head = LinearParamsReplicated(head.pae_head, self.device_mesh) + self.pde_ln = LayerNormParamsReplicated(head.pde_ln, self.device_mesh) + self.pde_head = LinearParamsReplicated(head.pde_head, self.device_mesh) + + def forward_sharded( + self, + z_dt: DTensor, + n_orig: int, + *, + s_inputs: torch.Tensor, + x_pred: torch.Tensor, + distogram_atom_idx: torch.Tensor, + token_attention_mask: torch.Tensor, + atom_to_token: torch.Tensor, + atom_attention_mask: torch.Tensor, + asym_id: torch.Tensor, + mol_type: torch.Tensor, + num_diffusion_samples: int = 1, + relative_position_encoding: torch.Tensor | None = None, + token_bonds_encoding: torch.Tensor | None = None, + ) -> dict[str, torch.Tensor]: + if num_diffusion_samples != 1: + raise NotImplementedError( + "distributed confidence head supports num_diffusion_samples==1" + ) + mesh = self.device_mesh + Lpad = z_dt.shape[1] + pad = Lpad - n_orig + pair_dtype = torch.float32 + + # --- distogram bins: sharded block cdist (never the full [B, N, N]) ----- + rep_idx = distogram_atom_idx.long() + rep_coords = gather_rep_atom_coords( + x_pred, rep_idx + ) # (B, n, 3) — replicated, cheap + bins_p = build_sharded_distogram_bins( + rep_coords, self.head.boundaries, mesh + ) # (S0, S1, S2) [B, Lpad, Lpad] + + # --- pad aux tensors to the shard factor (z_dt is already padded) ------- + def _pad_pair(t): + # An already-sharded DTensor is padded (built at the shard factor) — + # pass it through; only a full tensor needs padding here. + if isinstance(t, DTensor): + return t + return F.pad(t, (0, 0, 0, pad, 0, pad)) if pad else t + + s_in_p = F.pad(s_inputs, (0, 0, 0, pad)) if pad else s_inputs + relpos_p = ( + _pad_pair(relative_position_encoding) + if relative_position_encoding is not None + else None + ) + tokb_p = ( + _pad_pair(token_bonds_encoding) + if token_bonds_encoding is not None + else None + ) + mask_p = ( + F.pad(token_attention_mask.float(), (0, pad)) + if pad + else (token_attention_mask.float()) + ) + + # --- z_base (sharded) --------------------------------------------------- + pair_dt = self.zbase( + z_dt.to(pair_dtype), s_in_p, bins_p, relpos_p, tokb_p + ) # (S0, S1, S2) fp32, [B, Lpad, Lpad, d_pair] + + # --- nested FoldingTrunk (add-back, like serial pair.add_(trunk(pair))) - + # Pair mask built SHARDED from the (padded) token mask — the per-block outer + # product, never a full [B, Lpad, Lpad] (matches the recycle / MSA masks). + with torch.amp.autocast("cuda", enabled=True, dtype=torch.bfloat16): + pair_mask_dt = build_sharded_pair_mask(mask_p, mesh).to(torch.bfloat16) + delta_dt = self.head.folding_trunk.forward_sharded( + pair_dt.to(torch.bfloat16), pair_attention_mask=pair_mask_dt + ) + pair_dt = pair_dt + delta_dt.to(pair_dtype) + + # --- row pooling -> single (gather small) ------------------------------- + single_dt = self.row_pool(pair_dt, mask_p) # (S0,S1,R) [B,Lpad,d_single] + single = single_dt.full_tensor()[:, :n_orig, :].contiguous() + + # --- PAE / PDE heads on the sharded pair; gather (sliced) logits -------- + pae_logits_dt = self.pae_head(self.pae_ln(pair_dt)) + pde_logits_dt = self.pde_head(self.pde_ln(pair_dt)) + del pair_dt + pae_logits = pae_logits_dt.full_tensor()[:, :n_orig, :n_orig, :].contiguous() + del pae_logits_dt + pde_logits = pde_logits_dt.full_tensor()[:, :n_orig, :n_orig, :].contiguous() + del pde_logits_dt + + # --- serial back-half (atom-space + pTM/ipTM), shared with forward ------ + return self.head._finish( + single=single, + pae_logits=pae_logits, + pde_logits=pde_logits, + x_pred=x_pred, + distogram_atom_idx=distogram_atom_idx, + token_attention_mask=token_attention_mask, + atom_to_token=atom_to_token, + atom_attention_mask=atom_attention_mask, + asym_id=asym_id, + mol_type=mol_type, + num_diffusion_samples=num_diffusion_samples, + ) diff --git a/esm/models/esmfold2/distributed/distogram_wrapper.py b/esm/models/esmfold2/distributed/distogram_wrapper.py new file mode 100644 index 00000000..32226435 --- /dev/null +++ b/esm/models/esmfold2/distributed/distogram_wrapper.py @@ -0,0 +1,93 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: MIT +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. + +"""Distributed distogram head for ESMFold2 (inference-only). + +The serial op is a single channel-wise Linear over the symmetrized pair: + + distogram_logits = distogram_head(z + z.transpose(-2, -3)) # (B, L, L, bins) + +``z.transpose(-2, -3)`` swaps the two token axes (rows i <-> cols j). Under 2D CP +the pair is sharded ``(Shard(0), Shard(1), Shard(2))``; the (p, q) rank's +transposed tile ``zᵀ[i∈p, j∈q] = z[j∈q, i∈p]`` lives on the transpose peer +``(q, p)`` — one ``TransposeComm`` fetches that tile, and a local axis-swap +arranges it. The Linear is per-pair channel-wise → local. Only the small +``bins``-channel logits are gathered (vs. the full ``d_pair`` pair), so the full +``z`` (256 ch) is never re-gathered for the distogram. +""" + +import torch +import torch.nn as nn +from torch.distributed.tensor import DTensor, Shard + +from esm.models.esmfold2.distributed.comm import TransposeComm +from esm.models.esmfold2.distributed.model.layers.linear import LinearParamsReplicated + +_PAIR = [Shard(0), Shard(1), Shard(2)] + + +def _contiguous_strides(shape: tuple[int, ...]) -> tuple[int, ...]: + strides = [1] * len(shape) + for i in range(len(shape) - 2, -1, -1): + strides[i] = strides[i + 1] * shape[i + 1] + return tuple(strides) + + +class DistogramHeadCPWrapper(nn.Module): + """Distributed distogram head. ``forward_sharded`` takes the sharded pair + DTensor (padded) + the original length and returns the full (sliced) + ``distogram_logits`` — gathering only the ``bins``-channel output.""" + + def __init__(self, distogram_head: nn.Linear, dist_manager) -> None: + super().__init__() + if not isinstance(distogram_head, nn.Linear): + raise TypeError( + f"distogram_head must be nn.Linear, got {type(distogram_head).__name__}" + ) + self.device_mesh = dist_manager.device_mesh_subgroups + self.head = LinearParamsReplicated(distogram_head, self.device_mesh) + self.transpose = TransposeComm( + dist_manager.group["cp"], dist_manager.layout_subgroups["cp"] + ) + + def forward_sharded(self, z_dt: DTensor, n_orig: int) -> torch.Tensor: + mesh = self.device_mesh + # Match serial: distogram runs on z.float(). + z_local = z_dt.to_local().float().contiguous() # (B, sLi, sLj, c) = z[i,j] + + recv = self.transpose.enqueue_to_dispatch(z_local) + self.transpose.wait_until_finished() + # recv = z's (q,p) tile = z[a∈q, b∈p]; zᵀ[i∈p, j∈q] = z[j,i] = recv[j, i]. + zT_local = recv.transpose(1, 2).contiguous() # (B, sLi, sLj, c) + + sym_local = z_local + zT_local + L = z_dt.shape[1] + b_size, _, _, c = sym_local.shape + full_shape = torch.Size((b_size, L, L, c)) + sym_dt = DTensor.from_local( + sym_local, + device_mesh=mesh, + placements=_PAIR, + shape=full_shape, + stride=_contiguous_strides(tuple(full_shape)), + ) + logits_dt = self.head(sym_dt) # (S0, S1, S2) [B, L, L, bins] + return logits_dt.full_tensor()[:, :n_orig, :n_orig, :].contiguous() diff --git a/esm/models/esmfold2/distributed/esmc_tp.py b/esm/models/esmfold2/distributed/esmc_tp.py new file mode 100644 index 00000000..8fdb7260 --- /dev/null +++ b/esm/models/esmfold2/distributed/esmc_tp.py @@ -0,0 +1,129 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: MIT +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. + +"""Tensor-parallel ESM-C over the CP ranks (inference-only). + +The replicated ESM-C 6B forward is the dominant per-rank floor (T0 re-hook: +~19.8 GB live, identical on every rank). Tensor parallelism shards its weights +AND intra-block activations across the CP ranks. + +**Phase 1 (this module): the SwiGLU MLP** — ~67% of ESM-C's params. Each +``UnifiedTransformerBlock.ffn`` is a Transformer Engine ``LayerNormMLP``; TE has +native TP (``set_parallel_mode=True`` → column-parallel fc1 with the SwiGLU +gate/up split handled internally + row-parallel fc2 + an internal all-reduce), so +the wrap is: construct a TP ``LayerNormMLP`` over the CP process group and copy the +full weights in sharded. The MLP's ``ffn_hidden`` (6912) divides 4/9/16, so this +works at every CP grid (unlike head-parallel attention, blocked by n_heads=40). + +Validated bit-exact (modulo bf16 rounding) under ``torch.inference_mode`` — the +column/row shard + internal all-reduce reproduce the serial output; the ESM-C +forward already runs under the model's ``@torch.inference_mode`` so TE-TP (plain +sharded matmuls + NCCL, no autograd hooks) is inference-safe. + +Attention TP (qkv column + out_proj row + the full-width QK-LayerNorm all-reduce, +constrained to TP=4 by the 40 heads) is a later phase — see ``fix_9x_ESMC_TP.md``. +""" + +import torch +import torch.nn as nn + + +def _tp_layernorm_mlp(old, tp_group, tp_size: int, rank: int): + """Build a TE LayerNormMLP TP-shard of ``old`` (a serial te.LayerNormMLP), + holding only this rank's slice. Bit-exact (bf16) vs the full module.""" + import transformer_engine.pytorch as te + + hidden = old.layer_norm_weight.shape[0] + ffn = old.fc2_weight.shape[1] # fc2 = [hidden, ffn] + if ffn % tp_size: + raise ValueError(f"ESM-C ffn_hidden={ffn} not divisible by tp_size={tp_size}") + dtype = old.fc1_weight.dtype + device = old.fc1_weight.device + + new = te.LayerNormMLP( + hidden, + ffn, + eps=getattr(old, "eps", 1e-5), + normalization=getattr(old, "normalization", "LayerNorm"), + activation=getattr(old, "activation", "swiglu"), + bias=False, # ESM-C trained with bias=False + zero_centered_gamma=getattr(old, "zero_centered_gamma", False), + set_parallel_mode=True, + tp_group=tp_group, + tp_size=tp_size, + sequence_parallel=False, + params_dtype=dtype, + ).to(device=device, dtype=dtype) + new.eval() + + fl = ffn // tp_size + with torch.no_grad(): + new.layer_norm_weight.copy_(old.layer_norm_weight) # ty:ignore[call-non-callable] + new.layer_norm_bias.copy_(old.layer_norm_bias) # ty:ignore[unresolved-attribute] + # fc1 full = [2*ffn, hidden] laid out [gate(ffn); up(ffn)]; column-parallel + # SwiGLU shard = cat([gate[r], up[r]]) (confirmed bit-exact by the spike). + gate, up = old.fc1_weight[:ffn], old.fc1_weight[ffn:] + new.fc1_weight.copy_( + torch.cat( + [gate[rank * fl : (rank + 1) * fl], up[rank * fl : (rank + 1) * fl]], + dim=0, + ) + ) # ty:ignore[call-non-callable] + # fc2 full = [hidden, ffn]; row-parallel shards the input (ffn) dim. + new.fc2_weight.copy_(old.fc2_weight[:, rank * fl : (rank + 1) * fl]) # ty:ignore[call-non-callable] + return new + + +def tp_shard_esmc_mlp(model: nn.Module, dist_manager, tp_group=None) -> int: + """Replace every ESM-C block's SwiGLU MLP with a TE tensor-parallel shard + over ``tp_group`` (default: the full CP process group). Returns the number of + blocks sharded. No-op (returns 0) if ESM-C isn't loaded. + + Each rank ends up holding ``1/tp_size`` of every MLP weight; the per-block + forward all-reduces internally so the block output stays replicated — the + ESM-C hidden states reach all CP ranks unchanged for the #14 lm_z builder. + """ + esmc = getattr(model, "esmc", None) + if esmc is None: + return 0 + blocks = esmc.transformer.blocks + + if tp_group is None: + tp_group = dist_manager.group["cp"] + tp_size = torch.distributed.get_world_size(tp_group) + rank = torch.distributed.get_rank(tp_group) + if tp_size == 1: + return 0 + + n = 0 + for block in blocks: + ffn = getattr(block, "ffn", None) + # Only the TE LayerNormMLP path is supported (the accelerated build); + # the pure-PyTorch fallback would need a DTensor TP and isn't on this path. + if ffn is None or type(ffn).__name__ != "LayerNormMLP": + raise TypeError( + f"expected block.ffn to be a TE LayerNormMLP, got " + f"{type(ffn).__name__ if ffn is not None else None}" + ) + block.ffn = _tp_layernorm_mlp(ffn, tp_group, tp_size, rank) + del ffn + n += 1 + return n diff --git a/esm/models/esmfold2/distributed/manager.py b/esm/models/esmfold2/distributed/manager.py new file mode 100644 index 00000000..4128c57e --- /dev/null +++ b/esm/models/esmfold2/distributed/manager.py @@ -0,0 +1,586 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: MIT +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. + + +import os +from math import prod +from typing import Any, Dict, Optional, OrderedDict, Union +from warnings import warn + +import torch + +from esm.models.esmfold2.distributed.utils import LayoutMap, LayoutRightMap + +# grid_group_sizes objects must have +# (1) .values() attribute, (2) .items() attribute +_GridGroupSizesType = OrderedDict[str, Union[int, tuple[int, ...]]] + + +class DistributedManager: + """Borg-style singleton class for managing distributed state. + + Manages the device mesh, process groups, and subgroups for 2D context + parallelism. Initialized with e.g.:: + + DistributedManager.initialize(OrderedDict([("dp", 1), ("cp", (N, N))])) + + to create a 2D CP grid of size N×N. + """ + + # Borg-style shared state. Every instance's ``__dict__`` is rebound to + # ``_state`` in ``__new__``; one-time defaults are seeded via ``setdefault`` + # (rather than ``instance._foo = ...`` which pyright flags as "Cannot + # assign to attribute" on a class without those names declared). + _state: dict = {} + _DEFAULT_STATE: dict = { + "_initialized": False, + "_has_dist": False, + "_rank": 0, + "_world_size": 1, + "_local_rank": 0, + "_device": torch.device("cpu"), + "_backend": None, + "_device_mesh": None, + "_layout_device_mesh": None, + "_has_subgroups": False, + "_device_mesh_subgroups": None, + "_layout_device_mesh_subgroups": None, + "_group": {}, + "_group_rank": {}, + "_group_ranks": {}, + "_subgroups": {}, + "_subgroups_rank": {}, + "_subgroups_ranks": {}, + "_layout_subgroups": {}, + "_method_init": None, + } + + def __new__(cls): + instance = super().__new__(cls) + instance.__dict__ = cls._state + for key, default in cls._DEFAULT_STATE.items(): + cls._state.setdefault(key, default) + return instance + + @classmethod + def methods_init_available(cls) -> set[str]: + return {"ENV", "SLURM"} + + @classmethod + def backend_for_device(cls) -> Dict[str, Optional[str]]: + return { + "cuda": "nccl" if torch.distributed.is_nccl_available() else None, + "cpu": "gloo" if torch.distributed.is_gloo_available() else None, + } + + @classmethod + def is_initialized(cls) -> bool: + return cls._state.get("_initialized", False) + + def __init__(self): + if not self._initialized: + raise RuntimeError( + "A DistributedManager instance is being instantiated before " + "the singleton class is initialized. " + "Please call DistributedManager.initialize() first." + ) + super().__init__() + + def __getattr__(self, name: str) -> Any: + key_state = f"_{name}" + has_key_shared_state = key_state in self.__dict__ + has_key = name in self.__dict__ + if has_key_shared_state: + return self.__dict__[key_state] + elif has_key: + return self.__dict__[name] + else: + raise AttributeError(f'Attribute "{name}" or "_{name}" not found.') + + def __str__(self): + return ( + f"Initialized process {self.rank} of {self.world_size} using " + f"method '{self.method_init}'. Device set to {str(self.device)}. " + f"Backend is {self.backend}" + ) + + @staticmethod + def _setup( + grid_group_sizes: Optional[_GridGroupSizesType] = None, + device_type: str = "cuda", + backend: Optional[str] = None, + rank: int = -1, + node_rank: int = -1, + world_size: int = -1, + local_rank: Optional[int] = None, + addr: str = "localhost", + port: str = "29500", + method_init: str = "ENV", + **kwargs_init_pg, + ): + if device_type == "cuda" and not torch.cuda.is_available(): + raise RuntimeError( + f"Input device type {device_type} but torch.cuda is not available" + ) + + if world_size != -1 and grid_group_sizes is not None: + total_size = 1 + assert hasattr(grid_group_sizes, "values") + for value in grid_group_sizes.values(): + if isinstance(value, tuple) and all(isinstance(v, int) for v in value): + total_size *= prod(value) + elif isinstance(value, int): + total_size *= value + else: + raise RuntimeError( + f"Values in grid_group_sizes must be either int or tuple[int, ...], got {type(value)}" + ) + if world_size != total_size: + raise RuntimeError( + f"Non-default world_size {world_size} != product of grid_group_sizes values ({total_size})" + ) + + backend_for_device = DistributedManager.backend_for_device() + + if backend_for_device["cpu"] is None and backend_for_device["cuda"] is None: + raise RuntimeError( + f"No backend available for the supported device types: {backend_for_device.keys()}" + ) + + if device_type not in backend_for_device: + raise RuntimeError( + f"Invalid input device type {device_type}: only supports {backend_for_device.keys()}" + ) + + if backend is None: + backend = backend_for_device[device_type] + elif backend != backend_for_device[device_type]: + raise RuntimeError( + f"Invalid input backend {backend} for input device type {device_type}" + ) + + os.environ["MASTER_ADDR"] = addr + os.environ["MASTER_PORT"] = str(port) + + DistributedManager._state["_initialized"] = True + manager = DistributedManager() + + manager._has_dist = torch.distributed.is_available() # ty:ignore[unresolved-attribute] + manager._rank = rank # ty:ignore[unresolved-attribute] + manager._world_size = world_size # ty:ignore[unresolved-attribute] + manager._node_rank = node_rank # ty:ignore[unresolved-attribute] + + if device_type == "cuda": + if ( + manager.world_size > torch.cuda.device_count() + and manager.world_size % torch.cuda.device_count() + ): + warn("world_size is not a multiple of torch.cuda.device_count()") + if local_rank is None: + manager._local_rank = manager.rank % torch.cuda.device_count() # ty:ignore[unresolved-attribute] + else: + manager._local_rank = local_rank # ty:ignore[unresolved-attribute] + manager._device = torch.device(f"cuda:{manager.local_rank}") # ty:ignore[unresolved-attribute] + else: + if local_rank is not None: + manager._local_rank = local_rank # ty:ignore[unresolved-attribute] + manager._device = torch.device("cpu") # ty:ignore[unresolved-attribute] + + if not manager.has_dist: + warn("DistributedManager initialized without torch.distributed package") + return + + if manager.device.type == "cuda": + torch.cuda.set_device(manager.device) + torch.cuda.device(manager.device) + torch.cuda.empty_cache() + + manager._backend = backend # ty:ignore[unresolved-attribute] + + if manager.device.type == "cuda" and backend == "nccl": + # Disable NVLS (NVLink SHARP multicast) by default — it requires + # NVSwitch hardware + Fabric Manager, and probing for it via + # cuMulticastCreate raises a hard CUDA error (401) when unavailable. + # Users on NVSwitch systems can opt in by setting NCCL_NVLS_ENABLE=1. + os.environ.setdefault("NCCL_NVLS_ENABLE", "0") + try: + torch.distributed.init_process_group( + manager.backend, + rank=manager.rank, + world_size=manager.world_size, + device_id=manager.device, + **kwargs_init_pg, + ) + except (TypeError, RuntimeError): + # TypeError: older PyTorch doesn't accept device_id. + # RuntimeError: device_id triggers eager NCCL connect which + # can fail (e.g. NVLS/NVSwitch not available, CUDA error 401). + # Fall back to lazy (non-eager) init in both cases. + torch.distributed.init_process_group( + manager.backend, + rank=manager.rank, + world_size=manager.world_size, + **kwargs_init_pg, + ) + else: + torch.distributed.init_process_group( + manager.backend, + rank=manager.rank, + world_size=manager.world_size, + **kwargs_init_pg, + ) + + manager._group["world"] = torch.distributed.group.WORLD + manager._group_rank["world"] = manager.rank + manager._group_ranks["world"] = torch.distributed.get_process_group_ranks( + manager.group["world"] + ) + manager._method_init = method_init # ty:ignore[unresolved-attribute] + + if grid_group_sizes is not None: + DistributedManager.create_grid_group(grid_group_sizes) + + @staticmethod + def _create_device_mesh_and_groups( + name: list[str], shape: list[int], suffix_mesh: Optional[str] = None + ) -> None: + if not DistributedManager.is_initialized(): + raise RuntimeError("DistributedManager is not initialized") + if ( + not DistributedManager._state["_has_dist"] + or not torch.distributed.is_available() + ): + raise RuntimeError( + "_create_device_mesh_and_groups requires torch.distributed" + ) + if ( + DistributedManager._state["_method_init"] is None + or DistributedManager._state["_method_init"] + not in DistributedManager.methods_init_available() + ): + raise RuntimeError( + f"Invalid DistributedManager method_init {DistributedManager._state['_method_init']}" + ) + if ( + DistributedManager._state["_backend"] is None + or DistributedManager._state["_backend"] + not in DistributedManager.backend_for_device().values() + ): + raise RuntimeError( + f"Invalid DistributedManager backend {DistributedManager._state['_backend']}" + ) + if ( + DistributedManager._state["_device"] is None + or DistributedManager._state["_device"].type + not in DistributedManager.backend_for_device().keys() + ): + raise RuntimeError( + f"Invalid DistributedManager device type {DistributedManager._state['_device'].type}" + ) + + world_size_expected = prod(shape) + if world_size_expected != DistributedManager._state["_world_size"]: + raise RuntimeError( + f"world_size {DistributedManager._state['_world_size']} does not match " + f"expected {world_size_expected} from shape {shape}" + ) + + device_type = DistributedManager._state["_device"].type + name_mesh = ( + f"_device_mesh_{suffix_mesh}" if suffix_mesh is not None else "_device_mesh" + ) + layout = LayoutRightMap(tuple(shape)) + DistributedManager._state[f"_layout{name_mesh}"] = layout + + grid2rank = torch.as_strided( + torch.arange(world_size_expected), size=layout.shape, stride=layout.strides + ) + DistributedManager._state[name_mesh] = torch.distributed.device_mesh.DeviceMesh( + device_type, grid2rank, mesh_dim_names=tuple(name) + ) + + for i_group in range(len(name)): + name_group = name[i_group] + if name_group in DistributedManager._state["_group"]: + continue + DistributedManager._state["_group"][name_group] = DistributedManager._state[ + name_mesh + ].get_group(name_group) + DistributedManager._state["_group_rank"][name_group] = ( + torch.distributed.get_group_rank( + DistributedManager._state["_group"][name_group], + DistributedManager._state["_rank"], + ) + ) + DistributedManager._state["_group_ranks"][name_group] = ( + torch.distributed.get_process_group_ranks( + DistributedManager._state["_group"][name_group] + ) + ) + + @staticmethod + def create_grid_group(grid_group_sizes: _GridGroupSizesType) -> None: + """Create a grid group for 2D context parallelism. + + Example:: + + from collections import OrderedDict + + DistributedManager.initialize(OrderedDict([("dp", 1), ("cp", (2, 2))])) + """ + shape_groups = [] + name_groups = [] + shape_subgroups = [] + name_subgroups = [] + group2subgroup = {} + group2subgroup_axes = {} + assert hasattr(grid_group_sizes, "items") + for k, v in grid_group_sizes.items(): + if isinstance(v, tuple) and all(isinstance(v_i, int) for v_i in v): + shape_groups.append(prod(v)) + name_groups.append(k) + shape_subgroups.extend(v) + names_this_subgroup = [f"{k}_axis_{i}" for i in range(len(v))] + name_subgroups.extend(names_this_subgroup) + group2subgroup[k] = names_this_subgroup + group2subgroup_axes[k] = list( + range(len(name_subgroups) - len(v), len(name_subgroups)) + ) + elif isinstance(v, int): + shape_groups.append(v) + name_groups.append(k) + shape_subgroups.append(v) + name_subgroups.append(k) + else: + raise RuntimeError( + f"Values in grid_group_sizes must be int or tuple[int, ...], got {type(v)}" + ) + + DistributedManager._create_device_mesh_and_groups(name_groups, shape_groups) + if (name_groups == name_subgroups) != (shape_groups == shape_subgroups): + raise RuntimeError("Inconsistent group and subgroup settings") + + DistributedManager._state["_has_subgroups"] = name_groups != name_subgroups + if DistributedManager._state["_has_subgroups"]: + if len(group2subgroup) == 0: + raise RuntimeError( + "group2subgroup is empty while _has_subgroups is True" + ) + DistributedManager._create_device_mesh_and_groups( + name_subgroups, shape_subgroups, suffix_mesh="subgroups" + ) + layout = DistributedManager._state["_layout_device_mesh_subgroups"] + # get_coordinate() returned a list in older PyTorch and a tuple in + # newer versions; normalise to list so axis assignment works. + coords = list( + DistributedManager._state["_device_mesh_subgroups"].get_coordinate() + ) + for name_group, subgroup_names in group2subgroup.items(): + DistributedManager._state["_subgroups"][name_group] = [ + DistributedManager._state["_group"][n] for n in subgroup_names + ] + DistributedManager._state["_subgroups_ranks"][name_group] = [ + DistributedManager._state["_group_ranks"][n] for n in subgroup_names + ] + DistributedManager._state["_subgroups_rank"][name_group] = [ + DistributedManager._state["_group_rank"][n] for n in subgroup_names + ] + axes_subgroup = group2subgroup_axes[name_group] + slices = coords.copy() + for axis in axes_subgroup: + slices[axis] = slice(None) + layout_subgroup = layout[tuple(slices)] + DistributedManager._state["_layout_subgroups"][name_group] = LayoutMap( + layout_subgroup.strides, layout_subgroup.shape, offset=0 + ) + + @staticmethod + def create_group(name: str, ranks: list[int], **kwargs_dist_ng) -> None: + DistributedManager._state["_group"][name] = torch.distributed.new_group( + ranks=ranks, **kwargs_dist_ng + ) + DistributedManager._state["_group_ranks"][name] = ranks + DistributedManager._state["_group_rank"][name] = ( + torch.distributed.get_group_rank( + DistributedManager._state["_group"][name], + DistributedManager._state["_rank"], + ) + ) + + @staticmethod + def _initialize_env(*args, **kwargs): + if not ("RANK" in os.environ and "WORLD_SIZE" in os.environ): + raise RuntimeError( + "environment variables RANK and WORLD_SIZE must be set for env:// initialization" + ) + rank = os.environ.get("RANK") + world_size = os.environ.get("WORLD_SIZE") + local_rank = os.environ.get("LOCAL_RANK") + group_rank = os.environ.get("GROUP_RANK", 0) + node_rank = int(os.environ.get("NODE_RANK", group_rank)) + try: + rank = int(rank) # ty:ignore[invalid-argument-type] + world_size = int(world_size) # ty:ignore[invalid-argument-type] + if local_rank is not None: + local_rank = int(local_rank) + except TypeError: + raise RuntimeError( + "environment variables RANK, LOCAL_RANK and WORLD_SIZE must be integers" + ) + DistributedManager._setup( + *args, + rank=rank, + node_rank=node_rank, + world_size=world_size, + local_rank=local_rank, + addr=os.environ.get("MASTER_ADDR"), # ty:ignore[invalid-argument-type] + port=os.environ.get("MASTER_PORT"), # ty:ignore[invalid-argument-type] + method_init="ENV", + **kwargs, + ) + + @staticmethod + def _initialize_slurm(*args, **kwargs): + keys = ( + "SLURM_PROCID", + "SLURM_NPROCS", + "SLURM_LOCALID", + "SLURM_LAUNCH_NODE_IPADDR", + ) + if not all(k in os.environ for k in keys): + raise RuntimeError( + f"environment variables {keys} must be set for SLURM initialization" + ) + rank = os.environ.get("SLURM_PROCID") + node_rank = int(os.environ.get("SLURM_NODEID", 0)) + world_size = os.environ.get("SLURM_NPROCS") + local_rank = os.environ.get("SLURM_LOCALID") + addr = os.environ.get("SLURM_LAUNCH_NODE_IPADDR") + try: + rank = int(rank) # ty:ignore[invalid-argument-type] + world_size = int(world_size) # ty:ignore[invalid-argument-type] + if local_rank is not None: + local_rank = int(local_rank) + except TypeError: + raise RuntimeError( + "environment variables SLURM_{PROCID,NPROCS,LOCALID} must be integers" + ) + DistributedManager._setup( + *args, + rank=rank, + node_rank=node_rank, + world_size=world_size, + local_rank=local_rank, + addr=addr, # ty:ignore[invalid-argument-type] + method_init="SLURM", + **kwargs, + ) + + @staticmethod + def initialize( + grid_group_sizes: Optional[OrderedDict[str, int | tuple[int, ...]]] = None, + device_type: str = "cuda", + backend: Optional[str] = None, + **kwargs_init_pg, + ): + """Initialize the DistributedManager singleton. + + Parameters + ---------- + grid_group_sizes: + E.g. ``OrderedDict([("dp", 1), ("cp", (2, 2))])`` for a 2×2 CP grid. + device_type: + "cuda" (default) or "cpu". + backend: + Defaults to nccl for cuda, gloo for cpu. + """ + if DistributedManager.is_initialized(): + warn("DistributedManager is already initialized. Skip initialize()") + return + if backend == "nccl": + os.environ["TORCH_NCCL_ASYNC_ERROR_HANDLING"] = "0" + method_init = os.getenv("ESMCFOLD_DISTRIBUTED_INIT_METHOD") + if ( + method_init is not None + and method_init not in DistributedManager.methods_init_available() + ): + raise ValueError( + f"Unknown ESMCFOLD_DISTRIBUTED_INIT_METHOD={method_init}. " + f"Allowed: {DistributedManager.methods_init_available()}" + ) + if method_init is None: + try: + DistributedManager._initialize_env( + grid_group_sizes, + device_type=device_type, + backend=backend, + **kwargs_init_pg, + ) + except RuntimeError as except_env: + try: + DistributedManager._initialize_slurm( + grid_group_sizes, + device_type=device_type, + backend=backend, + **kwargs_init_pg, + ) + except RuntimeError as except_slurm: + warn( + "Could not initialize DistributedManager with env:// nor slurm.\n" + f"Error env://: {except_env}\n" + f"Error slurm: {except_slurm}\n" + "Will default initialize DistributedManager" + ) + DistributedManager._state["_initialized"] = True + elif method_init == "ENV": + DistributedManager._initialize_env( + grid_group_sizes, + device_type=device_type, + backend=backend, + **kwargs_init_pg, + ) + elif method_init == "SLURM": + DistributedManager._initialize_slurm( + grid_group_sizes, + device_type=device_type, + backend=backend, + **kwargs_init_pg, + ) + + @staticmethod + def cleanup(): + if DistributedManager._state.get("_group", {}) != {}: + if torch.distributed.is_initialized(): + try: + if ( + DistributedManager._state["_device"].type == "cuda" + and torch.cuda.is_available() + ): + torch.distributed.barrier( + device_ids=[DistributedManager._state["_local_rank"]] + ) + else: + torch.distributed.barrier() + except Exception: + pass + torch.distributed.destroy_process_group() + else: + DistributedManager._state = {} diff --git a/esm/models/esmfold2/distributed/model/__init__.py b/esm/models/esmfold2/distributed/model/__init__.py new file mode 100644 index 00000000..eda8d5e6 --- /dev/null +++ b/esm/models/esmfold2/distributed/model/__init__.py @@ -0,0 +1,2 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: MIT diff --git a/esm/models/esmfold2/distributed/model/layers/__init__.py b/esm/models/esmfold2/distributed/model/layers/__init__.py new file mode 100644 index 00000000..eda8d5e6 --- /dev/null +++ b/esm/models/esmfold2/distributed/model/layers/__init__.py @@ -0,0 +1,2 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: MIT diff --git a/esm/models/esmfold2/distributed/model/layers/atom_to_token.py b/esm/models/esmfold2/distributed/model/layers/atom_to_token.py new file mode 100644 index 00000000..e0ffae74 --- /dev/null +++ b/esm/models/esmfold2/distributed/model/layers/atom_to_token.py @@ -0,0 +1,192 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: MIT +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. + +"""Distributed atom<->token gather/scatter for ESMFold2's diffusion module. + +ESMFold2 maps atoms to tokens with an **index** (``atom_to_token: [B, A]`` int64), +not boltz-cp's one-hot ``[B, A, n_tokens]`` matrix, so these are index-based +gather/scatter (``torch.gather`` / ``scatter_reduce``) rather than a one-hot +matmul — strictly less memory (no ``A×L`` materialisation). + +Sharding convention (matches boltz-cp's 1D sequence reprs): a sequence tensor +(atoms or tokens) on the ``(dp, cp_axis_0, cp_axis_1)`` mesh has placements +``(Shard(0), Shard(1), Replicate())`` — the sequence axis (dim 1) is split across +``cp_axis_0`` and replicated across ``cp_axis_1``. (The pair stays 2-D sharded; +the diffusion's heavy L×L work is handled separately by the ring attention.) + +Inference-only (no autograd) — ESMFold2's CP path runs under +``@torch.inference_mode``. + +These are the atom<->token data-movement primitives. The atom encoder/decoder's +sliding-window attention (``swa_window_size``) imposes a separate constraint when +sharding the atom axis — atom shards must align to window boundaries (shard at a +multiple of the window) or exchange halos — handled where the atom transformer is +distributed, not here. +""" + +import torch +import torch.distributed as dist +from torch.distributed.tensor import DTensor, Replicate, Shard + +from esm.models.esmfold2.layers import gather_token_to_atom + +_SEQ_PL = [Shard(0), Shard(1), Replicate()] # (dp, cp_axis_0=seq, cp_axis_1) +_SEQ_GATHERED_PL = [Shard(0), Replicate(), Replicate()] # seq gathered on cp_axis_0 + + +def _contiguous_strides(shape: tuple[int, ...]) -> tuple[int, ...]: + strides = [1] * len(shape) + for i in range(len(shape) - 2, -1, -1): + strides[i] = strides[i + 1] * shape[i + 1] + return tuple(strides) + + +def dist_gather_token_to_atom( + token_dt: DTensor, atom_to_token_idx_dt: DTensor +) -> DTensor: + """Broadcast per-token features to per-atom features (sharded). + + Parameters + ---------- + token_dt: + Token features ``(B, L, d)`` with placements ``(Shard(0), Shard(1), + Replicate())`` (token axis split on ``cp_axis_0``). + atom_to_token_idx_dt: + Per-atom global token index ``(B, A)`` int64, same placements (atom axis + split on ``cp_axis_0``). + + Returns + ------- + Atom features ``(B, A, d)`` with placements ``(Shard(0), Shard(1), + Replicate())`` (atom axis split on ``cp_axis_0``). + """ + mesh = token_dt.device_mesh + # Gather the full token axis onto every cp_axis_0 rank (token repr is the + # cheap 1-D L×d, not L×L), so each rank can index its local atoms. + token_full = token_dt.redistribute(mesh, _SEQ_GATHERED_PL).to_local().contiguous() + idx_local = atom_to_token_idx_dt.to_local() + atom_local = gather_token_to_atom(token_full, idx_local) + + b = atom_local.shape[0] + a_global = atom_to_token_idx_dt.shape[1] + d = atom_local.shape[-1] + shape = torch.Size((b, a_global, d)) + return DTensor.from_local( + atom_local.contiguous(), + device_mesh=mesh, + placements=_SEQ_PL, + shape=shape, + stride=_contiguous_strides(tuple(shape)), + ) + + +def _local_scatter_sum_count( + atom_local: torch.Tensor, + idx_local: torch.Tensor, + n_tokens: int, + mask_local: torch.Tensor | None, +) -> tuple[torch.Tensor, torch.Tensor]: + """Local (per-rank) scatter-add of atom features into token bins, plus a + per-token contributing-atom count. Masked atoms are routed to a throwaway + bin ``n_tokens`` and dropped.""" + b, _, d = atom_local.shape + idx_use = idx_local + n_out = n_tokens + if mask_local is not None: + idx_use = torch.where(mask_local.bool(), idx_local, n_tokens) + n_out = n_tokens + 1 + + idx_e = idx_use.unsqueeze(-1).expand(b, idx_use.shape[1], d) + s = torch.zeros(b, n_out, d, device=atom_local.device, dtype=atom_local.dtype) + s.scatter_add_(1, idx_e, atom_local) + c = torch.zeros(b, n_out, 1, device=atom_local.device, dtype=atom_local.dtype) + c.scatter_add_( + 1, + idx_use.unsqueeze(-1), + torch.ones( + b, idx_use.shape[1], 1, device=atom_local.device, dtype=atom_local.dtype + ), + ) + return s[:, :n_tokens], c[:, :n_tokens] + + +def dist_scatter_atom_to_token( + atom_dt: DTensor, + atom_to_token_idx_dt: DTensor, + n_tokens: int, + atom_mask_dt: DTensor | None = None, +) -> DTensor: + """Aggregate per-atom features to per-token features (mean), sharded. + + Bit-equivalent to the serial ``scatter_atom_to_token`` (mean over the atoms + of each token, empty tokens → 0): each rank scatter-adds its local atom shard + into a full-L sum + count, the two are all-reduced across ``cp_axis_0``, then + divided to a mean and re-sharded onto the token axis. + + Parameters + ---------- + atom_dt: + Atom features ``(B, A, d)`` with placements ``(Shard(0), Shard(1), + Replicate())`` (atom axis split on ``cp_axis_0``). + atom_to_token_idx_dt: + Per-atom global token index ``(B, A)`` int64, same placements. + n_tokens: + Global token count ``L``. + atom_mask_dt: + Optional per-atom bool mask ``(B, A)``, same placements. + + Returns + ------- + Token features ``(B, L, d)`` with placements ``(Shard(0), Shard(1), + Replicate())`` (token axis split on ``cp_axis_0``). + """ + mesh = atom_dt.device_mesh + atom_local = atom_dt.to_local() + idx_local = atom_to_token_idx_dt.to_local() + mask_local = atom_mask_dt.to_local() if atom_mask_dt is not None else None + + s_local, c_local = _local_scatter_sum_count( + atom_local, idx_local, n_tokens, mask_local + ) + + # Sum the per-rank partials across the cp_axis_0 sequence-shard group. The + # mesh-dim-1 process group is exactly that axis; cp_axis_1 holds replicas, so + # each cp_axis_1 column independently reconstructs the same full token tensor. + cp0_group = mesh.get_group(1) + if dist.is_initialized() and dist.get_world_size(cp0_group) > 1: + dist.all_reduce(s_local, op=dist.ReduceOp.SUM, group=cp0_group) + dist.all_reduce(c_local, op=dist.ReduceOp.SUM, group=cp0_group) + + mean_full = s_local / c_local.clamp(min=1.0) # empty tokens -> 0 + + # mean_full is the full (B, L, d) tensor, identical on every cp_axis_0 rank in + # a column. Wrap as gathered-on-cp_axis_0, then reshard to the token axis + # (Replicate -> Shard is a local chunk, no further communication). + b, d = mean_full.shape[0], mean_full.shape[-1] + shape = torch.Size((b, n_tokens, d)) + full_dt = DTensor.from_local( + mean_full.contiguous(), + device_mesh=mesh, + placements=_SEQ_GATHERED_PL, + shape=shape, + stride=_contiguous_strides(tuple(shape)), + ) + return full_dt.redistribute(mesh, _SEQ_PL) diff --git a/esm/models/esmfold2/distributed/model/layers/attention_pair_bias.py b/esm/models/esmfold2/distributed/model/layers/attention_pair_bias.py new file mode 100644 index 00000000..75ad8a3c --- /dev/null +++ b/esm/models/esmfold2/distributed/model/layers/attention_pair_bias.py @@ -0,0 +1,118 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: MIT +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. + +"""Distributed token self-attention with a 2-D-sharded pair bias (diffusion). + +This is the *core attention mechanism* for distributing ESMFold2's diffusion +token transformer (``AttentionPairBias``). It shards the expensive ``L×L`` +attention over the query-row (``cp_axis_0``) axis, which is the diffusion's +memory ceiling; the cheap 1-D token reprs and the atom encoder/decoder stay +replicated (see ``fix_9x_#2.md``). + +Two strategies, mirroring the MSA pair-averaging design: + +* ``"gather"`` (implemented here): query rows are sharded on ``cp_axis_0``; + ``k``/``v`` and the bias *row* are gathered along the key/column axis, then a + single local softmax attention runs over the full key axis — **bit-exact** + with the serial op. Per-rank attention/bias is ``L²·H / P`` (sharded on + ``cp_axis_0``), down from the full ``L²``. The gathered ``k``/``v`` are the + cheap 1-D ``L·H·D``. +* ``"ring"`` (future): also shard the key axis on ``cp_axis_1`` and ring k/v/z + with online softmax (``AttentionPairBiasComm`` + ``tiled_softmax_attention_update``), + removing the full-key-axis gather. A memory optimisation over ``"gather"``. + +Inference-only. The ``q``/``k``/``v`` token reprs use placements +``(Shard(0), Shard(1), Replicate())`` (rows on ``cp_axis_0``); the pair bias uses +``(Shard(0), Shard(1), Shard(2))`` (the trunk's 2-D pair sharding). +""" + +import torch +from torch.distributed.tensor import DTensor, Replicate, Shard + +_ROW_PL = [Shard(0), Shard(1), Replicate()] # token rows on cp_axis_0 +_ROW_GATHERED_PL = [Shard(0), Replicate(), Replicate()] # key axis gathered +_PAIR_PL = [Shard(0), Shard(1), Shard(2)] # 2-D pair sharding +_PAIR_ROWS_PL = [Shard(0), Shard(1), Replicate()] # bias rows on cp0, cols gathered + + +def attention_pair_bias_gather( + q_dt: DTensor, + k_dt: DTensor, + v_dt: DTensor, + bias_dt: DTensor, + scale: float, + key_mask_dt: DTensor | None = None, +) -> DTensor: + """Self-attention with an additive per-head pair bias, "gather" strategy. + + Bit-exact with the serial reference:: + + logits[b,i,j,h] = scale * (q[b,i,h,:] . k[b,j,h,:]) + bias[b,i,j,h] + attn = softmax_j(logits) # over keys j + o[b,i,h,:] = sum_j attn[b,i,j,h] v[b,j,h,:] + + Parameters + ---------- + q_dt, k_dt, v_dt: + Token reprs ``(B, L, H, D)`` with placements ``(Shard(0), Shard(1), + Replicate())`` — query/key rows sharded on ``cp_axis_0``. + bias_dt: + Per-head pair bias ``(B, L, L, H)`` with placements ``(Shard(0), + Shard(1), Shard(2))`` — the trunk's 2-D pair sharding. + scale: + Attention logit scale (``head_dim**-0.5``). + key_mask_dt: + Optional key mask ``(B, L)`` (``True`` = keep), placements ``(Shard(0), + Shard(1), Replicate())``. Masked keys get ``-inf`` logit. + + Returns + ------- + Output ``(B, L, H, D)`` with placements ``(Shard(0), Shard(1), Replicate())`` + (query rows on ``cp_axis_0``). + """ + mesh = q_dt.device_mesh + + # Query rows stay sharded on cp_axis_0; gather the key axis (cheap 1-D k/v) + # and the bias *row* (rows block stays on cp_axis_0, columns gathered). + q_local = q_dt.to_local() # (B, Lq_local, H, D) + k_full = k_dt.redistribute(mesh, _ROW_GATHERED_PL).to_local() # (B, L, H, D) + v_full = v_dt.redistribute(mesh, _ROW_GATHERED_PL).to_local() # (B, L, H, D) + bias_row = bias_dt.redistribute( + mesh, _PAIR_ROWS_PL + ).to_local() # (B, Lq_local, L, H) + + # logits (B, Lq_local, L, H) + logits = torch.einsum("bihd,bjhd->bijh", q_local, k_full) * scale + logits = logits + bias_row.to(logits.dtype) + + if key_mask_dt is not None: + key_mask = key_mask_dt.redistribute(mesh, _ROW_GATHERED_PL).to_local() # (B, L) + neg = torch.finfo(logits.dtype).min + logits = logits + torch.where(key_mask.bool()[:, None, :, None], 0.0, neg).to( + logits.dtype + ) + + attn = torch.softmax(logits, dim=2).to(v_full.dtype) # over keys j + o_local = torch.einsum("bijh,bjhd->bihd", attn, v_full) # (B, Lq_local, H, D) + + return DTensor.from_local( + o_local.contiguous(), device_mesh=mesh, placements=_ROW_PL + ) diff --git a/esm/models/esmfold2/distributed/model/layers/confidence_zbase.py b/esm/models/esmfold2/distributed/model/layers/confidence_zbase.py new file mode 100644 index 00000000..85d31bc8 --- /dev/null +++ b/esm/models/esmfold2/distributed/model/layers/confidence_zbase.py @@ -0,0 +1,145 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: MIT +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. + +"""Distributed z_base / s->z builder for the ESMFold2 confidence head (#6). + +Reproduces the serial pre-trunk pair construction +(``modeling_esmfold2.py:ConfidenceHead.forward`` lines 187-216) keeping the pair a +sharded ``(Shard(0), Shard(1), Shard(2))`` DTensor: + + z_base = z_norm(z) [+ rel_pos] [+ token_bonds] + + s_to_z(s)[:, :, None] # row i, broadcast over j + + s_to_z_transpose(s)[:, None, :] # col j, broadcast over i + + s_to_z_prod_out(prod_in1(s)[:, :, None] * prod_in2(s)[:, None, :]) + pair = z_base + dist_bin_pairwise_embed(distogram_bins) + +Because ``s_inputs`` (hence ``s = s_inputs_norm(s_inputs)`` and all the per-token +projections) is replicated, the row term needs only the local cp_axis_0 rows and +the column term only the local cp_axis_1 cols — both obtained by slicing the cheap +full ``[B, L, d_pair]`` projection two ways (``distribute_tensor`` of a replicated +tensor = a local slice, no communication). So the whole build is **local and +bit-exact** (no transpose, no reduction): every term lands on the same +``(sLi, sLj)`` tile the pair owns. +""" + +import torch +import torch.nn as nn +import torch.nn.functional as F +from torch.distributed.tensor import DTensor, Replicate, Shard, distribute_tensor + +from esm.models.esmfold2.distributed.model.layers.layernorm import ( + LayerNormParamsReplicated, +) + +_PAIR = [Shard(0), Shard(1), Shard(2)] +_ROW = [Shard(0), Shard(1), Replicate()] # token L sharded on cp_axis_0 (rows i) +_COL = [Shard(0), Replicate(), Shard(1)] # token L sharded on cp_axis_1 (cols j) + + +def _contiguous_strides(shape: tuple[int, ...]) -> tuple[int, ...]: + strides = [1] * len(shape) + for i in range(len(shape) - 2, -1, -1): + strides[i] = strides[i + 1] * shape[i + 1] + return tuple(strides) + + +class ConfidenceZBaseDistributed(nn.Module): + """Builds the confidence head's pre-trunk pair as a sharded DTensor. + + Reads the projection submodules off the serial ``ConfidenceHead``; only the + pair-channel ``z_norm`` is wrapped (it runs on the sharded ``z``). The + per-token projections run on the replicated ``s_inputs`` exactly as serial. + """ + + def __init__(self, layer, dist_manager) -> None: + super().__init__() + self.device_mesh = dist_manager.device_mesh_subgroups + self.z_norm = LayerNormParamsReplicated(layer.z_norm, self.device_mesh) + # Per-token / pair-channel ops: replicated params, run as-is. + self.s_inputs_norm = layer.s_inputs_norm + self.s_to_z = layer.s_to_z + self.s_to_z_transpose = layer.s_to_z_transpose + self.s_to_z_prod_in1 = layer.s_to_z_prod_in1 + self.s_to_z_prod_in2 = layer.s_to_z_prod_in2 + self.s_to_z_prod_out = layer.s_to_z_prod_out + self.dist_bin_pairwise_embed = layer.dist_bin_pairwise_embed + + def forward( + self, + z: DTensor, + s_inputs: torch.Tensor, + distogram_bins: torch.Tensor, + relative_position_encoding: torch.Tensor | None = None, + token_bonds_encoding: torch.Tensor | None = None, + ) -> DTensor: + mesh = self.device_mesh + L = z.shape[1] + + s = self.s_inputs_norm(s_inputs) # full [B, L, d_inputs], replicated + + def row(t): # local cp_axis_0 rows + return distribute_tensor(t.contiguous(), mesh, _ROW).to_local() + + def col(t): # local cp_axis_1 cols + return distribute_tensor(t.contiguous(), mesh, _COL).to_local() + + a_row = row(self.s_to_z(s)) # (B, sLi, d_pair) + b_col = col(self.s_to_z_transpose(s)) # (B, sLj, d_pair) + p1_row = row(self.s_to_z_prod_in1(s)) # (B, sLi, d_pair) + p2_col = col(self.s_to_z_prod_in2(s)) # (B, sLj, d_pair) + + # rel_pos / token_bonds may arrive already sharded (built block-local by + # PairInitDistributed) — use the local shard directly; only a full tensor + # needs the distribute-then-take-local round trip. + def _pair_local(t): + if isinstance(t, DTensor): + return t.to_local() + return distribute_tensor(t.contiguous(), mesh, _PAIR).to_local() + + zb = self.z_norm(z).to_local() # (B, sLi, sLj, d_pair) + if relative_position_encoding is not None: + zb = zb + _pair_local(relative_position_encoding) + if token_bonds_encoding is not None: + zb = zb + _pair_local(token_bonds_encoding) + + zb = zb + a_row[:, :, None, :] + b_col[:, None, :, :] + prod = p1_row[:, :, None, :] * p2_col[:, None, :, :] # (B, sLi, sLj, d_pair) + zb = zb + F.linear(prod, self.s_to_z_prod_out.weight) + + # distogram_bins may arrive already sharded (block cdist by + # build_sharded_distogram_bins) — take the local shard directly. + bins_local = ( + distogram_bins.to_local() + if isinstance(distogram_bins, DTensor) + else distribute_tensor(distogram_bins.contiguous(), mesh, _PAIR).to_local() + ) # (B, sLi, sLj) + zb = zb + self.dist_bin_pairwise_embed(bins_local) + + b_size = zb.shape[0] + d_pair = zb.shape[-1] + full_shape = torch.Size((b_size, L, L, d_pair)) + return DTensor.from_local( + zb.contiguous(), + device_mesh=mesh, + placements=_PAIR, + shape=full_shape, + stride=_contiguous_strides(tuple(full_shape)), + ) diff --git a/esm/models/esmfold2/distributed/model/layers/diffusion_conditioning.py b/esm/models/esmfold2/distributed/model/layers/diffusion_conditioning.py new file mode 100644 index 00000000..cae0f2db --- /dev/null +++ b/esm/models/esmfold2/distributed/model/layers/diffusion_conditioning.py @@ -0,0 +1,127 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: MIT +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. + +"""Distributed DiffusionConditioning for ESMFold2's structure head. + +The conditioning produces, from the trunk pair ``z_trunk``, the per-head bias +source ``z`` (``B, L, L, c_z``) that biases the diffusion token attention — the +**dominant** diffusion tensor (``L²·c_z`` ≈ 4.6 GB at L=3000, far above the +per-layer attention scores). To get any diffusion memory win this ``z`` must be +sharded, so this module produces it as a 2-D-sharded DTensor +``(Shard(0), Shard(1), Shard(2))`` rather than the full tensor. + +The ``z`` path (``z_input_norm`` → ``z_proj`` → ``z_transitions``) is entirely +pair-channel-wise (``TransitionLayer`` is LayerNorm + SwiGLU over the last dim), +so it runs **as-is on the local pair shard** using the serial submodule's +identically-replicated params. The ``s`` path (token single repr, cheap 1-D) runs +**replicated/full** exactly as serial. + +Caller owns padding: ``z_trunk`` / ``rel_pos`` must already be padded so the token +axis is a multiple of the CP shard factor. Inference-only. +""" + +import torch +import torch.nn as nn +from torch.distributed.tensor import DTensor, Shard + +from esm.models.esmfold2.layers import ( + DiffusionConditioning as SerialDiffusionConditioning, +) + +_PAIR_PL = [Shard(0), Shard(1), Shard(2)] + + +class DiffusionConditioningDistributed(nn.Module): + """Distributed ``DiffusionConditioning``: full ``s``, 2-D-sharded ``z``. + + Thin shell over the serial module — reuses its submodules, running the ``z`` + path on the local pair shard and the ``s`` path on the full tensor. Bit-exact + with serial (the ``z`` path is pointwise in the pair indices). + """ + + def __init__(self, layer: SerialDiffusionConditioning, device_mesh) -> None: + super().__init__() + if not isinstance(layer, SerialDiffusionConditioning): + raise TypeError( + f"layer must be DiffusionConditioning, got {type(layer).__name__}" + ) + self.layer = layer + self.device_mesh = device_mesh + + def forward( + self, + t_hat: torch.Tensor, + s_inputs: torch.Tensor, + z_trunk_dt: DTensor, + rel_pos_dt: DTensor, + sigma_data: float | None = None, + num_diffusion_samples: int = 1, + inference_cache: dict | None = None, + ) -> tuple[torch.Tensor, DTensor]: + """Mirrors ``DiffusionConditioning.forward`` (z cached across rollout). + + ``z_trunk_dt`` / ``rel_pos_dt`` are 2-D-sharded DTensors (already padded); + returns ``(s_full, z_sharded_dt)``. + """ + layer = self.layer + mesh = self.device_mesh + sigma = layer.sigma_data if sigma_data is None else float(sigma_data) + # base (unexpanded) batch from s_inputs — z_trunk_dt is None on the cached + # path, and s_inputs always carries the base batch. + base_batch = s_inputs.shape[0] + target_batch = base_batch * num_diffusion_samples + + # --- z path (cached), on the local pair shard --- + if inference_cache is not None and "z_cp" in inference_cache: + z_dt = inference_cache["z_cp"] + else: + zt_local = z_trunk_dt.to_local().to(torch.float32) + zr_local = rel_pos_dt.to_local().to(torch.float32) + z_local = torch.cat([zt_local, zr_local], dim=-1) + z_local = layer.z_proj(layer.z_input_norm(z_local)) + with torch.autocast(device_type="cuda", dtype=torch.bfloat16): + for block in layer.z_transitions: + z_local = z_local + block(z_local) + z_dt = DTensor.from_local( + z_local.contiguous(), device_mesh=mesh, placements=_PAIR_PL + ) + if inference_cache is not None: + inference_cache["z_cp"] = z_dt + + # --- s path (full / replicated), identical to serial --- + s_inputs_eff = s_inputs + if s_inputs_eff.shape[0] != target_batch: + s_inputs_eff = s_inputs_eff.repeat_interleave(num_diffusion_samples, 0) + s = layer.s_proj(layer.s_input_norm(s_inputs_eff.to(torch.float32))) + + t = torch.as_tensor(t_hat, dtype=torch.float32, device=s.device).reshape(-1) + if t.numel() == 1: + t = t.expand(target_batch) + elif t.shape[0] != target_batch: + t = t.repeat_interleave(num_diffusion_samples, 0) + t_noise = 0.25 * torch.log((t / sigma).clamp(min=1e-20)) + n = layer.fourier(t_noise) + n = layer.noise_proj(layer.noise_norm(n)) + s = s + n.unsqueeze(1) + for block in layer.s_transitions: + s = s + block(s) + + return s, z_dt diff --git a/esm/models/esmfold2/distributed/model/layers/diffusion_transformer.py b/esm/models/esmfold2/distributed/model/layers/diffusion_transformer.py new file mode 100644 index 00000000..ee5ca214 --- /dev/null +++ b/esm/models/esmfold2/distributed/model/layers/diffusion_transformer.py @@ -0,0 +1,184 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: MIT +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. + +"""Distributed diffusion token transformer for ESMFold2. + +Distributes ``DiffusionTransformer`` (``AttentionPairBias`` + +``ConditionedTransitionBlock`` per block) by sharding the token row axis on +``cp_axis_0`` (the diffusion memory ceiling is the L×L token attention). + +Design: every per-token operation — AdaLN, the q/k/v/gate/out projections, the +output gate, and the whole transition block — is *channel-wise* on the +row-sharded token repr, so it runs **as-is on the local shard** using the serial +submodule's (identically replicated) parameters. Only the attention score/context +(``q·k`` over the full key axis + the 2-D-sharded pair bias) needs cross-rank +communication, handled by :func:`attention_pair_bias_gather`. This keeps the code +a thin shell over the serial layer and bit-exact with it (modulo the gather, +which is lossless). + +Token reprs (``a``, ``s``) are DTensors ``(B, L, d)`` with placements +``(Shard(0), Shard(1), Replicate())`` (rows on ``cp_axis_0``); the pair ``z`` is +``(B, L, L, d_pair)`` with ``(Shard(0), Shard(1), Shard(2))``. Inference-only, +single diffusion sample (``num_diffusion_samples`` handled by the caller/sampler). +""" + +import torch +import torch.nn as nn +from torch.distributed.tensor import DTensor, Replicate, Shard + +from esm.models.esmfold2.distributed.model.layers.attention_pair_bias import ( + attention_pair_bias_gather, +) +from esm.models.esmfold2.layers import AttentionPairBias as SerialAttentionPairBias +from esm.models.esmfold2.layers import ( + ConditionedTransitionBlock as SerialConditionedTransitionBlock, +) +from esm.models.esmfold2.layers import ( + DiffusionTransformer as SerialDiffusionTransformer, +) + +_ROW_PL = [Shard(0), Shard(1), Replicate()] +_PAIR_PL = [Shard(0), Shard(1), Shard(2)] + + +def _row_dtensor(local: torch.Tensor, mesh) -> DTensor: + return DTensor.from_local(local.contiguous(), device_mesh=mesh, placements=_ROW_PL) + + +class AttentionPairBiasDistributed(nn.Module): + """Distributed ``AttentionPairBias`` (standard, non-fused path). + + Holds the serial layer; runs its per-token ops on local shards and routes the + attention through :func:`attention_pair_bias_gather`. + """ + + def __init__(self, layer: SerialAttentionPairBias) -> None: + super().__init__() + if not isinstance(layer, SerialAttentionPairBias): + raise TypeError( + f"layer must be AttentionPairBias, got {type(layer).__name__}" + ) + self.layer = layer + self.num_heads = layer.num_heads + self.head_dim = layer.head_dim + self.scale = layer.scale + self.use_conditioning = hasattr(layer, "adaln") + self.has_pair = hasattr(layer, "pair_bias_proj") + + def forward( + self, + a_dt: DTensor, + s_dt: DTensor | None, + z_dt: DTensor, + key_mask_dt: DTensor | None = None, + ) -> DTensor: + mesh = a_dt.device_mesh + layer = self.layer + h, hd = self.num_heads, self.head_dim + + a_local = a_dt.to_local() + b, l_loc = a_local.shape[0], a_local.shape[1] + + # --- per-token ops on the local shard (channel-wise) --- + if s_dt is not None and self.use_conditioning: + s_local = s_dt.to_local() + x_local = layer.adaln(a_local, s_local) + else: + s_local = None + x_local = layer.pre_norm(a_local) + + q_local = layer.q_proj(x_local).view(b, l_loc, h, hd) + k_local, v_local = layer.kv_proj(x_local).chunk(2, dim=-1) + k_local = k_local.reshape(b, l_loc, h, hd) + v_local = v_local.reshape(b, l_loc, h, hd) + g_local = torch.sigmoid(layer.g_proj(x_local)).view(b, l_loc, h, hd) + + # --- pair bias on the local (2-D-sharded) pair shard --- + z_local = z_dt.to_local() # (B, rows_loc, cols_loc, d_pair) + bias_local = layer.pair_bias_proj(layer.pair_norm(z_local)) # (.., H) + bias_dt = DTensor.from_local( + bias_local.contiguous(), device_mesh=mesh, placements=_PAIR_PL + ) + + # --- distributed attention (gather strategy) --- + q_dt = _row_dtensor(q_local, mesh) + k_dt = _row_dtensor(k_local, mesh) + v_dt = _row_dtensor(v_local, mesh) + o_dt = attention_pair_bias_gather( + q_dt, k_dt, v_dt, bias_dt, self.scale, key_mask_dt=key_mask_dt + ) + + # --- gate + output projection on the local shard --- + ctx_local = (g_local * o_dt.to_local()).reshape(b, l_loc, h * hd) + out_local = layer.out_proj(ctx_local) + if s_local is not None and self.use_conditioning: + out_local = torch.sigmoid(layer.out_gate(s_local)) * out_local + return _row_dtensor(out_local, mesh) + + +class ConditionedTransitionDistributed(nn.Module): + """Distributed ``ConditionedTransitionBlock`` — purely per-token, so just the + serial block run on the local shard.""" + + def __init__(self, layer: SerialConditionedTransitionBlock) -> None: + super().__init__() + if not isinstance(layer, SerialConditionedTransitionBlock): + raise TypeError( + f"layer must be ConditionedTransitionBlock, got {type(layer).__name__}" + ) + self.layer = layer + + def forward(self, a_dt: DTensor, s_dt: DTensor | None) -> DTensor: + mesh = a_dt.device_mesh + a_local = a_dt.to_local() + s_local = s_dt.to_local() if s_dt is not None else None + return _row_dtensor(self.layer(a_local, s_local), mesh) + + +class DiffusionTransformerDistributed(nn.Module): + """Distributed ``DiffusionTransformer``: per block ``x = x + attn(x,s,z); x = + x + transition(x,s)``, matching the serial loop.""" + + def __init__(self, transformer: SerialDiffusionTransformer) -> None: + super().__init__() + if not isinstance(transformer, SerialDiffusionTransformer): + raise TypeError( + f"transformer must be DiffusionTransformer, got {type(transformer).__name__}" + ) + self.attn_blocks = nn.ModuleList( + [AttentionPairBiasDistributed(b) for b in transformer.attn_blocks] # ty:ignore[invalid-argument-type] + ) + self.transition_blocks = nn.ModuleList( + [ConditionedTransitionDistributed(b) for b in transformer.transition_blocks] # ty:ignore[invalid-argument-type] + ) + + def forward( + self, + a_dt: DTensor, + s_dt: DTensor | None, + z_dt: DTensor, + key_mask_dt: DTensor | None = None, + ) -> DTensor: + x = a_dt + for attn, transition in zip(self.attn_blocks, self.transition_blocks): + x = x + attn(x, s_dt, z_dt, key_mask_dt=key_mask_dt) + x = x + transition(x, s_dt) + return x diff --git a/esm/models/esmfold2/distributed/model/layers/layernorm.py b/esm/models/esmfold2/distributed/model/layers/layernorm.py new file mode 100644 index 00000000..659e10d0 --- /dev/null +++ b/esm/models/esmfold2/distributed/model/layers/layernorm.py @@ -0,0 +1,234 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: MIT +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. + +"""Distributed LayerNorm with parameters replicated across the device mesh.""" + +from typing import Optional, Union + +import torch +import torch.distributed as dist +import torch.nn as nn +import torch.nn.functional as F +from torch.distributed.tensor import DeviceMesh, DTensor, Replicate, distribute_tensor + +_shape_t = Union[int, list[int], torch.Size] + + +class _LayerNormParamsReplicatedImpl(torch.autograd.Function): + """LayerNorm with replicated parameters and arbitrary DTensor input placement.""" + + @staticmethod + def forward( + ctx, + x: DTensor, + normalized_shape: list[int], + weight: Optional[DTensor], + bias: Optional[DTensor], + eps: float, + reduce_group: dist.ProcessGroup, + ) -> DTensor: + if not isinstance(x, DTensor): + raise TypeError(f"x must be DTensor, got {type(x)}") + + x_local = x.to_local() + weight_local = weight.to_local() if weight is not None else None + bias_local = bias.to_local() if bias is not None else None + + ctx.reduce_group = reduce_group + ctx.eps = eps + ctx.normalized_shape = normalized_shape + ctx.x_shape = x.shape + ctx.x_stride = x.stride() + ctx.x_placements = x.placements + ctx.device_mesh = x.device_mesh + ctx.save_for_backward(x_local, weight_local) + + out_local = F.layer_norm( + x_local, normalized_shape, weight_local, bias_local, eps + ) + return DTensor.from_local( + out_local, + device_mesh=x.device_mesh, + placements=x.placements, + shape=x.shape, + stride=x.stride(), + ) + + @staticmethod + def backward(ctx, d_out: DTensor): + if not isinstance(d_out, DTensor): + raise TypeError(f"d_out must be DTensor, got {type(d_out)}") + + x_saved, weight_saved = ctx.saved_tensors + d_out_local = d_out.to_local() + eps = ctx.eps + normalized_shape = ctx.normalized_shape + + dx_dtensor: Optional[DTensor] = None + dw_dtensor: Optional[DTensor] = None + db_dtensor: Optional[DTensor] = None + + if ctx.needs_input_grad[0] or ctx.needs_input_grad[2]: + dims = tuple(-(i + 1) for i in range(len(normalized_shape))) + mean = x_saved.mean(dim=dims, keepdim=True) + var = x_saved.var(dim=dims, unbiased=False, keepdim=True) + x_norm = (x_saved - mean) / torch.sqrt(var + eps) + + if ctx.needs_input_grad[0]: + if weight_saved is not None: + dy = d_out_local * weight_saved.view( + *([1] * (d_out_local.ndim - len(normalized_shape))), + *weight_saved.shape, + ) + else: + dy = d_out_local + dims = tuple(-(i + 1) for i in range(len(normalized_shape))) + dy_mean = dy.mean(dim=dims, keepdim=True) + dy_x_norm_mean = (dy * x_norm).mean(dim=dims, keepdim=True) + dx_local = (dy - dy_mean - x_norm * dy_x_norm_mean) / torch.sqrt(var + eps) + dx_dtensor = DTensor.from_local( + dx_local, + device_mesh=ctx.device_mesh, + placements=ctx.x_placements, + shape=ctx.x_shape, + stride=ctx.x_stride, + ) + + if ctx.needs_input_grad[2]: + reduce_dims = list(range(d_out_local.ndim - len(normalized_shape))) + dw = (d_out_local * x_norm).sum(dim=reduce_dims).contiguous() + dw_work = dist.all_reduce( + dw, op=dist.ReduceOp.SUM, group=ctx.reduce_group, async_op=True + ) + + if ctx.needs_input_grad[3]: + reduce_dims = list(range(d_out_local.ndim - len(normalized_shape))) + db = d_out_local.sum(dim=reduce_dims).contiguous() + db_work = dist.all_reduce( + db, op=dist.ReduceOp.SUM, group=ctx.reduce_group, async_op=True + ) + + replicate = [Replicate()] * ctx.device_mesh.ndim + if ctx.needs_input_grad[2]: + dw_work.wait() + dw_dtensor = DTensor.from_local( + dw, + device_mesh=ctx.device_mesh, + placements=replicate, + shape=dw.shape, + stride=dw.stride(), + ) + if ctx.needs_input_grad[3]: + db_work.wait() + db_dtensor = DTensor.from_local( + db, + device_mesh=ctx.device_mesh, + placements=replicate, + shape=db.shape, + stride=db.stride(), + ) + + return dx_dtensor, None, dw_dtensor, db_dtensor, None, None + + +class LayerNormParamsReplicated(nn.Module): + """nn.LayerNorm wrapper with parameters replicated over the device mesh. + + Accepts DTensor inputs with arbitrary placements and outputs DTensors + with the same placements. + + Parameters + ---------- + layer_local: + The serial nn.LayerNorm layer whose parameters to replicate. + device_mesh: + The device mesh for distributing tensors. + """ + + def __init__(self, layer_local: nn.LayerNorm, device_mesh: DeviceMesh) -> None: + super().__init__() + if not isinstance(layer_local, nn.LayerNorm): + raise TypeError( + f"layer_local must be nn.LayerNorm, got {type(layer_local)}" + ) + + self.device_mesh = device_mesh + self.normalized_shape = list(layer_local.normalized_shape) + self.eps = layer_local.eps + replicate_placements = [Replicate()] * device_mesh.ndim + + if layer_local.weight is not None: + self.weight = nn.Parameter( + distribute_tensor(layer_local.weight, device_mesh, replicate_placements) + ) + else: + self.weight = None + + if layer_local.bias is not None: + self.bias = nn.Parameter( + distribute_tensor(layer_local.bias, device_mesh, replicate_placements) + ) + else: + self.bias = None + + if "cp" in device_mesh.mesh_dim_names: # ty:ignore[unsupported-operator] + self._reduce_group = device_mesh.get_group("cp") + else: + self._reduce_group = dist.group.WORLD + + # Local-tensor cache for the inference fast path (see LinearParamsReplicated). + self._w_local: Optional[torch.Tensor] = None + self._b_local: Optional[torch.Tensor] = None + + def _inference_locals(self): + w = self.weight + cur_dtype = None if w is None else w.dtype + if (w is not None and self._w_local is None) or ( + self._w_local is not None and self._w_local.dtype != cur_dtype + ): + self._w_local = w.to_local() # ty:ignore[unresolved-attribute] + self._b_local = self.bias.to_local() if self.bias is not None else None # ty:ignore[unresolved-attribute] + return self._w_local, self._b_local + + def forward(self, x: DTensor) -> DTensor: + if torch.is_grad_enabled(): + return _LayerNormParamsReplicatedImpl.apply( + x, + self.normalized_shape, + self.weight, + self.bias, + self.eps, + self._reduce_group, + ) + # Inference fast path: skip the autograd.Function machinery + ctx stores, + # reuse cached replicated param locals. Bit-identical to the Function's + # forward (same to_local → F.layer_norm → from_local). + w_local, b_local = self._inference_locals() + out_local = F.layer_norm( + x.to_local(), self.normalized_shape, w_local, b_local, self.eps + ) + return DTensor.from_local( + out_local, + device_mesh=x.device_mesh, + placements=x.placements, + shape=x.shape, + stride=x.stride(), + ) diff --git a/esm/models/esmfold2/distributed/model/layers/linear.py b/esm/models/esmfold2/distributed/model/layers/linear.py new file mode 100644 index 00000000..d7bc0d3c --- /dev/null +++ b/esm/models/esmfold2/distributed/model/layers/linear.py @@ -0,0 +1,226 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: MIT +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. + +"""Distributed linear layer with parameters replicated across the device mesh.""" + +from typing import Optional + +import torch +import torch.distributed as dist +import torch.nn as nn +import torch.nn.functional as F +from torch import Tensor +from torch.distributed.device_mesh import DeviceMesh +from torch.distributed.tensor import DTensor, Replicate, distribute_tensor + +from esm.models.esmfold2.distributed.utils import update_exhaustive_strides + + +class _LinearParamsReplicatedImpl(torch.autograd.Function): + """Linear layer with replicated parameters and arbitrary input placements. + + Parameters are replicated on all mesh dimensions; inputs can be sharded + (e.g. Shard(0), Shard(1), Shard(2)) or replicated. Backward uses + all-reduce over the cp group to accumulate parameter gradients. + """ + + @staticmethod + def forward( + ctx, + x: DTensor, + weight: DTensor, + bias: Optional[DTensor], + reduce_group: dist.ProcessGroup, + avg_reduce: bool, + ) -> DTensor: + if not isinstance(x, DTensor): + raise TypeError(f"x must be DTensor, got {type(x)}") + if not isinstance(weight, DTensor): + raise TypeError(f"weight must be DTensor, got {type(weight)}") + + ctx.reduce_group = reduce_group + ctx.avg_reduce = avg_reduce + + x_local = x.to_local() + weight_local = weight.to_local() + bias_local = bias.to_local() if bias is not None else None + + ctx.save_for_backward(x_local, weight_local) + ctx.x_shape = x.shape + ctx.x_stride = x.stride() + ctx.x_placements = x.placements + ctx.device_mesh = x.device_mesh + + out_local = F.linear(x_local, weight_local, bias_local) + + shape_output = x.shape[:-1] + (weight.shape[0],) + stride_output = update_exhaustive_strides(x.shape, x.stride(), shape_output) + return DTensor.from_local( + out_local, + device_mesh=x.device_mesh, + placements=x.placements, + shape=shape_output, + stride=stride_output, + ) + + @staticmethod + def backward(ctx, d_out: DTensor): + if not isinstance(d_out, DTensor): + raise TypeError(f"d_out must be DTensor, got {type(d_out)}") + + x_saved, weight_saved = ctx.saved_tensors + d_out_local = d_out.to_local() + + dw: Optional[Tensor] = None + dw_work = None + if ctx.needs_input_grad[1]: + # Aggregate over all but the last two dims (batch + seq dims) + dw = torch.einsum("...i,...j->ij", d_out_local, x_saved) + dw = dw.contiguous() + op = dist.ReduceOp.AVG if ctx.avg_reduce else dist.ReduceOp.SUM + dw_work = dist.all_reduce(dw, op=op, group=ctx.reduce_group, async_op=True) + + db: Optional[Tensor] = None + db_work = None + if ctx.needs_input_grad[2]: + reduce_dims = list(range(d_out_local.ndim - 1)) + db = d_out_local.sum(dim=reduce_dims).contiguous() + op = dist.ReduceOp.AVG if ctx.avg_reduce else dist.ReduceOp.SUM + db_work = dist.all_reduce(db, op=op, group=ctx.reduce_group, async_op=True) + + dx_dtensor: Optional[DTensor] = None + if ctx.needs_input_grad[0]: + dx_local = F.linear(d_out_local, weight_saved.t()) + shape_dx = ctx.x_shape + stride_dx = ctx.x_stride + dx_dtensor = DTensor.from_local( + dx_local, + device_mesh=ctx.device_mesh, + placements=ctx.x_placements, + shape=shape_dx, + stride=stride_dx, + ) + + # Wrap parameter gradients as DTensors with Replicate placement + replicate = [Replicate()] * ctx.device_mesh.ndim + dw_dtensor: Optional[DTensor] = None + if dw_work is not None: + dw_work.wait() + dw_dtensor = DTensor.from_local( + dw, # ty:ignore[invalid-argument-type] + device_mesh=ctx.device_mesh, + placements=replicate, + shape=dw.shape, # ty:ignore[unresolved-attribute] + stride=dw.stride(), # ty:ignore[unresolved-attribute] + ) + + db_dtensor: Optional[DTensor] = None + if db_work is not None: + db_work.wait() + db_dtensor = DTensor.from_local( + db, # ty:ignore[invalid-argument-type] + device_mesh=ctx.device_mesh, + placements=replicate, + shape=db.shape, # ty:ignore[unresolved-attribute] + stride=db.stride(), # ty:ignore[unresolved-attribute] + ) + + return dx_dtensor, dw_dtensor, db_dtensor, None, None + + +class LinearParamsReplicated(nn.Module): + """nn.Linear wrapper with parameters replicated over the device mesh. + + Accepts DTensor inputs with arbitrary placements and outputs DTensors + with the same placements. Parameter gradients are all-reduced across + the CP group so every rank accumulates the full gradient. + + Parameters + ---------- + layer_local: + The serial nn.Linear layer whose parameters to replicate. + device_mesh: + The device mesh for distributing tensors. + avg_reduce: + If True, use AVG instead of SUM for all-reduce (useful when the + effective batch is already averaged). + """ + + def __init__( + self, layer_local: nn.Linear, device_mesh: DeviceMesh, avg_reduce: bool = False + ) -> None: + super().__init__() + if not isinstance(layer_local, nn.Linear): + raise TypeError(f"layer_local must be nn.Linear, got {type(layer_local)}") + + self.device_mesh = device_mesh + self.avg_reduce = avg_reduce + replicate_placements = [Replicate()] * device_mesh.ndim + + self.weight = nn.Parameter( + distribute_tensor(layer_local.weight, device_mesh, replicate_placements) + ) + if layer_local.bias is not None: + self.bias = nn.Parameter( + distribute_tensor(layer_local.bias, device_mesh, replicate_placements) + ) + else: + self.bias = None + + # Choose reduce group: use cp group if present, otherwise world + if "cp" in device_mesh.mesh_dim_names: # ty:ignore[unsupported-operator] + self._reduce_group = device_mesh.get_group("cp") + else: + self._reduce_group = dist.group.WORLD + + # Cache of the replicated params' local tensors for the inference fast + # path (populated lazily on first inference forward, after any post- + # construction dtype cast such as TrunkCPWrapper's .to(bf16)). + self._w_local: Optional[Tensor] = None + self._b_local: Optional[Tensor] = None + + def _inference_locals(self) -> tuple[Tensor, Optional[Tensor]]: + # Refresh if dtype changes (covers a later .to(bf16)); weights are static + # within an inference run so this caches after the first call. + if self._w_local is None or self._w_local.dtype != self.weight.dtype: + self._w_local = self.weight.to_local() # ty:ignore[unresolved-attribute] + self._b_local = self.bias.to_local() if self.bias is not None else None # ty:ignore[unresolved-attribute] + return self._w_local, self._b_local + + def forward(self, x: DTensor) -> DTensor: + if torch.is_grad_enabled(): + return _LinearParamsReplicatedImpl.apply( + x, self.weight, self.bias, self._reduce_group, self.avg_reduce + ) + # Inference fast path: skip the autograd.Function.apply machinery + ctx + # bookkeeping and reuse cached replicated weight locals. Bit-identical to + # the Function's forward (same to_local → F.linear → from_local). + w_local, b_local = self._inference_locals() + out_local = F.linear(x.to_local(), w_local, b_local) + shape_output = x.shape[:-1] + (self.weight.shape[0],) + stride_output = update_exhaustive_strides(x.shape, x.stride(), shape_output) + return DTensor.from_local( + out_local, + device_mesh=x.device_mesh, + placements=x.placements, + shape=shape_output, + stride=stride_output, + ) diff --git a/esm/models/esmfold2/distributed/model/layers/msa_encoder.py b/esm/models/esmfold2/distributed/model/layers/msa_encoder.py new file mode 100644 index 00000000..90f98ebc --- /dev/null +++ b/esm/models/esmfold2/distributed/model/layers/msa_encoder.py @@ -0,0 +1,198 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: MIT +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. + +"""Distributed MSAEncoder block / stack for ESMFold2 (inference-only). + +Mirrors the serial ``MSAEncoderBlock`` (modeling_esmfold2.py) exactly: + + pair = pair + outer_product_mean(m, msa_mask) + if not final: + m = m + msa_pair_weighted_averaging(m, pair, pair_mask) + m = m + msa_transition(m) + pair = pair + tri_mul_out(pair, mask=pair_mask) + pair = pair + tri_mul_in(pair, mask=pair_mask) + pair = pair + pair_transition(pair) + +All residuals are explicit (added here). The serial block uses +``modeling_esmfold2.PairTransition`` for both ``msa_transition`` and +``pair_transition`` — that class returns ``ffn(norm(x))`` WITHOUT a residual, +so we use the bare ``MSATransitionDistributed`` (not ``TransitionDistributed``, +which folds the residual in) and add the residual in the block. +""" + +from typing import Optional + +import torch.nn as nn +import torch.nn.functional as F +from torch.distributed.device_mesh import DeviceMesh +from torch.distributed.tensor import DTensor + +from esm.models.esmfold2.distributed.comm import Ring2DComm +from esm.models.esmfold2.distributed.manager import DistributedManager +from esm.models.esmfold2.distributed.model.layers.layernorm import ( + LayerNormParamsReplicated, +) +from esm.models.esmfold2.distributed.model.layers.linear import LinearParamsReplicated +from esm.models.esmfold2.distributed.model.layers.outer_product_mean import ( + OuterProductMeanDistributed, +) +from esm.models.esmfold2.distributed.model.layers.pair_averaging import ( + MSAPairWeightedAveragingDistributed, +) +from esm.models.esmfold2.distributed.model.layers.triangular_mult import ( + TriangleMultiplicativeBlockDistributed, +) +from esm.models.esmfold2.model import MSAEncoder as SerialMSAEncoder +from esm.models.esmfold2.model import MSAEncoderBlock as SerialMSAEncoderBlock +from esm.models.esmfold2.model import PairTransition as SerialPairTransition + + +class MSATransitionDistributed(nn.Module): + """Bare LayerNorm + SwiGLU FFN (no residual) on a sharded representation. + + Matches ``modeling_esmfold2.PairTransition.forward`` which returns + ``ffn(norm(x))``; the residual is added by the calling block. + """ + + def __init__(self, layer: SerialPairTransition, device_mesh: DeviceMesh) -> None: + super().__init__() + if not isinstance(layer, SerialPairTransition): + raise TypeError(f"layer must be PairTransition, got {type(layer).__name__}") + self.norm = LayerNormParamsReplicated(layer.norm, device_mesh) + self.w12 = LinearParamsReplicated(layer.ffn.w12, device_mesh) + self.w3 = LinearParamsReplicated(layer.ffn.w3, device_mesh) + self.hidden_features = layer.ffn.hidden_features + + def forward(self, x: DTensor) -> DTensor: + normed = self.norm(x) + x12 = self.w12(normed) + x1, x2 = x12.split(self.hidden_features, dim=-1) + hidden = F.silu(x1) * x2 + return self.w3(hidden) + + +class MSAEncoderBlockDistributed(nn.Module): + """Distributed MSAEncoderBlock. + + Parameters + ---------- + layer: + Serial MSAEncoderBlock to distribute. + dist_manager: + DistributedManager with the CP group and subgroups set up. + """ + + def __init__( + self, + layer: SerialMSAEncoderBlock, + dist_manager: DistributedManager, + comm: str = "gather", + ) -> None: + super().__init__() + if not isinstance(layer, SerialMSAEncoderBlock): + raise TypeError( + f"layer must be MSAEncoderBlock, got {type(layer).__name__}" + ) + mesh = dist_manager.device_mesh_subgroups + self.is_final_block = layer.is_final_block + + self.outer_product_mean = OuterProductMeanDistributed( + layer.outer_product_mean, dist_manager + ) + if not self.is_final_block: + self.msa_pair_weighted_averaging = MSAPairWeightedAveragingDistributed( + layer.msa_pair_weighted_averaging, dist_manager, comm=comm + ) + self.msa_transition = MSATransitionDistributed(layer.msa_transition, mesh) + + ring_comm_out = Ring2DComm( + dist_manager.group["cp"], + dist_manager.subgroups["cp"][0], + dist_manager.layout_subgroups["cp"], + ) + ring_comm_in = Ring2DComm( + dist_manager.group["cp"], + dist_manager.subgroups["cp"][0], + dist_manager.layout_subgroups["cp"], + ) + self.tri_mul_out = TriangleMultiplicativeBlockDistributed( + layer.tri_mul_out._engine, mesh, ring_comm_out + ) + self.tri_mul_in = TriangleMultiplicativeBlockDistributed( + layer.tri_mul_in._engine, mesh, ring_comm_in + ) + # Serial pair_transition is a (residual-free) PairTransition. + self.pair_transition = MSATransitionDistributed(layer.pair_transition, mesh) + + def forward( + self, + m: DTensor, + pair: DTensor, + msa_attention_mask: DTensor, + pair_attention_mask: Optional[DTensor] = None, + ) -> tuple[DTensor, DTensor]: + pair = pair + self.outer_product_mean(m, msa_attention_mask) + if not self.is_final_block: + m = m + self.msa_pair_weighted_averaging(m, pair, pair_attention_mask) + m = m + self.msa_transition(m) + pair = pair + self.tri_mul_out(pair, mask=pair_attention_mask) + pair = pair + self.tri_mul_in(pair, mask=pair_attention_mask) + pair = pair + self.pair_transition(pair) + return m, pair + + +class MSAEncoderDistributed(nn.Module): + """Distributed MSAEncoder: ModuleList of MSAEncoderBlockDistributed. + + Parameters + ---------- + encoder: + Serial MSAEncoder module. + dist_manager: + DistributedManager with the CP group and subgroups set up. + """ + + def __init__( + self, + encoder: SerialMSAEncoder, + dist_manager: DistributedManager, + comm: str = "gather", + ) -> None: + super().__init__() + if not isinstance(encoder, SerialMSAEncoder): + raise TypeError(f"encoder must be MSAEncoder, got {type(encoder).__name__}") + self.blocks = nn.ModuleList( + [ + MSAEncoderBlockDistributed(block, dist_manager, comm=comm) # ty:ignore[invalid-argument-type] + for block in encoder.blocks # type: ignore[arg-type] + ] + ) + + def forward( + self, + m: DTensor, + pair: DTensor, + msa_attention_mask: DTensor, + pair_attention_mask: DTensor, + ) -> DTensor: + for block in self.blocks: + m, pair = block(m, pair, msa_attention_mask, pair_attention_mask) + return pair diff --git a/esm/models/esmfold2/distributed/model/layers/outer_product_mean.py b/esm/models/esmfold2/distributed/model/layers/outer_product_mean.py new file mode 100644 index 00000000..707b6a85 --- /dev/null +++ b/esm/models/esmfold2/distributed/model/layers/outer_product_mean.py @@ -0,0 +1,142 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: MIT +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. + +"""Distributed OuterProductMean for ESMFold2's MSA encoder (inference-only). + +The serial op maps an MSA representation m (B, L, M, d_msa) into a pair update +(B, L, L, d_pair): + + outer[b,i,j] = sum_m a[b,i,m] (x) b[b,j,m] # einsum "bimc,bjmd->bijcd" + z = Wout(outer) / n_valid # divide_outer_before_proj=False + +Under 2D context parallelism the pair z is sharded ``(Shard(0), Shard(1), +Shard(2))`` — row token i on cp_axis_0, col token j on cp_axis_1. The MSA m is +sharded ``(Shard(0), Shard(1), Replicate())`` — token L on cp_axis_0, MSA depth +M replicated on cp_axis_1. + +Because M is replicated, the contraction over m is fully local. The tile owned +by rank (p, q) needs row block p (held locally) and col block q. Col block q is +exactly the row block held by the transpose peer (q, p), so a single transpose +of the b operand (and the mask) suffices — no ring rotation is required (unlike +boltz-cp, which shards the contracted dimension and therefore must ring-reduce). +""" + +import torch +import torch.nn as nn +from torch.distributed.tensor import DTensor, Shard + +from esm.models.esmfold2.distributed.comm import TransposeComm +from esm.models.esmfold2.distributed.model.layers.layernorm import ( + LayerNormParamsReplicated, +) +from esm.models.esmfold2.distributed.model.layers.linear import LinearParamsReplicated +from esm.models.esmfold2.layers import OuterProductMean as SerialOuterProductMean + + +def _contiguous_strides(shape: tuple[int, ...]) -> tuple[int, ...]: + strides = [1] * len(shape) + for i in range(len(shape) - 2, -1, -1): + strides[i] = strides[i + 1] * shape[i + 1] + return tuple(strides) + + +class OuterProductMeanDistributed(nn.Module): + """Distributed (transpose-based) OuterProductMean. + + Parameters + ---------- + layer: + The serial OuterProductMean to distribute. + dist_manager: + DistributedManager with the CP group / subgroups set up. + """ + + def __init__(self, layer: SerialOuterProductMean, dist_manager) -> None: + super().__init__() + if not isinstance(layer, SerialOuterProductMean): + raise TypeError( + f"layer must be OuterProductMean, got {type(layer).__name__}" + ) + self.device_mesh = dist_manager.device_mesh_subgroups + self.d_hidden = layer.d_hidden + self.divide_outer_before_proj = layer.divide_outer_before_proj + + self.norm = LayerNormParamsReplicated(layer.norm, self.device_mesh) + self.W = LinearParamsReplicated(layer.W, self.device_mesh) + self.Wout = LinearParamsReplicated(layer.Wout, self.device_mesh) + + # Transpose (i,j) <-> (j,i) to fetch the column-token block. + self.transpose = TransposeComm( + dist_manager.group["cp"], dist_manager.layout_subgroups["cp"] + ) + + def forward(self, m: DTensor, msa_attention_mask: DTensor) -> DTensor: + mesh = self.device_mesh + dh = self.d_hidden + L = m.shape[1] + + m_norm = self.norm(m) + x = self.W(m_norm) # (B, L, M, 2*d_hidden), placements (S0, S1, R) + x = x * msa_attention_mask.unsqueeze(-1).to(x.dtype) + + x_local = x.to_local() + a_local, b_local = torch.chunk(x_local, 2, dim=-1) + a_local = a_local.contiguous() + b_local = b_local.contiguous() + mask_local = msa_attention_mask.to_local().to(a_local.dtype) # (B, sL, M) + + # Fetch the column-token block of b and the mask via a single transpose. + packed = torch.cat([b_local, mask_local.unsqueeze(-1)], dim=-1).contiguous() + recv = self.transpose.enqueue_to_dispatch(packed) + self.transpose.wait_until_finished() + b_q = recv[..., :dh].contiguous() + mask_q = recv[..., dh].contiguous() # (B, sL, M) + + outer = torch.einsum("bimc,bjmd->bijcd", a_local, b_q).flatten(-2).contiguous() + n_valid_local = ( + torch.einsum("bim,bjm->bij", mask_local, mask_q) + .unsqueeze(-1) + .clamp(min=1.0) + .contiguous() + ) + + b_size = outer.shape[0] + c_out = outer.shape[-1] + z_shape = torch.Size((b_size, L, L, c_out)) + z_dt = DTensor.from_local( + outer, + device_mesh=mesh, + placements=[Shard(0), Shard(1), Shard(2)], + shape=z_shape, + stride=_contiguous_strides(tuple(z_shape)), + ) + nv_shape = torch.Size((b_size, L, L, 1)) + n_valid_dt = DTensor.from_local( + n_valid_local, + device_mesh=mesh, + placements=[Shard(0), Shard(1), Shard(2)], + shape=nv_shape, + stride=_contiguous_strides(tuple(nv_shape)), + ) + + if self.divide_outer_before_proj: + return self.Wout(z_dt / n_valid_dt) + return self.Wout(z_dt) / n_valid_dt diff --git a/esm/models/esmfold2/distributed/model/layers/pair_averaging.py b/esm/models/esmfold2/distributed/model/layers/pair_averaging.py new file mode 100644 index 00000000..c6efa631 --- /dev/null +++ b/esm/models/esmfold2/distributed/model/layers/pair_averaging.py @@ -0,0 +1,265 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: MIT +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. + +"""Distributed MSAPairWeightedAveraging for ESMFold2's MSA encoder (inference). + +The serial op (AF3 Algorithm 10) updates the MSA representation m using the +pair representation as attention bias: + + bias = compute_bias(pair) # (B, L, L, h) + attn = softmax_j(masked(bias)) # over col token j + out[b,i,m,h,d] = (sum_j attn[b,i,j,h] v[b,j,m,h,d]) * gate[b,i,m,h,d] + return Wout(out) + +Under 2D CP the pair (and bias) is sharded ``(Shard(0), Shard(1), Shard(2))`` +and m / v / gate are sharded ``(Shard(0), Shard(1), Replicate())`` (token L on +cp_axis_0, MSA depth M replicated). The softmax is over j = cp_axis_1, which is +sharded, and the value index is the (replicated) MSA depth with token on +cp_axis_0. + +Two communication strategies (selectable via ``comm``): + +* ``"gather"`` (default): ``DTensor.redistribute`` gathers the full j of the + bias/mask and the full token axis of v, then the attention is computed + locally over the full j in natural order — bit-exact with the serial op. The + gathered buffers are small (``L²·h`` and ``L·M·c``, far below the sharded + pair), so this is the better choice for typical grids / MSA depths. + +* ``"ring"`` (boltz-style): never materialises the full j. ``v`` is transposed + so block ``q`` aligns with ``bias[block p, block q]``, then both ring along + the column axis (``comm_row``) accumulating with an online softmax + (``tiled_softmax_attention_update``). Pays off only at larger grids (n >= 3) + with deep MSAs, where the gathered buffers would be large. The online-softmax + reassociation makes it close-but-not-bit-exact vs. the serial op. +""" + +import torch +import torch.nn as nn +from torch.distributed.tensor import DTensor, Replicate, Shard + +from esm.models.esmfold2.distributed.comm import Ring2DComm +from esm.models.esmfold2.distributed.model.layers.layernorm import ( + LayerNormParamsReplicated, +) +from esm.models.esmfold2.distributed.model.layers.linear import LinearParamsReplicated +from esm.models.esmfold2.distributed.utils import tiled_softmax_attention_update +from esm.models.esmfold2.layers import ( + MSAPairWeightedAveraging as SerialMSAPairWeightedAveraging, +) + + +def _contiguous_strides(shape: tuple[int, ...]) -> tuple[int, ...]: + strides = [1] * len(shape) + for i in range(len(shape) - 2, -1, -1): + strides[i] = strides[i + 1] * shape[i + 1] + return tuple(strides) + + +class MSAPairWeightedAveragingDistributed(nn.Module): + """Distributed MSAPairWeightedAveraging (inference-only). + + Parameters + ---------- + layer: + The serial MSAPairWeightedAveraging to distribute. + dist_manager: + DistributedManager with the CP group / subgroups set up. + comm: + ``"gather"`` (default, bit-exact all-gather) or ``"ring"`` (boltz-style + online-softmax ring). + """ + + def __init__( + self, layer: SerialMSAPairWeightedAveraging, dist_manager, comm: str = "gather" + ) -> None: + super().__init__() + if not isinstance(layer, SerialMSAPairWeightedAveraging): + raise TypeError( + f"layer must be MSAPairWeightedAveraging, got {type(layer).__name__}" + ) + if comm not in ("gather", "ring"): + raise ValueError(f"comm must be 'gather' or 'ring', got {comm!r}") + self.device_mesh = dist_manager.device_mesh_subgroups + self.comm_mode = comm + self.n_heads = layer.n_heads + self.head_width = layer.head_width + + self.norm_single = LayerNormParamsReplicated( + layer.norm_single, self.device_mesh + ) + # compute_bias is nn.Sequential(LayerNorm(d_pair), Linear(d_pair, n_heads)) + self.bias_norm = LayerNormParamsReplicated( + layer.compute_bias[0], # ty:ignore[invalid-argument-type] + self.device_mesh, + ) + self.bias_lin = LinearParamsReplicated(layer.compute_bias[1], self.device_mesh) # ty:ignore[invalid-argument-type] + self.Wv = LinearParamsReplicated(layer.Wv, self.device_mesh) + self.Wgate = LinearParamsReplicated(layer.Wgate, self.device_mesh) + self.Wout = LinearParamsReplicated(layer.Wout, self.device_mesh) + + if comm == "ring": + # ring_v owns the value transpose + value column-ring; ring_bias + # owns an independent bias column-ring (same schedule, separate + # comm handles so both can be in flight together). + self.ring_v = Ring2DComm( + dist_manager.group["cp"], + dist_manager.subgroups["cp"][0], + dist_manager.layout_subgroups["cp"], + ) + self.ring_bias = Ring2DComm( + dist_manager.group["cp"], + dist_manager.subgroups["cp"][0], + dist_manager.layout_subgroups["cp"], + ) + + def forward( + self, m: DTensor, pair: DTensor, pair_attention_mask: DTensor + ) -> DTensor: + if self.comm_mode == "ring": + return self._forward_ring(m, pair, pair_attention_mask) + return self._forward_gather(m, pair, pair_attention_mask) + + # -- shared projections ------------------------------------------------ + def _project(self, m: DTensor, pair: DTensor): + msa_normed = self.norm_single(m) + v_dt = self.Wv(msa_normed) # (B, L, M, h*dh), (S0, S1, R) + gate_dt = torch.sigmoid(self.Wgate(msa_normed)) # (B, L, M, h*dh) + bias_dt = self.bias_lin(self.bias_norm(pair)) # (B, L, L, h), (S0, S1, S2) + return v_dt, gate_dt, bias_dt + + def _finish(self, m: DTensor, o_local: torch.Tensor, gate_local: torch.Tensor): + h, dh = self.n_heads, self.head_width + b_size, s_l, m_depth = o_local.shape[0], o_local.shape[1], o_local.shape[2] + o_local = ( + (o_local * gate_local).reshape(b_size, s_l, m_depth, h * dh).contiguous() + ) + out_shape = torch.Size((b_size, m.shape[1], m_depth, h * dh)) + out_dt = DTensor.from_local( + o_local, + device_mesh=self.device_mesh, + placements=[Shard(0), Shard(1), Replicate()], + shape=out_shape, + stride=_contiguous_strides(tuple(out_shape)), + ) + return self.Wout(out_dt) + + # -- gather strategy (default, bit-exact) ------------------------------ + def _forward_gather( + self, m: DTensor, pair: DTensor, pair_attention_mask: DTensor + ) -> DTensor: + mesh = self.device_mesh + h, dh = self.n_heads, self.head_width + v_dt, gate_dt, bias_dt = self._project(m, pair) + + bias_full = ( + bias_dt.redistribute(mesh, [Shard(0), Shard(1), Replicate()]) + .to_local() + .contiguous() + ) # (B, sL, L, h) + mask_full = ( + pair_attention_mask.redistribute(mesh, [Shard(0), Shard(1), Replicate()]) + .to_local() + .contiguous() + ) # (B, sL, L) + bias_full = bias_full.masked_fill(~mask_full.unsqueeze(-1).bool(), -1e5) + attn = torch.softmax(bias_full, dim=-2) # softmax over j + + v_full = ( + v_dt.redistribute(mesh, [Shard(0), Replicate(), Replicate()]) + .to_local() + .contiguous() + ) # (B, L, M, h*dh) + + b_size, s_l = attn.shape[0], attn.shape[1] + l_full, m_depth = v_full.shape[1], v_full.shape[2] + v_full = v_full.reshape(b_size, l_full, m_depth, h, dh) + gate_local = gate_dt.to_local().reshape(b_size, s_l, m_depth, h, dh) + + o_local = torch.einsum("bijh,bjmhd->bimhd", attn, v_full) + return self._finish(m, o_local, gate_local) + + # -- ring strategy (boltz-style online softmax) ------------------------ + def _forward_ring( + self, m: DTensor, pair: DTensor, pair_attention_mask: DTensor + ) -> DTensor: + h, dh = self.n_heads, self.head_width + n = self.device_mesh.size(1) # CP axis size + v_dt, gate_dt, bias_dt = self._project(m, pair) + + # Local blocks: bias[i in p, j in q], v[block p], mask[i in p, j in q]. + bias_local = bias_dt.to_local() # (B, sLi, sLj, h) + mask_local = pair_attention_mask.to_local() # (B, sLi, sLj) + # Pre-mask: each block's mask is co-located with its bias and rings + # along with it, so masking once before the ring is correct. + bias_local = bias_local.masked_fill( + ~mask_local.unsqueeze(-1).bool(), -1e5 + ).contiguous() + v_local = v_dt.to_local().contiguous() # (B, sLp, M, h*dh) + + # Transpose v: block p -> block q (aligns v with the local bias block). + v_q = self.ring_v.comm_2d_trans.enqueue_to_dispatch(v_local) + self.ring_v.comm_2d_trans.wait_until_finished() + + b_size, s_li = bias_local.shape[0], bias_local.shape[1] + m_depth = v_q.shape[2] + bias_buf = [bias_local, torch.empty_like(bias_local)] + v_buf = [v_q.contiguous(), torch.empty_like(v_q)] + i_ready, i_recv = 0, 1 + o = lse = amax = None + + for k in range(n): + b_blk = bias_buf[i_ready] + v_blk = v_buf[i_ready] + if k < n - 1: + bias_buf[i_recv] = self.ring_bias.comm_row.enqueue_to_dispatch( + b_blk, bias_buf[i_recv] + ) + v_buf[i_recv] = self.ring_v.comm_row.enqueue_to_dispatch( + v_blk, v_buf[i_recv] + ) + + amax_blk = b_blk.amax(dim=2, keepdim=True) # (B, sLi, 1, h) + lse_blk = torch.logsumexp(b_blk - amax_blk, dim=2, keepdim=True) + p = torch.softmax(b_blk, dim=2) # (B, sLi, sLj, h) + v_blk_r = v_blk.reshape(b_size, v_blk.shape[1], m_depth, h, dh) + o_blk = torch.einsum("bijh,bjmhd->bimhd", p, v_blk_r) # (B, sLi, M, h, dh) + + # Arrange so the softmax-reduced axes (M, dh) are the trailing + # feature and the per-(i, h) lse/amax broadcast over them. + o_blk2 = o_blk.permute(0, 1, 3, 2, 4).reshape(b_size, s_li, h, m_depth * dh) + lse_blk2 = lse_blk.permute(0, 1, 3, 2).reshape(b_size, s_li, h, 1) + amax_blk2 = amax_blk.permute(0, 1, 3, 2).reshape(b_size, s_li, h, 1) + o, lse, amax = tiled_softmax_attention_update( + o_blk2, lse_blk2, amax_blk2, o, lse, amax + ) + + if k < n - 1: + self.ring_bias.comm_row.wait_until_finished() + self.ring_v.comm_row.wait_until_finished() + i_ready ^= 1 + i_recv ^= 1 + + # (B, sLi, h, M*dh) -> (B, sLi, M, h, dh) + o_local = ( + o.reshape(b_size, s_li, h, m_depth, dh).permute(0, 1, 3, 2, 4).contiguous() # ty:ignore[unresolved-attribute] + ) + gate_local = gate_dt.to_local().reshape(b_size, s_li, m_depth, h, dh) + return self._finish(m, o_local, gate_local) diff --git a/esm/models/esmfold2/distributed/model/layers/pair_init.py b/esm/models/esmfold2/distributed/model/layers/pair_init.py new file mode 100644 index 00000000..122c4557 --- /dev/null +++ b/esm/models/esmfold2/distributed/model/layers/pair_init.py @@ -0,0 +1,383 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: MIT +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. + +"""Distributed sharded pair-initialisation for ESMFold2 (inference-only). + +The serial forward builds the initial pair tensors — ``z_init`` (an outer sum of +two per-token projections), the relative-position encoding, and the token-bond +encoding — as full ``[B, L, L, d_pair]`` tensors on *every* rank, then only +shards them at the recycle-loop boundary (``distribute_tensor`` inside +``CPRecycleEngine.run_loop``). That makes the per-rank init peak scale like a +single GPU (several full L×L tensors resident at once, plus ``rel_pos`` / +``token_bonds`` held full for the whole forward), so CP buys no memory relief for +the init phase and OOMs at short L. + +``PairInitDistributed`` builds each of these directly as a 2-D-sharded +``(Shard(0), Shard(1), Shard(2))`` DTensor by computing only this rank's +``(row, col)`` block — the full L×L tensor is never resident. Per-token inputs +(``x_inputs`` and the index tensors) are cheap ``[B, L, *]`` and stay replicated; +only their O(L²) outer combinations are sharded. Each is wrapped as a DTensor so +the recycle engine / structure-head wrapper consume them with no gather. + +The channel-wise submodules (``z_init_1`` / ``z_init_2``, the ``token_bonds`` +Linear, and ``rel_pos.embed``) are the model's own layers, run on the local +block — their weights are identical on every rank, so the block result equals the +corresponding slice of the serial full tensor. +""" + +from math import lcm, sqrt + +import torch +import torch.nn as nn +import torch.nn.functional as F +from torch.distributed.tensor import DTensor, Shard + +from esm.models.esmfold2.layers import ResIdxAsymIdSymIdEntityIdEncoding + +_PAIR_PL = [Shard(0), Shard(1), Shard(2)] + +# Per-rank RNG seed base for the sharded initial pair-state draw. Kept off the +# global generator so the recycle loop's MSA-subsample RNG stays synchronized +# across ranks (see ``init_pair_state``). +_Z_INIT_SEED = 0x2717 + + +def _contiguous_strides(shape: tuple[int, ...]) -> tuple[int, ...]: + strides = [1] * len(shape) + for i in range(len(shape) - 2, -1, -1): + strides[i] = strides[i + 1] * shape[i + 1] + return tuple(strides) + + +def _block_geometry(n_orig: int, device_mesh): + """This rank's pair-block geometry: ``(lpad, s0, s1, row_slice, col_slice)``. + + The token axis is padded to ``shard_factor = lcm(cp_axis_0, cp_axis_1)``; the + rank holds rows ``row_slice`` (its cp_axis_0 coordinate) × cols ``col_slice`` + (its cp_axis_1 coordinate) of the ``[lpad, lpad]`` pair. + """ + cp0 = device_mesh.size(1) + cp1 = device_mesh.size(2) + shard_factor = lcm(cp0, cp1) + coord = device_mesh.get_coordinate() + assert coord is not None, "device mesh has no coordinate for this rank" + row_rank, col_rank = int(coord[1]), int(coord[2]) + pad = (shard_factor - n_orig % shard_factor) % shard_factor + lpad = n_orig + pad + s0 = lpad // cp0 + s1 = lpad // cp1 + row = slice(row_rank * s0, (row_rank + 1) * s0) + col = slice(col_rank * s1, (col_rank + 1) * s1) + return lpad, s0, s1, row, col + + +def build_sharded_pair_mask(tok_mask: torch.Tensor, device_mesh) -> DTensor: + """This rank's ``[B, s0, s1]`` block of the outer-product pair mask as a + ``(Shard0, Shard1, Shard2)`` DTensor — never the full ``[B, L, L]``. + + Outer product of the + row / col slices of the per-token mask (``tok_mask[:, row]`` ⊗ + ``tok_mask[:, col]``), value-identical to + ``distribute_tensor(tok_mask[:,:,None] * tok_mask[:,None,:])`` but with no full + L×L intermediate. The token axis is padded to + ``shard_factor = lcm(cp_axis_0, cp_axis_1)`` (padded positions → 0, matching + the serial zero-pad). Shared by ``PairInitDistributed.pair_mask`` (recycle + pair mask) and the MSA encoder's ``forward_sharded`` (per-iteration mask). + + ``tok_mask`` is ``[B, L]`` (any numeric / bool dtype); the returned DTensor is + float32 — cast at the call site to the consumer dtype. + """ + b = tok_mask.shape[0] + lpad, _, _, row, col = _block_geometry(tok_mask.shape[1], device_mesh) + pad = lpad - tok_mask.shape[1] + + tm = tok_mask.float() + if pad: + tm = F.pad(tm, (0, pad)) + block = tm[:, row].unsqueeze(2) * tm[:, col].unsqueeze(1) # [B, s0, s1] + + shape = torch.Size((b, lpad, lpad)) + return DTensor.from_local( + block.contiguous(), + device_mesh=device_mesh, + placements=_PAIR_PL, + stride=_contiguous_strides(tuple(shape)), + shape=shape, + ) + + +def build_sharded_distogram_bins( + rep_coords: torch.Tensor, boundaries: torch.Tensor, device_mesh +) -> DTensor: + """This rank's ``[B, s0, s1]`` block of the distance-distogram bins as a + ``(Shard0, Shard1, Shard2)`` DTensor — ``cdist`` on the row / col coordinate + blocks only, never the full ``[B, N, N]`` distance matrix. + + ``rep_coords`` is ``[B, N, 3]`` per-token representative-atom coordinates + (replicated, cheap). ``boundaries`` is the distance-bin edge buffer. Padded + rows/cols → bin 0, matching the serial ``F.pad(distogram_bins)`` zero-fill. + This has no serial counterpart: the serial confidence head computes the full + ``[B, N, N]`` cdist instead. + """ + b = rep_coords.shape[0] + n = rep_coords.shape[1] + lpad, _, _, row, col = _block_geometry(n, device_mesh) + pad = lpad - n + + coords = F.pad(rep_coords, (0, 0, 0, pad)) if pad else rep_coords + row_c = coords[:, row] # [B, s0, 3] + col_c = coords[:, col] # [B, s1, 3] + dist = torch.cdist(row_c, col_c, compute_mode="donot_use_mm_for_euclid_dist") + bins = (dist.unsqueeze(-1) > boundaries).sum(dim=-1).long() # [B, s0, s1] + if pad: + rr = torch.arange(row.start, row.stop, device=rep_coords.device) < n + cc = torch.arange(col.start, col.stop, device=rep_coords.device) < n + bins = bins * (rr[:, None] & cc[None, :]) # padded positions → bin 0 + + shape = torch.Size((b, lpad, lpad)) + return DTensor.from_local( + bins.contiguous(), + device_mesh=device_mesh, + placements=_PAIR_PL, + stride=_contiguous_strides(tuple(shape)), + shape=shape, + ) + + +class PairInitDistributed(nn.Module): + """Builds ``z_init`` / ``rel_pos`` / ``token_bonds`` as 2-D-sharded DTensors. + + ``forward`` returns ``(z_init_dt, rel_pos_dt, token_bonds_dt, n_orig)``: the + three pair tensors as padded ``(Shard0, Shard1, Shard2)`` DTensors (token axis + padded to ``shard_factor = lcm(cp_axis_0, cp_axis_1)`` — the same padding + contract the recycle engine and LM->pair builder use) plus the original + (pre-pad) token count. ``z_init_dt`` already folds in ``rel_pos`` + + ``token_bonds`` (matching the serial ``z_init`` sum); ``rel_pos_dt`` / + ``token_bonds_dt`` are returned separately for the structure / confidence + heads. + """ + + def __init__(self, model, dist_manager) -> None: + super().__init__() + self.device_mesh = dist_manager.device_mesh_subgroups + # device_mesh is (dp, cp_axis_0, cp_axis_1); rows shard on cp_axis_0, cols on cp_axis_1. + self.cp_axis_0 = self.device_mesh.size(1) + self.cp_axis_1 = self.device_mesh.size(2) + self.shard_factor = lcm(self.cp_axis_0, self.cp_axis_1) + coord = self.device_mesh.get_coordinate() + assert coord is not None, "device mesh has no coordinate for this rank" + self.row_rank = int(coord[1]) + self.col_rank = int(coord[2]) + + # Model's own channel-wise layers (run on the local block; weights replicated). + self.z_init_1 = model.z_init_1 + self.z_init_2 = model.z_init_2 + self.token_bonds = model.token_bonds + self.rel_pos: ResIdxAsymIdSymIdEntityIdEncoding = model.rel_pos + + def _block_valid_mask( + self, n_orig: int, lpad: int, device: torch.device + ) -> torch.Tensor | None: + """This rank's ``[s0, s1]`` boolean mask marking (row, col) positions that + are *real* tokens (global index < ``n_orig``), or ``None`` if there is no + CP padding. Used to zero the padded rows/cols so the sharded pair matches + the serial ``_pad_pair`` (zero-fill) contract — padded positions must not + leak nonzero values into the masked trunk. Note this is a *position* mask + (CP padding only), NOT the token attention mask: in-range but attention- + masked tokens stay nonzero here, exactly as in the serial full tensor.""" + if lpad - n_orig <= 0: + return None + s0 = lpad // self.cp_axis_0 + s1 = lpad // self.cp_axis_1 + r0 = self.row_rank * s0 + c0 = self.col_rank * s1 + row_valid = torch.arange(r0, r0 + s0, device=device) < n_orig + col_valid = torch.arange(c0, c0 + s1, device=device) < n_orig + return row_valid[:, None] & col_valid[None, :] # [s0, s1] bool + + def _wrap(self, local: torch.Tensor, b: int, lpad: int, d: int) -> DTensor: + shape = torch.Size((b, lpad, lpad, d)) + return DTensor.from_local( + local.contiguous(), + device_mesh=self.device_mesh, + placements=_PAIR_PL, + shape=shape, + stride=_contiguous_strides(tuple(shape)), + ) + + def _sharded_rel_pos( + self, + residue_index: torch.Tensor, + asym_id: torch.Tensor, + sym_id: torch.Tensor, + entity_id: torch.Tensor, + token_index: torch.Tensor, + row: slice, + col: slice, + ) -> torch.Tensor: + """This rank's ``[B, s0, s1, d_pair]`` block of the relative-position + encoding. Byte-for-byte the arithmetic of + ``ResIdxAsymIdSymIdEntityIdEncoding.forward`` but with the index tensors + sliced to the row / col block before the outer differences (so no full + ``[B, L, L, *]`` one-hot intermediate).""" + rp = self.rel_pos + bins_r = rp.n_relative_residx_bins + bins_c = rp.n_relative_chain_bins + + ri_r, ri_c = residue_index[:, row], residue_index[:, col] + ai_r, ai_c = asym_id[:, row], asym_id[:, col] + si_r, si_c = sym_id[:, row], sym_id[:, col] + ei_r, ei_c = entity_id[:, row], entity_id[:, col] + ti_r, ti_c = token_index[:, row], token_index[:, col] + + same_chain = ai_r.unsqueeze(2) == ai_c.unsqueeze(1) + same_res = ri_r.unsqueeze(2) == ri_c.unsqueeze(1) + same_ent = ei_r.unsqueeze(2) == ei_c.unsqueeze(1) + + d_res = torch.clip( + ri_r.unsqueeze(2) - ri_c.unsqueeze(1) + bins_r, 0, 2 * bins_r + ) + d_res = torch.where(same_chain, d_res, 2 * bins_r + 1) + a_res = F.one_hot(d_res, 2 * bins_r + 2) + + d_tok = torch.clip( + ti_r.unsqueeze(2) - ti_c.unsqueeze(1) + bins_r, 0, 2 * bins_r + ) + d_tok = torch.where(same_chain & same_res, d_tok, 2 * bins_r + 1) + a_tok = F.one_hot(d_tok, 2 * bins_r + 2) + + d_ch = torch.clip(si_r.unsqueeze(2) - si_c.unsqueeze(1) + bins_c, 0, 2 * bins_c) + d_ch = torch.where(same_chain, 2 * bins_c + 1, d_ch) + a_ch = F.one_hot(d_ch, 2 * bins_c + 2) + + feats = torch.cat( + [ + a_res.float(), + a_tok.float(), + same_ent.float().unsqueeze(-1), + a_ch.float(), + ], + dim=-1, + ) + return rp.embed(feats) # [B, s0, s1, d_pair] + + def forward( + self, + x_inputs: torch.Tensor, + residue_index: torch.Tensor, + asym_id: torch.Tensor, + sym_id: torch.Tensor, + entity_id: torch.Tensor, + token_index: torch.Tensor, + token_bonds: torch.Tensor, + ) -> tuple[DTensor, DTensor, DTensor, int]: + b = x_inputs.shape[0] + n_orig = x_inputs.shape[1] + pad = (self.shard_factor - n_orig % self.shard_factor) % self.shard_factor + lpad = n_orig + pad + s0 = lpad // self.cp_axis_0 + s1 = lpad // self.cp_axis_1 + row = slice(self.row_rank * s0, (self.row_rank + 1) * s0) + col = slice(self.col_rank * s1, (self.col_rank + 1) * s1) + + # --- z_init: outer SUM of two per-token projections, sliced to the block. + # z1/z2 are [B, L, d] (per-token, cheap — not L×L), so building them full and + # slicing costs O(L*d), not O(L²). + z1 = self.z_init_1(x_inputs) + z2 = self.z_init_2(x_inputs) + d = z1.shape[-1] + if pad: + z1 = F.pad(z1, (0, 0, 0, pad)) + z2 = F.pad(z2, (0, 0, 0, pad)) + z_init_local = z1[:, row].unsqueeze(2) + z2[:, col].unsqueeze( + 1 + ) # [B, s0, s1, d] + + # --- relative-position encoding (index tensors sliced to row/col block) --- + def _pad_idx(idx: torch.Tensor) -> torch.Tensor: + return F.pad(idx, (0, pad)) if pad else idx + + rel_local = self._sharded_rel_pos( + _pad_idx(residue_index), + _pad_idx(asym_id), + _pad_idx(sym_id), + _pad_idx(entity_id), + _pad_idx(token_index), + row, + col, + ) # [B, s0, s1, d] + + # --- token-bond encoding: slice the (valid) input block, then pad the small + # block to [s0, s1] — never materialises a full padded L×L×1 tensor. + r0 = self.row_rank * s0 + c0 = self.col_rank * s1 + tb_v = token_bonds[ + :, r0 : min(r0 + s0, n_orig), c0 : min(c0 + s1, n_orig), : + ].float() + pr = s0 - tb_v.shape[1] + pc = s1 - tb_v.shape[2] + if pr or pc: + tb_v = F.pad(tb_v, (0, 0, 0, pc, 0, pr)) + tb_local = self.token_bonds(tb_v) # [B, s0, s1, d] + + z_comb = z_init_local + rel_local + tb_local + # Zero the CP-padded rows/cols so the sharded pair matches the serial + # zero-padding (else e.g. z_init_local[pad_row, j] = z2[j] would leak). + vmask = self._block_valid_mask(n_orig, lpad, x_inputs.device) + if vmask is not None: + z_comb = z_comb * vmask[None, :, :, None].to(z_comb.dtype) + rel_local = rel_local * vmask[None, :, :, None].to(rel_local.dtype) + tb_local = tb_local * vmask[None, :, :, None].to(tb_local.dtype) + + z_init_dt = self._wrap(z_comb, b, lpad, d) + rel_pos_dt = self._wrap(rel_local, b, lpad, d) + token_bonds_dt = self._wrap(tb_local, b, lpad, d) + return z_init_dt, rel_pos_dt, token_bonds_dt, n_orig + + def pair_mask(self, tok_mask: torch.Tensor) -> DTensor: + """This rank's sharded ``[B, s0, s1]`` block of the pair attention mask + (see :func:`build_sharded_pair_mask`).""" + return build_sharded_pair_mask(tok_mask, self.device_mesh) + + def init_pair_state(self, z_init_dt: DTensor, n_orig: int) -> DTensor: + """Sharded random initial pair state (the serial ``_init_pair_state``). + + Draws this rank's local block from a *per-rank* generator (distinct seed) + so (a) the full L×L state is never resident and (b) the global RNG — which + the recycle loop's MSA subsample relies on to stay in lockstep across + ranks — is untouched. Not bit-exact with the serial full-tensor + ``trunc_normal_`` draw (a valid-but-different iid sample, and ``clamp`` vs + resample at the ±3σ bound), matching the shard-local-dropout contract + already used in the recycle engine; validate by end-to-end pLDDT/pTM + parity. CP-padded rows/cols are zeroed to match the serial ``_pad_pair``. + """ + ref = z_init_dt.to_local() + b, lpad, _, d = z_init_dt.shape + std = sqrt(2.0 / (5.0 * ref.shape[-1])) + gen = torch.Generator(device=ref.device) + gen.manual_seed(_Z_INIT_SEED + torch.distributed.get_rank()) + state = torch.empty(ref.shape, dtype=torch.float32, device=ref.device) + state.normal_(0.0, std, generator=gen).clamp_(-3 * std, 3 * std) + state = state.to(dtype=ref.dtype) + vmask = self._block_valid_mask(int(n_orig), int(lpad), ref.device) + if vmask is not None: + state = state * vmask[None, :, :, None].to(state.dtype) + return self._wrap(state, int(b), int(lpad), int(d)) diff --git a/esm/models/esmfold2/distributed/model/layers/pairformer.py b/esm/models/esmfold2/distributed/model/layers/pairformer.py new file mode 100644 index 00000000..e63678d7 --- /dev/null +++ b/esm/models/esmfold2/distributed/model/layers/pairformer.py @@ -0,0 +1,169 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: MIT +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. + +"""Distributed PairUpdateBlock and Pairformer for ESMFold2. + +ESMFold2's Pairformer is pair-only (no single/sequence track), with each block: + PairUpdateBlock: + pair = pair + tri_mul_out(pair) + pair = pair + tri_mul_in(pair) + pair = pair_transition(pair) + +All three operations are distributed across the 2D CP grid. +""" + +from typing import Optional + +import torch.nn as nn +import torch.nn.functional as F +from torch.distributed.device_mesh import DeviceMesh +from torch.distributed.tensor import DTensor + +from esm.models.esmfold2.distributed.comm import Ring2DComm +from esm.models.esmfold2.distributed.manager import DistributedManager +from esm.models.esmfold2.distributed.model.layers.layernorm import ( + LayerNormParamsReplicated, +) +from esm.models.esmfold2.distributed.model.layers.linear import LinearParamsReplicated +from esm.models.esmfold2.distributed.model.layers.triangular_mult import ( + TriangleMultiplicativeBlockDistributed, +) +from esm.models.esmfold2.layers import FoldingTrunk as SerialFoldingTrunk +from esm.models.esmfold2.layers import PairUpdateBlock as SerialPairUpdateBlock +from esm.models.esmfold2.layers import Transition as SerialTransition + + +class TransitionDistributed(nn.Module): + """Distributed Transition block (LayerNorm + SwiGLU FFN). + + Parameters are replicated; the LayerNorm and both Linear layers + use DTensor with replicated params. + """ + + def __init__(self, layer: SerialTransition, device_mesh: DeviceMesh) -> None: + super().__init__() + if not isinstance(layer, SerialTransition): + raise TypeError(f"layer must be Transition, got {type(layer).__name__}") + self.norm = LayerNormParamsReplicated(layer.norm, device_mesh) + # SwiGLUMLP has w12 and w3 + self.w12 = LinearParamsReplicated(layer.ffn.w12, device_mesh) + self.w3 = LinearParamsReplicated(layer.ffn.w3, device_mesh) + self.hidden_features = layer.ffn.hidden_features + + def forward(self, x: DTensor) -> DTensor: + normed = self.norm(x) + x12 = self.w12(normed) + x1, x2 = x12.split(self.hidden_features, dim=-1) + hidden = F.silu(x1) * x2 + out = self.w3(hidden) + return x + out + + +class PairUpdateBlockDistributed(nn.Module): + """Distributed PairUpdateBlock. + + Computes: + pair = pair + tri_mul_out(pair, mask) + pair = pair + tri_mul_in(pair, mask) + pair = pair_transition(pair) + + All pair operations are distributed via 2D CP ring communication. + + Parameters + ---------- + layer: + Serial PairUpdateBlock to distribute. + dist_manager: + DistributedManager with the CP group and subgroups set up. + """ + + def __init__( + self, layer: SerialPairUpdateBlock, dist_manager: DistributedManager + ) -> None: + super().__init__() + if not isinstance(layer, SerialPairUpdateBlock): + raise TypeError( + f"layer must be PairUpdateBlock, got {type(layer).__name__}" + ) + + self.dist_manager = dist_manager + self.device_mesh = dist_manager.device_mesh_subgroups + + ring_comm_out = Ring2DComm( + dist_manager.group["cp"], + dist_manager.subgroups["cp"][0], + dist_manager.layout_subgroups["cp"], + ) + ring_comm_in = Ring2DComm( + dist_manager.group["cp"], + dist_manager.subgroups["cp"][0], + dist_manager.layout_subgroups["cp"], + ) + + self.tri_mul_out = TriangleMultiplicativeBlockDistributed( + layer.tri_mul_out._engine, self.device_mesh, ring_comm_out + ) + self.tri_mul_in = TriangleMultiplicativeBlockDistributed( + layer.tri_mul_in._engine, self.device_mesh, ring_comm_in + ) + self.pair_transition = TransitionDistributed( + layer.pair_transition, self.device_mesh + ) + + def forward( + self, pair: DTensor, pair_attention_mask: Optional[DTensor] = None + ) -> DTensor: + pair = pair + self.tri_mul_out(pair, mask=pair_attention_mask) + pair = pair + self.tri_mul_in(pair, mask=pair_attention_mask) + pair = self.pair_transition(pair) + return pair + + +class FoldingTrunkDistributed(nn.Module): + """Distributed Pairformer: ModuleList of PairUpdateBlockDistributed. + + Wraps the serial Pairformer by distributing each block. + + Parameters + ---------- + pairformer: + Serial Pairformer module. + dist_manager: + DistributedManager with the CP group and subgroups set up. + """ + + def __init__( + self, trunk: SerialFoldingTrunk, dist_manager: DistributedManager + ) -> None: + super().__init__() + if not isinstance(trunk, SerialFoldingTrunk): + raise TypeError(f"trunk must be FoldingTrunk, got {type(trunk).__name__}") + + self.blocks = nn.ModuleList( + [PairUpdateBlockDistributed(block, dist_manager) for block in trunk.blocks] # ty:ignore[invalid-argument-type] + ) + + def forward( + self, pair: DTensor, pair_attention_mask: Optional[DTensor] = None + ) -> DTensor: + for block in self.blocks: + pair = block(pair, pair_attention_mask=pair_attention_mask) + return pair diff --git a/esm/models/esmfold2/distributed/model/layers/row_attention_pooling.py b/esm/models/esmfold2/distributed/model/layers/row_attention_pooling.py new file mode 100644 index 00000000..000261ba --- /dev/null +++ b/esm/models/esmfold2/distributed/model/layers/row_attention_pooling.py @@ -0,0 +1,116 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: MIT +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. + +"""Distributed RowAttentionPooling for the ESMFold2 confidence head (#6, inference). + +The serial op (``modeling_esmfold2_common.py:RowAttentionPooling``) pools the pair +``z`` (B, L, L, d_pair) over the column axis j into a single repr (B, L, d_single): + + scores = attn_proj(z).squeeze(-1) # (B, L, L) + scores = scores + mask_bias_over_j # -1e9 where col j is padding + weights = softmax(scores, dim=-1) # over j + pooled = einsum("bnm,bnmd->bnd", weights, z) # sum over j + return out_proj(pooled) # (B, L, d_single) + +Under 2D CP the pair is sharded ``(Shard(0), Shard(1), Shard(2))`` (row i on +cp_axis_0, col j on cp_axis_1). The softmax and the weighted sum are both over +j = cp_axis_1, so we do a **distributed softmax + weighted-sum reduction along the +column group** — never gathering ``z``'s columns: + + * attn_proj is channel-wise → local scores (B, sLi, sLj); + * softmax over the full j: all-reduce MAX then SUM over cp_axis_1 (online stats); + * pooled: local ``einsum`` over the local j, then all-reduce SUM over cp_axis_1. + +The result is row-sharded on cp_axis_0, replicated on cp_axis_1 — returned as a +``(Shard(0), Shard(1), Replicate())`` single-repr DTensor. +""" + +import torch +import torch.distributed as dist +import torch.nn as nn +from torch.distributed.tensor import DTensor, Replicate, Shard, distribute_tensor + +from esm.models.esmfold2.distributed.model.layers.linear import LinearParamsReplicated +from esm.models.esmfold2.layers import RowAttentionPooling as SerialRowAttentionPooling + + +def _contiguous_strides(shape: tuple[int, ...]) -> tuple[int, ...]: + strides = [1] * len(shape) + for i in range(len(shape) - 2, -1, -1): + strides[i] = strides[i + 1] * shape[i + 1] + return tuple(strides) + + +class RowAttentionPoolingDistributed(nn.Module): + """Distributed RowAttentionPooling (inference-only). + + ``forward`` takes the pair DTensor ``z`` ``(Shard(0), Shard(1), Shard(2))`` and + the full token mask ``(B, L)``; returns the single-repr DTensor + ``(Shard(0), Shard(1), Replicate())`` of shape ``(B, L, d_single)``. + """ + + def __init__(self, layer: SerialRowAttentionPooling, dist_manager) -> None: + super().__init__() + if not isinstance(layer, SerialRowAttentionPooling): + raise TypeError( + f"layer must be RowAttentionPooling, got {type(layer).__name__}" + ) + self.device_mesh = dist_manager.device_mesh_subgroups + self.attn_proj = LinearParamsReplicated(layer.attn_proj, self.device_mesh) + self.out_proj = LinearParamsReplicated(layer.out_proj, self.device_mesh) + # device_mesh dims: (dp, cp_axis_0, cp_axis_1); j is cp_axis_1 (col group). + self.col_group = self.device_mesh.get_group(2) + + def forward(self, z: DTensor, mask: torch.Tensor) -> DTensor: + mesh = self.device_mesh + # Column-shard the token mask to match z's local columns j (cp_axis_1). + mask_cols = distribute_tensor( + mask.contiguous(), mesh, [Shard(0), Replicate(), Shard(1)] + ).to_local() # (B, sLj) + + scores = self.attn_proj(z).to_local().squeeze(-1) # (B, sLi, sLj) + neg = torch.full_like(scores, -1e9) + scores = torch.where(mask_cols[:, None, :].bool(), scores, neg) + + # Distributed softmax over the full j (cp_axis_1). + local_max = scores.amax(dim=-1, keepdim=True) # (B, sLi, 1) + dist.all_reduce(local_max, op=dist.ReduceOp.MAX, group=self.col_group) + e = torch.exp(scores - local_max) + local_sum = e.sum(dim=-1, keepdim=True) # (B, sLi, 1) + dist.all_reduce(local_sum, op=dist.ReduceOp.SUM, group=self.col_group) + weights = e / local_sum # (B, sLi, sLj) — globally normalized over j + + # Weighted sum over the local j, then reduce across the column group. + z_local = z.to_local() # (B, sLi, sLj, d_pair) + pooled = torch.einsum("bij,bijd->bid", weights, z_local).contiguous() + dist.all_reduce(pooled, op=dist.ReduceOp.SUM, group=self.col_group) + # pooled is now the full-j pool: row-sharded on cp_axis_0, identical across + # cp_axis_1 -> wrap as (Shard(0), Shard(1), Replicate()). + b_size, s_li, d_pair = pooled.shape + full_shape = torch.Size((b_size, z.shape[1], d_pair)) + pooled_dt = DTensor.from_local( + pooled, + device_mesh=mesh, + placements=[Shard(0), Shard(1), Replicate()], + shape=full_shape, + stride=_contiguous_strides(tuple(full_shape)), + ) + return self.out_proj(pooled_dt) # (B, L, d_single), (S0, S1, R) diff --git a/esm/models/esmfold2/distributed/model/layers/single_to_pair.py b/esm/models/esmfold2/distributed/model/layers/single_to_pair.py new file mode 100644 index 00000000..ee0785ea --- /dev/null +++ b/esm/models/esmfold2/distributed/model/layers/single_to_pair.py @@ -0,0 +1,190 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: MIT +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. + +"""Distributed LM->pair builder for ESMFold2 (Fix #14, inference-only). + +The serial ``LanguageModelShim`` turns the ESM-C hidden states ``[B, L, 81, +d_model]`` into the pair representation ``lm_z`` ``[B, L, L, d_z]``. The L×L +blow-up happens entirely inside ``SingleToPair``: + + x = downproject(x) # [B, L, dp] + feat = cat([x_i * x_j, x_i - x_j], dim=-1) # [B, L, L, 2*dp] <-- the floor + lm_z = output_mlp(feat) # [B, L, L, d_z] + +Materialising that full ``[B, L, L, *]`` on every rank is the binding per-rank +peak measured by the T0 phase sweep (the ``language_model`` phase, identical at +2×2 and 4×4 -> unsharded). This module builds ``lm_z`` directly as a 2-D-sharded +``(Shard(0), Shard(1), Shard(2))`` DTensor (row token i on cp_axis_0, col token j +on cp_axis_1), so the L×L tensor is never resident full on any rank. + +The outer op ``f(x_i, x_j)`` mirrors ``OuterProductMeanDistributed``: the row +block ``x_i`` is held locally; the column block ``x_j`` is fetched with a single +``TransposeComm`` (the (q,p) peer's row block is exactly rank (p,q)'s column +block on a square grid). All channel-wise ops (downproject, output_mlp, the +trailing LayerNorm) run on the local shard via replicated params — only the +outer op needs communication. +""" + +from math import lcm + +import torch +import torch.nn as nn +import torch.nn.functional as F +from torch.distributed.tensor import DTensor, Replicate, Shard, distribute_tensor + +from esm.models.esmfold2.distributed.comm import TransposeComm +from esm.models.esmfold2.distributed.model.layers.layernorm import ( + LayerNormParamsReplicated, +) +from esm.models.esmfold2.distributed.model.layers.linear import LinearParamsReplicated +from esm.models.esmfold2.layers import LanguageModelShim as SerialLanguageModelShim +from esm.models.esmfold2.layers import SingleToPair as SerialSingleToPair + +_PAIR_PLACEMENTS = [Shard(0), Shard(1), Shard(2)] +_TOKEN_PLACEMENTS = [Shard(0), Shard(1), Replicate()] + + +def _contiguous_strides(shape: tuple[int, ...]) -> tuple[int, ...]: + strides = [1] * len(shape) + for i in range(len(shape) - 2, -1, -1): + strides[i] = strides[i + 1] * shape[i + 1] + return tuple(strides) + + +class SingleToPairDistributed(nn.Module): + """Distributed (transpose-based) ``SingleToPair``. + + ``forward`` takes a per-token DTensor ``x`` with placements + ``(Shard(0), Shard(1), Replicate())`` (token L sharded on cp_axis_0, + replicated on cp_axis_1) and returns the pair DTensor with placements + ``(Shard(0), Shard(1), Shard(2))``. + """ + + def __init__(self, layer: SerialSingleToPair, dist_manager) -> None: + super().__init__() + if not isinstance(layer, SerialSingleToPair): + raise TypeError(f"layer must be SingleToPair, got {type(layer).__name__}") + self.device_mesh = dist_manager.device_mesh_subgroups + + self.downproject = LinearParamsReplicated(layer.downproject, self.device_mesh) + # output_mlp = Sequential(Linear, GELU, Linear) + self.out_in = LinearParamsReplicated(layer.output_mlp[0], self.device_mesh) # ty:ignore[invalid-argument-type] + self.gelu = layer.output_mlp[1] + self.out_proj = LinearParamsReplicated(layer.output_mlp[2], self.device_mesh) # ty:ignore[invalid-argument-type] + + # (i,j) <-> (j,i) transpose to fetch the column-token block. + self.transpose = TransposeComm( + dist_manager.group["cp"], dist_manager.layout_subgroups["cp"] + ) + + def forward(self, x: DTensor) -> DTensor: + mesh = self.device_mesh + L = x.shape[1] + + x = self.downproject(x) # (B, L, dp), placements (S0, S1, R) + + # Row block i is local; fetch column block j from the transpose peer. + a_local = x.to_local().contiguous() # (B, sL_i, dp) + b_q = self.transpose.enqueue_to_dispatch(a_local) + self.transpose.wait_until_finished() + b_q = b_q.contiguous() # (B, sL_j, dp) + + # cat([x_i * x_j, x_i - x_j]) on the local (sL_i, sL_j) tile — mirrors the + # serial cat([x.unsqueeze(2) * x.unsqueeze(1), x.unsqueeze(2) - x.unsqueeze(1)]). + ai = a_local.unsqueeze(2) # (B, sL_i, 1, dp) + bj = b_q.unsqueeze(1) # (B, 1, sL_j, dp) + feat_local = torch.cat([ai * bj, ai - bj], dim=-1).contiguous() + + b_size = feat_local.shape[0] + c_feat = feat_local.shape[-1] + feat_shape = torch.Size((b_size, L, L, c_feat)) + feat_dt = DTensor.from_local( + feat_local, + device_mesh=mesh, + placements=_PAIR_PLACEMENTS, + shape=feat_shape, + stride=_contiguous_strides(tuple(feat_shape)), + ) + + out = self.out_in(feat_dt) # (B, L, L, out_dim), (S0, S1, S2) + # GELU is pointwise → run on the local shard, preserve the DTensor metadata. + gelu_local = self.gelu(out.to_local()) + out = DTensor.from_local( + gelu_local.contiguous(), + device_mesh=mesh, + placements=_PAIR_PLACEMENTS, + shape=out.shape, + stride=out.stride(), + ) + return self.out_proj(out) + + +class LanguageModelShimDistributed(nn.Module): + """Distributed ``LanguageModelShim`` producing a sharded ``lm_z`` DTensor. + + The per-token front end (``base_z_linear`` + the ``base_z_combine`` softmax + mix) is cheap (per-token, ~MB) and runs replicated on each rank exactly as in + the serial shim. Only the L×L ``SingleToPair`` blow-up and the trailing + LayerNorm are distributed. + + ``forward`` returns ``(lm_z_dt, n_orig)``: a padded, 2-D-sharded pair DTensor + and the original (pre-pad) token count. The token axis is padded to + ``shard_factor = lcm(cp_axis_0, cp_axis_1)`` so the pair shards evenly — the + same padding contract the recycle engine uses for ``z`` / ``z_init``. Padded + rows/cols carry no signal (masked downstream by the padded pair mask; sliced + to ``n_orig`` after any gather), so they need not be explicitly zeroed. + """ + + def __init__(self, layer: SerialLanguageModelShim, dist_manager) -> None: + super().__init__() + if not isinstance(layer, SerialLanguageModelShim): + raise TypeError( + f"layer must be LanguageModelShim, got {type(layer).__name__}" + ) + self.device_mesh = dist_manager.device_mesh_subgroups + # device_mesh is (dp, cp_axis_0, cp_axis_1); pad to lcm so the pair shards evenly. + self.shard_factor = lcm(self.device_mesh.size(1), self.device_mesh.size(2)) + + # Per-token front end runs replicated (kept as the serial submodules). + self.base_z_linear = layer.base_z_linear + self.base_z_combine = layer.base_z_combine + + # base_z_mlp = Sequential(SingleToPair, LayerNorm(d_z)) + self.single_to_pair = SingleToPairDistributed(layer.base_z_mlp[0], dist_manager) # ty:ignore[invalid-argument-type] + self.norm = LayerNormParamsReplicated(layer.base_z_mlp[1], self.device_mesh) # ty:ignore[invalid-argument-type] + + def forward(self, hidden_states: torch.Tensor) -> tuple[DTensor, int]: + # Per-token front end (replicated, full L — cheap, no L×L here). + lm_single = self.base_z_linear(hidden_states) # [B, L, 81, d_z] + weights = self.base_z_combine.softmax(0) # [81] + lm_single = (weights @ lm_single).squeeze(-2) # [B, L, d_z] + + n_orig = lm_single.shape[1] + pad = (self.shard_factor - n_orig % self.shard_factor) % self.shard_factor + if pad: + lm_single = F.pad(lm_single, (0, 0, 0, pad)) + + x_dt = distribute_tensor( + lm_single.contiguous(), self.device_mesh, _TOKEN_PLACEMENTS + ) + lm_z = self.single_to_pair(x_dt) # (S0, S1, S2), [B, Lpad, Lpad, d_z] + lm_z = self.norm(lm_z) + return lm_z, n_orig diff --git a/esm/models/esmfold2/distributed/model/layers/triangular_mult.py b/esm/models/esmfold2/distributed/model/layers/triangular_mult.py new file mode 100644 index 00000000..62c0bfe4 --- /dev/null +++ b/esm/models/esmfold2/distributed/model/layers/triangular_mult.py @@ -0,0 +1,446 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: MIT +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. + +"""Distributed TriangleMultiplicativeBlock for ESMFold2's pair representation. + +The pair tensor z has shape (B, N, N, d_pair) and is distributed across a 2D +CP grid as DTensor with placements (Shard(0), Shard(1), Shard(2)). GPU (i, j) +owns the shard z[..., i_start:i_end, j_start:j_end, :]. + +Triangle multiplication patterns: + Outgoing: contracted[b,n,m,d] = sum_k a[b,n,k,d] * b[b,m,k,d] (einsum "bnkd,bmkd->bnmd") + Incoming: contracted[b,n,m,d] = sum_k a[b,k,n,d] * b[b,k,m,d] (einsum "bknd,bkmd->bnmd") + +The distributed BMM uses ring communication to accumulate partial results: + - One operand is transposed across the 2D grid (so (i,j) gets the chunk from (j,i)) + - Both operands are ring-shifted (row-wise and column-wise respectively) + - Each step computes a local matmul; results are accumulated +""" + +import os +from enum import Enum, auto +from typing import Tuple + +import torch +import torch.distributed as dist +from torch import nn +from torch.distributed.device_mesh import DeviceMesh +from torch.distributed.tensor import DTensor, Shard + +from esm.models.esmfold2.distributed.comm import Ring2DComm +from esm.models.esmfold2.distributed.model.layers.layernorm import ( + LayerNormParamsReplicated, +) +from esm.models.esmfold2.distributed.model.layers.linear import LinearParamsReplicated +from esm.models.esmfold2.distributed.utils import update_exhaustive_strides +from esm.models.esmfold2.layers import ( + TriangleMultiplicativeBlock as SerialTriangleMultiplicativeBlock, +) + +# Optional cross-rank re-alignment before the ring comm (cuda.synchronize + +# barrier on the CP group at each tri-mul). Motivated by the hypothesis that the +# CP trunk's time is NCCL P2P spin-wait from rank desync. A controlled A/B +# (CP_RANK_SYNC 0 vs 1, 5 loops) showed only a ~9% mean change, WITHIN the +# run-to-run noise (trunk ~45–66 s/call) — i.e. desync is NOT the dominant cost. +# Kept as an opt-in experiment knob, DEFAULT OFF. Correctness-neutral (touches no +# tensors). Enable with CP_RANK_SYNC=1. +_CP_RANK_SYNC = os.environ.get("CP_RANK_SYNC", "0") == "1" + + +class _Direction(Enum): + Outgoing = auto() + Incoming = auto() + + +# --------------------------------------------------------------------------- +# Core distributed batch matmul +# --------------------------------------------------------------------------- + + +class _XposeArgs(Enum): + lhs = auto() + rhs = auto() + + +def _distributed_bmm( + lhs: torch.Tensor, + rhs: torch.Tensor, + comm: Ring2DComm, + permute_lhs: tuple[int, ...] | None = None, + permute_rhs: tuple[int, ...] | None = None, + permute_out: tuple[int, ...] | None = None, + xpose_args: _XposeArgs | None = None, +) -> torch.Tensor: + """Distributed batch matmul using ring communication on a 2D process grid. + + See boltz-cp's comm.py diagrams for the full algorithm description. + + Parameters + ---------- + lhs, rhs: + Local tensor shards. Shape: (B, ..., N, K) after optional permute. + comm: + Ring2DComm object providing all the communication handles. + permute_lhs, permute_rhs: + Optional permutations applied before computation. + permute_out: + Optional permutation applied to the output. + xpose_args: + Which operand to transpose across the grid first. + """ + if permute_lhs is not None: + lhs = lhs.permute(permute_lhs) + lhs = lhs.clone(memory_format=torch.contiguous_format) + if permute_rhs is not None: + rhs = rhs.permute(permute_rhs) + rhs = rhs.clone(memory_format=torch.contiguous_format) + + if xpose_args == _XposeArgs.lhs: + lhs_recv = comm.comm_2d_trans.enqueue_to_dispatch(lhs) + rhs_recv = rhs + rhs = torch.empty_like(rhs_recv) + elif xpose_args == _XposeArgs.rhs: + rhs_recv = comm.comm_2d_trans.enqueue_to_dispatch(rhs) + lhs_recv = lhs + lhs = torch.empty_like(lhs_recv) + elif xpose_args is None: + lhs_recv = lhs + lhs = torch.empty_like(lhs_recv) + rhs_recv = rhs + rhs = torch.empty_like(rhs_recv) + else: + raise ValueError(f"Invalid xpose_args: {xpose_args}") + + i_ready = 0 + i_recv = i_ready ^ 1 + lhs_buffer = [lhs_recv, lhs] + rhs_buffer = [rhs_recv, rhs] + + if xpose_args is not None: + comm.comm_2d_trans.wait_until_finished() + + lhs_buffer[i_recv] = comm.comm_row_init.enqueue_to_dispatch( + lhs_buffer[i_ready], lhs_buffer[i_recv] + ) + rhs_buffer[i_recv] = comm.comm_col_init.enqueue_to_dispatch( + rhs_buffer[i_ready], rhs_buffer[i_recv] + ) + + i_ready ^= 1 + i_recv ^= 1 + + # Accumulator initialised from the first matmul rather than a pre-allocated + # zeros buffer: avoids one large per-call allocation (~370 MB bf16) and is + # more correct — the matmul output is (B,D,n,m), whereas zeros_like(lhs) is + # (B,D,n,k) and only matched when the local shard was square (n==m==k). + out: torch.Tensor | None = None + + comm.comm_row_init.wait_until_finished() + comm.comm_col_init.wait_until_finished() + + for k_step in range(comm.group_layout.shape[1]): + lhs_ready = lhs_buffer[i_ready] + rhs_ready = rhs_buffer[i_ready] + if k_step < comm.group_layout.shape[1] - 1: + lhs_buffer[i_recv] = comm.comm_row.enqueue_to_dispatch( + lhs_ready, lhs_buffer[i_recv] + ) + rhs_buffer[i_recv] = comm.comm_col.enqueue_to_dispatch( + rhs_ready, rhs_buffer[i_recv] + ) + prod = torch.matmul(lhs_ready, rhs_ready) + out = prod if out is None else out + prod + if k_step < comm.group_layout.shape[1] - 1: + comm.comm_row.wait_until_finished() + comm.comm_col.wait_until_finished() + i_ready ^= 1 + i_recv ^= 1 + + if permute_out is not None: + out = out.permute(permute_out) # ty:ignore[unresolved-attribute] + return out # ty:ignore[invalid-return-type] + + +# --------------------------------------------------------------------------- +# Autograd function for distributed triangle multiplication +# --------------------------------------------------------------------------- + + +class _TriangleMultiplicativeBlockImpl(torch.autograd.Function): + """Distributed triangle multiplication autograd function. + + Forward + ------- + Input x has shape (B, N_local_row, N_local_col, 2*d) and is already: + x = signal * sigmoid(gate_logits) * visibility_mask (pre-combined inner gate + mask) + + The function splits x into a = x[..., :d] and b = x[..., d:], then computes + the triangle multiplication using ring communication. + + Backward + -------- + Propagates gradients back through the distributed BMM. + """ + + @staticmethod + @torch.amp.custom_fwd(device_type="cuda") + def forward(ctx, x: DTensor, comm: Ring2DComm, direction: _Direction) -> DTensor: + if not isinstance(x, DTensor): + raise TypeError(f"x must be DTensor, got {type(x)}") + if x.ndim != 4: + raise ValueError(f"x must be 4D, got {x.ndim}D") + if x.shape[-1] % 2 != 0: + raise ValueError(f"Last dim of x must be even, got {x.shape[-1]}") + placements = x.placements + if placements != (Shard(0), Shard(1), Shard(2)): + raise ValueError( + f"x must have placements (Shard(0), Shard(1), Shard(2)), got {placements}" + ) + + x_local = x.to_local() + a_local, b_local = torch.chunk(x_local, 2, dim=-1) + a_local = a_local.clone(memory_format=torch.contiguous_format) + b_local = b_local.clone(memory_format=torch.contiguous_format) + + if x.requires_grad: + ctx.save_for_backward(a_local, b_local) + ctx.comm = comm + ctx.shape_x = x.shape + ctx.stride_x = x.stride() + ctx.placements = placements + ctx.device_mesh = x.device_mesh + ctx.direction = direction + + if direction == _Direction.Outgoing: + # contracted[b,n,m,d] = sum_k a[b,n,k,d] * b[b,m,k,d] + # a: (B,n,k,d) → permute to (B,D,n,k) + # b: (B,m,k,d) → permute to (B,D,k,m) (needs transpose of (B,D,n,k)) + permute_lhs = (0, 3, 1, 2) + permute_rhs = (0, 3, 2, 1) + permute_out = (0, 2, 3, 1) + xpose_args = _XposeArgs.rhs + elif direction == _Direction.Incoming: + # contracted[b,n,m,d] = sum_k a[b,k,n,d] * b[b,k,m,d] + # a: (B,k,n,d) → permute to (B,D,n,k) => need (0,3,2,1) + # b: (B,k,m,d) → permute to (B,D,k,m) => need (0,3,1,2) + permute_lhs = (0, 3, 2, 1) + permute_rhs = (0, 3, 1, 2) + permute_out = (0, 2, 3, 1) + xpose_args = _XposeArgs.lhs + else: + raise ValueError(f"Invalid direction: {direction}") + + out_local = _distributed_bmm( + a_local, + b_local, + comm, + permute_lhs=permute_lhs, + permute_rhs=permute_rhs, + permute_out=permute_out, + xpose_args=xpose_args, + ).contiguous() + + # Output has shape (B, N_local_row, N_local_col, d) + shape_output = x.shape[:-1] + (out_local.shape[-1],) + stride_output = update_exhaustive_strides(x.shape, x.stride(), shape_output) + return DTensor.from_local( + out_local, + device_mesh=x.device_mesh, + placements=placements, + shape=shape_output, + stride=stride_output, + ) + + @staticmethod + @torch.amp.custom_bwd(device_type="cuda") + def backward(ctx, d_out: DTensor) -> Tuple[DTensor, None, None]: + if not isinstance(d_out, DTensor): + raise TypeError(f"d_out must be DTensor, got {type(d_out)}") + + a, b = ctx.saved_tensors + comm = ctx.comm + direction = ctx.direction + d_out_local = d_out.to_local().to(dtype=a.dtype) + + if direction == _Direction.Outgoing: + # d_a: d_out[b,n,m,d] * b[b,m,k,d] -> d_a[b,n,k,d] + # permute: d_out (B,n,m,D)->(B,D,n,m); b (B,m,k,D)->(B,D,m,k); out (B,D,n,k)->(B,n,k,D) + lhs_da, rhs_da = d_out_local, b + permute_lhs_da = (0, 3, 1, 2) + permute_rhs_da = (0, 3, 1, 2) + permute_out_da = (0, 2, 3, 1) + xpose_da = None + + # d_b: d_out[b,n,m,d] * a[b,n,k,d] -> d_b[b,m,k,d] + # permute: d_out (B,n,m,D)->(B,D,m,n); a (B,n,k,D)->(B,D,n,k); out (B,D,m,k)->(B,m,k,D) + lhs_db, rhs_db = d_out_local, a + permute_lhs_db = (0, 3, 2, 1) + permute_rhs_db = (0, 3, 1, 2) + permute_out_db = (0, 2, 3, 1) + xpose_db = _XposeArgs.lhs + + elif direction == _Direction.Incoming: + # d_a: d_out[b,n,m,d] * b[b,k,m,d] -> d_a[b,k,n,d] + lhs_da, rhs_da = b, d_out_local + permute_lhs_da = (0, 3, 1, 2) + permute_rhs_da = (0, 3, 2, 1) + permute_out_da = (0, 2, 3, 1) + xpose_da = _XposeArgs.rhs + + # d_b: d_out[b,n,m,d] * a[b,k,n,d] -> d_b[b,k,m,d] + lhs_db, rhs_db = a, d_out_local + permute_lhs_db = (0, 3, 1, 2) + permute_rhs_db = (0, 3, 1, 2) + permute_out_db = (0, 2, 3, 1) + xpose_db = None + else: + raise ValueError(f"Invalid direction: {direction}") + + da_local = _distributed_bmm( + lhs_da, + rhs_da, + comm, + permute_lhs=permute_lhs_da, + permute_rhs=permute_rhs_da, + permute_out=permute_out_da, + xpose_args=xpose_da, + ).contiguous() + db_local = _distributed_bmm( + lhs_db, + rhs_db, + comm, + permute_lhs=permute_lhs_db, + permute_rhs=permute_rhs_db, + permute_out=permute_out_db, + xpose_args=xpose_db, + ).contiguous() + + dab_local = torch.cat([da_local, db_local], dim=-1) + dx = DTensor.from_local( + dab_local, + device_mesh=ctx.device_mesh, + placements=ctx.placements, + shape=ctx.shape_x, + stride=ctx.stride_x, + ) + return dx, None, None + + +# --------------------------------------------------------------------------- +# Public distributed module +# --------------------------------------------------------------------------- + + +class TriangleMultiplicativeBlockDistributed(nn.Module): + """Distributed TriangleMultiplicativeBlock for ESMFold2's pair representation. + + Replaces the serial layer in a model by: + 1. Replacing all parameters with DTensor replicated parameters. + 2. Implementing the forward pass using distributed ring-communication BMM. + + The pair tensor z is expected as a DTensor with placements + (Shard(0), Shard(1), Shard(2)) on a 3D mesh (dp, cp_axis_0, cp_axis_1). + + Parameters + ---------- + layer: + The serial TriangleMultiplicativeBlock to distribute. + device_mesh: + The device mesh (should be the subgroups mesh: dp × cp_axis_0 × cp_axis_1). + comm: + Ring2DComm for the CP group. + """ + + def __init__( + self, + layer: SerialTriangleMultiplicativeBlock, + device_mesh: DeviceMesh, + comm: Ring2DComm, + ) -> None: + super().__init__() + if not isinstance(layer, SerialTriangleMultiplicativeBlock): + raise TypeError( + f"layer must be TriangleMultiplicativeBlock, got {type(layer).__name__}" + ) + self.device_mesh = device_mesh + self.ring_comm = comm + + self._direction = ( + _Direction.Outgoing if layer.flow == "outgoing" else _Direction.Incoming + ) + self._latent_channels = layer.latent_channels + + self.norm_start = LayerNormParamsReplicated(layer.norm_start, device_mesh) + self.norm_mix = LayerNormParamsReplicated(layer.norm_mix, device_mesh) + self.proj_bundle = LinearParamsReplicated(layer.proj_bundle, device_mesh) + self.proj_emit = LinearParamsReplicated(layer.proj_emit, device_mesh) + self.proj_gate = LinearParamsReplicated(layer.proj_gate, device_mesh) + + def forward(self, pair: DTensor, mask: DTensor | None = None) -> DTensor: + """Forward pass. + + Parameters + ---------- + pair: + Pair tensor (B, N, N, d_pair) as DTensor(Shard(0), Shard(1), Shard(2)). + mask: + Visibility mask (B, N, N) as DTensor(Shard(0), Shard(1), Shard(2)). + If None, no masking is applied. + + Returns + ------- + DTensor of same shape and placements as pair. + """ + # Re-align ranks before the ring comm to prevent NCCL P2P spin-wait + # (see _CP_RANK_SYNC above). The cuda.synchronize() throttles the CPU so + # it can't race ahead enqueuing unbounded async P2P + large allocations + # (the dominant cause of the pile-up); the barrier then aligns ranks + # cross-process. Correctness-neutral (no tensors touched). + if _CP_RANK_SYNC and dist.is_initialized(): + torch.cuda.synchronize() + dist.barrier(group=self.ring_comm.group_2d) + + # 1. Layer-normalise input + normalized = self.norm_start(pair) + + # 2. Compute bundled projection: (d → 4*d) + bundled = self.proj_bundle(normalized) + + # 3. Split into signal (2*d) and inner gate logits (2*d); apply inner gate + latent = self._latent_channels + signal, gate_logits = bundled.split(2 * latent, dim=-1) # DTensor ops + x = signal * gate_logits.sigmoid() + + # 4. Apply visibility mask + if mask is not None: + x = x * mask.unsqueeze(-1) + + # 5. Distributed triangle multiplication + contracted = _TriangleMultiplicativeBlockImpl.apply( + x, self.ring_comm, self._direction + ) + + # 6. Norm + output projection + out = self.proj_emit(self.norm_mix(contracted)) + + # 7. Output gate (applied to pre-norm input) + output_gate = self.proj_gate(normalized).sigmoid() + return out * output_gate diff --git a/esm/models/esmfold2/distributed/msa_wrapper.py b/esm/models/esmfold2/distributed/msa_wrapper.py new file mode 100644 index 00000000..16378899 --- /dev/null +++ b/esm/models/esmfold2/distributed/msa_wrapper.py @@ -0,0 +1,354 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: MIT +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. + +"""End-to-end CP runtime for ESMFold2's MSAEncoder. + +``MSAEncoderCPWrapper`` is a drop-in replacement for the serial ``MSAEncoder`` +(same plain-tensor in/out signature) that shards the MSA encoder's pair-space +work across the 2D CP grid, mirroring ``TrunkCPWrapper`` for the folding trunk. +Without it, the serial MSA encoder materialises the full L×L pair on every rank +(the dominant cost for large complexes) before the trunk even runs. +""" + +from math import lcm + +import torch +import torch.nn as nn +import torch.nn.functional as F +from torch.distributed.tensor import DTensor, Replicate, Shard, distribute_tensor + + +class MSAEncoderCPWrapper(nn.Module): + """Drop-in replacement for ``MSAEncoder`` that runs distributed. + + The small per-residue embedding (``embed`` + ``project_inputs``) runs serial + per rank (cheap), then m / pair / masks are zero-padded to a multiple of the + CP shard factor, distributed as DTensors, processed by + ``MSAEncoderDistributed``, and the gathered pair is sliced back. + """ + + def __init__( + self, + serial_encoder: nn.Module, + dist_manager, + comm: str = "gather", + bf16: bool = True, + ) -> None: + super().__init__() + from esm.models.esmfold2.distributed.manager import DistributedManager + from esm.models.esmfold2.distributed.model.layers.msa_encoder import ( + MSAEncoderDistributed, + ) + from esm.models.esmfold2.model import MSAEncoder as SerialMSAEncoder + + if not isinstance(serial_encoder, SerialMSAEncoder): + raise TypeError(f"expected MSAEncoder, got {type(serial_encoder).__name__}") + if not isinstance(dist_manager, DistributedManager): + raise TypeError( + f"expected DistributedManager, got {type(dist_manager).__name__}" + ) + + # The distributed path ignores the serial chunk-size knob. + serial_encoder.set_chunk_size(None) + + # The embedding projections stay serial (tiny); reference them directly. + self.embed = serial_encoder.embed + self.project_inputs = serial_encoder.project_inputs + self.dist_encoder = MSAEncoderDistributed( + serial_encoder, dist_manager, comm=comm + ) + + # bf16 MSA encoder: the recycle pair is fp32 and the MSA encoder otherwise + # runs fp32 (its distributed tri-mul's custom_fwd disables autocast), which + # made it the largest recycle-loop peak contributor (~16 GB @ L1422). Cast + # the heavy dist_encoder to bf16 (mirrors TrunkCPWrapper's bf16=True, which + # was validated quality-neutral); the pair is cast to bf16 at the boundary + # in forward[_sharded] and the output cast back to the caller's dtype. The + # tiny embed/project_inputs are left fp32 (shared with the serial model; + # m is cast to bf16 after embedding). + self._bf16 = bf16 + if self._bf16: + self.dist_encoder = self.dist_encoder.to(torch.bfloat16) + + self.dist_manager = dist_manager + self.device_mesh = dist_manager.device_mesh_subgroups + # device_mesh is (dp, cp_axis_0, cp_axis_1) + self.cp_axis_0 = self.device_mesh.size(1) + self.cp_axis_1 = self.device_mesh.size(2) + self.shard_factor = lcm(self.cp_axis_0, self.cp_axis_1) + + def set_chunk_size(self, _chunk_size: int | None) -> None: + return + + def set_kernel_backend(self, _backend: str | None) -> None: + return + + def forward_sharded( + self, + x_pair_dt: DTensor, + x_inputs: torch.Tensor, + msa_oh: torch.Tensor, + has_deletion: torch.Tensor, + deletion_value: torch.Tensor, + msa_attention_mask: torch.Tensor, + ) -> DTensor: + """Run the distributed MSA encoder on an already-sharded pair DTensor. + + ``x_pair_dt`` must already be padded to a multiple of + ``self.shard_factor`` and distributed ``[Shard(0), Shard(1), Shard(2)]`` + on ``self.device_mesh``. The MSA inputs (``m`` and the masks) are still + embedded serially per rank (cheap), padded to match the pair's padded + length, and distributed here. Returns the pair-space output as a DTensor + — **no** ``full_tensor()`` gather. + + This is the entry point a CP orchestrator uses to keep the pair sharded + across the MSA→trunk boundary; the plain-tensor :meth:`forward` is a thin + adapter around it. + """ + # Serial embedding (matches MSAEncoder.forward), per rank on full tensors. + m_feat = torch.cat( + [msa_oh, has_deletion.unsqueeze(-1), deletion_value.unsqueeze(-1)], dim=-1 + ) + m = self.embed(m_feat) + self.project_inputs(x_inputs).unsqueeze(2) + + # The incoming pair is already padded; pad m / masks to match it so the + # token (L) axis lines up shard-for-shard. + n = m.shape[1] + n_padded = x_pair_dt.shape[1] + pad = n_padded - n + if pad < 0: + raise ValueError(f"sharded pair length {n_padded} < MSA token length {n}") + if pad: + m = F.pad(m, (0, 0, 0, 0, 0, pad)) # pad L (dim 1) + msa_attention_mask = F.pad(msa_attention_mask, (0, 0, 0, pad)) # pad L + + # Cast the heavy inputs to bf16 so the (bf16) dist_encoder runs bf16 end to + # end; restore the caller's dtype on the output. + orig_dtype = x_pair_dt.dtype + if self._bf16: + m = m.to(torch.bfloat16) + x_pair_dt = x_pair_dt.to(torch.bfloat16) # ty:ignore[invalid-assignment] + msa_attention_mask = msa_attention_mask.to(torch.bfloat16) + + mesh = self.device_mesh + m_dt = distribute_tensor( + m.contiguous(), mesh, [Shard(0), Shard(1), Replicate()] + ) + msa_mask_dt = distribute_tensor( + msa_attention_mask.to(m.dtype).contiguous(), + mesh, + [Shard(0), Shard(1), Replicate()], + ) + # Pair attention mask built SHARDED from the (padded) token mask — the + # per-block outer product, never a full [B, L, L]. Padded rows/cols are + # zero, so they contribute nothing. + from esm.models.esmfold2.distributed.model.layers.pair_init import ( + build_sharded_pair_mask, + ) + + pair_mask_dt = build_sharded_pair_mask(msa_attention_mask[:, :, 0], mesh).to( + m.dtype + ) + + out_dt = self.dist_encoder(m_dt, x_pair_dt, msa_mask_dt, pair_mask_dt) + if self._bf16: + out_dt = out_dt.to(orig_dtype) + return out_dt + + def forward( + self, + x_pair: torch.Tensor, + x_inputs: torch.Tensor, + msa_oh: torch.Tensor, + has_deletion: torch.Tensor, + deletion_value: torch.Tensor, + msa_attention_mask: torch.Tensor, + ) -> torch.Tensor: + n = x_pair.shape[1] + pad = (self.shard_factor - n % self.shard_factor) % self.shard_factor + if pad: + x_pair = F.pad(x_pair, (0, 0, 0, pad, 0, pad)) # pad both L dims + + pair_dt = distribute_tensor( + x_pair.contiguous(), self.device_mesh, [Shard(0), Shard(1), Shard(2)] + ) + + out_dt = self.forward_sharded( + pair_dt, x_inputs, msa_oh, has_deletion, deletion_value, msa_attention_mask + ) + out = out_dt.full_tensor() + if pad: + out = out[:, :n, :n, :] + return out + + +def wrap_model_with_cp_msa_encoder( + model: nn.Module, dist_manager, comm: str = "gather", bf16: bool = True +) -> list[str]: + """Replace every ``MSAEncoder`` submodule with ``MSAEncoderCPWrapper``. + + Walks ``model.named_modules()`` and rebinds each attribute that points at a + serial ``MSAEncoder``. Returns the list of replaced submodule paths. Models + without an MSA encoder (``model.msa_encoder is None``) are left untouched. + + ``comm`` selects the MSA pair-weighted-averaging communication strategy: + ``"gather"`` (default, bit-exact all-gather) or ``"ring"`` (boltz-style + online-softmax ring, for large grids / deep MSAs). + """ + mesh = dist_manager.device_mesh_subgroups + cp0, cp1 = mesh.size(1), mesh.size(2) + if cp0 != cp1: + raise ValueError( + f"CP grid must be square (cp_axis_0 == cp_axis_1), got {cp0}×{cp1}" + ) + + from esm.models.esmfold2.model import MSAEncoder as SerialMSAEncoder + + targets: list[tuple[str, nn.Module, str, nn.Module]] = [] + for parent_name, parent in model.named_modules(): + for child_name, child in parent.named_children(): + if child is None: + continue + if isinstance(child, SerialMSAEncoder): + full = f"{parent_name}.{child_name}" if parent_name else child_name + targets.append((full, parent, child_name, child)) + + replaced: list[str] = [] + for full, parent, child_name, child in targets: + wrapped = MSAEncoderCPWrapper(child, dist_manager, comm=comm, bf16=bf16).to( + device=dist_manager.device, dtype=next(child.parameters()).dtype + ) + setattr(parent, child_name, wrapped) + replaced.append(full) + return replaced + + +def wrap_model_with_cp( + model: nn.Module, + dist_manager, + comm: str = "gather", + bf16: bool = True, + offload_esmc: bool = True, + wrap_structure: bool = True, + tp_esmc: bool = False, +) -> list[str]: + """Wrap both the folding trunk(s) and the MSA encoder for 2D CP. + + Convenience over calling ``wrap_model_with_cp_trunks`` and + ``wrap_model_with_cp_msa_encoder`` separately. Returns the combined list of + replaced submodule paths. + + ``comm`` selects the MSA pair-weighted-averaging communication strategy: + ``"gather"`` (default, bit-exact all-gather) or ``"ring"`` (boltz-style + online-softmax ring). It affects only the MSA encoder; the folding trunk + and the OuterProductMean (whose contracted MSA-depth axis is replicated) + are unaffected. + + ``bf16`` (default True): run the distributed trunk in bf16 — quality-neutral, + ~1.3-1.6x faster, lower peak VRAM (see ``wrap_model_with_cp_trunks``). + + ``offload_esmc`` (default True): offload the ESM-C LM (~12 GB) to CPU after + its one-shot use, freeing it for the trunk/diffusion. Sets + ``model._offload_esmc``. Validated with bf16: 2.14x end-to-end + 18.5 GB lower + peak vs fp32, pLDDT unchanged. Set False for high-throughput tiny-fold or + concurrent-fold use (the per-fold ~12 GB CPU<->GPU transfer isn't worth it). + """ + from esm.models.esmfold2.distributed.utils import wrap_model_with_cp_trunks + + model._offload_esmc = offload_esmc # ty:ignore[unresolved-attribute] + replaced = wrap_model_with_cp_trunks(model, dist_manager, bf16=bf16) + replaced += wrap_model_with_cp_msa_encoder( + model, dist_manager, comm=comm, bf16=bf16 + ) + + # Install the CP recycle orchestrator so the pair stays sharded across the + # recycle loop (no per-iteration full_tensor() round-trips). The model's + # _run_one_loop delegates to this when present (CP-agnostic seam). Only the + # main model owns the loop (parcae_input_norm + _run_one_loop); guard so + # wrapping a bare submodule is a no-op. + if hasattr(model, "_run_one_loop") and hasattr(model, "parcae_input_norm"): + from esm.models.esmfold2.distributed.recycle import CPRecycleEngine + + model._cp_recycle_engine = CPRecycleEngine(model, dist_manager) # ty:ignore[unresolved-attribute] + + # Sharded pair-init builder: constructs z_init / rel_pos / token_bonds as + # 2-D-sharded DTensors so the full L×L pair is never resident on a rank + # during the init phase (the dominant per-rank peak that made CP OOM at + # short L). The model's forward delegates via the _cp_pair_init seam. + from esm.models.esmfold2.distributed.model.layers.pair_init import ( + PairInitDistributed, + ) + + model._cp_pair_init = PairInitDistributed(model, dist_manager) + + # Install the distributed LM->pair builder (#14) so lm_z is produced as a + # sharded DTensor instead of a full L×L tensor on every rank (the binding + # per-rank peak per the T0 phase sweep). The model's forward delegates via + # the _cp_language_model seam when present. + lm = getattr(model, "language_model", None) + if lm is not None: + from esm.models.esmfold2.distributed.model.layers.single_to_pair import ( + LanguageModelShimDistributed, + ) + + model._cp_language_model = LanguageModelShimDistributed(lm, dist_manager) + + # Distributed confidence head (#6): consumes the sharded pair from the CP + # tail (no full-L×L z re-gather). Its nested FoldingTrunk is already + # CP-wrapped by wrap_model_with_cp_trunks above. + conf = getattr(model, "confidence_head", None) + if conf is not None: + from esm.models.esmfold2.distributed.confidence_wrapper import ( + ConfidenceHeadCPWrapper, + ) + + model._cp_confidence_head = ConfidenceHeadCPWrapper(conf, dist_manager) + + # Distributed distogram head: distogram_head(z + zᵀ) off the sharded pair + # (one transpose + local Linear), gathering only the bins-channel logits. + disto = getattr(model, "distogram_head", None) + if disto is not None: + from esm.models.esmfold2.distributed.distogram_wrapper import ( + DistogramHeadCPWrapper, + ) + + model._cp_distogram_head = DistogramHeadCPWrapper(disto, dist_manager) + + # Distribute the diffusion structure head (conditioned-z + token attention + # sharded; atom encoder/decoder replicated). The serial sample() loop keeps + # driving the (now wrapped) diffusion_module. + if wrap_structure: + from esm.models.esmfold2.distributed.structure_wrapper import ( + wrap_model_with_cp_structure_head, + ) + + replaced += wrap_model_with_cp_structure_head(model, dist_manager) + + # Tensor-parallel ESM-C (the dominant replicated floor). Phase 1: shard the + # SwiGLU MLP (~67% of ESM-C weights) over the CP ranks via TE-native TP. + if tp_esmc: + from esm.models.esmfold2.distributed.esmc_tp import tp_shard_esmc_mlp + + n_tp = tp_shard_esmc_mlp(model, dist_manager) + if n_tp: + replaced.append(f"esmc.transformer.blocks[*].ffn (TP x{n_tp})") + + return replaced diff --git a/esm/models/esmfold2/distributed/recycle.py b/esm/models/esmfold2/distributed/recycle.py new file mode 100644 index 00000000..fbbded66 --- /dev/null +++ b/esm/models/esmfold2/distributed/recycle.py @@ -0,0 +1,358 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: MIT +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. + +"""Context-parallel orchestrator for ESMFold2's recycle loop. + +``CPRecycleEngine`` owns the sharded version of ``EsmFold2Model._run_one_loop``. +The model file stays CP-agnostic: ``_run_one_loop`` simply delegates to an +installed engine (see the ``_cp_recycle_engine`` seam there), which is set by +``wrap_model_with_cp``. + +Why an orchestrator: the serial loop materialises the full ``L×L`` pair on every +rank at *each* module boundary (MSA encoder, LM encoder, trunk) via the wrappers' +``forward()`` (``distribute_tensor`` → … → ``full_tensor``). Keeping the running +pair sharded across the whole loop — calling each wrapper's ``forward_sharded`` +and running the parcae combine on local shards — removes those per-iteration +gathers. The pair is gathered exactly once, when the loop returns (until the +diffusion / confidence heads are also distributed, at which point even that +final gather can go). + +All per-iteration tensors stay DTensors with placements +``(Shard(0), Shard(1), Shard(2))``; the channel-dim parcae combine is computed on +``to_local()`` shards (channel is replicated, so it needs no communication) and +re-wrapped — mirroring ``LinearParamsReplicated.forward``. Validated bit-exact +vs. the serial path on a 2×2 grid. +""" + +from math import lcm + +import torch +import torch.nn.functional as F +from torch.distributed.tensor import DTensor, Shard, distribute_tensor + +from esm.models.esmfold2.distributed.model.layers.layernorm import ( + LayerNormParamsReplicated, +) +from esm.models.esmfold2.layers import NUM_RES_TYPES, maybe_subsample_msa + +_PLACEMENTS = [Shard(0), Shard(1), Shard(2)] + + +class CPRecycleEngine: + """Sharded drop-in for ``EsmFold2Model._run_one_loop``. + + Parameters + ---------- + model: + The ESMFold2 model being wrapped. Used to read ``parcae_input_norm`` (to + build a replicated-param copy) at construction; the (already CP-wrapped) + ``folding_trunk`` / ``msa_encoder`` / ``lm_encoder`` and config are read + per call so the engine always sees the current submodules. + dist_manager: + The DistributedManager providing the CP device mesh. + """ + + def __init__(self, model, dist_manager) -> None: + self.dist_manager = dist_manager + self.device_mesh = dist_manager.device_mesh_subgroups + # device_mesh is (dp, cp_axis_0, cp_axis_1) + self.cp_axis_0 = self.device_mesh.size(1) + self.cp_axis_1 = self.device_mesh.size(2) + self.shard_factor = lcm(self.cp_axis_0, self.cp_axis_1) + + # Replicated-param copy of the channel-dim LayerNorm: a plain nn.LayerNorm + # cannot run on a DTensor (mixed plain-tensor/DTensor params raise), so the + # parcae input norm is wrapped to hold replicated DTensor params. + self.parcae_input_norm = LayerNormParamsReplicated( + model.parcae_input_norm, self.device_mesh + ) + + def parcae_finish(self, model, z_dt: DTensor, pair_mask: torch.Tensor) -> DTensor: + """Sharded post-recycle parcae tail, keeping the pair sharded. + + Mirrors the serial ``z = parcae_readout(z); z = parcae_coda(z, mask)``: + ``parcae_readout`` is a channel-wise Linear run on the local shard + (weight replicated, no token mixing); ``parcae_coda`` is a CP-wrapped + ``FoldingTrunk`` driven via ``forward_sharded`` (no gather/re-distribute). + ``z_dt`` is the 2-D-sharded (padded) pair from ``run_loop(gather=False)``; + ``pair_mask`` is either the full (unpadded) token pair mask or an already + padded + sharded DTensor (from ``PairInitDistributed``). Returns sharded z.""" + mesh = self.device_mesh + + # parcae_readout: channel Linear on the local shard. + z_local = z_dt.to_local() + z_local = F.linear(z_local, model.parcae_readout.weight) + z_dt = DTensor.from_local(z_local.contiguous(), mesh, _PLACEMENTS) + + # parcae_coda: CP-wrapped trunk → forward_sharded (needs a padded, sharded mask). + if isinstance(pair_mask, DTensor): + pair_mask_dt = pair_mask.to(z_dt.dtype) + else: + n = pair_mask.shape[-1] + pad = (self.shard_factor - n % self.shard_factor) % self.shard_factor + pm = F.pad(pair_mask, (0, pad, 0, pad)) if pad else pair_mask + pair_mask_dt = distribute_tensor( + pm.to(z_dt.dtype).contiguous(), mesh, _PLACEMENTS + ) + return model.parcae_coda.forward_sharded(z_dt, pair_attention_mask=pair_mask_dt) + + def _prepare_msa_iter(self, model, msa_inputs, tok_mask): + """Per-iteration MSA feature prep — a faithful copy of the serial block + in ``EsmFold2Model._run_one_loop`` (kept here so the engine doesn't + mutate the serial inference path). Returns plain tensors; sharding + happens inside ``MSAEncoderCPWrapper.forward_sharded``.""" + msa_i, mask_i, hd_i, dv_i = maybe_subsample_msa( + msa_inputs["msa"], + msa_inputs["msa_attention_mask"], + msa_inputs["has_deletion"], + msa_inputs["deletion_value"], + max_depth=msa_inputs["max_depth"], + enabled=msa_inputs["subsample_enabled"], + ) + B_msa, M, L_msa = msa_i.shape + msa_oh = F.one_hot( + msa_i.permute(0, 2, 1).long(), num_classes=NUM_RES_TYPES + ).float() + msa_attn = ( + mask_i.permute(0, 2, 1).float() + if mask_i is not None + else tok_mask[:, :, None].expand(-1, -1, M).float() + ) + # Bias-free MSAEncoder.embed requires zeroed padding. + msa_oh = msa_oh * msa_attn.unsqueeze(-1) + hd = ( + hd_i.permute(0, 2, 1).float() + if hd_i is not None + else torch.zeros(B_msa, L_msa, M, device=msa_i.device) + ) + dv = ( + dv_i.permute(0, 2, 1).float() + if dv_i is not None + else torch.zeros(B_msa, L_msa, M, device=msa_i.device) + ) + return msa_oh, msa_attn, hd, dv, msa_inputs["x_inputs"] + + def run_loop( + self, + model, + *, + z: torch.Tensor, + z_init: torch.Tensor, + lm_z: torch.Tensor | None, + msa_inputs: dict | None, + pair_mask: torch.Tensor, + a: torch.Tensor, + b_mat: torch.Tensor, + tok_mask: torch.Tensor, + total_steps: int, + gather: bool = True, + n_orig: int | None = None, + ): + """Sharded equivalent of ``_run_one_loop``. + + ``z`` / ``z_init`` may arrive either as full plain tensors (distributed + here, the serial-fallback contract) or as already-padded, already-sharded + ``(Shard0, Shard1, Shard2)`` DTensors built by ``PairInitDistributed`` — in + which case the full L×L pair is never resident on any rank and ``n_orig`` + (the pre-pad token count) must be supplied so the returned length is the + original, not the padded, count. + + With ``gather=True`` (default) returns the gathered full tensor (drop-in + for the serial loop). With ``gather=False`` returns + ``(z_dt, n_orig)`` — the **sharded** ``(Shard0, Shard1, Shard2)`` pair + DTensor (still padded to the shard factor) and the original token length + ``n_orig`` — so a CP tail can keep the pair sharded through + parcae / distogram / structure / confidence instead of re-gathering. The + padded rows/cols carry no signal (masked); the caller slices to + ``n_orig`` after any later gather.""" + lm_cfg = model.config.lm_encoder + lm_dropout_p = getattr(lm_cfg, "lm_dropout", 0.0) + # Per-loop LM dropout: the serial path applies a fresh F.dropout over the + # *full* lm_z each iteration (training=True even under eval). We reproduce + # it the same way — dropout on the full (unpadded) lm_z, then pad + + # distribute — so with the run's synchronized RNG (all ranks share the + # seed and draw in the same order) every rank gets the identical mask, and + # it matches the serial / gather-fallback path. Drawing the mask on a + # per-rank shard instead would desync the ranks. + per_loop_lm_dropout = ( + lm_z is not None + and getattr(lm_cfg, "per_loop_lm_dropout", False) + and lm_dropout_p > 0.0 + ) + + mesh = self.device_mesh + loop_dtype = z.dtype + # z / z_init may be pre-sharded DTensors (PairInitDistributed) or full + # tensors (serial fallback). For the DTensor case N is the ORIGINAL token + # count (passed as n_orig); z.shape[1] would be the padded length. + z_is_dt = isinstance(z, DTensor) + z_init_is_dt = isinstance(z_init, DTensor) + if z_is_dt: + assert n_orig is not None, "n_orig required when z is a pre-sharded DTensor" + N = n_orig + else: + N = z.shape[1] + pad = (self.shard_factor - N % self.shard_factor) % self.shard_factor + + def _pad_pair(t: torch.Tensor) -> torch.Tensor: # (B, L, L, C) + return F.pad(t, (0, 0, 0, pad, 0, pad)) if pad else t + + def _pad_mask(t: torch.Tensor) -> torch.Tensor: # (B, L, L) + return F.pad(t, (0, pad, 0, pad)) if pad else t + + def _distribute_lm(lm_full: torch.Tensor) -> DTensor: + return distribute_tensor( + _pad_pair(lm_full.to(z_init.dtype)).contiguous(), mesh, _PLACEMENTS + ) + + # Distribute the persistent loop tensors ONCE (vs. per-boundary in serial). + # Pre-sharded DTensors (already padded to shard_factor) are used as-is — + # no full L×L is ever materialised on a rank. + z_dt = ( + z + if z_is_dt + else distribute_tensor(_pad_pair(z).contiguous(), mesh, _PLACEMENTS) + ) + z_init_dt = ( + z_init + if z_init_is_dt + else distribute_tensor(_pad_pair(z_init).contiguous(), mesh, _PLACEMENTS) + ) + # pair_mask may arrive as a pre-sharded (padded) DTensor (PairInitDistributed) + # or a full tensor (serial fallback). + if isinstance(pair_mask, DTensor): + pair_mask_dt = pair_mask.to(loop_dtype) + else: + pair_mask_dt = distribute_tensor( + _pad_mask(pair_mask).to(loop_dtype).contiguous(), mesh, _PLACEMENTS + ) + + # lm_z may arrive as a sharded (Shard0,Shard1,Shard2) DTensor (#14 — + # produced full-L×L-free by LanguageModelShimDistributed) or as a full + # tensor (serial-produced, e.g. the unit tests). Either way the loop needs a + # per-iteration sharded DTensor. + lm_is_dt = isinstance(lm_z, DTensor) + # Per-rank generator for SHARD-LOCAL dropout (sharded lm_z only): each rank + # drops its own tile with a distinct, fresh-per-loop mask. This is NOT + # bit-exact with the serial full-L×L dropout mask (torch's sequential philox + # can't be cheaply tiled) — a valid-but-different stochastic sample, like the + # ring MSA path; validated by end-to-end pLDDT/pTM parity. Crucially it uses + # a *separate* generator, leaving the global RNG untouched so the MSA + # subsample below stays synchronized ACROSS ranks (all ranks pick the same + # subset — required for the distributed MSA encoder to be consistent). + lm_local_static: torch.Tensor | None = None + lm_shape = lm_stride = None + lm_drop_gen: torch.Generator | None = None + lm_z_dt_static: DTensor | None = None + if lm_is_dt: + lm_cast = lm_z.to(z_init.dtype) + if per_loop_lm_dropout: + lm_local_static = lm_cast.to_local() # ty:ignore[unresolved-attribute] + lm_shape, lm_stride = lm_cast.shape, lm_cast.stride() + lm_drop_gen = torch.Generator(device=lm_local_static.device) + lm_drop_gen.manual_seed(0x5F14 + torch.distributed.get_rank()) + else: + lm_z_dt_static = lm_cast # ty:ignore[invalid-assignment] + elif lm_z is not None and not per_loop_lm_dropout: + lm_z_dt_static = _distribute_lm(lm_z) + + overwrite = model.config.msa_encoder.overwrite + + for _ in range(total_steps): + # (1) LM dropout FIRST, matching the serial loop's RNG-draw order + # (dropout before MSA subsample) so the synchronized RNG stays in + # lockstep with the serial / gather-fallback path. + if lm_z is None: + lm_z_i = None + elif not per_loop_lm_dropout: + lm_z_i = lm_z_dt_static + elif lm_is_dt: + # Shard-local dropout on the local tile (see generator note above): + # keep with prob (1-p), scale by 1/(1-p) — matches F.dropout. + keep = 1.0 - lm_dropout_p + mask = ( + torch.rand( + lm_local_static.shape, # ty:ignore[unresolved-attribute] + generator=lm_drop_gen, + device=lm_local_static.device, # ty:ignore[unresolved-attribute] + dtype=lm_local_static.dtype, # ty:ignore[unresolved-attribute] + ) + >= lm_dropout_p + ) + drop_local = lm_local_static * mask / keep # ty:ignore[unsupported-operator] + lm_z_i = DTensor.from_local( + drop_local.contiguous(), + device_mesh=mesh, + placements=_PLACEMENTS, + shape=lm_shape, + stride=lm_stride, + ) + else: + lm_i_full = F.dropout(lm_z, p=lm_dropout_p, training=True) + lm_z_i = _distribute_lm(lm_i_full) + + refined_lm_z: DTensor | None = None + if lm_z_i is not None and model.lm_encoder is not None: + refined_lm_z = model.lm_encoder.forward_sharded( + lm_z_i, pair_attention_mask=pair_mask_dt + ) + + z_inject = z_init_dt + if lm_z_i is not None and model.lm_encoder is None: + z_inject = z_inject + lm_z_i.to(z_inject.dtype) + + # (2) MSA prep (RNG draw) AFTER LM dropout, matching serial order. + if model.msa_encoder is not None and msa_inputs is not None: + msa_oh, msa_attn, hd, dv, x_inputs = self._prepare_msa_iter( + model, msa_inputs, tok_mask + ) + msa_pair = model.msa_encoder.forward_sharded( + z_inject, x_inputs, msa_oh, hd, dv, msa_attn + ).to(z_inject.dtype) + z_inject = msa_pair if overwrite else (z_inject + msa_pair) + + if refined_lm_z is not None: + z_inject = z_inject + refined_lm_z.to(z_inject.dtype) + + # parcae combine (channel-dim): norm via replicated params, then the + # affine recurrence on local shards (channel is replicated, so a*z and + # F.linear are purely local — raw F.linear on a DTensor would try to + # flatten the sharded token axis and fail). + inj_dt = self.parcae_input_norm(z_inject) + z_local = z_dt.to_local() + out_local = a * z_local + F.linear( + inj_dt.to_local().to(z_local.dtype), b_mat + ) + z_dt = DTensor.from_local(out_local, mesh, _PLACEMENTS) + + z_dt = model.folding_trunk.forward_sharded( + z_dt, pair_attention_mask=pair_mask_dt + ) + + if not gather: + # Keep the pair sharded for an end-to-end sharded tail; padded to the + # shard factor. Caller slices to N after any later gather. + return z_dt, N + + z_full = z_dt.full_tensor() + if pad: + z_full = z_full[:, :N, :N, :] + return z_full diff --git a/esm/models/esmfold2/distributed/structure_wrapper.py b/esm/models/esmfold2/distributed/structure_wrapper.py new file mode 100644 index 00000000..f5b01699 --- /dev/null +++ b/esm/models/esmfold2/distributed/structure_wrapper.py @@ -0,0 +1,270 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: MIT +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. + +"""Distributed diffusion structure head for ESMFold2. + +``DiffusionModuleCPWrapper`` is a drop-in for ``DiffusionModule`` that shards the +per-step diffusion working set: the conditioned pair ``z`` (via +``DiffusionConditioningDistributed``) and the token transformer's L×L attention +(via ``DiffusionTransformerDistributed``). The atom encoder/decoder and the +single-repr steps run **replicated** on full tensors (cheap 1-D / windowed work). +``wrap_model_with_cp_structure_head`` installs it by swapping +``structure_head.diffusion_module`` — the serial ``sample()`` loop (noise +schedule, augmentation) is untouched and keeps driving it. + +Scope: ``num_diffusion_samples == 1`` (multi-sample batch expansion of the pair +bias is a TODO). The trunk pair ``z_trunk`` still arrives **full** from the +recycle gather, so it remains a per-step floor; the conditioned ``z`` and the +attention are what get sharded here. Removing the ``z_trunk`` floor needs the +pair kept sharded across the recycle→parcae→structure-head boundary (follow-on). +Inference-only. +""" + +from math import lcm + +import torch +import torch.nn as nn +import torch.nn.functional as F +from torch.distributed.tensor import DTensor, Replicate, Shard, distribute_tensor + +_PAIR_PL = [Shard(0), Shard(1), Shard(2)] +_ROW_PL = [Shard(0), Shard(1), Replicate()] + + +class DiffusionModuleCPWrapper(nn.Module): + """Drop-in for ``DiffusionModule`` that distributes conditioning + token attn.""" + + def __init__(self, serial_module: nn.Module, dist_manager) -> None: + super().__init__() + from esm.models.esmfold2.distributed.model.layers.diffusion_conditioning import ( + DiffusionConditioningDistributed, + ) + from esm.models.esmfold2.distributed.model.layers.diffusion_transformer import ( + DiffusionTransformerDistributed, + ) + from esm.models.esmfold2.layers import DiffusionModule as SerialDiffusionModule + + if not isinstance(serial_module, SerialDiffusionModule): + raise TypeError( + f"expected DiffusionModule, got {type(serial_module).__name__}" + ) + self.serial = serial_module + self.device_mesh = dist_manager.device_mesh_subgroups + self.shard_factor = lcm(self.device_mesh.size(1), self.device_mesh.size(2)) + self.cond = DiffusionConditioningDistributed( + serial_module.conditioning, self.device_mesh + ) + self.token_transformer = DiffusionTransformerDistributed( + serial_module.token_transformer + ) + + # ``DiffusionStructureHead.set_kernel_backend`` forwards to its + # ``diffusion_module``, so this wrapper needs the same no-op hook the trunk + # and MSA-encoder wrappers have: the distributed path runs its own kernels, + # and a single-GPU backend choice must not reach it. Without this, calling + # ``model.set_kernel_backend(...)`` after ``wrap_model_with_cp`` raises + # AttributeError. + def set_kernel_backend(self, _backend: str | None) -> None: + return + + def set_chunk_size(self, _chunk_size: int | None) -> None: + return + + def forward( + self, + *, + x_noisy, + t_hat, + ref_pos, + ref_charge, + ref_mask, + ref_element, + ref_atom_name_chars, + ref_space_uid, + tok_idx, + s_inputs, + s_trunk, + z_trunk, + relative_position_encoding, + asym_id, + residue_index, + entity_id, + token_index, + sym_id, + sigma_data=None, + token_attention_mask=None, + num_diffusion_samples: int = 1, + return_token_repr: bool = False, + return_atom_repr: bool = False, + inference_cache=None, + ): + if num_diffusion_samples != 1: + raise NotImplementedError( + "DiffusionModuleCPWrapper supports num_diffusion_samples==1 " + "(pair-bias batch expansion not yet sharded)." + ) + m = self.serial + mesh = self.device_mesh + bsz = x_noisy.shape[0] + sigma = m.sigma_data if sigma_data is None else float(sigma_data) + t = torch.as_tensor(t_hat, dtype=torch.float32, device=x_noisy.device).reshape( + -1 + ) + if t.numel() == 1: + t = t.expand(bsz) + + # z_trunk may arrive as a full tensor (pad here) or as an already-padded, + # already-sharded DTensor from the CP tail (#7) — in which case the token + # axis is its padded length and the other (token-space) tensors are padded + # to match. Padded token rows carry no atoms (atom_to_token < L), so they + # don't affect the atom-space coordinate output. + zt_is_dtensor = isinstance(z_trunk, DTensor) + if zt_is_dtensor: + L = s_inputs.shape[1] # original token length + pad = z_trunk.shape[1] - L # to the pre-padded sharded length + else: + L = z_trunk.shape[1] + pad = (self.shard_factor - L % self.shard_factor) % self.shard_factor + + def _pad_pair(x): # (B, L, L, C) + return F.pad(x, (0, 0, 0, pad, 0, pad)) if pad else x + + def _pad_tok(x): # (B, L, C) + return F.pad(x, (0, 0, 0, pad)) if pad else x + + def _pad_mask(x): # (B, L) + return F.pad(x, (0, pad)) if pad else x + + # Step 1: conditioning -> full s, sharded (padded) z. Only materialise + + # distribute the full z_trunk on the cache miss (first step); afterwards + # the sharded z is reused from the cache. + # relative_position_encoding may arrive already padded + sharded (from + # PairInitDistributed) — use it directly instead of re-materialising the + # full L×L tensor to distribute it. + def _rp_to_dt(): + if isinstance(relative_position_encoding, DTensor): + return relative_position_encoding + return distribute_tensor( + _pad_pair(relative_position_encoding).contiguous(), mesh, _PAIR_PL + ) + + z_cached = inference_cache is not None and "z_cp" in inference_cache + if z_cached: + zt_dt = rp_dt = None + elif zt_is_dtensor: + zt_dt = z_trunk # already padded + sharded; do not re-distribute + rp_dt = _rp_to_dt() + else: + zt_dt = distribute_tensor(_pad_pair(z_trunk).contiguous(), mesh, _PAIR_PL) + rp_dt = _rp_to_dt() + s, z_dt = self.cond( + t_hat=t, + s_inputs=s_inputs, + z_trunk_dt=zt_dt, + rel_pos_dt=rp_dt, + sigma_data=sigma, + num_diffusion_samples=num_diffusion_samples, + inference_cache=inference_cache, + ) + + # Step 2: normalise noisy coords (replicated) + denom = torch.sqrt(t * t + sigma * sigma) + r_noisy = x_noisy / denom[:, None, None] + + # Step 3: atom encoder (replicated, full) + a, q_skip, c_skip, p_skip, enc_int = m.atom_encoder( + ref_pos=ref_pos, + atom_attention_mask=ref_mask, + ref_space_uid=ref_space_uid, + ref_charge=ref_charge, + ref_element=ref_element, + ref_atom_name_chars=ref_atom_name_chars, + atom_to_token=tok_idx, + r_l=r_noisy, + s_i=s_trunk, + num_diffusion_samples=num_diffusion_samples, + return_intermediates=return_atom_repr, + inference_cache=inference_cache, + ) + + # Step 4: add conditioned s (replicated) + a = a + m.s_to_token(m.s_step_norm(s)) + + # Step 5: token transformer (distributed). Pad token axis; row-shard a/s/ + # mask; z_dt already 2-D sharded at padded length; gather + slice a back. + a_dt = distribute_tensor(_pad_tok(a).contiguous(), mesh, _ROW_PL) + s_dt = distribute_tensor(_pad_tok(s).contiguous(), mesh, _ROW_PL) + mask_dt = None + if token_attention_mask is not None: + mask_dt = distribute_tensor( + _pad_mask(token_attention_mask.to(a.dtype)).contiguous(), mesh, _ROW_PL + ) + a_dt = self.token_transformer(a_dt, s_dt, z_dt, key_mask_dt=mask_dt) + a = a_dt.full_tensor() + if pad: + a = a[:, :L, :] + + # Step 6: token norm (replicated) + a = m.token_norm(a) + + # Step 7: atom decoder (replicated, full) + r_update, dec_int = m.atom_decoder( + a_i=a, + q_l=q_skip, + c_l=c_skip, + p_lm=p_skip, + atom_to_token=tok_idx, + atom_attention_mask=ref_mask, + num_diffusion_samples=num_diffusion_samples, + return_intermediates=return_atom_repr, + ) + + # Step 8: denoised output (replicated) + sigma2, t2 = sigma * sigma, t * t + out = (sigma2 / (sigma2 + t2))[:, None, None] * x_noisy + out = out + ((sigma * t) / torch.sqrt(sigma2 + t2))[:, None, None] * r_update + + atom_intermediates = None + if return_atom_repr: + all_ints = enc_int + dec_int + if all_ints: + atom_intermediates = torch.stack(all_ints, dim=2) + return { + "x_denoised": out, + "token_repr": a if return_token_repr else None, + "atom_intermediates": atom_intermediates, + } + + +def wrap_model_with_cp_structure_head(model: nn.Module, dist_manager) -> list[str]: + """Replace each ``DiffusionStructureHead``'s ``diffusion_module`` with the CP + wrapper. The serial ``sample()`` loop keeps driving it. Returns replaced paths.""" + from esm.models.esmfold2.layers import DiffusionModule as SerialDiffusionModule + + replaced: list[str] = [] + for name, module in model.named_modules(): + dm_child = getattr(module, "diffusion_module", None) + if isinstance(dm_child, SerialDiffusionModule): + module.diffusion_module = DiffusionModuleCPWrapper( + dm_child, dist_manager + ).to(device=dist_manager.device, dtype=next(dm_child.parameters()).dtype) + replaced.append(f"{name}.diffusion_module" if name else "diffusion_module") + return replaced diff --git a/esm/models/esmfold2/distributed/utils.py b/esm/models/esmfold2/distributed/utils.py new file mode 100644 index 00000000..84803898 --- /dev/null +++ b/esm/models/esmfold2/distributed/utils.py @@ -0,0 +1,542 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: MIT +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. + +"""Layout / sharding helpers and the user-facing model-wrap entry point for +2D context parallelism.""" + +from math import isqrt, lcm +from typing import Sequence + +import numpy as np +import torch +import torch.nn as nn +import torch.nn.functional as F +from torch.distributed.tensor import DTensor, Shard, distribute_tensor + + +class LayoutMap: + """Bijective mapping between multi-dimensional indices and flat indices. + + Analogous to C++ std::layout_stride::mapping. + + Parameters + ---------- + strides: + Per-dimension strides (must be positive integers). + shape: + Per-dimension sizes (must be positive integers). + offset: + Offset added to every flat index (default 0). + """ + + def __init__( + self, strides: tuple[int, ...], shape: tuple[int, ...], offset: int = 0 + ): + if not all(isinstance(s, (int, np.int64)) and s > 0 for s in strides): + raise ValueError(f"Strides must be positive integers: {strides}") + if any(s < 0 for s in shape): + raise ValueError(f"Shape must be non-negative: {shape}") + if any(s == 0 for s in shape): + raise ValueError(f"Shape must not contain zeros: {shape}") + + self._strides = strides + self._n_axes = len(strides) + if len(shape) != self._n_axes: + raise ValueError( + f"Shape {shape} and strides {strides} must have the same length" + ) + + self._shape = shape + self._numel = int(np.prod(self._shape)) + self._offset = offset + + shape_and_strides = np.array( + list(zip(self._shape, self._strides)), + dtype=np.dtype([("shape", int), ("strides", int)]), + ) + argsort_ascend = np.argsort(shape_and_strides, order=["strides", "shape"]) + + self.is_unique = self._is_unique(argsort_ascend) + self.is_exhaustive = self._is_exhaustive(argsort_ascend) + + if not self.is_unique: + raise ValueError( + f"Strides {strides} and shape {shape} do not give a unique layout." + ) + + self._required_span_size = self._compute_required_span_size() + self._argsort_descend_strides = argsort_ascend[::-1] + self._argsort_ascend_strides = argsort_ascend + + def _compute_required_span_size(self) -> int: + if self._n_axes == 0: + return 1 + return 1 + sum( + (self._shape[i] - 1) * self._strides[i] for i in range(self._n_axes) + ) + + def _strides_exhaustive(self, permutation: np.ndarray): + strides = np.array(self._strides) + shape = np.array(self._shape) + shape_permuted = shape[permutation] + strides_permuted = strides[permutation] + shape_shifted = np.concatenate([[1], shape_permuted[:-1]]) + strides_shifted = np.concatenate([[1], strides_permuted[:-1]]) + return strides_permuted, strides_shifted * shape_shifted + + def _is_unique(self, permutation: np.ndarray) -> bool: + if self._n_axes == 0: + return True + strides, strides_exhaustive = self._strides_exhaustive(permutation) + return bool(np.all(strides >= strides_exhaustive)) + + def _is_exhaustive(self, permutation: np.ndarray) -> bool: + if self._n_axes == 0: + return True + strides, strides_exhaustive = self._strides_exhaustive(permutation) + return bool(np.all(strides == strides_exhaustive)) + + @property + def offset(self) -> int: + return self._offset + + @property + def required_span_size(self) -> int: + return self._required_span_size + + @property + def numel(self) -> int: + return self._numel + + @property + def shape(self) -> tuple[int, ...]: + return self._shape + + @property + def strides(self) -> tuple[int, ...]: + return self._strides + + def __call__(self, ids: tuple[int, ...]) -> int: + if len(ids) != self._n_axes: + raise ValueError( + f"Expected {self._n_axes} elements in ids but got {len(ids)}" + ) + if len(ids) == 0: + return self._offset + if self._shape is not None: + for axis, idx in enumerate(ids): + if idx < 0 or idx >= self._shape[axis]: + raise ValueError( + f"ids[{axis}] == {idx} out of range [0, {self._shape[axis] - 1}]" + ) + return int(np.dot(ids, self._strides)) + self._offset + + def unravel(self, flat_index: int) -> tuple[int, ...]: + if not self.is_unique: + raise ValueError(f"Layout is not unique, cannot unravel {flat_index}") + if not isinstance(flat_index, (int, np.integer)): + raise TypeError(f"Expected int, got {type(flat_index)}") + remaining = flat_index - self._offset + if remaining < 0 or remaining >= self._required_span_size: + raise ValueError( + f"flat_index {flat_index} out of range [{self._offset}, " + f"{self._offset + self._required_span_size - 1}]" + ) + indices = [0] * self._n_axes + for i_dim in self._argsort_descend_strides: + stride = self._strides[i_dim] + size = self._shape[i_dim] + indices[i_dim] = (remaining // stride) % size + remaining -= indices[i_dim] * stride + if remaining != 0: + raise ValueError(f"flat_index {flat_index} is out of the valid span range.") + return tuple(indices) + + def __getitem__(self, slices) -> "LayoutMap": + if not isinstance(slices, tuple) and isinstance(slices, (slice, int)): + slices = (slices,) + if len(slices) < self._n_axes: + slices = slices + (slice(None),) * (self._n_axes - len(slices)) + + new_shape = [] + new_strides = [] + new_offset = self.offset + + for axis, s in enumerate(slices): + if isinstance(s, (int, np.int64)): + new_offset += s * self.strides[axis] + elif isinstance(s, slice): + start, stop, step = s.indices(self.shape[axis]) + if step <= 0: + raise ValueError("Unsupported slicing: negative or zero steps") + if start >= stop: + raise ValueError("Unsupported slicing: start not smaller than stop") + dim_len = max(0, (stop - start + step - 1) // step) + new_shape.append(dim_len) + new_strides.append(self.strides[axis] * step) + new_offset += start * self.strides[axis] + else: + raise TypeError(f"Unsupported slice type: {type(s)}") + + return LayoutMap(tuple(new_strides), tuple(new_shape), new_offset) + + +class LayoutRightMap(LayoutMap): + """Row-major (C-contiguous) layout.""" + + def __init__(self, shape: tuple[int, ...]): + strides = np.ones_like(shape) + strides[1:] = shape[:0:-1] + strides = np.cumprod(strides)[::-1] + super().__init__(tuple(strides), shape=shape) + + +class LayoutLeftMap(LayoutMap): + """Column-major (Fortran-contiguous) layout.""" + + def __init__(self, shape: tuple[int, ...]): + strides = np.ones_like(shape) + strides[1:] = shape[:-1] + strides = np.cumprod(strides) + super().__init__(tuple(strides), shape=shape) + + +def get_group_rank_from_axial_shift( + coord: tuple[int, ...], axis: int, delta: int, layout_group: LayoutMap +) -> int: + """Return the rank obtained by shifting coord along axis by delta (wrapping).""" + if len(coord) != len(layout_group.shape): + raise ValueError( + f"Incompatible coord {coord} and layout_group shape {layout_group.shape}" + ) + if axis >= len(coord): + raise ValueError(f"Axis {axis} out of range for coord {coord}") + coord_shifted = list(coord) + coord_shifted[axis] = (coord_shifted[axis] + delta) % layout_group.shape[axis] + return layout_group(coord_shifted) # ty:ignore[invalid-argument-type] + + +def update_exhaustive_strides( + shape_original: Sequence[int], + strides_original: Sequence[int], + shape_new: Sequence[int], +) -> tuple[int, ...]: + """Compute strides for shape_new that preserve the same axis-ordering as + the exhaustive layout (shape_original, strides_original).""" + layout_original = LayoutMap(tuple(strides_original), tuple(shape_original)) + if not layout_original.is_exhaustive: + raise ValueError( + f"Layout (shape={shape_original}, strides={strides_original}) is not exhaustive" + ) + shape_new_ascending = np.array(shape_new)[layout_original._argsort_ascend_strides] + argsort_output = np.argsort(layout_original._argsort_ascend_strides) + strides_new_ascending = np.concatenate(([1], shape_new_ascending[:-1])).cumprod() + strides_new = strides_new_ascending[argsort_output] + return tuple(strides_new.tolist()) + + +def slice_repr_mask( + s: torch.Tensor, + z: torch.Tensor, + mask: torch.Tensor, + pair_mask: torch.Tensor, + n_ranks: int, + layout_group: LayoutMap, +) -> tuple[ + list[torch.Tensor], list[torch.Tensor], list[torch.Tensor], list[torch.Tensor] +]: + """Slice s, z, mask, pair_mask into n_ranks shards for 2D CP distribution.""" + if z.shape[-2] != z.shape[-3]: + raise ValueError(f"z is not square in the middle two axes: {z.shape}") + if s.shape[-2] != z.shape[-3]: + raise ValueError(f"Incompatible s {s.shape} and z {z.shape}") + if mask.shape != s.shape[:-1]: + raise ValueError(f"Incompatible s {s.shape} and mask {mask.shape}") + if pair_mask.shape != z.shape[:-1]: + raise ValueError(f"Incompatible z {z.shape} and pair_mask {pair_mask.shape}") + + n_tokens = s.shape[-2] + coords = [layout_group.unravel(rank) for rank in range(n_ranks)] + n_ranks_axis = isqrt(n_ranks) + if n_ranks_axis * n_ranks_axis != n_ranks: + raise ValueError(f"n_ranks is not a perfect square: {n_ranks}") + if n_tokens % n_ranks_axis: + raise ValueError( + f"Token dim {n_tokens} not divisible by sqrt(n_ranks) = {n_ranks_axis}" + ) + stride = n_tokens // n_ranks_axis + s_slices, z_slices, mask_slices, pair_mask_slices = [], [], [], [] + for i_row, j_col in coords: + i0, i1 = i_row * stride, (i_row + 1) * stride + j0, j1 = j_col * stride, (j_col + 1) * stride + s_slices.append(s[..., i0:i1, :].contiguous()) + mask_slices.append(mask[..., i0:i1].contiguous()) + z_slices.append(z[..., i0:i1, j0:j1, :].contiguous()) + pair_mask_slices.append(pair_mask[..., i0:i1, j0:j1].contiguous()) + return s_slices, z_slices, mask_slices, pair_mask_slices + + +def tiled_softmax_attention_update( + o_chunk: torch.Tensor, + lse_m_chunk: torch.Tensor, + amax_chunk: torch.Tensor | None, + o: torch.Tensor | None = None, + lse_m: torch.Tensor | None = None, + amax: torch.Tensor | None = None, +) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor | None]: + """Numerically-stable online softmax accumulation for ring attention. + + Updates running (o, lse_m, amax) with a new (o_chunk, lse_m_chunk, amax_chunk). + When amax_chunk is None the function operates without amax tracking. + """ + if not ((o is None) == (lse_m is None)): + raise ValueError("o and lse_m must both be None or both provided") + + has_amax = amax_chunk is not None + is_initial_chunk = o is None + + if lse_m_chunk.shape[-1] != 1: + raise ValueError("lse_m_chunk must have shape (..., 1)") + if o_chunk.ndim != lse_m_chunk.ndim: + raise ValueError("o_chunk and lse_m_chunk must have the same ndim") + if lse_m_chunk.shape[:-1] != o_chunk.shape[:-1]: + raise ValueError("o_chunk and lse_m_chunk must match except in the last dim") + + if is_initial_chunk: + return o_chunk, lse_m_chunk, amax_chunk + + assert o is not None and lse_m is not None + + if has_amax: + assert amax is not None + d_lse_m = lse_m - lse_m_chunk + amax_next = torch.maximum(amax_chunk, amax) + delta_lse = amax_chunk - amax - d_lse_m + o_new = o - torch.sigmoid(delta_lse) * (o - o_chunk) + lse_m_new = lse_m_chunk + torch.logsumexp( + torch.cat([(amax - amax_next) + d_lse_m, amax_chunk - amax_next], dim=-1), + dim=-1, + keepdim=True, + ).to(dtype=lse_m_chunk.dtype) + return o_new, lse_m_new, amax_next + else: + d_lse_m = lse_m - lse_m_chunk + o_new = o - torch.sigmoid(-d_lse_m) * (o - o_chunk) + lse_m_new = lse_m_chunk + torch.log1p(torch.exp(d_lse_m)).to( + dtype=lse_m_chunk.dtype + ) + return o_new, lse_m_new, None + + +# --------------------------------------------------------------------------- +# End-to-end CP runtime: drop-in replacement for the serial ``FoldingTrunk``. +# --------------------------------------------------------------------------- + + +class TrunkCPWrapper(nn.Module): + """Drop-in replacement for ``FoldingTrunk`` that runs distributed. + + Accepts and returns plain tensors so the rest of an ESMFold2 model + (LM, MSA encoder, diffusion sampler) is untouched. Pair tensors whose + ``N`` is not a multiple of the CP axes are zero-padded; the gathered + output is sliced back to the original length, and the mask is padded + the same way so padded rows/cols contribute nothing. + + Typical use on an ``N×N`` CP grid (e.g. 4 ranks via + ``torch.multiprocessing.spawn``):: + + from collections import OrderedDict + + from transformers.models.esmc import EsmFold2Model + from transformers.models.esmc.distributed import ( + DistributedManager, + wrap_model_with_cp_trunks, + ) + + DistributedManager.initialize(OrderedDict([("dp", 1), ("cp", (2, 2))])) + dm = DistributedManager() + + model = EsmFold2Model.from_pretrained(...).cuda().eval() + wrap_model_with_cp_trunks(model, dm) + # ``model.forward`` (and ``processor.fold``) now run the Pairformer + # across the CP grid; everything else stays serial per rank. + """ + + def __init__( + self, serial_trunk: nn.Module, dist_manager, bf16: bool = True + ) -> None: + super().__init__() + # Lazy imports: this module is imported by manager.py and the + # distributed layers, so importing pairformer / model_common at + # module level would create a cycle. + from esm.models.esmfold2.distributed.manager import DistributedManager + from esm.models.esmfold2.distributed.model.layers.pairformer import ( + FoldingTrunkDistributed, + ) + from esm.models.esmfold2.layers import FoldingTrunk as SerialFoldingTrunk + + if not isinstance(serial_trunk, SerialFoldingTrunk): + raise TypeError(f"expected FoldingTrunk, got {type(serial_trunk).__name__}") + if not isinstance(dist_manager, DistributedManager): + raise TypeError( + f"expected DistributedManager, got {type(dist_manager).__name__}" + ) + + # ``FoldingTrunkDistributed`` requires the serial trunk's tri-mul + # kernels off and chunking disabled (it composes its own ring loop). + # ``serial_trunk`` is typed ``nn.Module`` (the SerialFoldingTrunk + # symbol is imported lazily inside the function to avoid a circular + # import with pairformer.py); pyright can't narrow through the + # lazy-import isinstance check. + serial_trunk.set_kernel_backend(None) + serial_trunk.set_chunk_size(None) + + self.dist_trunk = FoldingTrunkDistributed(serial_trunk, dist_manager) + + # bf16 trunk: the serial recycle path runs the trunk with no autocast on + # an fp32 pair, and the distributed ring does plain fp32 torch.matmul + # (TF32 off) → fp32 tensor-core-less matmul + huge fp32 intermediates at + # ~92% memory occupancy dominate wall-clock. boltz-cp runs its CP trunk + # entirely in bf16 (bf16-mixed; tri-mul contraction accumulated in bf16), + # so bf16 here has direct production precedent. We cast params to bf16 + # ("bf16-true") rather than autocast, because the distributed tri-mul's + # custom_fwd disables autocast. The pair is cast to bf16 at the boundary + # in forward() and the output cast back to the caller's dtype. + # NOTE: ESMFold2's *serial* reference upcasts the tri-mul contraction to + # fp32; validated quality-neutral (pLDDT 0.643 vs 0.639). Controlled by + # the ``bf16`` arg (threaded from wrap_model_with_cp[_trunks]). + self._trunk_bf16 = bf16 + if self._trunk_bf16: + self.dist_trunk = self.dist_trunk.to(torch.bfloat16) + + self.dist_manager = dist_manager + self.device_mesh = dist_manager.device_mesh_subgroups + # device_mesh is (dp, cp_axis_0, cp_axis_1) + self.cp_axis_0 = self.device_mesh.size(1) + self.cp_axis_1 = self.device_mesh.size(2) + self.shard_factor = lcm(self.cp_axis_0, self.cp_axis_1) + + # The serial trunk exposes these knobs to the parent model. The + # distributed path doesn't support kernels/chunking, but the parent + # ``set_kernel_backend`` / ``set_chunk_size`` calls still need a no-op + # hook so they don't blow up. + def set_kernel_backend(self, _backend: str | None) -> None: + return + + def set_chunk_size(self, _chunk_size: int | None) -> None: + return + + def forward_sharded( + self, pair_dt: DTensor, pair_attention_mask: DTensor | None = None + ) -> DTensor: + """Run the distributed trunk on an already-sharded pair DTensor. + + ``pair_dt`` (and the optional mask) must already be padded to a multiple + of ``self.shard_factor``, cast to the trunk dtype, and distributed with + placements ``[Shard(0), Shard(1), Shard(2)]`` on ``self.device_mesh``. + Returns a DTensor with the same placements — **no** ``full_tensor()`` + gather. + + This is the entry point a CP orchestrator uses to keep the pair sharded + across module boundaries (e.g. the recycle loop), instead of paying a + ``full_tensor()`` + ``distribute_tensor`` round-trip on every call. The + plain-tensor :meth:`forward` is a thin adapter around it. + """ + return self.dist_trunk(pair_dt, pair_attention_mask=pair_attention_mask) + + def forward( + self, pair: torch.Tensor, pair_attention_mask: torch.Tensor | None = None + ) -> torch.Tensor: + N = pair.shape[1] + orig_dtype = pair.dtype + # Run the distributed trunk in bf16 (params cast in __init__). Cast the + # pair and visibility mask to bf16 so the tri-mul ops stay bf16×bf16 + # (no fp32 promotion); the gathered output is cast back to orig_dtype. + if self._trunk_bf16: + pair = pair.to(torch.bfloat16) + if pair_attention_mask is not None: + pair_attention_mask = pair_attention_mask.to(torch.bfloat16) + + pad = (self.shard_factor - N % self.shard_factor) % self.shard_factor + if pad: + # F.pad pads from the last dim backward; pair is (B, N, N, d_pair). + pair = F.pad(pair, (0, 0, 0, pad, 0, pad)) + if pair_attention_mask is not None: + pair_attention_mask = F.pad(pair_attention_mask, (0, pad, 0, pad)) + + pair_dt = distribute_tensor( + pair.contiguous(), self.device_mesh, [Shard(0), Shard(1), Shard(2)] + ) + mask_dt = None + if pair_attention_mask is not None: + mask_dt = distribute_tensor( + pair_attention_mask.contiguous(), + self.device_mesh, + [Shard(0), Shard(1), Shard(2)], + ) + + out_dt = self.forward_sharded(pair_dt, pair_attention_mask=mask_dt) + out = out_dt.full_tensor() + if self._trunk_bf16: + out = out.to(orig_dtype) + if pad: + out = out[:, :N, :N, :] + return out + + +def wrap_model_with_cp_trunks( + model: nn.Module, dist_manager, bf16: bool = True +) -> list[str]: + """Replace every ``FoldingTrunk`` submodule with ``TrunkCPWrapper``. + + Walks ``model.named_modules()`` and rebinds each attribute that points + at a serial ``FoldingTrunk``. Returns the list of replaced submodule + paths so callers (e.g. spawned workers) can log what got wrapped. + + ``bf16`` (default True): run the distributed trunk in bf16 (params + pair cast + to bf16, output back to the caller's dtype) — ~1.3-1.6x faster and lower peak + VRAM, quality-neutral (pLDDT 0.643 vs 0.639). Pass False for an fp32 trunk + (e.g. tight bit-exact parity checks). + """ + mesh = dist_manager.device_mesh_subgroups + cp0, cp1 = mesh.size(1), mesh.size(2) + if cp0 != cp1: + raise ValueError( + f"CP grid must be square (cp_axis_0 == cp_axis_1), got {cp0}×{cp1}" + ) + + from esm.models.esmfold2.layers import FoldingTrunk as SerialFoldingTrunk + + targets: list[tuple[str, nn.Module, str, nn.Module]] = [] + for parent_name, parent in model.named_modules(): + for child_name, child in parent.named_children(): + if isinstance(child, SerialFoldingTrunk): + full = f"{parent_name}.{child_name}" if parent_name else child_name + targets.append((full, parent, child_name, child)) + + replaced: list[str] = [] + for full, parent, child_name, child in targets: + wrapped = TrunkCPWrapper(child, dist_manager, bf16=bf16).to( + device=dist_manager.device, dtype=next(child.parameters()).dtype + ) + setattr(parent, child_name, wrapped) + replaced.append(full) + return replaced diff --git a/esm/models/esmfold2/experimental.py b/esm/models/esmfold2/experimental.py index 174c3cda..96992c2b 100644 --- a/esm/models/esmfold2/experimental.py +++ b/esm/models/esmfold2/experimental.py @@ -654,7 +654,9 @@ def load_esmc(self, esmc_model_path: str) -> None: """ from esm.models.esmc import EsmcModel - self.esmc = EsmcModel.from_pretrained(esmc_model_path) + self.esmc = EsmcModel.from_pretrained( + esmc_model_path, device=self.device, dtype=torch.bfloat16 + ) self.finalize_esmc() def finalize_esmc(self) -> None: @@ -890,6 +892,7 @@ def forward( max_inference_sigma: float | None = 256.0, seed: int | None = None, provide_soft_sequence_to_msa_and_profile: bool = True, + include_embeddings: bool = False, **unused_features: Tensor, ) -> dict[str, Tensor]: """Full ESMFold2 inference pipeline. @@ -1115,6 +1118,12 @@ def forward( # 6. Distogram (inside the trunk autocast so z stays bf16) distogram_logits = self.distogram_head(z + z.transpose(-2, -3)) + embeddings: dict[str, Tensor] = {} + if include_embeddings: + embeddings["output_embedding_pair_pooled"] = z.detach().mean( + dim=1, dtype=torch.float32 + ) + # 7. Diffusion sampling (always no_grad; optional seed for parity) with torch.no_grad(), _seed_context(seed): structure_output = self.structure_head.sample( @@ -1171,6 +1180,7 @@ def forward( ) output["residue_index"] = residue_index output["entity_id"] = entity_id + output.update(embeddings) return output diff --git a/esm/models/esmfold2/hf_adapter.py b/esm/models/esmfold2/hf_adapter.py index cf7945a8..6bb71927 100644 --- a/esm/models/esmfold2/hf_adapter.py +++ b/esm/models/esmfold2/hf_adapter.py @@ -244,6 +244,7 @@ def forward( max_inference_sigma: float | None = 256.0, disto_cond: Tensor | None = None, disto_cond_mask: Tensor | None = None, + include_embeddings: bool = False, **unused_features: Tensor, ) -> dict[str, Tensor]: """Our end-to-end ``forward``, dispatched to their ``fold``. @@ -265,6 +266,13 @@ def forward( "fold without it." ) + if include_embeddings: + raise NotImplementedError( + "include_embeddings is not supported through the HF adapter: the " + "upstream fold() does not return the trunk pair state. Use " + "EsmFold2Model directly." + ) + if lm_mask_pct: input_ids = self._mask_input_ids(input_ids, lm_mask_pct) diff --git a/esm/models/esmfold2/layers.py b/esm/models/esmfold2/layers.py index 7027f4fa..d4a796e7 100644 --- a/esm/models/esmfold2/layers.py +++ b/esm/models/esmfold2/layers.py @@ -1765,7 +1765,17 @@ def _weighted_rigid_align( xgt_c = x_gt - mu_gt H = torch.einsum("bni,bnj->bij", w * xgt_c, x_c) H32 = H.float() - U, _, Vh = torch.linalg.svd(H32, driver="gesvd" if H32.is_cuda else None) + try: + U, _, Vh = torch.linalg.svd(H32, driver="gesvd" if H32.is_cuda else None) + except RuntimeError: + # Near the OOM boundary cuSOLVER can fail to allocate its handle + # (cusolverDnCreate) even though this 3x3 SVD is trivial. Fall back to + # a CPU SVD of the tiny matrix — instant, and only reached when the GPU + # path would otherwise crash. driver=None (gesvd is cuSOLVER-only). + if not H32.is_cuda: + raise + Ucpu, _, Vhcpu = torch.linalg.svd(H32.cpu(), driver=None) + U, Vh = Ucpu.to(H32.device), Vhcpu.to(H32.device) det = torch.linalg.det(U @ Vh) ones = torch.ones_like(det) R = (U @ torch.diag_embed(torch.stack([ones, ones, det], dim=-1)) @ Vh).to( @@ -1959,7 +1969,11 @@ def __init__(self, config: EsmFold2Config) -> None: self.atom_attention_encoder = EsmFold2AtomEncoder( d_atom=swa_cfg.hidden_size, - d_token=swa_cfg.output_dim, + d_token=( + swa_cfg.token_hidden_size + if swa_cfg.token_hidden_size is not None + else swa_cfg.output_dim + ), n_blocks=swa_cfg.num_hidden_layers, n_heads=swa_cfg.num_attention_heads, swa_window_size=config.sliding_window, diff --git a/esm/models/esmfold2/model.py b/esm/models/esmfold2/model.py index 69102b6c..c09e60a9 100644 --- a/esm/models/esmfold2/model.py +++ b/esm/models/esmfold2/model.py @@ -207,11 +207,8 @@ def forward( pair = self._repeat_batch(z_base, num_diffusion_samples) x_pred_flat = self._flatten_sample_axis(x_pred) - atom_to_token_m = self._repeat_batch(atom_to_token, num_diffusion_samples) - atom_mask_m = self._repeat_batch(atom_attention_mask, num_diffusion_samples) rep_idx_m = self._repeat_batch(distogram_atom_idx, num_diffusion_samples).long() mask = self._repeat_batch(token_attention_mask, num_diffusion_samples) - Bm = pair.shape[0] rep_coords = gather_rep_atom_coords(x_pred_flat, rep_idx_m) rep_distances = torch.cdist( @@ -233,6 +230,55 @@ def forward( del pair_delta single = self.row_attention_pooling(pair, mask) + pae_logits = self.pae_head(self.pae_ln(pair)) + pde_logits = self.pde_head(self.pde_ln(pair)) + return self._finish( + single=single, + pae_logits=pae_logits, + pde_logits=pde_logits, + x_pred=x_pred, + distogram_atom_idx=distogram_atom_idx, + token_attention_mask=token_attention_mask, + atom_to_token=atom_to_token, + atom_attention_mask=atom_attention_mask, + asym_id=asym_id, + mol_type=mol_type, + num_diffusion_samples=num_diffusion_samples, + ) + + def _finish( + self, + *, + single: Tensor, + pae_logits: Tensor, + pde_logits: Tensor, + x_pred: Tensor, + distogram_atom_idx: Tensor, + token_attention_mask: Tensor, + atom_to_token: Tensor, + atom_attention_mask: Tensor, + asym_id: Tensor, + mol_type: Tensor, + num_diffusion_samples: int, + ) -> dict[str, Tensor]: + """Atom-space (pLDDT/resolved) + PAE/PDE + pTM/ipTM back-half. + + Split out of ``forward`` so the CP wrapper (#6) can run the distributed + pair front (z_base → trunk → row-pool → pae/pde heads), gather the small + ``single`` / ``pae_logits`` / ``pde_logits``, and reuse this exact + (serial, replicated) reduction code — one source of truth for the fiddly + pTM/ipTM/pair_chains logic.""" + x_pred_flat = self._flatten_sample_axis(x_pred) + atom_to_token_m = self._repeat_batch(atom_to_token, num_diffusion_samples) + atom_mask_m = self._repeat_batch(atom_attention_mask, num_diffusion_samples) + rep_idx_m = self._repeat_batch(distogram_atom_idx, num_diffusion_samples).long() + mask = self._repeat_batch(token_attention_mask, num_diffusion_samples) + Bm = single.shape[0] + rep_coords = gather_rep_atom_coords(x_pred_flat, rep_idx_m) + rep_distances = torch.cdist( + rep_coords, rep_coords, compute_mode="donot_use_mm_for_euclid_dist" + ) + atom_mask_f = atom_mask_m.float() s_at_atoms = gather_token_to_atom(single, atom_to_token_m) s_at_atoms_ln = self.plddt_ln(s_at_atoms) @@ -282,12 +328,8 @@ def forward( plddt_ca = plddt_per_atom.gather(1, rep_idx_m) - # PAE - pae_logits = self.pae_head(self.pae_ln(pair)) + # PAE / PDE (logits computed by the caller — serial front or CP wrapper). pae = _categorical_mean(pae_logits, start=0.0, end=32.0).detach() - - # PDE - pde_logits = self.pde_head(self.pde_ln(pair)) pde = _categorical_mean(pde_logits, start=0.0, end=32.0).detach() # Resolved (per-atom binary). @@ -576,6 +618,14 @@ def __init__(self, config: EsmFold2Config) -> None: EsmcModel(config.esmc_config) if config.esmc_config is not None else None ) self._esmc_fp8: bool = False # set by load_esmc(fp8=True) + # When True, ESM-C is offloaded to CPU after its one-shot hidden-state + # computation (restored on the next call), freeing its ~12 GB for the + # recycling trunk + diffusion — the freed blocks stay in the caching pool + # so the trunk reuses them without cudaMalloc, relieving the allocator + # pressure that drove rank desync / spin-wait. Opt-in (the CP entry point + # ``wrap_model_with_cp(offload_esmc=...)`` sets it). Validated with CP: + # 2.14x end-to-end + 18.5 GB lower peak vs fp32, pLDDT unchanged. + self._offload_esmc: bool = False self.folding_trunk = FoldingTrunk( n_layers=config.folding_trunk_num_hidden_layers, @@ -637,7 +687,11 @@ def load_esmc(self, esmc_model_path: str, precision: str = "bf16") -> None: """ from esm.models.esmc import EsmcModel - self.esmc = EsmcModel.from_pretrained(esmc_model_path) + self.esmc = EsmcModel.from_pretrained( + esmc_model_path, + device=self.device, + dtype=torch.float32 if precision == "fp32" else torch.bfloat16, + ) self.set_esmc_precision(precision) def set_esmc_precision(self, precision: str = "bf16") -> None: @@ -803,6 +857,10 @@ def _compute_lm_hidden_states( lm_mask_pct: float = 0.0, ) -> Tensor: assert self.esmc is not None + # Restore ESM-C to the compute device if it was offloaded to CPU after a + # previous fold (see self._offload_esmc). No-op when already resident. + if self._offload_esmc: + self.esmc.to(self.device) # fp8 TE kernels require prod(shape[:-1]) % 8 == 0. pad_to = 8 if self._esmc_fp8 else None with _lm_precision_context(self._esmc_fp8): @@ -841,6 +899,27 @@ def _run_one_loop( tok_mask: Tensor, total_steps: int, ) -> Tensor: + # CP orchestrator seam: when a context-parallel recycle engine has been + # installed (by ``wrap_model_with_cp``), delegate the whole loop to it so + # the pair stays sharded across iterations instead of round-tripping + # through ``full_tensor()`` at every module boundary. Plain ``getattr`` + # default keeps this file CP-agnostic (no distributed import). The engine + # returns a gathered full tensor, so the caller is unchanged. + _cp_engine = getattr(self, "_cp_recycle_engine", None) + if _cp_engine is not None: + return _cp_engine.run_loop( + self, + z=z, + z_init=z_init, + lm_z=lm_z, + msa_inputs=_msa_inputs, + pair_mask=pair_mask, + a=a, + b_mat=b_mat, + tok_mask=tok_mask, + total_steps=total_steps, + ) + # Helper method (not inline) so per-iter locals free on return — # otherwise leaks ~2 GB L²×c_z into distogram/sample scope. # training=True forces dropout under eval(), matching the per-loop @@ -963,6 +1042,7 @@ def forward( max_inference_sigma: float | None = 256.0, disto_cond: Tensor | None = None, disto_cond_mask: Tensor | None = None, + include_embeddings: bool = False, **unused_features: Tensor, ) -> dict[str, Tensor]: unexpected = sorted(set(unused_features) - _IGNORED_FEATURE_KEYS) @@ -1045,19 +1125,41 @@ def forward( atom_to_token=atom_to_token, ) - z_init = self.z_init_1(x_inputs).unsqueeze(2) + self.z_init_2( - x_inputs - ).unsqueeze(1) - - relative_position_encoding = self.rel_pos( - residue_index=residue_index, - asym_id=asym_id, - sym_id=sym_id, - entity_id=entity_id, - token_index=token_index, - ) - token_bonds_encoding = self.token_bonds(token_bonds.float()) - z_init = z_init + relative_position_encoding + token_bonds_encoding + # CP (sharded pair-init): build z_init / rel_pos / token_bonds directly + # as 2-D-sharded DTensors so the full L×L pair is never resident on any + # rank (the init-phase peak that made CP OOM at short L). Installed with + # the recycle engine; plain getattr keeps this file CP-agnostic. + _cp_pair_init = getattr(self, "_cp_pair_init", None) + if _cp_pair_init is not None: + ( + z_init, + relative_position_encoding, + token_bonds_encoding, + _pi_n_orig, + ) = _cp_pair_init( + x_inputs, + residue_index, + asym_id, + sym_id, + entity_id, + token_index, + token_bonds, + ) + else: + z_init = self.z_init_1(x_inputs).unsqueeze(2) + self.z_init_2( + x_inputs + ).unsqueeze(1) + + relative_position_encoding = self.rel_pos( + residue_index=residue_index, + asym_id=asym_id, + sym_id=sym_id, + entity_id=entity_id, + token_index=token_index, + ) + token_bonds_encoding = self.token_bonds(token_bonds.float()) + z_init = z_init + relative_position_encoding + token_bonds_encoding + _pi_n_orig = z_init.shape[1] if ( lm_hidden_states is None @@ -1078,12 +1180,45 @@ def forward( ) lm_z: Tensor | None = None if lm_hidden_states is not None: - lm_z = self.language_model(lm_hidden_states.detach()) + # CP (#14): when a distributed LM->pair builder is installed, emit + # lm_z as a sharded (Shard0,Shard1,Shard2) DTensor so the full L×L + # pair is never resident on any rank (the binding peak per the T0 + # phase sweep). Consumed by the recycle engine, installed together + # with it. Plain getattr keeps this file CP-agnostic. + # Only emit a sharded lm_z when the recycle engine (its sole + # consumer) is active; the serial _run_one_loop expects a full + # tensor. Keeps the gather-per-iteration fallback path valid. + _cp_lm = getattr(self, "_cp_language_model", None) + if ( + _cp_lm is not None + and getattr(self, "_cp_recycle_engine", None) is not None + ): + lm_z, _ = _cp_lm(lm_hidden_states.detach()) + else: + lm_z = self.language_model(lm_hidden_states.detach()) del lm_hidden_states + # ESM-C is done for this forward — offload to free its ~12 GB for the + # trunk/diffusion. Return the freed blocks to the driver (not just the + # caching pool) so cuSOLVER's out-of-pool cudaMalloc — for its handle in + # the diffusion rigid-align SVD — can succeed near the OOM boundary. + if self._offload_esmc and self.esmc is not None: + self.esmc.to("cpu") + torch.cuda.empty_cache() - pair_mask = tok_mask[:, :, None].float() * tok_mask[:, None, :].float() + # Pair attention mask: sharded block under CP (outer product of the + # sliced row/col token-mask — never the full L×L), else the serial + # full outer product. + if _cp_pair_init is not None: + pair_mask = _cp_pair_init.pair_mask(tok_mask) + else: + pair_mask = tok_mask[:, :, None].float() * tok_mask[:, None, :].float() - z = self._init_pair_state(z_init) + # Initial pair state: sharded random block under CP (never full L×L), + # else the serial full draw. + if _cp_pair_init is not None: + z = _cp_pair_init.init_pair_state(z_init, _pi_n_orig) + else: + z = self._init_pair_state(z_init) a, b = self._discretized_dynamics() a = a.view(1, 1, 1, -1).to(device=z.device, dtype=z.dtype) @@ -1115,28 +1250,81 @@ def forward( subsample_enabled=depth is not None, ) - # Method call (not inline loop) frees per-iter L²×c_z locals. - z = self._run_one_loop( - z=z, - z_init=z_init, - lm_z=lm_z, - _msa_inputs=_msa_inputs, - pair_mask=pair_mask, - a=a, - b_mat=b_mat, - tok_mask=tok_mask, - total_steps=total_steps, - ) - del z_init, lm_z, _msa_inputs, a, b_mat + # CP tail (#7): when a recycle engine is installed, keep the pair + # SHARDED through the recycle loop + parcae so the full L×L pair is + # not resident during the expensive structure phase. The pair lives as + # ``_z_dt`` (DTensor); it is gathered only transiently for the heads + # that don't yet consume sharded input (distogram, confidence). Plain + # ``getattr`` keeps this file CP-agnostic. + _cp_engine = getattr(self, "_cp_recycle_engine", None) + _z_dt = None + # z may be a sharded DTensor (padded) under CP, so use the pre-pad + # token count tracked at pair-init rather than z.shape[1]. + _n_orig = _pi_n_orig + if _cp_engine is not None: + _z_dt, _n_orig = _cp_engine.run_loop( + self, + z=z, + z_init=z_init, + lm_z=lm_z, + msa_inputs=_msa_inputs, + pair_mask=pair_mask, + a=a, + b_mat=b_mat, + tok_mask=tok_mask, + total_steps=total_steps, + gather=False, + n_orig=_pi_n_orig, + ) + del z_init, lm_z, _msa_inputs, a, b_mat + _z_dt = _cp_engine.parcae_finish(self, _z_dt, pair_mask) + z = None # pair lives sharded in _z_dt; gathered transiently below + else: + # Method call (not inline loop) frees per-iter L²×c_z locals. + z = self._run_one_loop( + z=z, + z_init=z_init, + lm_z=lm_z, + _msa_inputs=_msa_inputs, + pair_mask=pair_mask, + a=a, + b_mat=b_mat, + tok_mask=tok_mask, + total_steps=total_steps, + ) + del z_init, lm_z, _msa_inputs, a, b_mat + + z = self.parcae_readout(z) + z = self.parcae_coda(z, pair_attention_mask=pair_mask) + + z = z.float() - z = self.parcae_readout(z) - z = self.parcae_coda(z, pair_attention_mask=pair_mask) + embeddings: dict[str, Tensor] = {} + if include_embeddings: + _z_emb = ( + _z_dt.full_tensor()[:, :_n_orig, :_n_orig, :].float() + if _z_dt is not None + else z + ) + assert _z_emb is not None + embeddings["output_embedding_pair_pooled"] = _z_emb.detach().mean( + dim=1, dtype=torch.float32 + ) + del _z_emb - z = z.float() - distogram_logits = self.distogram_head(z + z.transpose(-2, -3)) + _cp_disto = getattr(self, "_cp_distogram_head", None) + if _z_dt is not None and _cp_disto is not None: + # Distributed: symmetrize + head off the sharded pair, no full-z gather. + distogram_logits = _cp_disto.forward_sharded(_z_dt, _n_orig) + else: + if _z_dt is not None: + z = _z_dt.full_tensor()[:, :_n_orig, :_n_orig, :].float() + distogram_logits = self.distogram_head(z + z.transpose(-2, -3)) # ty:ignore[unresolved-attribute] + if _z_dt is not None: + del z # free the full pair before the (sharded) structure phase structure_output = self.structure_head.sample( - z_trunk=z, + z_trunk=(_z_dt if _z_dt is not None else z), # ty:ignore[invalid-argument-type] s_inputs=x_inputs, s_trunk=None, relative_position_encoding=relative_position_encoding, @@ -1166,26 +1354,50 @@ def forward( output: dict[str, Tensor] = {"distogram_logits": distogram_logits} output["sample_atom_coords"] = sample_coords - confidence_output = self.confidence_head( - s_inputs=x_inputs.detach(), - z=z.detach().float(), - x_pred=sample_coords.detach(), - distogram_atom_idx=disto_idx, - token_attention_mask=tok_mask, - atom_to_token=atom_to_token, - atom_attention_mask=atm_mask, - asym_id=asym_id, - mol_type=mol_type, - num_diffusion_samples=n_samples, - relative_position_encoding=relative_position_encoding.detach(), - token_bonds_encoding=token_bonds_encoding.detach(), - ) + # Confidence head (#6): on the CP tail, run it distributed straight off the + # sharded pair (no full-L×L re-gather of z) when the wrapper is installed; + # otherwise re-gather transiently and run serially. CP-agnostic getattr. + _cp_conf = getattr(self, "_cp_confidence_head", None) + if _z_dt is not None and _cp_conf is not None: + confidence_output = _cp_conf.forward_sharded( + _z_dt, + _n_orig, + s_inputs=x_inputs.detach(), + x_pred=sample_coords.detach(), + distogram_atom_idx=disto_idx, + token_attention_mask=tok_mask, + atom_to_token=atom_to_token, + atom_attention_mask=atm_mask, + asym_id=asym_id, + mol_type=mol_type, + num_diffusion_samples=n_samples, + relative_position_encoding=relative_position_encoding.detach(), + token_bonds_encoding=token_bonds_encoding.detach(), + ) + else: + if _z_dt is not None: + z = _z_dt.full_tensor()[:, :_n_orig, :_n_orig, :].float() + confidence_output = self.confidence_head( + s_inputs=x_inputs.detach(), + z=z.detach().float(), # ty:ignore[unresolved-attribute] + x_pred=sample_coords.detach(), + distogram_atom_idx=disto_idx, + token_attention_mask=tok_mask, + atom_to_token=atom_to_token, + atom_attention_mask=atm_mask, + asym_id=asym_id, + mol_type=mol_type, + num_diffusion_samples=n_samples, + relative_position_encoding=relative_position_encoding.detach(), + token_bonds_encoding=token_bonds_encoding.detach(), + ) output.update(confidence_output) output["atom_pad_mask"] = ( atm_mask.unsqueeze(0) if atm_mask.dim() == 1 else atm_mask ) output["residue_index"] = residue_index output["entity_id"] = entity_id + output.update(embeddings) return output @torch.no_grad() diff --git a/esm/models/esmfold2/processor.py b/esm/models/esmfold2/processor.py index e4a87863..47ff27d0 100644 --- a/esm/models/esmfold2/processor.py +++ b/esm/models/esmfold2/processor.py @@ -279,6 +279,8 @@ def decode( pair_chains_t = output.get("pair_chains_iptm") residue_index_t = output.get("residue_index") entity_id_t = output.get("entity_id") + emb_seq_t = output.get("output_embedding_sequence") + emb_pair_pooled_t = output.get("output_embedding_pair_pooled") results: list[MolecularComplexResult] = [] for i in range(Bm): @@ -318,6 +320,14 @@ def decode( if entity_id_t is not None else None ), + output_embedding_sequence=( + emb_seq_t[0].detach().cpu() if emb_seq_t is not None else None + ), + output_embedding_pair_pooled=( + emb_pair_pooled_t[0].detach().cpu() + if emb_pair_pooled_t is not None + else None + ), ) ) @@ -343,6 +353,7 @@ def fold( msa_max_depth: int | None = 1024, msa_column_mask_rate: float = 0.1, msa_subsample_at_inference: bool | None = None, + include_embeddings: bool = False, complex_id: str = "pred", ) -> MolecularComplexResult | list[MolecularComplexResult]: """Fold a structure end-to-end: encode → model → decode. @@ -375,6 +386,13 @@ def fold( (shared across loops). Defaults to ``config.msa_encoder.column_mask_rate`` when ``None``. Only affects inputs that carry an MSA. + include_embeddings : bool + Populate ``output_embedding_pair_pooled`` on each result: the trunk's + final pair state averaged over the first token axis, ``[L, d_pair]`` + fp32 on CPU. This is the same tensor, computed the same way, that the + Forge ``FoldingConfig.include_embeddings`` flag returns. + ``output_embedding_sequence`` stays None, as it does over the API - + neither released architecture keeps a single trunk state. complex_id : str Identifier assigned to the predicted MolecularComplex(es). @@ -415,6 +433,7 @@ def fold( num_diffusion_samples=num_diffusion_samples, msa_max_depth=msa_max_depth, msa_column_mask_rate=msa_column_mask_rate, + include_embeddings=include_embeddings, **sampler_kwargs, ) diff --git a/esm/models/hub.py b/esm/models/hub.py index 058e4348..7452ac0b 100644 --- a/esm/models/hub.py +++ b/esm/models/hub.py @@ -200,8 +200,9 @@ def _load_pretrained( ) model._materialize_uninitialized(device="cpu") - model.to(device) + del raw, adapted if dtype is not None: model.to(dtype) + model.to(device) model.eval() return model diff --git a/esm/sdk/api.py b/esm/sdk/api.py index 806c434e..df1c28e1 100644 --- a/esm/sdk/api.py +++ b/esm/sdk/api.py @@ -28,7 +28,7 @@ class ESMProtein(ProteinType): # Tracks sequence: str | None = None secondary_structure: str | None = None - sasa: list[float | None] | None = None + sasa: list[float | str | None] | None = None function_annotations: list[FunctionAnnotation] | None = None coordinates: torch.Tensor | None = None diff --git a/esm/sdk/base_forge_client.py b/esm/sdk/base_forge_client.py index 459b4cb4..714f7243 100644 --- a/esm/sdk/base_forge_client.py +++ b/esm/sdk/base_forge_client.py @@ -1,6 +1,7 @@ import asyncio import time from abc import ABC, abstractmethod +from collections.abc import Iterator from contextlib import suppress from typing import Any, Generic, Literal, TypeVar, overload from urllib.parse import urljoin @@ -11,6 +12,42 @@ from esm.sdk.retry import retry_decorator from esm.utils.decoding import assemble_message +POLL_BACKOFF_FACTOR = 2.0 +POLLS_PER_INTERVAL = 3 +DEFAULT_POLL_INTERVAL = 2 + + +def _check_polling(interval: int | None, ceiling: int | None) -> None: + """Reject a pair one caller supplied that cannot both be honored.""" + if interval is not None and ceiling is not None and ceiling < interval: + raise ValueError( + f"poll_max_interval ({ceiling}) is below poll_interval ({interval}); the " + "backoff ceiling cannot be lower than the interval it grows from" + ) + + +def _resolve_polling( + intervals: tuple[int | None, ...], ceilings: tuple[int | None, ...] +) -> tuple[int, int]: + """Settle the poll interval and its backoff ceiling, most specific preference first.""" + interval = next((v for v in intervals if v is not None), DEFAULT_POLL_INTERVAL) + ceiling = next((v for v in ceilings if v is not None), interval) + return interval, max(interval, ceiling) + + +def _poll_intervals(initial: float, maximum: float | None = None) -> Iterator[float]: + """Staircased exponential polling backoff. + + Poll at `initial` for `POLLS_PER_INTERVAL` times, then multiply the interval by + `POLL_BACKOFF_FACTOR` and repeat, up to `maximum`. + """ + ceiling = initial if maximum is None else maximum + interval = min(initial, ceiling) + while True: + for _ in range(POLLS_PER_INTERVAL): + yield interval + interval = min(interval * POLL_BACKOFF_FACTOR, ceiling) + class _BaseForgeInferenceClient: def __init__( @@ -131,6 +168,13 @@ async def _async_post( return data except ESMProteinError as e: raise e + except (TypeError, ValueError) as e: + # Serializing the request or parsing the reply failed deterministically. + # 500 would be retried + raise ESMProteinError( + error_code=400, + error_msg=f"Failed to submit request to {endpoint}. Error: {str(e)}", + ) except Exception as e: raise ESMProteinError( error_code=500, @@ -162,6 +206,13 @@ def _post( return data except ESMProteinError as e: raise e + except (TypeError, ValueError) as e: + # Serializing the request or parsing the reply failed deterministically. + # 500 would be retried + raise ESMProteinError( + error_code=400, + error_msg=f"Failed to submit request to {endpoint}. Error: {str(e)}", + ) except Exception as e: raise ESMProteinError( error_code=500, @@ -182,7 +233,8 @@ def __init__( min_retry_wait: int = 1, max_retry_wait: int = 10, max_retry_attempts: int = 5, - poll_interval: int = 2, + poll_interval: int | None = None, + poll_max_interval: int | None = None, transfer_timeout: int | None = 60, ): super().__init__( @@ -194,8 +246,9 @@ def __init__( max_retry_wait=max_retry_wait, max_retry_attempts=max_retry_attempts, ) - # How often to poll for status + _check_polling(poll_interval, poll_max_interval) self.poll_interval = poll_interval + self.poll_max_interval = poll_max_interval # Separate (longer) timeout for the payload-sized transfers (submit upload + # S3 result download) self.transfer_timeout = transfer_timeout @@ -243,12 +296,22 @@ async def async_get_status(self, task_id: str) -> dict[str, Any]: return await self._async_post("batch/status", {"task_id": task_id}) def wait_for_completion( - self, task_id: str, timeout: int, poll_interval: int | None = None + self, + task_id: str, + timeout: int, + poll_interval: int | None = None, + poll_max_interval: int | None = None, ) -> dict: - start_time = time.time() - interval = poll_interval if poll_interval is not None else self.poll_interval + _check_polling(poll_interval, poll_max_interval) + deadline = time.monotonic() + timeout + intervals = _poll_intervals( + *_resolve_polling( + (poll_interval, self.poll_interval), + (poll_max_interval, self.poll_max_interval), + ) + ) - while time.time() - start_time < timeout: + while True: response = self.get_status(task_id) job_status = response.get("status") if job_status == "done": @@ -262,7 +325,10 @@ def wait_for_completion( error_code=500, error_msg=f"Job {task_id} failed with error: '{response.get('error')}'.", ) - time.sleep(interval) + remaining = deadline - time.monotonic() + if remaining <= 0: + break + time.sleep(min(next(intervals), remaining)) raise ESMProteinError( error_code=500, @@ -270,12 +336,22 @@ def wait_for_completion( ) async def async_wait_for_completion( - self, task_id: str, timeout: int, poll_interval: int | None = None + self, + task_id: str, + timeout: int, + poll_interval: int | None = None, + poll_max_interval: int | None = None, ) -> dict: - start_time = time.time() - interval = poll_interval if poll_interval is not None else self.poll_interval + _check_polling(poll_interval, poll_max_interval) + deadline = time.monotonic() + timeout + intervals = _poll_intervals( + *_resolve_polling( + (poll_interval, self.poll_interval), + (poll_max_interval, self.poll_max_interval), + ) + ) - while time.time() - start_time < timeout: + while True: response = await self.async_get_status(task_id) job_status = response.get("status") if job_status == "done": @@ -289,7 +365,10 @@ async def async_wait_for_completion( error_code=500, error_msg=f"Job {task_id} failed with error: '{response.get('error')}'.", ) - await asyncio.sleep(interval) + remaining = deadline - time.monotonic() + if remaining <= 0: + break + await asyncio.sleep(min(next(intervals), remaining)) raise ESMProteinError( error_code=500, @@ -358,6 +437,8 @@ async def async_get_result_from_s3( class EndpointHandler(ABC, Generic[TResponse]): poll_interval: int | None = None + poll_max_interval: int = 30 + default_timeout: int = 30 * 60 # 30 minutes def __init__(self, batch_client: _BaseForgeBatchClient): self._batch_client = batch_client @@ -365,6 +446,19 @@ def __init__(self, batch_client: _BaseForgeBatchClient): self.max_retry_wait = batch_client.max_retry_wait self.max_retry_attempts = batch_client.max_retry_attempts + def _resolved_polling( + self, poll_interval: int | None = None, poll_max_interval: int | None = None + ) -> tuple[int, int]: + _check_polling(poll_interval, poll_max_interval) + return _resolve_polling( + (poll_interval, self._batch_client.poll_interval, self.poll_interval), + ( + poll_max_interval, + self._batch_client.poll_max_interval, + self.poll_max_interval, + ), + ) + @property @abstractmethod def endpoint_name(self) -> str: @@ -383,25 +477,36 @@ async def _async_process_response(self, response: dict, **kwargs) -> TResponse: pass def run( - self, timeout: int = 300, cancel_on_timeout: bool = True, **kwargs + self, + timeout: int | None = None, + cancel_on_timeout: bool = True, + poll_interval: int | None = None, + poll_max_interval: int | None = None, + **kwargs, ) -> TResponse | ESMProteinError: """ Submit and execute a batch job, waiting for completion by polling the status of the job. Args: timeout: Maximum time to wait for job completion, in seconds. + poll_interval: Seconds between status polls + poll_max_interval: Ceiling the backoff may grow `poll_interval` to. cancel_on_timeout: If True, cancels the batch job if it times out or is interrupted. **kwargs: Arguments to pass to the batch job. Returns: The response from the batch job or an ESMProteinError if the job fails. """ + timeout = self.default_timeout if timeout is None else timeout task_id = None task_timed_out = False keyboard_interrupted = False + interval, max_interval = self._resolved_polling( + poll_interval, poll_max_interval + ) try: request = self._prepare_request(**kwargs) task_id = self._batch_client.submit(self.endpoint_name, request) response = self._batch_client.wait_for_completion( - task_id, timeout, poll_interval=self.poll_interval + task_id, timeout, poll_interval=interval, poll_max_interval=max_interval ) return self._process_response(response, **kwargs) except KeyboardInterrupt: @@ -424,16 +529,25 @@ def run( self._batch_client.cancel(task_id) async def async_run( - self, timeout: int = 300, cancel_on_timeout: bool = True, **kwargs + self, + timeout: int | None = None, + cancel_on_timeout: bool = True, + poll_interval: int | None = None, + poll_max_interval: int | None = None, + **kwargs, ) -> TResponse | ESMProteinError: + timeout = self.default_timeout if timeout is None else timeout task_id = None task_timed_out = False keyboard_interrupted = False + interval, max_interval = self._resolved_polling( + poll_interval, poll_max_interval + ) try: request = self._prepare_request(**kwargs) task_id = await self._batch_client.async_submit(self.endpoint_name, request) response = await self._batch_client.async_wait_for_completion( - task_id, timeout, poll_interval=self.poll_interval + task_id, timeout, poll_interval=interval, poll_max_interval=max_interval ) return await self._async_process_response(response, **kwargs) except KeyboardInterrupt: diff --git a/esm/sdk/base_forge_client_test.py b/esm/sdk/base_forge_client_test.py new file mode 100644 index 00000000..31bcd1c8 --- /dev/null +++ b/esm/sdk/base_forge_client_test.py @@ -0,0 +1,210 @@ +"""The batch client's polling backoff and per-endpoint wait defaults.""" + +import itertools + +import pytest + +from esm.sdk import base_forge_client +from esm.sdk.api import ESMProteinError +from esm.sdk.base_forge_client import ( + DEFAULT_POLL_INTERVAL, + EndpointHandler, + _poll_intervals, +) +from esm.sdk.forge import ForgeBatchClient + + +class _QuietHandler(EndpointHandler[dict]): + """An endpoint that states no pacing of its own, so the layers above it decide.""" + + @property + def endpoint_name(self) -> str: + return "quiet" + + def _prepare_request(self, **kwargs) -> list[dict]: + return [] + + def _process_response(self, response: dict, **kwargs) -> dict: + return response + + async def _async_process_response(self, response: dict, **kwargs) -> dict: + return response + + +@pytest.fixture +def client() -> ForgeBatchClient: + return ForgeBatchClient(url="http://localhost", token="unused") + + +def quiet(client: ForgeBatchClient) -> _QuietHandler: + return _QuietHandler(client._batch_client) + + +def test_the_staircase_holds_each_rung_then_doubles_up_to_the_ceiling(): + got = list(itertools.islice(_poll_intervals(2, 30), 15)) + assert got == [2, 2, 2, 4, 4, 4, 8, 8, 8, 16, 16, 16, 30, 30, 30] + + +def test_an_endpoint_paces_itself_when_the_user_says_nothing(client): + assert client._batch_client.poll_interval is None + assert client.fold_max_accuracy._resolved_polling() == (10, 60) + assert quiet(client)._resolved_polling() == (DEFAULT_POLL_INTERVAL, 30) + + +def test_the_users_interval_beats_the_endpoints_own(): + client = ForgeBatchClient(url="http://localhost", token="unused", poll_interval=45) + assert client.fold_max_accuracy.poll_interval == 10, ( + "the endpoint still asks for 10" + ) + assert client.fold_max_accuracy._resolved_polling() == (45, 60) + assert quiet(client)._resolved_polling() == (45, 45) + + +def test_the_ceiling_never_clamps_the_interval_below_what_was_asked_for(): + """Asking for 90s against `fold_max_accuracy`'s ceiling of 60 has to mean 90.""" + client = ForgeBatchClient(url="http://localhost", token="unused", poll_interval=90) + assert client.fold_max_accuracy._resolved_polling() == (90, 90) + assert quiet(client)._resolved_polling() == (90, 90) + + +def test_a_ceiling_above_the_interval_is_what_the_backoff_grows_into(): + client = ForgeBatchClient( + url="http://localhost", token="unused", poll_interval=90, poll_max_interval=120 + ) + assert client.fold_max_accuracy._resolved_polling() == (90, 120) + + +def test_a_user_ceiling_applies_without_touching_the_endpoints_interval(): + client = ForgeBatchClient( + url="http://localhost", token="unused", poll_max_interval=15 + ) + assert client.fold_max_accuracy._resolved_polling() == (10, 15) + + +def test_a_per_call_interval_beats_every_standing_setting(client): + """Named on `run` rather than left to `**kwargs`, which forwards to the payload.""" + configured = ForgeBatchClient( + url="http://localhost", token="unused", poll_interval=45 + ) + assert client.fold_max_accuracy._resolved_polling(poll_interval=5) == (5, 60) + assert configured.fold_max_accuracy._resolved_polling(poll_interval=90) == (90, 90) + assert client.fold_max_accuracy._resolved_polling( + poll_interval=90, poll_max_interval=100 + ) == (90, 100) + + +def test_a_ceiling_below_its_interval_from_the_same_caller_is_rejected(client): + with pytest.raises(ValueError, match="poll_max_interval"): + ForgeBatchClient( + url="http://localhost", + token="unused", + poll_interval=200, + poll_max_interval=120, + ) + with pytest.raises(ValueError, match="below poll_interval"): + client.fold_max_accuracy._resolved_polling( + poll_interval=200, poll_max_interval=120 + ) + with pytest.raises(ValueError, match="below poll_interval"): + client._batch_client.wait_for_completion( + "task-9", timeout=10, poll_interval=200, poll_max_interval=120 + ) + + +class _FakeClock: + """A monotonic clock that only advances when something sleeps.""" + + def __init__(self) -> None: + self.now = 1000.0 + self.sleeps: list[float] = [] + + def monotonic(self) -> float: + return self.now + + def sleep(self, seconds: float) -> None: + self.sleeps.append(seconds) + self.now += seconds + + +@pytest.fixture +def clock(monkeypatch: pytest.MonkeyPatch) -> _FakeClock: + fake = _FakeClock() + monkeypatch.setattr(base_forge_client.time, "monotonic", fake.monotonic) + monkeypatch.setattr(base_forge_client.time, "sleep", fake.sleep) + return fake + + +def _client_returning(statuses: list[dict]) -> ForgeBatchClient: + """A client whose `get_status` walks `statuses`, then repeats the last one.""" + client = ForgeBatchClient(url="http://localhost", token="unused") + remaining = list(statuses) + + def get_status(task_id: str) -> dict: + return remaining.pop(0) if len(remaining) > 1 else remaining[0] + + client._batch_client.get_status = get_status + return client + + +def test_a_sleep_never_overshoots_the_deadline(clock): + client = _client_returning([{"status": "in_progress"}]) + with pytest.raises(ESMProteinError, match="timed out"): + client._batch_client.wait_for_completion( + "task-1", timeout=10, poll_interval=60, poll_max_interval=60 + ) + assert clock.sleeps == [10] + + +def test_a_job_finishing_during_the_final_sleep_is_returned(clock): + """Waking past the deadline must not report a timeout and cancel a finished job.""" + client = _client_returning([{"status": "in_progress"}, {"status": "done"}]) + response = client._batch_client.wait_for_completion( + "task-2", timeout=5, poll_interval=5, poll_max_interval=5 + ) + assert response == {"status": "done"} + assert clock.sleeps == [5] + + +def test_a_terminal_status_is_returned_without_sleeping(clock): + client = _client_returning([{"status": "done"}]) + assert client._batch_client.wait_for_completion("task-3", timeout=600) == { + "status": "done" + } + assert clock.sleeps == [] + + +def test_a_failed_job_raises_with_the_server_reason(clock): + client = _client_returning([{"status": "failed", "error": "CUDA out of memory"}]) + with pytest.raises(ESMProteinError, match="CUDA out of memory"): + client._batch_client.wait_for_completion("task-4", timeout=600) + + +def _stub_submission( + monkeypatch: pytest.MonkeyPatch, handler: EndpointHandler, task_id: str +) -> None: + monkeypatch.setattr(handler, "_prepare_request", lambda **kwargs: []) + monkeypatch.setattr(handler._batch_client, "submit", lambda *a, **kw: task_id) + + +def test_the_users_interval_reaches_the_wait_loop(clock, monkeypatch): + client = _client_returning([{"status": "in_progress"}]) + client._batch_client.poll_interval = 45 + handler = client.fold_max_accuracy + _stub_submission(monkeypatch, handler, "task-7") + + result = handler.run(timeout=90, cancel_on_timeout=False) + + assert isinstance(result, ESMProteinError) + assert clock.sleeps == [45, 45] + + +def test_a_per_call_interval_reaches_the_wait_loop(clock, monkeypatch): + client = _client_returning([{"status": "in_progress"}]) + client._batch_client.poll_interval = 45 + handler = client.fold_max_accuracy + _stub_submission(monkeypatch, handler, "task-8") + + result = handler.run(timeout=60, cancel_on_timeout=False, poll_interval=20) + + assert isinstance(result, ESMProteinError) + assert clock.sleeps == [20, 20, 20] diff --git a/esm/sdk/forge.py b/esm/sdk/forge.py index 2081f874..21ca420e 100644 --- a/esm/sdk/forge.py +++ b/esm/sdk/forge.py @@ -36,6 +36,7 @@ ) from esm.sdk.retry import retry_decorator from esm.sdk.validation import validate_fold_max_accuracy_input +from esm.utils.compression import compress_state_dict from esm.utils.constants.api import MIMETYPE_ES_PICKLE from esm.utils.constants.models import ( DEFAULT_ESMFOLD2_FAST_LM_MASK_PCT, @@ -168,14 +169,12 @@ def _process_fold_request( stacklevel=4, ) elif len(msa.sequences) > ESMFOLD2_MAX_MSA_SEQS: - warnings.warn( + raise ValueError( f"MSA depth ({len(msa.sequences)}) exceeds the maximum of " - f"{ESMFOLD2_MAX_MSA_SEQS}. The MSA will be truncated to " - f"{ESMFOLD2_MAX_MSA_SEQS} sequences by the server.", - UserWarning, - stacklevel=4, + f"{ESMFOLD2_MAX_MSA_SEQS}. Truncate the alignment using " + "`select_sequences` before submitting." ) - request["msa"] = msa.state_dict(json_serializable=True) + request["msa"] = compress_state_dict(msa) else: error_msg = f"MSA must be None or MSA. Got {msa} instead." raise AttributeError(error_msg) @@ -403,12 +402,10 @@ def _process_fold_all_atom_request( and len(seq.msa.sequences) > ESMFOLD2_MAX_MSA_SEQS ): chain_id = seq.id if seq.id is not None else "unknown" - warnings.warn( + raise ValueError( f"MSA depth ({len(seq.msa.sequences)}) for chain '{chain_id}' " - f"exceeds the maximum of {ESMFOLD2_MAX_MSA_SEQS}. The MSA will " - f"be truncated to {ESMFOLD2_MAX_MSA_SEQS} sequences by the server.", - UserWarning, - stacklevel=4, + f"exceeds the maximum of {ESMFOLD2_MAX_MSA_SEQS}. Truncate the " + "alignment using `select_sequences` before submitting." ) request: dict[str, Any] = { @@ -1392,6 +1389,8 @@ class FoldMaxAccuracyHandler(EndpointHandler[MolecularComplexResult]): """Fold a molecular complex at the highest-accuracy settings.""" poll_interval = 10 + poll_max_interval = 60 + default_timeout = 2 * 60 * 60 # 2 hours def __init__(self, batch_client: _BaseForgeBatchClient): super().__init__(batch_client) @@ -1438,7 +1437,8 @@ def __init__( token: str = "", request_timeout: int | None = 15, model: str | None = None, - poll_interval: int = 2, + poll_interval: int | None = None, + poll_max_interval: int | None = None, min_retry_wait: int = 2, max_retry_wait: int = 2, max_retry_attempts: int = 5, @@ -1452,6 +1452,7 @@ def __init__( max_retry_wait=max_retry_wait, max_retry_attempts=max_retry_attempts, poll_interval=poll_interval, + poll_max_interval=poll_max_interval, transfer_timeout=transfer_timeout, ) self.model = model diff --git a/esm/tokenization/sasa_tokenizer.py b/esm/tokenization/sasa_tokenizer.py index 00a9ea66..48e6f513 100644 --- a/esm/tokenization/sasa_tokenizer.py +++ b/esm/tokenization/sasa_tokenizer.py @@ -1,3 +1,4 @@ +import math from functools import cached_property import torch @@ -76,6 +77,12 @@ def encode( ids.append(self.vocab_to_index[""]) # BOS for value in values: if isinstance(value, (float, int)): + if math.isnan(value): + # torch.bucketize would silently bin NaN into the top SASA range. + raise ValueError( + "Cannot tokenize NaN SASA value. Use None or " + f"'{self.mask_token}' to mask a position." + ) bucket = torch.bucketize(value, torch.tensor(self._boundaries)) token_id = len(self.special_tokens) + bucket elif isinstance(value, str): @@ -88,13 +95,21 @@ def encode( return torch.tensor(ids, dtype=torch.int64) - def decode_float(self, encoded: torch.Tensor) -> list[float]: - """Decodes SASA token ids into float values.""" - decoded = self.midpoints_tensor[encoded.cpu()] - nan_mask = torch.isnan(decoded) - np_arr = decoded.numpy() - np_arr[nan_mask.numpy()] = None - return np_arr.tolist() + def decode_float(self, encoded: torch.Tensor) -> list[float | str | None]: + """Decodes SASA token ids into range midpoints. + + Special tokens have no midpoint: `` decodes to None and the others to + their vocab string, so that the result re-encodes to the tokens it came from. + """ + decoded: list[float | str | None] = [] + for token_id in encoded.cpu().tolist(): + if token_id == self.mask_token_id: + decoded.append(None) + elif token_id < len(self.special_tokens): + decoded.append(self.vocab[token_id]) + else: + decoded.append(self.midpoints_tensor[token_id].item()) + return decoded def decode(self, encoded: torch.Tensor) -> str: """Decodes SASA token ids.""" diff --git a/esm/utils/compression.py b/esm/utils/compression.py new file mode 100644 index 00000000..1c99aad4 --- /dev/null +++ b/esm/utils/compression.py @@ -0,0 +1,22 @@ +"""Compressed wire form for a large field carried inside a request payload.""" + +import base64 +import json +from typing import Any, Protocol + +import zstd + + +class SupportsStateDict(Protocol): + """The SDK's serialization convention, as used for the Forge/BHP wire form.""" + + def state_dict(self, json_serializable: bool = False) -> dict[str, Any]: ... + + +def compress_state_dict(obj: SupportsStateDict) -> str: + payload = json.dumps(obj.state_dict(json_serializable=True)).encode() + return base64.b64encode(zstd.compress(payload)).decode() + + +def decompress_state_dict(blob: str) -> dict[str, Any]: + return json.loads(zstd.decompress(base64.b64decode(blob))) diff --git a/esm/utils/decoding.py b/esm/utils/decoding.py index 38657171..6187a70d 100644 --- a/esm/utils/decoding.py +++ b/esm/utils/decoding.py @@ -101,7 +101,7 @@ def decode_protein_tensor( return ESMProtein( sequence=sequence, secondary_structure=secondary_structure, - sasa=sasa, # type: ignore + sasa=sasa, function_annotations=function_annotations if function_annotations else None, coordinates=coordinates, plddt=plddt, @@ -175,12 +175,15 @@ def decode_secondary_structure( _bos_eos_warn("Secondary structure", secondary_structure_tokens, ss_tokenizer) secondary_structure_tokens = secondary_structure_tokens[1:-1] secondary_structure = ss_tokenizer.decode(secondary_structure_tokens) + secondary_structure = secondary_structure.replace( + ss_tokenizer.mask_token, C.MASK_STR_SHORT + ) return secondary_structure def decode_sasa( sasa_tokens: torch.Tensor, sasa_tokenizer: SASADiscretizingTokenizer -) -> list[float]: +) -> list[float | str | None]: if sasa_tokens[0] != 0: raise ValueError("SASA does not start with 0 corresponding to BOS token") if sasa_tokens[-1] != 0: @@ -194,11 +197,13 @@ def decode_sasa( torch.long, ]: # Decode if int - # handles turning NaN's into None's + # handles turning special tokens into None's / vocab strings sasa = sasa_tokenizer.decode_float(sasa_tokens) else: # If already float, just convert to list - sasa = cast(list[float], maybe_list(sasa_tokens, convert_nan_to_none=True)) + sasa = cast( + list[float | str | None], maybe_list(sasa_tokens, convert_nan_to_none=True) + ) return sasa diff --git a/esm/utils/msa/msa.py b/esm/utils/msa/msa.py index e9d7bb05..760a4fa2 100644 --- a/esm/utils/msa/msa.py +++ b/esm/utils/msa/msa.py @@ -17,11 +17,13 @@ from esm.utils.sequential_dataclass import SequentialDataclass from esm.utils.system import PathOrBuffer -REMOVE_LOWERCASE_TRANSLATION = str.maketrans(dict.fromkeys(string.ascii_lowercase)) +REMOVE_A3M_INSERTION_TRANSLATION = str.maketrans( + dict.fromkeys(string.ascii_lowercase + ".") +) def remove_insertions_from_sequence(seq: str) -> str: - return seq.translate(REMOVE_LOWERCASE_TRANSLATION) + return seq.translate(REMOVE_A3M_INSERTION_TRANSLATION) def is_a3m_insertion(ch: str) -> bool: @@ -29,6 +31,13 @@ def is_a3m_insertion(ch: str) -> bool: return ch == "." or ch.islower() +def _as_index(indices: Sequence[int] | np.ndarray | slice) -> np.ndarray | slice: + """Indices numpy will accept, so an empty selection still indexes.""" + if isinstance(indices, slice): + return indices + return np.asarray(indices, dtype=int) + + def a3m_deletion_counts(seq: str) -> np.ndarray: """Per-match-column count of preceding a3m insertions (lowercase letters / ``.``). @@ -246,7 +255,8 @@ def depth(self) -> int: @property def seqlen(self) -> int: - return len(self.entries[0].sequence) + # 0 for an empty alignment, matching `depth` + return len(self.entries[0].sequence) if self.entries else 0 @cached_property def array(self) -> np.ndarray: @@ -261,7 +271,11 @@ def _aligned_deletions(self) -> np.ndarray | None: Misalignment means the sequences still carry insertions (length != match-column count), so the stored deletions no longer describe them.""" - if self.deletions is None or self.deletions.shape != (self.depth, self.seqlen): + if self.deletions is None: + return None + if self.depth == 0: + return self.deletions if self.deletions.shape[0] == 0 else None + if self.deletions.shape != (self.depth, self.seqlen): return None return self.deletions @@ -269,15 +283,17 @@ def _select_deletion_columns(self, indices) -> np.ndarray | None: """Column-subselect ``deletions`` to match a position subselect, or None when ``deletions`` is not column-aligned with the sequences (e.g. the sequences still carry insertions, so length != match-column count).""" - if self.deletions is None or self.deletions.shape[1] != self.seqlen: + if self.deletions is None: + return None + if self.depth > 0 and self.deletions.shape[1] != self.seqlen: return None - return self.deletions[:, indices] + return self.deletions[:, _as_index(indices)] def select_sequences(self, indices: Sequence[int] | np.ndarray) -> MSA: """Subselect rows of the MSA.""" entries = [self.entries[idx] for idx in indices] deletions = ( - None if self.deletions is None else self.deletions[np.asarray(indices)] + None if self.deletions is None else self.deletions[_as_index(indices)] ) return dataclasses.replace(self, entries=entries, deletions=deletions) diff --git a/esm/utils/structure/input_builder.py b/esm/utils/structure/input_builder.py index a5716441..63004c6b 100644 --- a/esm/utils/structure/input_builder.py +++ b/esm/utils/structure/input_builder.py @@ -3,6 +3,7 @@ import numpy as np +from esm.utils.compression import compress_state_dict, decompress_state_dict from esm.utils.msa import MSA # fmt: off @@ -136,7 +137,7 @@ def create_chain_data(seq_input, chain_type: str) -> dict[str, Any]: elif seq_input.msa is None: chain_data["msa"] = None elif isinstance(seq_input.msa, MSA): - chain_data["msa"] = seq_input.msa.state_dict(json_serializable=True) + chain_data["msa"] = compress_state_dict(seq_input.msa) else: error_msg = f"MSA must be None or MSA. Got {seq_input.msa} instead." raise AttributeError(error_msg) @@ -215,7 +216,9 @@ def _msa(chain: dict[str, Any]) -> MSAInput: return None msa_blk = chain["msa"] if isinstance(msa_blk, str): - raise ValueError(f"Unexpected MSA string value: {msa_blk!r}") + # `serialize_structure_prediction_input` always compresses, so expanding + # here is what keeps this function its inverse. + return MSA.from_state_dict(decompress_state_dict(msa_blk)) return MSA.from_sequences(msa_blk["sequences"]) sequences: list[ProteinInput | RNAInput | DNAInput | LigandInput] = [] diff --git a/esm/widgets/components/results_visualizer.py b/esm/widgets/components/results_visualizer.py index 95ff1f7d..3d1125ff 100644 --- a/esm/widgets/components/results_visualizer.py +++ b/esm/widgets/components/results_visualizer.py @@ -194,7 +194,7 @@ def create_sasa_results_page( if item.sasa is None: print("Solvent Accessible Surface Area (SASA) is not available.") else: - sasa = [s or 0 for s in item.sasa] + sasa = [s if isinstance(s, (int, float)) else 0 for s in item.sasa] draw_data_array(output, data_array=sasa, cmap="Reds") if copy_to_prompt_callback: diff --git a/esm/widgets/views/esm3_prompt_preview.py b/esm/widgets/views/esm3_prompt_preview.py index 60a57e1b..535b6a44 100644 --- a/esm/widgets/views/esm3_prompt_preview.py +++ b/esm/widgets/views/esm3_prompt_preview.py @@ -1,3 +1,5 @@ +from collections.abc import Sequence + import torch from ipywidgets import widgets @@ -21,7 +23,7 @@ def coordinates_to_text(coordinates: torch.Tensor | None) -> str: return "".join(coordinates_text) -def sasa_to_text(sasa: list[int | float | None] | None) -> str: +def sasa_to_text(sasa: Sequence[int | float | str | None] | None) -> str: if sasa is None: return "" @@ -36,14 +38,17 @@ def sasa_to_text(sasa: list[int | float | None] | None) -> str: return ",".join(sasa_text) -def text_to_sasa(sasa_text: str) -> list[int | float | None] | None: +def text_to_sasa(sasa_text: str) -> list[int | float | str | None] | None: if not sasa_text: return None - sasa = [] + sasa: list[int | float | str | None] = [] for value in sasa_text.split(","): if value == MASK_STR_SHORT: sasa.append(None) + elif value.startswith("<"): + # A SASA vocab token such as ""; the tokenizer validates it. + sasa.append(value) else: sasa.append(float(value)) diff --git a/pixi.lock b/pixi.lock index 84a21f66..ab87343a 100644 --- a/pixi.lock +++ b/pixi.lock @@ -9671,6 +9671,12 @@ packages: - accelerate - cuequivariance-torch>=0.8.1 ; platform_machine == 'x86_64' and sys_platform == 'linux' - cuequivariance-ops-torch-cu13>=0.8.1 ; platform_machine == 'x86_64' and sys_platform == 'linux' + - triton>=3,<4 ; extra == 'fused' + - transformer-engine[pytorch]>=2,<3 ; extra == 'fold-cp' + - cuequivariance-torch ; extra == 'cueq12' + - cuequivariance-ops-torch-cu12 ; extra == 'cueq12' + - cuequivariance-torch ; extra == 'cueq13' + - cuequivariance-ops-torch-cu13 ; extra == 'cueq13' requires_python: '>=3.12' - pypi: direct+https://download.pytorch.org/whl/cu130/xformers-0.0.35-py39-none-manylinux_2_28_x86_64.whl name: xformers diff --git a/pyproject.toml b/pyproject.toml index a761f53f..32fb15db 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "esm" -version = "3.4.0" +version = "3.4.1" description = "EvolutionaryScale open model repository" readme = "README.md" requires-python = ">=3.12" @@ -54,6 +54,17 @@ dependencies = [ "cuequivariance-torch>=0.8.1; sys_platform == 'linux' and platform_machine == 'x86_64'", "cuequivariance-ops-torch-cu13>=0.8.1; sys_platform == 'linux' and platform_machine == 'x86_64'", ] + +[project.optional-dependencies] +# ESMFold2 "fused" kernel backend (set_kernel_backend("fused")) +# inference kernels for tri-mul / LN+SwiGLU / dropout-residual. +fused = ["triton>=3,<4"] +# ESMFold2 context-parallel ESM-C tensor parallelism (wrap_model_with_cp(tp_esmc=True)). +fold-cp = ["transformer-engine[pytorch]>=2,<3"] +# "cuequivariance" kernel backend — pick the build matching your CUDA toolkit. +cueq12 = ["cuequivariance-torch", "cuequivariance-ops-torch-cu12"] +cueq13 = ["cuequivariance-torch", "cuequivariance-ops-torch-cu13"] + # Pytest [tool.pytest.ini_options] addopts = """ diff --git a/tests/models/esmfold2_api_test.py b/tests/models/esmfold2_api_test.py index a82e824a..4fde2696 100644 --- a/tests/models/esmfold2_api_test.py +++ b/tests/models/esmfold2_api_test.py @@ -362,3 +362,100 @@ def test_experimental_forward_rejects_an_unknown_keyword(tiny_experimental): num_recyles=1, **FAST_FOLD, ) + + +# --------------------------------------------------------------------------- +# include_embeddings +# --------------------------------------------------------------------------- + + +def _pair_width(config) -> int: + return config.pairwise_hidden_size + + +def test_fold_omits_embeddings_by_default(tiny_esmfold2, builder): + """The pair export is opt-in: it costs an L x L reduction and an L x D tensor.""" + result = builder.fold(tiny_esmfold2, protein_input(), seed=0, **FAST_FOLD) + + assert result.output_embedding_pair_pooled is None + assert result.output_embedding_sequence is None + + +def test_fold_include_embeddings_returns_the_pooled_pair( + tiny_esmfold2, tiny_esmfold2_config, builder +): + """Shape, dtype and device match what the SDK hands back from Forge.""" + result = builder.fold( + tiny_esmfold2, protein_input(), seed=0, include_embeddings=True, **FAST_FOLD + ) + + pooled = result.output_embedding_pair_pooled + assert pooled is not None + assert pooled.shape == (len(TINY_SEQUENCE), _pair_width(tiny_esmfold2_config)) + assert pooled.dtype == torch.float32 + assert pooled.device.type == "cpu" + assert torch.isfinite(pooled).all() + + assert result.output_embedding_sequence is None + + +def test_pooled_pair_is_the_post_coda_pair_averaged_over_the_first_axis(tiny_esmfold2): + """Pin the reduction axis against the tensor it is supposed to summarise.""" + captured: dict[str, torch.Tensor] = {} + handle = tiny_esmfold2.parcae_coda.register_forward_hook( + lambda module, args, out: captured.__setitem__("z", out.detach().float()) + ) + try: + features, lm_hidden_states = esmfold2_inputs(tiny_esmfold2, TINY_SEQUENCE) + with torch.no_grad(): + output = tiny_esmfold2( + **features, + lm_hidden_states=lm_hidden_states, + include_embeddings=True, + **FAST_FOLD, + ) + finally: + handle.remove() + + z = captured["z"] + pooled = output["output_embedding_pair_pooled"] + + assert torch.allclose(pooled, z.mean(dim=1), atol=1e-6) + assert not torch.allclose(pooled, z.mean(dim=2), atol=1e-4) + + +def test_pooled_pair_is_shared_across_diffusion_samples(tiny_esmfold2, builder): + """The trunk runs once, so every sample carries the same embedding.""" + results = builder.fold( + tiny_esmfold2, + protein_input(), + seed=0, + num_loops=1, + num_sampling_steps=2, + num_diffusion_samples=2, + include_embeddings=True, + ) + + assert isinstance(results, list) and len(results) == 2 + first, second = (item.output_embedding_pair_pooled for item in results) + assert first is not None and second is not None + assert torch.equal(first, second) + + +def test_experimental_forward_supports_include_embeddings( + tiny_experimental, tiny_esmfold2_config +): + """``fold`` takes either architecture, so the flag cannot be release-only.""" + features, lm_hidden_states = esmfold2_inputs(tiny_experimental, TINY_SEQUENCE) + + with torch.no_grad(): + output = tiny_experimental( + **features, + lm_hidden_states=lm_hidden_states, + include_embeddings=True, + **FAST_FOLD, + ) + + pooled = output["output_embedding_pair_pooled"] + assert pooled.shape == (1, len(TINY_SEQUENCE), _pair_width(tiny_esmfold2_config)) + assert pooled.dtype == torch.float32 diff --git a/tests/models/esmfold2_inputs_test.py b/tests/models/esmfold2_inputs_test.py index c350bdd2..3d307ee5 100644 --- a/tests/models/esmfold2_inputs_test.py +++ b/tests/models/esmfold2_inputs_test.py @@ -687,6 +687,9 @@ def atom_aligned_inputs(tiny_esmfold2): return features, lm_hidden_states +@pytest.mark.skip( + reason="Flaky: distogram_logits are not bitwise-equal across atom-axis padding" +) def test_atom_padding_does_not_move_the_deterministic_outputs( tiny_esmfold2, atom_aligned_inputs ): diff --git a/tests/oss_pytests/requirements.txt b/tests/oss_pytests/requirements.txt index 143baa7a..e725bcba 100644 --- a/tests/oss_pytests/requirements.txt +++ b/tests/oss_pytests/requirements.txt @@ -1,2 +1,2 @@ -esm >=3.2.1post1,<4.0.0 +esm >=3.4.0,<4.0.0 pytest diff --git a/tests/oss_pytests/test_oss_client.py b/tests/oss_pytests/test_oss_client.py index 746bf301..d534328a 100644 --- a/tests/oss_pytests/test_oss_client.py +++ b/tests/oss_pytests/test_oss_client.py @@ -3,7 +3,7 @@ import pytest import torch -from esm.sdk import client # pyright: ignore +from esm.sdk import client, esmc_client # pyright: ignore from esm.sdk.api import ( # pyright: ignore ESMProtein, ESMProteinTensor, @@ -58,19 +58,19 @@ def test_oss_esmc_client(): sequence = "MALWMRLLPLLALLALAVPDPAAA" model = "esmc-300m-2024-12" - esmc_client = client(model=model, url=URL, token=API_TOKEN) + esmc = esmc_client(model=model, url=URL, token=API_TOKEN) protein = ESMProtein(sequence) - encoded_protein = esmc_client.encode(input=protein) + encoded_protein = esmc.encode(input=protein) assert isinstance(encoded_protein, ESMProteinTensor) - decoded_protein = esmc_client.decode(input=encoded_protein) + decoded_protein = esmc.decode(input=encoded_protein) assert isinstance(decoded_protein, ESMProtein) logits_config = LogitsConfig( sequence=True, return_embeddings=True, return_hidden_states=True ) - result = esmc_client.logits(input=encoded_protein, config=logits_config) + result = esmc.logits(input=encoded_protein, config=logits_config) assert isinstance(result, LogitsOutput) assert result.logits is not None assert isinstance(result.logits.sequence, torch.Tensor) diff --git a/tests/reference/esmfold2_tiny_cpu_fp32.pkl.gz b/tests/reference/esmfold2_tiny_cpu_fp32.pkl.gz index 41dcb925d16df9b0b930463b1fc9f1c01dbbc8f3..54cd0481c3b25d681e4c14c37ea1987bc979419e 100644 GIT binary patch literal 37937 zcmV($K;yq3iwFpxvY2WD|7CM+W^Zg{GGBCQZh2p0aCKj1a5FM4aBFM;?0p4TRa@6C zAV?}H7@$~`bfd8McmY8qR7woGyX^o}x>QuU%kGxF?8I)YV|RDowGlj?^L>8T|G)oz z?sLvPi)T6Knscl<$DHFGW5(EXnIf|2s-mC(|C(8(kt^k&5EvGo6dah8oDf`;+iiGa zv|VgekfC#IP;gXHZr7mTN#TKPMwh_2R%`T8ftMz`ns#{>rZ2874>1_y<*<$4!s*b8)A z_Vhg>(;4a68By(w4C;(%bVjs0Bl?{Y!_J6tXT-EKV%`}U+8MFzjBq+59ef1m_WaIq zP(pY}65A-YFZs_!_S~Xr*X>s+OwW}IjExBi5A_XUyD>N+E+IT7sVJ9Bj0}xT{7uZ+ z%h1q2&`{sd+~3eFAdsILWD#hlA82A2sBau#6k-}+YGP<^8faz|sBdIq?#~kG`vKV-;+NjeIKb>qt_itTORc5v|NyRnb$ z!uOvVoX{@jejzN(_<~PL;0u+X*-=y{jbLv7f2HwHN(zoi;sYZr*gvSfr0$7qb~}FC z)7tr!`5izk56Mx%?Wq!p;i1v~zCr#;{zcz~l`Gqh6yHC!P%xySlORV3oczmHVtC zRXRK-Bsd|M5BhJs+f&fL&1$b=sOgP|$~HQ=lD}ug6@Ay%T>0OV+s#dfl}XN(Zbx@e zd(L;*hz7^_2Sl-!D>F=s6+(CZD{)alK}p{`OsYL8DELna{=X-Pv3#`~nxgOKgiZ6G z(wR2$TUwaDqgdDw78RmF39)gp$w|NWOLDGa9BY>Szp>1s#s3Fj@D=Mv2w8n&wO!vj z3*w&WY<(OHV7|fi{UJLuIaj#DjI$64_D^JEa*3>(MFn>N>YEbqQPn*$D!g3-Cx^!b z#ZE4AOv&x>N2UB1?sFAmeA#mSlfowjvysH`peY@FqsrR$-?IY4_{jQ}Bb2Ma_71=6 zeWMfoeM5r%`Rbflloe+Gd%3JNNMMEadrN$SSPf+v`7Yw3tXxSpJK8@n(l;*fN1h~u z6QcutI}Dl&D`);=IBVp(2ZYClB?ssYcXPJUF|{rV)A**BDY+7C4RQQRSp9bv=n^*2 zh3)7rtjL6OrGux$@pYV4%mlVEvHS+~NsMEo{!zXxZ!8$tUiA%02#!x?`I=hveH_SD zVJ*y$gnT=i=*t(A|CI3do{)+0O%4zGA-Bj;B3J2q(!Z3vC$j@IlKigw12nR7#bYN0 zCrnOY2Y43J{8q`btzs+tqp@QfCd@WWELVcB39PJxlEV1n5^slXa2)&OEFhdK6Brwn z93Atmf7yofm3C_h|0+$+?fMO}9hl6OieoiCk#7Rq4GM{Mv|pUB9UWRI_^pDDr>@DJ zlZkKO6k}z?hnitga;``q+v>!26%|ejkD1yY6HENo2`sIY^B4y^Pe*qrcY8;7c2W}a z4S31?d4p80Sj_jyfK+a`nD(haqVFVs7TQIQKCG5^q{R8Nox&#hGX1^+vusP?&%R1yo+FkRo|r0}RjR<~GY0+Pd{f_zzPo*0|pJISzU zAM-Agw` z$oYO3_TQQD0vcZYc}iSDY*H*M8(&t|d{D*l+%FTA1n^8ODDrn4Lw&cX9M{ zNs1R9!xrZqFYs@P1^spXI#gFvs*Z8W9&c4y9mdgeuqRiaK7?WUojw?tUwoCl^<8Y(H&n**2gL?HoFu=_ay-IUmr>yzj#0UfEv5OdFrcRZ!W& zySYb&nqF_TQm$Xo@-`%$Ioh_Amvd_eb9JFEcZ2Xv-q_8qw5zd!*~!**a4i4g!ZEO5|25bDgK)%tYIr(`S>cEY{O7As;McCs&KLWa&X?d1 z>+GC5A-c1#bp-$FR^GzG$Xx#yHKCLE2Td3$UPE&h*3hfFW%OHh1HIL#p&L^xNzk&4 zt}~@{IIEoc`xcY`=qhrkEu)_!TdHFw`?u;|cJ*#yp>O<)hNgq~hwe>SdVqGX+)YET z9w70}d#UQoA-ey1AGOXpK)usj+ruNzPJU1OdG|`RuVEAXgTli*Iyd>RK0`4y(l;yPY%ADp3=4e37m{OLUy~B zQO$cPh{b1M+}Z=QcDXAu^Ruzl=q^R{8blW@&CsT$4R19UJeIeE!Bka@k=aIvmXE>V z;A+U#2EqGb7Gjcn;^F8uh*dm9R?4$5MXL~s)kkUBBV{ZUufQUcVl-@&MUwg?oH{Ir zC81hKYYs=a+8&B7GQ}Vx0l3&jW0+WR5na)k*95~Ou2654!j%uw zI8lC$oc2jzls7}VZ4z2ON0i`NT5wwuR|m^upXo#-9aMsZb2=65lfwWFU37xf=U+f7!=L|mZ25I8 zp{)PiE?`*aAlL!r8GrTp2n$ZrKh3pe{u7c-zsa_o{T(!NLYw;4-KElkW!On-4K?I8Za z*s~Y+zjFWF*msFbie}gNb_0%m+hLGMOb+PW3F5EwV{`p}TzEUUI}P_7KHAOJ-C?+k zrz0F4#@IRzw|5vl+IGwsS0@KM*gKAK8g6Uv>h9!bJI=*njN@=Occ&3fo=&b#uJ-nh zo_6-GHp86&I6JzIuy?X`8CMh*@8}nnz}|_@Z-v5=_&YJl-5%zHS02NibBLF9znYizYy?SaA7%;y&ocq~OPJxd zPP|jwm6;{67Top9#CxhRfJ!%f(kcy1@TmHh`XWrY)|D77oE z_K`a8;ItsdXL%oH+M!)cXha^j{^nDw-9Dh!g?o6TEmtz)Y8x2lg*>+@O? z1@(Aq`wGyGFa4RBx_(T?xV?;|*lynPS2uaO>Kfde!)rLrF}+CgSU#hwy_s<{PG@Ex zSK>On+|AT1=*#6i8Oj`3ph^X@2Y8!D4qs2Ohux$u|Aha9{~w2)M#AsU zV}00yzs0VPftiuMiN5i#?fQHt{)JtiF9ZK`zR>?}*C(v(TL=H;H}p-7%=FF7jepGq zcM|_#f}OmpNV2?vHm8=*dc`K{T~$l-t(r*3zKZha*V39N)l?`}N4>-WDajtKqK(20 zH0g33IaXFu-qvQ4TTx9~2Wx1)`7)9=j|vK_tD}&EbtLOnMIJjV==kI+(wkUK^DC<8xpp0GyIf1>k{d{@q?yXy zYbiawjD}cTdU2(iTt8LQhxawq>q`lBORJ{|wsj;WT|p~H)Df9hQ?_yqbs1Vg zpY`kMx9DQZ^{=Lr&Q)|fovr6)F>Nx?|nVpYN?<%CrfGZ_vbUe%FkU?SeWYT z{~AI&iGM)oe|5KvPuH2pTe_#6o*yoEJ4;{JwNkLu-tTn$bTh%--^(*eQn(r$6w(;o8q<%ga1@Popti_`FwQNb;bExTlutn-a2o7-}34CH2kvp^n4mV zKYU*JJpCM@E59MMP3o_WPy&dg|8Wpq#KJd zVc|)-kS&Zl+5iRbT8zvor=?y3c;}#uZqpnfqnd*Hta;QrUJ6;0#zE|W6(sVvQ{t%z z96Q&9h$W$r-s6qyz7D9>$-o&QQS@WP5nHSd*Zqdr{YD78oQ%=`xgSF2y`z{JiV%FS z2-RiznEhoX3b!7o&5HAI=tD6QV+u*ZSrO3#E3sl+BerY5A+xC#m~}!L+w&%%xabB+ zXKdq+dN2-6_iV9ngb;3P>)^Cl53K4bgJ6MubVcwwN%cNStE}hK!)jsV%^rz+Y_D}iciYDl)q4=E&Q)P=U>?j3<{{~Q7^VoWp*P*k zan*GqMg$DPoZLHWd<2G0b^B0P1c-@kL_ zEnN$vvro_)uj6#`$Ss;{?}D%SjyU(=GN~I$p=jR(+?Bsg@~=HfYM>|#*|Mg-K0to` zo2X#0BBsxsh?H_6Z171WH-SnTqc9p*f{l^4GzB>yQXt?z7@uY@8z_xYA*Vp%EXaT*Qu|r zJcjwEBBv=C_e?+2vUj)W=5!t8wYfq{qZFk*Y$1NB4%(?|@O%)DC9!j%p!$joyYz<4 z&P7qYPyJJZ9MH&@30@25HA$svJDa<)W6^{qv*#5&bsrWKU`;SBgcP8pAj8XeK z3ZFk&L+$)B>ZWdrW0ScUbaMu#Z0Lo`S2D1%(tw@kbUZJc2)(y;v}{QsgoAt`TDS`$ zCcEfJb{e)#D}ZI)C7Q5VA7wA|(WveX<$lei6}q0r6xk!P^*$|x0kj55_Yo#%NUK=Ie835VtkH|ypHD$g!MBU=M zrC`8I9w|;2#~crJ*iYz&z-a-PJ>C(E zBSs=OMF%a@V(_$^Ek2edQnTnF6!(2XscQXkUdj}0z2jh2D}#ehX=wNwkGl~bs1q`Q zpSLkR3vZ*Xl7(ms)y34jH4s@f3*A$Lu%)02E>(u&%c=$_?Dc~BXC1s3O2CczWiUVd zm<)z+k$y@BYR3&Q?Qk<4*(-ui(JWSOWzyX_D%h8hfr6?>l$fT5+@~pE7TXhCib!Io z4FnAyQy)`ZgxQ$D{fs1*$&G>PUO8M^KOaHI1M$>#B;FVf!j19y^e{gkXZ+T{_O28X zK1QKoWIRSmE~gvH9OO+8#w)8J1WB>H&l!rd8|NbO;XQhmG6~}QqmiT5NNd+5;QRw; z1l+O54h1i~S00SX2|?KQawU#Vw#V+_2Mr+kp(h-<|DT6DU)n#O|r-Idu(a;+@1SPB8F{61O z1&^-9wB%qKy-Ax3VAJ7Kn4)N{*FrO*IFLmQlL6oKqT8I(89#@p52=q-PR;#x{5@%mKsSnq*Wc4C}gmjgy4 z5noHC@a}9H3Ur%DSuTSzR}#|nfc*CwxE61KIhU6qMLGpucl6MG4F|VY>%+>lh;C+= z(M1^{n7*sV;IRxj+G?Qd5JGFXrl|pIYEh=L{B*_!}k6W^IAutRp^bFr)6Mx4_L#>Pw~7&=BG$6*ep z>{>}DZs%d4s2@(fo&f9nqcG}@6p};=Fs4Ndi+_v6sPK3kic3V8(sSBb?2rBnmm^!x z5_EP5UUtdC{E|#8T6T@Xq#u*{K0}Q7RD#WMkuc%p!Ln!nJ5J)W8YU9F(S+Kf>Op0v>9 zF$tVrjAlVCTpi7WdGe(2J-KTKyeQ?^MVvW-_lPNaY| z3ui#h+aJs3-=j>e!Fb8tjE1whc%h|(#Oe}qLP9zk5nl}<7&i>}Zs=n4su9SvH$&?7`B**9 z1asSFL-0c=cI1tPVZdA*zF$j{rw-8JGbLEE(-I~9H)H6CL}a+XqMN^IVE;o~Y<*Zk zh9m81Zv+S7w+d-kLIC1E>9gaF1nPIBP@8Q7b#XdQ?wT@K8IXy0A4lT&^d?&6%Y}Jz z1(XGS(A7>0noYU5FIK{&rY6*CRhup%NI#UGZCzUN?APZ^E%0_*6_xd8NuoPgkFUo6V1!=zKH zIO9AIrXx!be0m>wtyG8n!!n5FrDJMFKYUI556yH>)AoqN2kT11sYMzI)N10@*+L~XZ3_ko`1qLYMSlFupcfi@g(Ipe&S zEqaFfq2bPEvQ{la+tvVVZVAKnrq?vhYA_aEvV*6<7Mi$f866Lnf!@2N2;VT5l)dwj z+H)W@Zw*D>fHdSk8%DdV>ZnVuJ0dPvL%zQpHsU7)_`}g;8+#8s&w#npAKh8_}xy#`j z@ehuBpR>ixq`g+$(M1tVk+UlK1s>vc^%%?Oh|Ob`MN9DZ4oqU=j_u-DR)NX$HYKyq zRm`p9#mxRg`yAO73is(kYcJE?Zwr*lR zw5(4k#zxD)eM=aHioP73fpt8w z6@5rsdo(j@c^2cWu$xJpK9+OItCm+?|74heq_5S|MaQ`Jz1A_yZv-&Hd+He5{;xQ< zJeM%5<+gBdf6C^qJE2V@2XEvxl_)V+Z6%lp7c*}D$(1~k9>?+c@R^sHBhIdIf|#p! zOc-;SW~StJ6YmIjDKm1uICq<93NJOF7s7ivaDHzm{RYN5?d@7|gr)x>0x|W(+ zYKWOoL+`8V*|3)S3DwbV-Fg~*pn=lXR+DxXo7cOVx?ip)UTO*bTm|d@J_s@U^G`7U z6GF^7eu7E=IF8%j`8h&59JepT{^E0eeFF<4Gt)mhZa)kDY_i`;%|hgJ$F75;ZYspW zi%2TT2dzz}m@I1qabYh^(in=3o9E(@gE>^D%tDH#Ey|y?(smD76i;|dv!gh$w;F~Q zHW_&4a)#dZNQ1J}2+SYEf#!#OwD+?JhNZ8eds_EMS7-pXjrByxRuPr#9XWov}6MHVP67>1!&FVOxK<8e-@30EdYVE4hf;1=rPvHA)ezR?SU z%VW{owFD`5mXLn50y1*vVOgK)*ffm;ZNnIPcjqQ~xDUjr(~9WAT}rq2TI28p8CdVO zLV?K<>aHJw?8?=U`K*bP&&K2ATW=)yPC{mMA}TW9Qa71#ko?j{B7+z@_MU^9K7P1R zst5tWUNBKLK;EPV6s1={O>8fnH&1{~+ak!1eMI92yrW^}HFzLbfJMA>w1D|ck|J;D zF5`|Ddq2>C@uj4;UmZQQG7;=^h`bWmcEt^W8aQ~!EujGQ&*ZOnl{O?75hbV~EUi1P zTG&8snH%OmYM}-tZFty>$Hg1bkS-~s(1B6NnB0JYZ#`gd`-1koT!yN1)mZxG3c=$%_C*x zIOd}HSQ>iIze00+8ewYB#TdEQ79SFv@uFS|BX;T|(o-9&UXDkhwH5*{E~eQYDKKB+ zkBigy(<~1z;&KLH-)B3VZ@o;;BaLBsbr4$XUr-+l4;&WJf#pRllx$c)9=*g+iruux zoJXFUWXa9d7+o$5!6DgNieHvalXyMJ{-7h=MZ@4_>59dJCxct*iKr!}u#+}J>0FkN zL4uIl-w*H4RHG?X7mj{AXwd@`6mL&Om5Vu)UoJv|MHEHcUIoX{6C|)Z6!~Y%@MLTu zDGLx@6->l=tqSOd_|p4WS%|AlKyOPeRwtSe92JU9(%Ue@q?ImDnT@zZ(T)N&90gtC)n~pAc+9*C~H~2GkyF!l-&IQqD@?YTjvbSKLE8b(fP( zbT#$3Y7Va+%Gfl>7%%QUr8f3BfGo9Y=#Dq-4|;Qa&g7%APwDP4E+;D$oq7Q z>@w8g+(!@=2j$Vfj*D66`yz925efv;vGkM)0`D7xPM)F@k&j7z7oj#_E!|y^hRJ%y zkouH`jCxOGZ<&Ka3mH5UdPwW#hN9)zRywml86}fUa3C@hud;5^&3&t(9+U{**>ZAbiQp!0qb+ zFf8r@zmQBs%+$x^JW+g=3!;3tUPx+dq5PeTN#mpgJTsCIIklALNIPKcy?OYuauq$$ zl%schUGdvvH85R`;ZWv_0Zt#t!Q>7Nm2-yPv?dH5I0g?Q-qPd9dvrof4@s>Fh;mtv z*SS2(=8i*t(R}235N#SKh|%Vgp`bMzk6Rl_zi2p=j;2A{+!DfJhiK1A6$qV3qr^*@ z7}$L@`YfN0l+aL|K75Eg-gv`hb^{VEte|;D5y}BgnA+xu9mD3+w$`ttJ!TqqYPg{^ zbO@OQ_|P%od(_9NCw6zU#ofn}c=5~?j`b>t^ESuPT?61Ky@?tU)3NxHFK(<9hUn$- z@ENXv%Q1;)%D74nD^rkl#Tz#^S)o_0HPpVU(ADVQ$aG%>OgW=5E_f-5pG9JG$Y^|u z^v3bIpDBEaC@wISG@@=8&Y5c=V`T)6iDXcwUM!Yol_TbABGd;1mV2Txi()XtA_Z#K z>WSB!h+a;)*qoXM!IRQ>Gf5GR^K2o!e;0X`Nx}4qIL-;CQ{a<@crrTyeS}t$r=1VF zvh7cJG6e57R>8X=h?GK>Qb0%;dZ~>-Lt-Cbqdn#gUkcf~ItXzb1I0>1cx@u|OMgqY zahGU3Ba7QZtI+TI9-4L64+4QID69~K@=Y&WUJ>LR`6Xgi1f| zq4SG%k?A3do;SFd$92Tuh4b)kTPbE-zeFP2#-m}+d^E%fLt01}pO4IiNoXo=jh{rX zBf3NGnK{nf6NGH+03@u)Xd26 zPY3R0e(EadApXIj)A7wTflD1>OvjX|up><92sJvwh7P(h9id)FDEp_agZ~MK&Y(_* z&R>0vW1w%uMof)=X^r!p_=h#lKMs|jt#J<9bGMos@BH>HrjAb)#<3Fb@TlPb_^2e` zrGl^o_D*!}!aobC`rV}h_V4fGQZY0$u`o9L(-GL8xm3Or|KL(No8`ucIyLbM-BPWN zT~VUyFB_SXq8-fpBNrGO=W*O6dsCTd>Z6!i`#heCkP5-$DdVl9vC~?&nUNl4!1WmC z!XRxgr!bhyBG7@4^%q@9=soIo(_YD(cc7?Iis9+8ZkmqjSw~}dfU)CbvHe#Ee zxi%H}CjM&!Gjd`U$J)%3d1q_DKvtD0f6>G}xGa;M<*w&A{ftH$7enFd&33O!A7cORoD(&R#*uS62e>9P+(dNJ# zHqDoM8YnRF-Tasy8LxTz?-y~Nr7mXfRX*pWtO~PQ7uc1S9rj~Jei_SzhZHcC8y<1C zpUYz2d8l*7WKQJS<|t6Yv=ruxUl~)%*NTC@`JX@2^>f!s2k|esR#N|`Tq}Zq`H%b=ni-k@$+d!?_HYOB52i>+u97}p ztf%-ZWhDQso`lZT(fw_8^eUu=9`33lGo1!{c%zQGWRy}ucsVI=uc5>D>qsfLgyO`i zh&fqG-KN%%%F=3bd)Pn+YwKyj?t0>UYM{8c)wFFz9r0W%C}nUJ^*UWiJ7w!grKFDP z?lq8vRV5j}t)MjVX8MQ*y11*BBI=rG=KCtT`n8sBPO75f=t?pZX`qjT>#4<~o*wKk zpQsOy;MaPwX2EBYG}p#3fe5(KtET*`o9NCOe_qHS@rrglzb=t0VV&O%WM4)DCuyq zNQnRCzjbC{YGUyRl(4IvpMUJ^8#N1)E*%T~LASgR-A@%8j{}<~OvhpGd!*;2gu&h; zuzGA9PTkT(Y4lv&lb3`>l?0-Cn;>gQ8x8wJ*v78KF5L>mWzCDEYhj46f!#33X(i3% zfDRs(!=9)IRH%NP>Us;pUSMd;#)BSCU^`I4<}sftc_T z)FujH@rgdz(8UB}Z7p%qOA0$(uaVsl2WTubLz8DR#+^J$nti8W^376IFL1>tV=Z_L zos7*_3y?NK4FlAyQNKtEYuppD%_N(W`VT?>#Wpw`DUIjCLKyPk6K!0bf{4IPICA0@ zok*F6m(Kav{6qx(n~#%^<|32~oC8T!hU}+`p;Tfb6h?;Nb5JBE%PgVEYkNX|mK{pf z&XLQCObCy4MoW$(IG<0D@Yg#e?_NPGE$wLg^xLFU_=KwM^iW%BjIRl+D5~BLDIS`L zpOudfQ<(WD|yk>Do{nnTvZPTWPb8Jl(A_ z#KTMt42#u3Xn{WTZ^@wFGcGQAR^Xzc6<%eALh{@>8dou$zV@&Mm8wAcxeKn{+fVZo z%1LAF5P15@qH1p}-VG8%N|6YZ+y=sJKn_k_x=Vu&_QRH>C8#7TY#G0wRJ3)lNpvwH zrI*3a#}9Wip3n#L5y=0LM@lVr7;sGnoC8AGJvA4u=`mQteN8f^0+3b^!mFW@u+Q|s zdktr-@BwsK^^a;A0nrI67?<7)tGhIkW}rMI7ayUK`p+mg{v|zFCWjr}wb8}i0sGf& zp{SQ-v{-Q@0;hQ4=vWUNbDo944N>ro*Fl)3FBXhBM~w$o(TmF7D4ezsI|DzF-U?}C zOW5IS-+IjQ8UU#s#n>U@iunzVP*{DL1|OM*7jG&sy7V=@U*dqJ;2 zIK?$P4T8N zn}qr9eX*mDDBLI7;%J-!HauNUV~(;i9L>R&+p}@Q(j4BG`{2%W8+gySOKwxxVrh5ujU0k;MexxhvSE=oDF1RCR!s*Le+?(}^Qq?D;ye$X!+J0l( zu7R^AD{#3W6@r7L(3I3nM$^QwYMc*j5=Pt1^a2xtO%&Tk-^2W`{>|;k))jb$&}F7-hE+hQpKjJW7BB^Og?Ib&t*24Z6uGa z4#!AIe+H)8jmKJsi=x}{P+YG9jSzWA4I2hssvys`y6_PkftPQ_LQ`7-JHO6{a9I*C z?*nD;RK>Q#OGzU|1Iwk&adUVKYI7ve>RFDg=+UU}VTJc&oG|{?Anv1C+i9D7CaV2` zs0Ga!Jt+n8maH9gvZ0_Wp|BXd32K6OsaeViGd_$%m-(#TY4(Kd`Z#PdG{JtJ6++Im z(DpuWNjOsqiz7DC-DetbnY|EJhG$6JWC(`uSqQ~QDX0YBpjjh4F;>$RM~>@4;)6My z*R7(Hv*ux}ST{JA&d1Sr321#Mje;xA7&3G#IbTbG`mhbyJK`aI%?g9qogAbKT_VNV zgHfhF50x5q;4R(3^}eBosgqJ65i%ATBW0mD%m`ciE+I)%J7mUi;rGD}`xlv_p{y_R z4>>~beFHwsPQc?fC2X^DLC;57XuHr2(wA*8Z@fO_d^GVvUj@R00x3;n6xJS}h^fa7 z*zsXE&A%pzGt(q7S6>~o-x{LoR5b~%nhT*3NtktHFnZfRp~wD8IPdQdnFJl!Ty4bH zcOgiS<-lu921Z@oPV1)%Vd~Qie7s_cER90ky%&qUqqWgJRS6YV>G-@APj}by-b|}WlFT+9QBqXcEqFE;$SBAtuxGf0QNm6i;;-dJ-B0RK+LiL!> zG)h+lZmZ`(ZJ`~ms5#NnNg0S3wTs?xim>0`2g|PBpnc=AFv!ybBishzK;;>w8;UXX(9$f2-2b(++@yCbB>A`E}m zL>2SLAz`03NbfS`_S#F{rERqO&0#8562|U_W~`0NMED&mu_SL7_2u;!JO&kn9v$TJ)ez)tjtt&EziQ}Fcq39XoJL$*buL1uZwmgVD??XGLSooA8!_L0gT545EG-~Lji ztNl1PCwrH%o;HA^-EcP-I~VscaCdQaaksU1bGPMtZ&Jb%*gMhrNf-Ws7~xLuO$z(> zH}c+?S{R%C`OMf)y*Hi2KX`9+H%jpYE*LYXPAp|YVnt}e=Ycz4iiqvp79q0pW^R?` zj_OSecV!_rR&pILHE}oz_ZY~CKc2-@v7eZmNOsHR&v~C zz2FV6l%%KG^}M+y{TVN3J?7xB2+o-)x0z!Xu5!xt`nGt8_|l0JbD8ly_AnW(3z`0* zJ2;kl$Cx{@N!-I1&+>9orqhLGHB3feD`Qkv$c(C5!I3=a!4vS#+mU3sni(Wi#O?Vc zgV&Z-(sE(E8zYqIW;J2zTBh&WOis!JE6cpP$K0-x#3b1)VczL0GV7#_xXLDuOy5iy zZkMNTc^9?S$aV8%#zi@nxi#P`qc;5n$G_`(W=Qo$PWQL_c=9(CY41dT&J0zBmKmjM zc!v+#^ERH6V5Ew-w@A%C#np(BEechKa+1o6*Sv?(jQ)jAD zez$du;`aGW=CHj?@hDGTl(-1f<{Qb)TBFH)esGyPVWB2xZEFT^RsTR{ROC>u^7dS& z!nTStxBq2cpDn{FX-zyM6_vq+atoO`vf8{+VcyKMv%DQ=W`r|477NqBb6c2OH=~)T zw6%=8#H(LC5F`AT|L+M5^(}tU#o0;xgOdE?z4#~mC;Z22m-2V@+L@WK69dy?`O*kEN6+T1JmAG|--j_4M*i z1AX0DLC5UsXw<7(YRj%CH@j-8y;4cPDOb_J&_?pUUP3R;DoJ~CDJkEmp&eVRXvUdl zDtlH(RjswuBvVJOG1YWrO$|N#t(-=kt|o(tbu`w$h6J>#Y1-&!s^nA??{yU|_HUw% zZZ#D8vW}j$)so}*N~(EKOBLpoq}Q{C1YgwCoIX_~Z&X9|5#=OO)j;lh>dD|-HT6@g zAkD8;Wa3s$0c=?=51UBlQXK`(t|!Z?dTP?FC8ydtx}953s~*?WwB~BM^Qnfsj@Htp zbEP!IU>RNF*3m7o3JMfyAh!+WbmLVGjZdy1UR)hn9bQBhjdf&wu8O?2*O2>@Qqo`5 zMALSaQR~=z_px=fH>aBP z3d`x8ZzClStfHCEYpL)4I^syM{i|P1pA{-7L%g20Zmc4s(Uo-Yeiik)SV?9EOXi;fB`wcD342^$LLpq6n(2##Tmj7?R zB-&XvKhEccpU;2h=jV?((q0$8{LV7@@Emte-m(^ z!VH?SF0flX9A77lh5Fu6NO~8FlZOSNHk_eq`v45KYNoWBbWE3W!>GmyXgVRjCc77|>pLQkFUZ3c5fGDoFb%odv9eh|f6c-1}g9)_8@i)UUBea!Xw2Gj0t`n@P zHDP!`AI@tXFv&a~vQ{DFY2pDX@k*>dJrcI=8Q?9|g6hW&(6;Z51J`Fmb!IMjVvk5+ zp9%)cvk%IPv`{WUQjB*X%lEsJuIvg#Aa?Lo*UL=P-l8QLAa|UKw zD8QhHIEqfrh3Mu>q!Jy3y|PWvGq8tdiYev|bA?u`9~_GwQAPS7yvp#y7TyY4bfJ~j zZTKy5j4C_ddC<9S86anKCowdcXPp%-Si>cOTP@HtHx1O2VB z>OdXUt#KmWU_~q*KO9d^PsD94F?ikRhx_cnXCLd0+51%?tf>bht0H&?d0@n%+jQHK z3oEfuWNyoY^L#GeyfMPsaBUndxk<8Tv&qv_3;UJ2V`xknM43o5iuT9(Dh_(>eNSBv zeW7#i{cx*pI2xH)1iAOY{P8m|_MSX4=gQ;D$n8{Pc!8k3oE-brQ<_l@$&Yl!jEiCD zc}AV;Qy6Z*sWtQOmU0=*}Q$+KmrDR?qi)m+Glexe`5EW62~B7Gr%dquLUd^>4^=Q3ib)u8xT&XK8&z4tmx{L+M%r8t3ewtfO+4QXA5RCkUsnLn z%@(cJYLIYufpb+kB+HbsVs$L?>(wCd6OV9_FLbQH1}!ReIPq2!GsK6$bb~4)67wM> zZwsm3M`($vF9t94g<-4}Dz3Odq*rfLxxOOW&+<3=HbsaGg`=S&UOtFJeYh_qO+rx? zsfJf~1W?p2r#x+v0oDFMzRxgBs z>|l0s82Ol5A73Wn{{?YFv2JcmFwd%!bb!J&VhF1!hBHVDqRqytmt;zpMy8Oi97eld>>5 z&}I8w$|G~ z&j+b%>~VeAcx(t;1w;S6bZcfBTJ}|9MdeCbp6Z0KIkQn$odK=USLxxE$K<80gGpR7 z+|d%jqLM7i;Yg65gd_&^=!YH49kEr<2NTr9Q4`h|S7y(`X@jYVH`T|6I%^D&wZQtZ zJ+Q}L2x=1D$zIR{^1}Vmo5{o@*)DL63y16NVyGC%VZ?#gBpw@xm1n~suW*s#-#TM@ zi#hxnSJ8T9KiH2NOy@=v;E|CJrlt->?Y#tayEYeJHk!lweFWBOZA7o#(OB`q9!aOd zuq4z8#v64}cIpdV8>)hW^$W0Y%@kbqU5eK|1;IIxh>Vt5G}<&AnG06Iw>1&^sp^m) zHv$qaS%}Q*0jvGfv0>N*3~L<^(_==sKm#dxFhdS`jfhxikF(`1SkxmL>qW9rVD*T? zQ(_>*U5X;!5QsFW;HXzRymImoFjf^Vr^~TzPX;bE%whTEKy^|8#fa~wwuB5Qt{aT4 zuXeyBA`v@Nm2o6l9AVzxBywRk3?sThB7rb1M;=N?^I-G78IL}1q`o=PkV`ATnW{sS zBKm=@2dCoQ+Z>EvF_vOJUZaB(o5+N zYvBshg=JLXd5<21b;seRbex#KgetYqlDy&|$VQlB(#HYF*57S~;MDi?0LWP70XRxDf2Gc4(De4)eO5n5MOd-i*!yLt_)(`d*;~ z3kL|-&qUYJ2(&8j$ht)j?}tjTD>@u63?lGsCIgk1#zTl#K~4=@u{X>H?_Mxryl(|^ zS?}qt%W`<{4#3(eCn%}Q6kT?Kh3+WoiN%@4JCJro8zVi8p=%U@beDeGvX%kcQRiuQbtXj1 zebJYhg5Z&Hm^V5Q6LjOiT6vyYo%vHjX%~c5ZbRucLBtjiw(HAd`M^y4@mUjdt6S)< zUo*XHaL3d2@_b)6665BY!NZgVgPGPanimBB$~oBaA{hP4@?m1eq1gP<=(o$j0uh4! z?qu}VMPsjz97t;g(V+p?AY~K{WuebK8A?=^c&Yvzv^9ShKz*gyvk1tFl2i5MYJPK=j57QE!0Ic0>aXE#H zaHok-Z{f@Sg(YONPm`_r9qPZWjo|RzSYf(=Zl5j2Y42i4H+P|7eG6=-*TL=HH##UW z96==(*gt~_U8^uei!R1@*J(I9R}l)|=V3yK1i$PK65CNukGExDB%aV4Z4o>@sRYi> z3OMS_L#W^ziu@u5jk6Y5!sXzPPzM}YS`V$$dnu#T0fJ6Bn4)}z#L6C%QC}AH7Hq-P zMFmu-IT9suA-EBy3st`*xHT;fN@}x7?6W9&T#dp=4 z=$kqWYcqu5*S(cqhYO%{O##Lvnq!Ar5Oyp(M>1c$@Hw{_*97xvdDJ$F&|HXtf>ju$ z#g7GVe9%8-EoP-Bp?R4X9`Epm+6y_@^?fDZ17b+Ds77)pmyFlg;hdl<_HCSo3G>xa zrfvwiF#|NK%^l}$=fcm>4>tUmQrF`H$5KxcUnz)=9R@g)mHykqw(4ky0J?Rvz+E2IY1L>=Qkoj%L8)VMi?ihfJ=MNP=~=DV#?{D zD7O@kdwe0fGZS_nQ_=NcHcpj$;YOz#CCvPTDw0CcH)0m9eQHDW;aGgLF@a;%7(AZp zjM}x+aX?=LuC}}A<`F%9Sz9nBSpZ+%a*OQf?87d5&>WNTz zybyDZGf*Qr2{HG%q_TJ=ibXSF@1_B(K~20~{G1k9WMg3l;^%BM zJf4US*ZZ`AXO7+uMJyJaiYFUu>FF9vIDPPkrnCU!YOJwG<0#cg=|TUYGBT3}$>dfp zMz!Vr9a;5C@P9#8bt(U+B(?v*;F$fzEp`9S{13>ghMzR_=W<8Zh`-TCKPmesog?($ zA3t3BiL`qD^Yp@>RQV?z^%oIV{|Z@E^7o|n;s5^oqyIlLyi)$f{^-9lyi$H6t9~E< zfGSJ*-%w@Ur|W(u^~dma|G)2l-2MMIs%);Kvx|qbo0q4rudUmUl4m=5dAhmSx;f9b z^@59wmzSNhH@tk@Y%yz=vyYdZqno3v8+?7>>*?lcJIm4A&d1{iG_5;w8UKiP{WjtU zG)?#yG_9Ne^B)9F(>F6WFf{zDsGFb6|KKQEE=%#GOglSUj=k=j+$&EyTlVnMg-`Rg zi->i_uLmPCaw+fSupQQMrTM%kOk>I^l;AC$)5vpsd4yLZV!?Q?vzN!-8)-e?NsOm$ zVotLI+dBPsh4VIiY2uX~^ggszvXnRWGP&S&;s^-bdcdfBreY3~Pxp_QI?Ay*8HvI%Iv!{}$ zs+GyOxg>(;S6gmfU#7x)@ZFU5WOI0NZ|!+915Lc$!x}oYd{*;Xx^);6+U0rTP6i~p z&YmaE+`tQPzR6pDZS-G*rm2hko4?4%%-B@_f4o?r`=372`N_;TlJ4KooZs{}{Y`(< z-}L_jgt7ZS6~d^eXKHHjKNb%}|M^D5e`Ed&5XPddf1|_x$@jv46~ZVb^l!h_v%ZPx zUmU9Zo%tVzDza8gQp~EORiDd3Ss% zyHP{uOE?s=tBUsStD|=wObQQVk>=F~st{t+{`V|WzFk8TWUHuqQ#A=4VUdY(HN|YL zr>bNwy|U-f2;B-gD^^dXoEqv_&A-;aoD@bf=?edPMmwAO7T3{(Rn=rRR6{aDb#$YO zLs7%{?epc*@EaTo99vC`OgJ=qC!4ZGD(SXpBOT}0*;K2PrhKZShnAHzKB1Z(x>b;1 z4x7ZZ%83=qp`*gp6mXG4(RWH|<@W~K=2cEd=dmbeu%31usi%rbY!XzfqVxV-n%2su zxA$vFcp;m#Cx0*D~IrQ!VmkiV@$aa1WEla4P^+!0AI;w^wSsdCBR7JanRg>Qv zHtn`#k!*7nJ(|m;63r^wcBY(emYGu-H)f&njVAD%)Hl4Dmpd0)) z=hsw`#{xF3v#O!nA^ds>^L3-6n$ADwkmx`;#oVf)?8&wCL7q)vifrm?WKzV_3SyX7 zknWl~>N;ITB?mcty{e)fRW5O+mQ&mYHXZS(rI@RwBzK-k0(-f%ZCV9=(5|6z9lOX# zxsJjv*3;4f7RB=WV1yN$riWG2)RQl=ZT~Mh{cQh_w*S(fAK(4D?w7oN&>u4Y@%ab; zFZuoY{Ht!i`s_zpKd$*Diyyo{+W*hJ#!sm0N4bCZwL>E#W4-_Wk~7zDBmTt}J^#hc zcN^b+KI6IU6^lY6+o~I)_ z)*5p^U8nBBaAc$&q&Eyvh&;1`zfCm09hi*C&kPazOdo3k-%!OmCp7EVLGM{G*5BET zDA5@g~Td4A9igGx0&b5Vw?TajwJz>axw$pP>tbOM1{&(!qSG zjTF$T0@p=;P~pzRNwq&HJHidhc5Eosx}a1v5*7UmvF~dR6i*7`WXCfatvdr*zVh(= zb2RoU=wqIV8~j!)A%Gi$(&5+X%;+?z9PdDT)iVmWO~X6Gt$5epLy6huD4dsv@dA~w z+ByvPrPbm2ov>%qL`W`D2V?#&Dx9l`yCJJ_`Pg^}2d+RLHv)e|n;|x>meSM2VezGu zI+a*tYWbOjEhb{>@u>)ZD}uKY_T*U-0iB(b@aUNxHhtejw}b)^zN;D6w=KqwB44z* zr$BHZ59#M$)6K|fxYXo_5*2kcy1l2j3)_hvDk3EmaJid?^fOvu=$4R_tp!XC>(Oy> zI;xZlP`641gA;Gi%G)}S7F`C3i*Z_qW3QV_%YvLKTeh}}(Pba1m3)-20Fcbz#Dx6Xl!)Od{k>Wa!8+W7d&9_cz# zFiC$%db3M0ZYT|?|3oGul(6>GIGjy8L1~3&X~u~85ZOB!SsMj$)h!<8m6}LVI7RMp zibxe1gNgD_$!PNH87s3=s<2#3|>?O9tOrKYphy68kG$;P@D9QBohvk zwLJ%CZ090i+kULisHgWmGjVUpCTLbv(&snUfLR%GHFMFz^1{smF1bou|~ zv3EAo;`Sm^_F0PC1}0cuuo}EJ5e#){Vx!kxdSH}=&@*R9+tLN})eJj~%rQzWl{_^B zpynil!R#j5ZaNb0yhlJrJp>7VwYh=CV3&Cq)B zG+ED2$Gt%#oGkLfV4EQV)m9<$_ArDzABUb|A;IZfd~52PS@c5EXn$ zdp{;%srFjDk5q(@SqOsHS>kb$Ee>rtMTQgJ(xoXvn6vjdg}F=O-iACZD-wgdi5SjB z`{Iv^ZhGf(k=o@_&_B)w=Z|~hrjQ%Tsd z5#vuqkL_U1w!{vROoTpK57w}26csWT61#Tr`_vCgJz5BMnvbLJR$|rbR$A}}-#0I^ z!h{Y<7KH4q}?g?W$iAvXC5J&lS$W#&$tDNBK>4g+JZ&B31Q zRWQ9;41rhr_|`BSyxCjO9+QgcH+A8=aEK=FxJowz&JZ*24!P!pK(1W?%?ag*Qt-s5 zv;xYTLWq~t!PQSIF}xrYZ|vPEO0bxkT2=+P7$j(W@sxSVCZ;ZiwyEb^IwbQ+tby#_B1N1bWP?WPWQmivDPoW&I zH7-#5iv8r}6ONS&E>PBhC*-|K=w;AB%9ZgzmjVMpSKaY=!4ilqZzI>fL26`f#Eyn2 z6w1xOi)>vqY7bD=15dDufjiuTjBBgSF=nS8D#lmg&vo_0_DV)|=ss{CL_#!;uonvIp`< zmcoUdirb^6V#U(s*q+r2kvuhY9yY{+Q_EpDwh8GL^YLj@6<&P~#EbwS@t!;KuiYlA z-cc}EpUw}WPO#E$MEcwa+)$7O0z@#<=ruV?uE*xy35bfFg2Lqw`F{lX7u5kGZJ1;hwFGFsN}g}iY;GnG-Z)rD-Sj6%@8n= z1=GHYZ!5vrSm+3KzF#tXwwbiH^w5^%DxB`KL2zy>X6=lH)yJh6@x>XdW{*euhvSq| zT1~B1i?Hi$7QIN6M&O)@5Q~Vy`{sw_U?NL#dyR1>K^+qWT@W6Y496us^g?z8CY-Lr zrKdqK89g2j1?G5`9R$(6e^4-geD?OD3C_-Hz|7HV7gwSzkfFcfaE&jvHyl2()hBjuMfkzWJyH7)<@2X z;pnZ;0QcBx7(f#8-+Uo)rVQf;2v-GsF)1q@v2#sOUpWfLdq-o$s}Wc*QwaCtCZR*? zJvFXt#69D+u;7G2{D}j~)xFWb$q*H`Vi>%6l)gAy;7NHExNUoAbj*4jS)7F6nyDBv zHiDgG1}#_5r$85i=Xz5Zh{$7MqYWHdw_?o)6~4`H!<`zw&9vA$ z=6Pb_N(I<6lu&pi1mBJ{Vn&BM7`p{9K3f``)+z{|uZ8soc?51;2%p$J)O$AyM|`$J z7R%9gb|&7{<>6zh2X3EuOOMUJ(5@YZ2&pcE*V$n7-*7~=h&NWhW?*iE8oE`!kjf+( zq?Xk}&SW=YJr$t;@H4%aSAc2nb9&C*PmGqk^zlj&MjU!dk&DnzYp)P_C^bmfk4EQ@(Gw?@$8nQxhCgT?U1|Imk${fo&3p4nGjY713aYxFPnQs(1BK?q|MU9uB7C6&hsmDGkjZM6j_D*lXyFA;o06 z!;d?;Z^mM(?g`q`*-npI)9BimDX?4NjgtNIAi<1--mH71CTfRegLROotU>z)DHx21 z#^88ew7Q32<|jEU`tXc$PY%&caZRZ8)nmy~bzIF_he}6P4DKNqtt72P&Dy&sh zq3ivGP^*HDJq>u-p^7Ww>F|}ehj^emY~M&?M4>J94i3>(0Vc8{9nm^-KOQloP_dN?9>^K&5Q3*>CiZMZ?MCS9tl*j z#G&~}7A8BDuse7Zyl1#$N&hL*{*nVRflP?FZDaugK%U=Kf*bZg8+Qh+xyv_vuY&;!O8MSF;ho(oa-D1=>@fFpE;KF5C{w}&&) zzpkGujEWJL6NGg_hiTZm^|*S{0#^3=(94~Jl^6b?i8>7w5gU!dhi-TwD1%86+R(em zAJZ96#`d?1aaMK%&7YY8k9%5Z_%jZJ;iGV@Ndw7KkCR*AHM%`j2kuVWX_;&ZY~JD|^B76aSH zaT#_!b3~4&6NWoj4^ugd3v=f z4^1W0ke+b>1qat6=tLzR9U70%j#7{urU09XI(VU%MZ&ofi1f(Ah{a0qtJ{zCsT-l> z)`e2qf{zIWNXp|tv`_G~9w+kS*hbmXvFBEa?)N#Y^IG z6Bj<4?vt7HdqPh;`6-Cuo4zB)q|0HZ-)>Bok3qTYSmX$OqPq!#xL}uwtq~%y>)i>P zjc%yh?1kW4z?YL3UE-EL5G{zmh!DnhqzIF2hTpwY$~ zj}y;Oiriz85}1U0q9G8nT#WZqhUnHpUre4Ek83+GlT;SJpQiKWja2vurop{X0Fv)@ z@yak2N1{tGaDjmxF>;V@U4l(+-sns>g=nM^9*NX4JhCe(+#v&*qpZ-eu?31(`R!0# zfhFaHjz`LPz?H*?*x?9hzC%}pEijK$4vuLyih7K(t0W#qVXc^|<%;qa8GMhN1N8%G zh#T#VF>(`dC%_z9_r8-(=w6b18v+g61dJc8fT(fWxUe=Hs>u>C`h15r9MeX05*JHc z#Zf=q1>#~pSm_vq=tVav(@qK7_l!UWgGU~3#9?#Q7zb8=qSOTexXHH{!FQTyu=5A^ znhC~^AB~G6rz2s~2!zd_L00*JaJg;+ACZ@&GAM(VhgtAhwG2wrlrX%?5XvG>$kunl zvM#<~E;hmLF|mj(*+o&VG1wij9~Z?H(a3RzU+y;KY>Y>b9K~XNeR%Ax!^N|}pj|NJ z$MVMx75aESb_?o^+;DET1bCHusO)kRRhdkJ+NmN86AZ(%nE5cXT#fRX0Wb^a$5!qX zIOdLlhFk(fCOs#Ikuo?mt{!L3hGO(Q53Jm$2Y*>bXbIe)qig+9Lk_suD2nbK>bS&S z2g5fpXieIO%vTyXC!PxT$WkO;@1X8DC)D!i>^DlcVX>J84(@J&Ax8q<$!l@oTRa|@ ze5bs-8i02&Qm2ODj)gYTm%gN)-X%!d6^c(K379jjiBz|rqDAJ-BzSc@ZT>b7RuaMp z>9oU%T^t;(Ho=F2O{lv*1IyduF_K1Ne76~@CKTbCe;O=~nc0i)KV>8>uqlvFVsGmsW>maf!v)6k>g zFdE&6l&>#ngMkaaH%g-C!&W>LUx|Z14`ZIV3z%R3TX-E>#`hlUGwnXF$`OK zTyVB&GH&)x25X`=uH64XA)GNNKE%c8fPOOG`-sk5XJEydQs_SLLj33^^h>GZ&Q3M> zzhb~7bdZ!ktf!_M=V(bN3j$iIa5_{4+ssWUUwZ&=HgJ$vlnH&F(YkZ~$3#TIE%MrSi=|@!~ z#^Sp@3(X(TQRKr0q!w?%D*kv)&8ZCe&IRz^I}Z#kXMEXhgN14aXps$uQIIyphlpb3 zr7f5cXn?0P_o4Pn9A6hK@J?U@%GP@$J&k{2yuQFoO#WqhHIxT!D;*-U#s`@&6O z50=m85OeV&u&x$?6Ol^;@8q!VQYOL{`lJ1x6cTFXpj0#&+WsQw49UgaR1+;aSQqyw_Og$fko`i0i6~};P&R8hKYGXS`9WQ1tpj(`LxMwfJjIV_rh6r*CYgXZ9CgLj05YA74WJ6VBrM6TV_4{x`}U{WtplXU@y)pXc>{ z!ek_VK3*CA6IZnIXF1z{mh147j{kpoIq|A{yF~de^Tm`o}Rw>|JZwENWA(R{?pp(vWin^(lxveSyE>@|s27{_$F z?g39}OfBzf7%|9{yjx$}7>8U&@?>ZGkm$tuyi2`tJhM>=ye8E%R+U?Gc(-M4S}(|+(s_5W zE{&RUgm>cPVP1FiPG0vIVr2FU^TeVQc->m^JfT5RqL4_Q!rGZU-=t$aQIB%#y3AJI z*>jhz9G$g!3X@&Q$tIQ;ccPW&AKk)};bvIR_*Tq2y<#jQcGRa%w@->Br;)^)d1o9i z*p|z?bGxB4e%M;x;~hbauF6B5rZ$Y zMrP(l{|!&fcLV>#O8(CM!%Bwz#`OQDzv*xKoBpQ1>2La*{-(d_|Didp{7J)qZc?kE zzL|m1UrlQL8}mOTwf;r8fH0f<6It|PbvGo$PF}f@1oDh?WysIhHfnT5Q8rr1JC7;Q)ByH3{ORP8~ zAyiBKo4Dj|SWCtGSv2NAHLdHcriC396gsbw2C_ILDp)~wXKLu6YZa+9aOm2qO7iAb z)43_zNbDkq)=#gdsQzja?*ssv(8* zERsrK(arWs3XtO0;Z8kWl4q0NN+#`*u6-m!)>r`P_Mp3FJLqGHuba`0dhn#;+l zuZ}(`R*_#9hwi&Ei8ZB`CT*)EJ$V)h-m9h4d&((rK@}a{TT9MID=5H|OM%>GiZ8FD zvkU6!RcbZGtmTqe7K@%#bLjAk8dB!#nM!*NHM}Sz<%5;)lwL@lGKv<=g!v>Ta!b7 z%GQ#78HdiB)siaT?p`jcptf7J6dBB>+4otLzNnfSPgT*LGB#OQ)Y1t_4k<*J(=)kh z5?NnLB7G~KlN>oL8a;2_J-?f}jU3Ojskwti4@OkcN+%9|zQ!W!clITSa%mPB0YDMpw>GDR%PHmISN5quls%W=%)I_eLt zpvzlWl+?r|?;aL~RaH?ZUvG|AaB0?~2D*5fMQpxb2}-IWuVyywd%>p2*L)k3=F*^Z z1>NYYq_;iw^k8HyO}xXVBAYTAb&W|YQ`w|{vX+{rRntVTD!Rh=Eh2nf+|Ks{cWP_t z?3qeh&Oe?CHBujoO=X=-^3vwFC6!4FHEYS?6O$Ir=F3u)-~J_RQkANpwS3vr6&CeB z;?N8+Hp$CzXzOt<#Tv0`hXRYvj%}a?t+f=zDJ3!M8hV+|x2a<+vg6lrb25uARqu6BLxqw|lC@z{M`0<=U&^ArYpTggfkQbnxKvwPNiu_5$zza1%9dR6A6HE} z-F*4YWRkculLCiTlfy|Sx!&edLSZdEN#gfmNCjnhvFP>jan5s;x2@v}7Rc*?Ppd z3!)^&8V3)l;G5eFB&uanG=HuYIPoLhEx$wK=V)TgtC@%$evHPwQ9|?Hd`KA`BwvX| zw5HP%^UNA(T8}A`gc$H&J_`Fr-6gdSMK1wR#>Bu0dt$_C=am1s=5hC^;E&lTb^)#HxV!PcthlXCbF)p!t0(W zhNnhiwKWSJzRh^~=_CouW@GmQ7KRn}(TLnxc#oc5(pbW+S zP1qEZOCF;}B64sxOdNHPQDguY$NA(GC5o}l^I)pQhSED<2yTtX+3uN;Q0yYzOg$vb ze@onV(a1cVL$zY-$Zh#71l_g90`o$g63C+OtXq^CxCrerECgu1BbiGYFiojO&-)@Q zPn(R!15?mdumPoBiP*O6DUGjGz`W28G;mW5<35FY?cFWYjGFPM3Y}(6QkU>06K?g!t`ym)}aO zijj9Wm*$0T>SzVJdx)ly}PF zc|;Y!P(||bH#FGi4E6&Ks`G;|LWG5e12O1iHXxZlPd|2KE@q_{qA__34R**Zl#H%TN9%G1v=y(@?g#g&z%7VmzmLKvAzhq2;{(m< zqcN#r6x#n3MaI<>e5jPb>)wSBKjevAX>Fu@Ur#+&H%WP&5X^TB(X7~RnpezBB0rUk6kSOmTK@ET$LB;qt>Rxb!drv27^`$_d3CF$=m=`Gz_I z%dl&&5foAz@wF=%sbX1JaHO5Kot^>Td(CKBW{epptg-W66!cU#;*b6JXbSmZg_AQx z9SGB_D=7Q922?hQ;`vZHOuMavb|Ck|lfv^+-KO~RtNg*fEU zNfpO*kb6HD<=@!&Xn%?V<~ZYuX+D0eLCkNF#+%Ecn4i!~)u#5?w8j?mxt=)DV}#x# zyXl- zMnDgjYMhYp{5?JD_Je>o3%$04%9dzs;Bvrh&%we;nfT-U3(8$3f)w36Tt1PD^f#Kg zkSmCx=uk}F_myg0<8Y~T7#=idlJctc5aj!;VVf_Kj^I3)aU*dyDHeXbb=dG(82Lrd z$mqO3vXXR>9wUVX1746?RS$u(P1s#E8^?CpVRXC}wm-W>{lY~w*gOTUGiqSN5eB2j z7fY7QA}z}Z8skc7@`O=Xa;u(3n~gwii#`_42?j4o8Qo*gQrsc~^qZ|G0nKo%@nJyf zY9KU{cH_z1yF^u42zs~?m0GiD^JQbi+LWPe)kch~?WLC+yJ)Mo2b3kpLuAE7?5;DS z6&Du}E5`=)ag#Bak%ZNs7vsdXV3II1!o2qdc)l+i=NNPFY1243H|fG|)Hv9MA0cIf z2znXsjn{jWP$%mJBi<+S8|RN(19{k2qy?=6N6agUh6%S20mBAp+UT`Nf1HlsxFSkC zkP6c`-ZXMb3@$mk;ZnUPz8p?M-trjSn>rc+-`t@bQjWA2j!1G6z&2hk8r63|Xpt+X&M1dMoeK6(nS}kOUf9OWz#%6i zxEv_JQpPHHwYxxITOXMv)R3tPUk{Z_Fnp>xCcoN-G_e#69nQcmE{}pQhCnHx8fi1! zk(Ta({k&q_7FvlVbM8@dQySQs*|>RsDU`LB;p?h0sx0lL-0e9?+u;S)r2}|%CIyRj z8smIKA6@-pH-#GSXbI>Ic5 zHp1~Z%$F+X>SFE2FXaBs8e2DVq4`)AKC^0Id9I8c`gEY?8I9>5r7&jr4D5f9g-DW&C*4lavq!>znUa7>6nR`LOCf9-|TeIH4AVGg?XM&d+^ zJ#7!|qA>^b(LEv;)@t#P4HCi;_ZJi^^o4?kZNTIiYY{KrK@Q7ZapSHGzS!g-d3ij{ zha&OPT!C^cr(?LL6&#nTgL^_1j3h=n;w$>-BKq!hXW5ER-h)p!?z~*u9ty zK><7Tv#jCe#3qICbF?-_4qH#u;ciYdMcfI8P0t)CPYH)&7OS_BuF0^(+-(8p)MY0`xEov}D4WQjdd zyP(k=0hjK5lyp-DhmA6^)o?dv?C?PEtWZR?JR+fpKxkd!qWP2~+G`YHl9&$Py(yTU z_Lx-1T4PzlFuXgN27j>y*eYfRO+RDU^nRz}%@^qFu>w+hG!9kDO)$FOjBVpIF|Jbr zQT%cO!=Dk;b0keMiNdKq9n=YKL zyFfCUN$0JVu*O*$VvD!ZM!pBwC@=@jRzXNlHG_#Bz{xvAV?twK9c~Oslc{i1)@NKE z@FiaSG+c}uhat&DaGhlj!9PZzQAHK6>?$!%whYpFrnn)#5n6Q`*jQnRq*hJD$^1c0 z#pT5KJ`UGin~@N$ihJs-uq2pCql_46Z8pH$g+!;y=Tc&95T@-*2V<`_&c8PTXMGAk z*80PH{&X;}O@jCAFBCcZ0A(0DK<5z$XC@BA?Njm?>lpA9KoxSJicrQyE`{YNB#vB=8r)`dnsxnjPN!% z1CLK8;nHj+j1$bpF6XP%-q=8%rDfYs|YRQN26k0 z9<>W7p=j-9Xz#v2q1&fnXv=UkMyWz`kt@wTt&b8*H3-fhq8DS|(b7MsAX8ojqZ+I* z+~+RYG_6NF(*qjL`uHfJgNrWG5TBik1WhlzJl{&aj_SzRnTrXr<&fAt0dW!9*wdp8 z7mevSnj3=`r$i8_;DDFgY;enKI_4a(l#Mhbr%9r$1d1}8Yfq^A~v z*&oR3<9y_>t)Libhs)~@k&fmt;I;@BGx%c@zO17qc2jad0@i9YKz06SD&Z`^ncxK& zH$Mv!cZXrGge6YOO~Hd1Z^*BIE*j(tsoPZsB+S4>e$0%i{XlWGap*|Xfy`rZ?2HMZ znvfNx(F8XjfC=fieLO3cJZ zpO>V}Rl(wL37j~ti=-{Wc-5VQ_WPSjDyjzGQnw?}e?PMBmZGFE1p%Ul7;E#I(@j%3>j=EMUO%_hDT!kMhyfFjz&$5KU|L$AW?B5+yzT;Or8*RL<4dYR$+miF|~H- zLDek}+4KC7xV9es)7Rm4i514QtKnF8FfvcMK>Cag9*?tx=nFl3y|RTo>ue$QhYCg+ zL}8rjElRR8gLQs6)-y)pz%C`6NPI@;kF0}ee*&)gjfT*r(KvQJ26Ih-QQJo0v#U8S zo+ySu*h&}-gkiq@IINj9nby?z&{^voL~OA@nE8GTC@0{|s{|-rv%~kXO2`m)gWSnH z%JV)#{8;duFW;Mb zGqH8n9Qw2+4b{cDka?v8?sz-+xX3_f>UMGv9EF7^Ct!6l7bo_9Aek;d=$_L;h_^Yu zXqJ=GiV@f`;yBq*FWK1Skhi=Pw42m%a4;3m_n)L?tr}2jDZovpE-toB!odkC7#R&*A|B+6jV#9|y-rGarVbFl8l zXY}7sg~^%JW>r zl{#bir8Ke=jK`fznb6hEK~no{l-jO?Rf-XnYF?n_l)?9T8^NAy3-=W-Xo=no2)1?5 zbh8iCX7GrdW`t7Us4|SncSGM}K?v%mV3fKIny1|)t?w7BAfcY&Nb2!H!NL}L6nmZC9MQtbk=3C8tGzRg>TwVI zy#}HTsSFiDQAkRmQP)SAN&`C~V~UU|gd(LhX(A~pgpkTiGGyGQEi;KSWR?uEGi#s! zI(wgIJ!_q3t@FG+=j<21b-lRnweI0}-PiT`zPP@37jI^!9OPu&ZE@B}RB(JeCWHTw zh{7J6iVMYd%?i5iOc3(+Fm&GqdGm4!O1l->ThV9L&-BFYqJl*}hOw%Ay{I0jl6@sJ zX`JoC+L&&bc6lxq)_N#4>_GXi>tdd-;OJB%etMho{z)Fm@3c8RH-)Hb9ondkrgYj= zDITL>JjX3`z7;{rs4p_%vN~QwR4+@wq{Vzrc52Sh-3$2H z>6G-Hm(9;26WoV%V%rm4+B-UMtM7YJB%>@2?9SN4ojAp`W7ee^lpo$IMF&@4d_IM@ z6RI6O3i?s+BaGJiC_mIDlIgt;oz5}nHk`xvhW12N+VQAw7iNTa<)hO>x$(R$Vc)Z9 zHsPGCHP&a2niJ0~mSgKGLUcFff}@w6tK zw3vmAf47^w-g|L~%pt8qrre8AMzysw{VdmUz;+h*H*DqS+yPt}^<8d0&_ipPH`z(s zi8uA;!HsQ9`pph2gBHA5+7-RVO4OfuBR4{tW7Tds6GkqT#a4TmpT3F!-6UGZWRswh z&4f||E@sCuc4;(|f4Jc|v_CqrZROthi!$0EhH@24EL$Gn_`Nwq<;>&2RU3Nm8^ho_ zUsl}A6YDQIl6TjPQuQ2)=MCWW4R6jxE}+c^GrmnWB|T)HFql5W?X0JTs$1Sin6 z;GBGJrG;KnYl3dvk#$~)e0{Hp|7b0KtXD(tRSWXc?#rqx_VlnC!?St~^oIp8ZcM)1 zJCG|sqtv;7FI!H1NWk&DD+RC4iIV$Sv8^78twA2O13yUGz-T7EcH%tROiEjZ^YS(v z`!$SiXS*UEbmQV#(rLhUXar!dkIL|g!02vEN{D&)FVrnS6wTHZvuGvY7$y|hOwr{ z4e_0>OzYYrSvILw!W67^`%yY8zm4P6o*`7em`?DiU()~VSGj&Pp1-Eqpm!>VW47~n z8=;MAr8yh&+c42)JO%lk*x%)uL^rje;&iec-`$F`<$lZ>*?~#>Ruexzk~80|soWBe zZ;CS84(yZ{3KqP*vn$T+_VCz$3a%yr#uGX0BWd~SvaF7nK@a65dMokaqw?28 zGQNe9oaG}|T4jl@ff>q0A0<1yFR5;BOizC#N6#vH)3e^}TC$Pt%C79XumHma-n2ir zjX@q~(_-WdIoom$T_5HWF}NLHg2z&ovYH`(r4Xa{OSYBt<<9l~%;cr)Pq`+?hK^v| zdp$ZE*NH*PZp@L-a&C&E#_lU&-<&ptbe^lww^_9In2kq} zJzr;X@v9MPYilI*+aj)5^+IiwIu;5YFs6$-i&aB-@O3Lmm2ot3c1QQ<3MMOW#N>Lq z7?o+EY&e6=Z>Ch4?v~l-A2=>)*^Z}IJF|XoFqM@`=sT$4LqGbJrLweeJo>G@2yL%m z@K(y01UZwLI2Vl{bBJ9XByLIf#5l%>n^*gD{PGGCyxXBa?Wy?X^<+v?B;N{u$}q7W}h_Mp-JQ$BTC2mGCg<{8(jOdrPzwadxEi6{V3rL z>Lt8yM@ksS(YKq zZ$hbgDlVOeCz_TN_XbxklElcU7F_N~OP^h9U=XpD2aFze@D(ROx|JywsT zbC41HSNQNf%>Z+E6B-p)$kA{6h+5i+0dvQ)wS6^RKo{;{O?^`LI0v#z(o64Eh3)vLWiw0Vwd0aCJt&)0iXlTX*yDV-!SEv1j z*|(_;$$93Msf(8*N48%}i91TNMc6{`_y)b5+aNA0HM+IUm8U?46P zBX~E|kTI`=NYcD6w;p;ExW5xgYqMn5&j1SS47nCDk@rUS?BBnbYw`2>*eiq}B}Jap zJ$KYKIwu3pt!3uip3HXLN!H;Y67!|)uKOKpZ;#)0mV#pJH6J8FD#i+qLzJ`3120I*xZ#t1bjeZFql^FQROD@-T zVo$mai>CG^XPPP|g@HKQDN}H+2WS1i$hSetq_j`rXz>!7f2dq6>L@s#s z#NKFP=2zH} zaAwdz_O4T>PR9V-UHT-Hy^yYJ2XX(_3Gvd(Bq4bL?hi`k)}fiiKleiY#55KaM$qHx zHo5n`Q1)#ZOpmP!J(b^>iV03s|GAedE+K4r70BN8U5VQ6!R8BZC1XvQ+{rM&+9-oQ zM#^Z8v7=(!H0rhAim7e3+`HbKIRmeX*2dHF;$eSQ*=Mp@$C>JkQfaa<97Ff9^5J3_ zt$U5+@B?>F%{+-~euXT1=|%m3~cRsgT9d8JMVMF$hz-C42BDUXK+8`eclH zDBsI_@^x@`LJWK1w9A3atKX%=?U7vQx*F^5vHWpHlhANSTwf>PG;)hfm^WLoZ@svg ztAu8>Jw?N&$eOw7)b|hIX47u`NSn(xzcI9}-Y&apBMClqh>undBsZ~Q&HhCEuEt{U z$c#?xl!`)QVk!%a8v2<|oX7(SH|57T)1EQOP0qiMX~fg>E33iA@Fy|SJzZ+oLJugejbym%?S^adeKe;DO0HPUm)JOz~0}G?qNe+2Xgl zj|6<{L9Z6-iv8|J-jVSfZt8?epfwM#UzWW^8Qh-tORUQk_0GnRE=P?h_vwkn>J}`y zRVFirXYqRKBnrHW(O5Q>q$30IT%<;`AZs3Z=qU8#D;d(zkDi%3B)_r+ueK|`wR;vE zTN^8`X6MB;W*$2$yEE_EWY({r!R2RPC0#j$>y6Xdq1y*tCmUKMEu`{<6HS)ilNWcL z8PUv(EAdG@>FPzRngeq9VL!|s8FAk>nvd%Q7o{p`;A=>(qTWmji>Fd+A~$!36OevV zE*Y6oxN0gkZy$)#2TKl@4Ip{(7G~eug>8`<&n^YvUYNrEhh@_JPAi&PDA?iQ1QIpZ z%iUv!7;3KK`_Pq)ZZQt8YI{mUvK?Pomr6CewcvH!2+BQ zh4bZWFK!rS5pjJZwzUdg?CH#>&0RRuQipSghLGN~P?m39i*ZQ+WhsfMZ*I-t?9s>t z8-^$~MK_=k&+`Ve`9K=0s?<4gc{`SAp5)qla(87a8Ce_TSB);(H+oQaVjz7-_GGAM ze@5IcWWs!H8V*^`yQb!N^cl@=z2Ar$lOU>D8M4fI6aI?YT~dEop#|m<^`#?Y?`vZ5 z!kYd?sZ1;SUF?%bVOeFvoAR#A`W4G=ZGX`oYCx3gY^LlmLjA5TlUomBwr&C1^A~cs z@fP{{Rg>N=GqG*ce7(bQ?w3^p?{V)1HZ=-uW?lL-G(!A zxH>Df7BcyMTe<}H#Pwr;oMI=k{&)j!%~GOvr3zJlWN*N&ENE`E@#*W**>6SSF80jHdcZp(Jk~hCN%Tp4gNp?-J;e zI)csf+(@ytM|HUx!;}Xx>(~@}G@Z*Mr#_5b)}4DB2GDqOJ}nCaaH}~lzu&YX``{;8 zZXJrPet%RQ&#-GlnRI*|NoKFrbgWO7!lk`YH{C*ioiIXt&dbNz-mJOmPv)l@DP7Tn z!*w=X`1(rD=*7$P99KFT&1Ge4W1==1ptfVJEZy&jdtx8d{!r-1oloSz|aL-rAhH zl1t%d9abE1rOUdcB91KfjFHZEcv|^qwE*6)z%m- z152D0&+WVXh|INBqOr?*uK(!9CUZybm+P_MaVthfKaq|_X3P#ZVsAHRG&-!KUjrjN zHnpdCWJe$n-v}HvELe3Zk{4(FnVZysl?onw6}?f8s9cw+Iepn`kWNCiJDL8yxN~C^b_y-f z!c~6P%Hc`5 z`1|~n$)DO|5SBKk7xh(+8O!5K80z z;Us)^VC0QP=vrTqFHcr*xZN0L-f$%CsXB9P=aR5$9c{j-&?nuQlQ~@pi_#_L(<7Pu z)0Mu)X;dh*yob>>(O7LyMVltr7km+u?&oEbq|2GwbW%-YabD4wZw8Ur22bMEbZ=ga z^<+>FRlF~TN|4=HDyz&Gyh4YkcYh=O`+kmgH{^Q5TJf3oUOZnU@oH6jDtj!%!%G{r zGJPB`j+gVLYS)$e%r>% znz{rhmosiA^_`%G2=_8Ho+cC8;6IIs)Ql=`7ovo6C znyG|FuA=#(2`tU*tk8Ayq@?h$m>Fkaecc|NNBOJ|oI!l60&-s&a=_=kxU}la{fY0S zVs^gxy*wlXJXc|3cT@^GZ-fI`n5pN|{e2wG)}<5gltcSGGj@IZDPF~Tlp0sb?;hu6 zR*Rcba3hD=3XX0Xbe_@|YdIf%9M7uptkYeKW$aV=8gN)PuC*kl=RKLR?S#bIj!|f- z1O}gPLiQ&3QvOz29cs;AVUOjhu?d@f^5~VLjzM%k9$Qw)jD>^o`}Ij)bzLiNt75sc z$&0v`w`JICJG}Zf=ep`Dj`=D0p~Eg5e#Mb{sXqgJ_4%{7lMJ4{TU?(eFsD~frmoya zX6J=m%-3Mti5~oHq|hg^z3BV=zO)~{jH~TGN_$sRPLw#(vbP<7J{-r0h#^urU@(8V z1QWQy2ki@nXee_1^0OjRA1-8orz##z>ZL<|GUGoDrO2xbtAn4*yD!Ul6dJ>fx+&DG zo{yhhgq+&_yVS%dGI*^6CmtW=V00+ikD>`zXrwX~A6}R`axJG5!L5SCb9`H_-OXiV zSf)EwHz<4c!hY4#izh$%lbAIc$>8u=WjNt|*s>K;Cq zGX)hgd}dGfc(+A;g#|%}cF2*p;ITfPJ}$Gkwd<~AYkZXiH#Ztzxi7_G{+tU~f$PgY zEUo$?huqVMc8VZ({Z08Z>adJT{UO!Df zh&kg&ak_Ohg|_1fGR~8_qv0$axt-nN%_)6u#_>DL`IZ}vvP~#UZCa6LWlY+q0={G{ z!*in}$}J`lXF8sr4t+@XO6J4%jbuOcw z5-%tbaIrn3bZpqLbuvDRUZqwWMV}01#&`CnyZ?GDhmYglrKapVq`~@S?y{!JiPnk&NNn=ubtj2TJ zN@_M}P*k;x%gL3ZUOkO1LspZQk|)-_9^6&*l?962ivOA+W}8&l<};lL12bA~v{m%? zgA&>6vowGDR>m6cmTQYuSsu`dF-6LscnZufdbE0S8m){(NrD4hver?%? z&5L!o>|ctW>Nn{$Xc2qs>*alM6JC{+i{8*H;;u9gQ_V!St z(a+MNpG9x;|Caaef1LcEymxC=|BiOLe@CmHf5#q|CzR1_|J9p{|~JuE$6&`>IoJND)3l;H{SV9?t=H!t0RVW&a>W1u literal 37946 zcmV(>K-j+@iwFp~a(`+9|7CM+W^Zg{GGBCQZh2p0aCKj1a5FM4aBFM;?7anCRa@6C z4hWKxiUo>F3rLr+_jm+EBvewRJEWv+1yn*nKv3x}yIc0M6T7<=ySwh%3Lf9{zJ7nd zd%yep|IWE*Zx>^(ImeoF%sHPiW{f?TsUizMDhdklKQoFobEN_kf+Hf6LW7f%6GDq~ zyV)hi43CQrF?5a#35_n!?HUrA5*f@UbP0}6F3uGRNtzPR-id}q#|0)C85ZYC#PAvY z6GKBoiX8=VrNZJ8CIu#h{D=@u42+474oxWL7bYH?9OEAy7Z;yc>?oKk&%gVY92pz# zpO_TNj~2@9!H*7!3=2z6jEsx@&JgW^-g8OiYRkPY8_hPl{us+2>?)`zD0O`v)e)#e7Q*j13O;4~mTS4-E-t^Yty( zv=`|3*wc@IY-gZXXF#PhpxPPG>bUfgAGhV z&5eS?OwHN8|DNx+o|BA-3<(L1{oeMDD!JX`10xeAMJ9&&|4V}GpSt+J=*ws}!T%e0 z`1y5XZ@)bmI3+aU8w~n~u`uKVBPoI3>HNfwqB?m5a|ir8Phe6~XlxR{u!zvWkZ(Cj zC$h=k@ck|BTYY5z5Q>!{IXd)PtVCjDcub&wNMKT6@sG~TmHUPy|39^WWLR8sLS$$H zTfG>5#bv*9@nr~#jtidfO+cAHB7ca}BZ=+6#K>k4fK(ur$5-P{~F&SMraf_~IYhoU8Ch^fzV`OaB^qd$nS9xhK^zpx-2S0LlWZR+xV!_2b~2@OnSL-L8Nnnj0p0P4FE@KGh5 z7#;ad11CkshQv)Oc1+Ff@vlnxZ_?)~#`?4Q1|~(OgtCFe$dJh$ZKJ~4i$9WrBlyVr zo+6anooyX{)BDFH2KtAE2J+Q8u{b-z;g5V-YmmTp)*m(T4`DTw734=37iZ^6vdJ-l zi4*+egMXGuGBhzJ*uTTzxv>4rzl{tj&Xo>|jEhJP8e-?>Y@=&BtT;mZyI!W|O0Xrw z^9y11-&vpw+sa+o{u0iW37s6zS8!G<6WE%>@ulyZ7|#X+qy1UISR}Ab>K~R6Ix(3Q zYD)2sK_XX~H7`Fi@{MewKfjLxCr5s32H9BuhP=+EMrUnM!VN^D{OYU$XD39}Uw%a!1(0o&0bNfG?M690zT(0KOF zSwJ{fHaIRiIVSddrXZEuE%w_yA<;i2ki~VeqaUl{9Wn8NY@@JI0lA8Q zm`T>+Me@}lB9Ik|ok=(fI!du$vC#j-6#Z=?C6&aYG(yimIVmzakyR{Kn4sjy=n#L_ zlqbd|_@@{a7qRkp`_9JZkq{WpJ|@d%!?u1{0y{zkO-W*9FDf23Lr6d%RcM|>^rwp9 z?^U8&M)%fB6>`4ch5b8YqJZXD{v;+oAucJ7?Hhl#ulb;yD9AF5?Z&1=M`>p2_Cw7@JQCM@V2pgFg5tooOQN&SbY!~*9onK7s>L@%`n4d>n zZ1_Y`N8#jTwoJ{jqI~A>B2N?u{#FHv;rywh+xJL`iGpKgSRMldeIpYC12c0oLkkNd z(}~|RW~Z1FyEq28Bux|^!)E6@QQ$u^3kK>1bm*;yR6SEOZ@hI`O$0~B!Ja(*mN9vs zJ}?#)O-zDT9(O~33uby-ZEMIu4x`gYo!noIVZPe?b8c~%onIQ5o*LJfg4@R2 zJ7y)!fR+cG+m6GTa1|4h3io1`252&p$96L zV+|u5wtyLS?Iy?W+$F|LE{A(-(+=L2fum`E*h5|;Z#2`rGLz}$w90zi!;{R@>F2pt zf)9CjKl;-63@OGd#DL+}<}=b)WEii270mk!D>zP{wzO^^*q3;g;~6QD#mt$Y7Uo?S zF8AuTT4w6_EUujLX5Nk6%CzV82J2V#%Uj=sWiaP1F5%6)y`8ySq{m$&e2eF`$(43B zH8KavbD3f7x0!$o`>ii^Kf`bglDMN4CNf(?I5c^fJma+H2~YO?az-br*1C9Zu`4^v z=R-b&9orlQGx#Q!MO!qxpkTqq0$g%b0&8+YLu?~igIiRri`v)_NZK)1yclp0e zfArm4*4WNvhtkZ>zUljKeZyZa1;-`FvUADron!SE?;HbTGXryze>FT}zv`S$X0~(0 z1pez!fxz#5IvZc?-x^V%liw$>H=i(7d!Q$q`LbMxQQgdfa5X@Zk@Eu|OL z((8NWbiJmL-fGs;%_&tRXjM)(m@+z)T|oo=ODJ%3H96Fk)31>&)iI6zJ2kI4J1`m> z8yWtlq3K}$sd*EY?5ABTc9G@P{Up9=4^^K&NcUgwrM8*-sZU1RH+uNli63#lZeEFR z3)qCfkjTi6#!ddq&rpob%`J?L|D|#LemO(wU}hWFucL8aEsDjq?M3t?-5QJZC!^;E zSxju|h9%WGu-2M}SMt(GREkEzSxdw&bcb6%14O)=K?APFK>468K1daUS$>Pk^n&r$ zV+1bFm51NPr?eqR0w?1S)9{^3srJ1Tq$g&=bIpEQzRVTb^XFl!(LD<4tx6ZH%y3>? z2fpeqc&acQrc+cfMs_P5UN#1YLTk{yE(E?0voSTf7aojWjX1@F#8H}w$=U@NSaXDy zK2pL0@k$h%l%Q^d9FjCraPp8m^24=}-V%vO_1zRxYzkE)0dR)LfU8wYQb(eZu33gV zRTrpRR97_RHN)VrEBdub;qnI=tgg65PJ1N~;LDKyMQ^O$s0-bTgW%An390-6D3Q#9 zQD!MbCmbc`%v2oITZk{Gyvg9x2^yN8h=Uh15GuHa3hzkbnwkRkn))E=Ku<_GXHfoL zc?{6hLnlc6{tcwE{P};!eqhHEYS@3;B8=!91UsNS{jWXbnVFiKnHu~F$+Evfawjv3 z1K-VW|gcAj>l-E7?*>|8t@;pi~N*3r)1Vf1L*F=Jev9EQW*ag39lt-Y(e zlbh{07l$#9c5d!YBb+>)T%BC)?HxUb+q>G>IRS8XbRA*uWa~1nI3m$8AR>vqlAYfQ zMWpbzVJ3ZFh6k`;vCxPq-@EhOj(=-ce@hto_8Ne)$2dVEw646hk2H7(riL&9%la}?5AI|lqVl-gZ#}i%?FZUh zw43K*wSo~-U(f7#p}K} z;WmVN^A>0fC#U=IjD~3=^Jy24d32w{d8BT|EVc3BjgRfeYmg8p&bO^xr?b<(2q+`ESdcaR47MoKX3EM!38(^$o0CI@`9eyJ17=3~WK_2CI`VY3;X+%}%M@3fHFV>yC3eP~YWJcV-R z?n48v_NxJ$%w$Uv%1mM+P1i9l3R@ZDuSYmPjOR}a^&hGnk@HNr;!Nw^H?A8ciHtZGB!6c)&I?1pAP2V*!B4`=)X=E`v2PX ziKzVEz<>1(eRBhI3qvDQ^WQYV9n3$OU`O9->Q>Q6o6|~ZonkZfsjj0!>t@omucrBh zb+r0P4HbyhQ*W_KDq_P=nrevFHB-8L87V%kqA2fjl1r$jsFFGoOnLYy4c$3ZOY-l_NcKoIZ4hpxl*{$xSXD*yx3rM_@*2`UP)mj8OG(bKmX3_7qiIVT z$>weimFCuwrAQ4uE~_P>k|sK^yq5a;*3*R^l@w85Phkh@NzSdBJa$ylu}RgW=Tk!q zDy!+GPCaeCTt{b<8%eCRg(}?ZC?lhu=IPbZ`^&ZT;z|vt5hE6zF)18cJ8gi?Iwizv@NT*5~(yfk) znwyFDzJYGHR??djWwda71C^erCXGZ^zCM+-wz!HGFRdlfbG0<-dJWBWDyNOF>PalP zk``XAp2%fgrG*9cwp9ibPc{_0zfhUTUw|GH7bu5N$1=h(r` zLiBCNICK8#0_-2Wi5!n|aiTl{gP!-s<*lx`dN>lIuJJf4Vu5xgE!_Glg|ci1EXdP> zrK%0ZtW?7xUkmtN%EP$Zk{Gih4QbVV(Av)jRkP|z=AtSr#12xgJ{L)1urfZhgrj6| zK0Oz4LCNkJusJ!HK3!>|wDo0e&{RzmBNz^aaN`pdxHXC)ZBo> zrwlM<+ggktc8DTZk4MtYMer*+LFeWPqk-0=yKfyv=2TG2SOL6qP=e%C2gs_Vq9J=O zwT+iTcFH)2?ze{U{B4wUG76{9He3mH2vI2!$juE3c7Y9F-AR)GZdN?a0a!?hT$2DP_ z&KuI2Vu4x5Wl)~yjnd+qB$K(7>+)b6n%iwrFhU4-baZi2tOu6%lEnmpy>vzJ21)lh zK`Vw8(!&~I}sDs~4x~sUB#1s)iR!B7tjNA!GZPCiGDS_iQyb z2Is-tU@ns0M<7LTHNEO)j%%(ya1K(%?A*JQlC*$wRK@V=>i{@+^~VAkF=RX+h{0}_ z_@X%i(gO}s?a8i4E{etV<{8L*t_bN&LUU3j_CJirbqOuV>ltI$QE}YAv!8S>dSJtw z{xHxuKpG*b;PgyGzIIQ{&d`SYyyNtC>@hlV_%_Y4cY#2@BhEayOd3W~$k^+RdkS|* z>$N9!8zc%NHm_-~_mks*X3AGn#MC)HNUIRSdcQ<+7pS5!-AChQs4?=Eq+;HOR6Gb! z!{=E&G4Gx==4{zPubOyNp4S)WX0xkDO!kANsMXi!}D7hXd z`z)iP>Nyy2G7E=C-JpJY3b6D~LvC|2?wfw5rSERjt!cW*yXXpO%`%ksu!Z=gdg!L9 z!{fn3ERLIl9xAWMuuC8G*s%~r-D}YPS{ei9U7%4DMj-K+9=cvQMEz$UrHaRcaB<%u znqG35WCBN`k~;%SER0eAIvQU-4TI*nrPNKs6h|j^04Qb&$GEl*aw@2S{5MaCqOj-@<5G{2?BhL>2u^o+9Fwi3*ma0mbV(hD`!GFEd-nMyWmPyI6kgy zgu_SwqVn~;h8 z>PHlxu8zE?so1f|o@`u-Npyz|gbW^2UsFAV+nB)Pv?SKbkAdnQd0bjoh>&B!c$OJ6ERwH8QoIiAa7bIURsABLP`%)XItXjhB=6O z*iO$pm9ml+=)t5Q42QSK!Dbd+ffWh(YYwaG`J( z4p(m>Jppe_QTs^x^DMD8Y&3V%f?3pajvho0njz{hG)}~$s{Z(p`G~ixtiR<06VYn*}?T_D6yS)=Ma?4TGy%C0nHaOcP0{IKFsA!sn7pr{HSK$i9 zwU$!ijVb81&I5``#OsW`$3%Sm4X^2GSqc8My*+6uAKA+yW5jI#_!R5Ub9_2P;O3o(g7Z3*_`EuM` z7enDEyThm=3#)I;MPZjJl6GJhAfNoPveF0}?LE+??ml<<@M>b@GqF@s4NsM0VHz9) z#j1tWPgo0jkwsY6rv~%-Ibv(?>$C<+Po6OWcCMgfck)mq8i2Of-WYa&6h_^ZLb6Cc#0SDok_NBa z{qae80Q6?I&^(g@q#Lh*f=E8u7fgq`Zy+`lwo{h28eVcYq2)|2UTEtgv8I$n)Bv0|JFGB3_T*AL;?mgfb-pgA~nzm7hhV3!`JOHs1J3Z(-! zfiofzneMOX_H|8mytc*Dhm~YJ(w_E2aS(C4fUFXN5cf%+9d9JCWP2)|vuz|vr(@)< zC5sh7S$O+#B#uvOrd9r2m?u}Fx1b-o4%dcOb1ohTS72Sq16r5e1Cm#bFCSk0#~m|<#6nPS z1|9v9iBL0bYPw3b`E;0kDN zXT%Eg8U4tm%!l%Uocd8~nVGs0yjyKgIFnWnru8CQnUlJA8CgAsxu_S-Z9ne798=rL zNv(X5`%G7*FQ#W@oCURea~)4&mpm1C z>NA#`r!&07D(3n%e@?Y?HIucjYpc5ICJDD9@HZq@D*D(FmcX6!e%wp=g6mvbb`ZLRJIMDWNV{@2Y%^*f_cRgc2;1%bV=VE5H{ATWrPxE-IkL!@V+6Lb8(w@w9TL~u8#f&@u z#0nlqW*o=q!)IPbjyM%x4q-0dHDOF-TbM<6nt2DgOPCRb;@lmgsk~W1y-DiTAf`n} zk}+Dnlu6=F`c22Z&|mwfX%>e1#s+5pswDKQ4bM(yR!Qg|$L&A#r`Z<$O&!-}7WyWJ zhQE1S;Ro|?IIe%pYyWqTD|9-pKmXN#L~m$hVq|V+_**9E2lG!R$n<!S5?#TXLaN%UP)c|RMM4K<@9-Z zJ&A9xCGpA4B$L`mg*zLlwRa;49I2sbw+50EE~8D2)#M^lLkq(iX|+=|ZB}n2hZ)tB zEnG>p1DDdC+qG1y*GRE9>&d#QhWz%|k(Xl?joeX5idz~<)1ZnDZK3h9zNUtBv)Qyh zH6(qxhM2Tc`n3u+{ap}ZXk@H!Vfvd0>0tf|A^)#Cj(?4i4#({avA;S(^bPd&P5;$# z`&sap5b5A%A@aFn*TF$A4Z@Md)Gf&mZOx`gku!pr@K_{kT4MdCIe6q?4yDO6G0Vyp zcXoqmR@G)TnZx`C%{Z`78p#tZ+u-E2L;zsOuxIBhQxG7UhZ71?>h|}r*fcY7)x*N-XeGR zL2x*w2qEqgy0>Q-j(W?2c3C6VY@5~^Z*=$?53hFx3;S+7Slbl^L( zG_S=y`Ft$kouy*tGl`45p?i!wp6~fU1IL$A`{H)9}OT25huqLDGF5d+?O zz|Qss?R>cum1k?P=FJr%`~L8*Gez5+Aowp1B3+Lybo^x>^mTB=%6(2)by5LgIg3d7 z#t`gUv=AGw5_W_pV8x2PzwylwOPAu*Vi35}a|XK?{L|s@!Gd)sax;PLQr|qNZ9$cj648)$#!*QnP#%`1&KUZT2oga*&a&;8HG=sc(y=eFWN4Sbcz}d5F+XR9}n z!ZBfNd$o>o^_*~^XFUAI4TH_yUN}CW7Y=UTLZ3s_u(?GSueJo@RWBu+yU`1QkEdd@ zt{z&sbLsTX4fJJg9!7{IAhBgNM(vWugu63QA2|o3)rv{zuqGsq&Bx)8LYxa823y)r zDV9#ykKP#MU5eD{&1B8Wo|~tE)jT6C9#DcX=k7RRrHhEHOSDgKGG@KFL3xFlP_8dA7w4v!v6Sg&e~5A9FsRbwI+`%Hj?v^&hEYl7zW z!=?jVTrxXAwA&a4$BR+;={DJAs$)c7L6{v-z<_!#W}fSZ`D(?OFPMSlCrvQ!zA@;; zNjg5^F{$q))C8@es|(VRJj57XKV>7M!4p}VXCvQ27Egs9(i(Y7>^i!IPAyPENs0-M zO~}H_>|1nm?<%N;B*J&raMV8NjuhK>H0RbU8oV_NNh610p!ybAKS;&#C#874uz~!# zx?$~yCFu2fDkj>h;!9R0?%W81aY+|Mhh<^h41J{JiQ=<-2<5r;M#9Bb%Gt4qG)_2R zRAv&Qr<75;j03#d=c3!nmDH{!PjB|P;@V?%w01RyeYro>oj#DM$z8ILcZTlNW(*uO z1`ng&(&Gv3bWCgr650|F<+2XXa(Oh5I}Y=U3o*}wsAZfWMwm~6jP@+NYilCIVml}v zNyk?+D+op$q=PGzA#^;Qk}hSTm-J}#S~d+c!^3gr&_Npe#uv`B8j)yW4Xx9P=oZwB zNf#Z_I<$~Bwtc0+W2S4qucgUK(*kJ^OWskc)v?CNHV_Q#TV`OFnI4a!LIHOJwd z1K}XEk?ImNP;|*3H&+Nl@bY+Y?YiS~Y$BR7S%b486**UYv3H|2lWAX2xi*ScMB#);CS?wZLrZoA zVy`AbRSmG*9fPS9i|H1r=zXn$wznjrk5eu-q@_dXgbd!KD58F@Erj;%B=2%57(Efk z8Nmz+d{Tr*vl7r(Xa%_r_k##q|M(|^@n%Ccd>TVYF>DF>hee=|`Uo^6_62u?J?7Xg zf$Tk9ggA~tk19iWZ6x&1cuQ&VmuNgAhg+7_P`R<2Cf^HyLa;K5Dh1K^)>usNxKDkr zouR6p4{6#{S1d3rz~#G#Y0<~sbZ(IzGCV|~c$15Ct|Qcn=HlJfGR(MfiMnhZkA~fa zSP?G_X(3^JI6MbN;b~|epF(e>YW~x`q&NUPiSG)l+qs# zozH^*!J%_q^l#$OF*h(bG5qr(W!YcY%Y5Q>KCrn$STtJ7{AE z-CpMhuW<}a%?!c#1hVg8VcX(9re|%Ju?@~cT5_=^(cj2FfRQcgj0sH+Mxl{~H4cSZo_gpHS%s;tQ z&StwYqE5}cEVnf4wktiU_R9vQxOhAB@$h-Z&UqYn@t!nhs>Ud$(ms#ZS4f%Q@s#n_ z)!YHS7DjfI0oQAs3&Ty{!^sWhGI;|AQBnE_o}oi2BURAGtgcA2ww~9-%&H#&s>M*@1DvGn-k4geQaR1w>4TTPddS*zb)maT(jlnJ7-Z;zxj+c zubsIlR>`C-m*bvZ7|Q5uu;$9xU*MgbszG|^O_@vUy_m$Wjf}ldHpkA)lX+!pz;NVL zn9>)`++9V<%-NeWnUDG-^B(zdRWCa5ET;NXFN5w(Qnvs`A@eoQ@cly0Z?;fRz!}Yl6E{!=V7i z_KO!26_(E|TK|aCdN!MR;Gw}8mgU2<$>~lBQ&X8w0p-jhCWo0h)S9dObuDA+F`N4) z&Vf1cYAAVTsxYaygPF9d7RJT2g%e+}l!;!yv~};|J-nW$Wyr&BB@>`Oml0ay%f#-I z=E$pSF*VsPR<^RoTVMHkkj2;qjM9Ux%x%9lOdsa<@3~ZjyZps}Kw)5J_~)Ze_~j2M zI+$4{!9OmAf9QX3t>EwKS}`>?GcdLIP1i~%^KZCT(*CDhD}sOZ@!H7T+`{l*w?%(x ziheNvWQv63tLWo}2AX)KoaCQ1kkHwBda$*gUWV1u!=2S+rrSslZ`PA=W*H?!R*=%R zS~_&Uo_gk%QoL9-?Kn|J-KNx%@{$^Id)P<^>l&zVR|9cAHPXblHMDhkJ@H&CDOIhS zdY!5wMy{TeOY5nwy^$oWtH}6mC8deC&_^`Vg`IU2Ro_fA-dEGruXS`YrJ71&s>n>F zkv^(5P^(7+J=|AHz2%xH_Ixc}mZ_yT?ImO@P(fD9YpF%Qngqh@=-!xmI-1=`DKl$H z)1;2V_BGJm;s$EFTt`bTRnrBX8lv)ATK>M0HVHSW?}8FjGZWK4Ej@mXk`Csd zQ1Z{Yyte-WB^@pn3Gu%=N{o%o{`A67cD3_Mm~?QnFzM2<&^Nq27Qy{hu>Kg(>^%)_ zzU^e-)DvpHBe2>l9w%>Wp)6(&+7%?BQ7wV!J|@TOIH3K9;S zv%nPsCfe|^oPMmFf@FM7|JAk(0ybWK7~xc1lh%u zvZfch&m4{l^|R!>JPTdCoUuH|5%l>u34gsy3htGZVKtnXX?IAk;0e_XAA-6vV{}Pe zNzo0%k?f&`i8JRzK%<#7c7?#tt_6PEb>OC=2_>nyxGY}@)jgG@ViSP{89L}SG8gS5 z+ep`6f$mlt;z5=sIB}W?%GZa%ZCNNkG=JEST zNkvj^`Ftqi7)BFQhBsW z>p3LNj6izJy z6a0zvm&+hWVmQ9`Yru@L10l7&1g#>jC}?bg!m7)pad;|Tys3mw*=u^g*a1t$>u}g^ zEKZpw(2ldmD6YjBH@%fGab*cMMaW=&>qZQ(`bgGEbMaJl0h&E($@!cs8j@=vTat&l zZbvC{nksU6DJXL9ht|HL7~^A$WAO%Pd%BE9AJIXN(Hv~PGYcoI%n^FIFYZmVf!p+Z zGJ1Xa7WC5=a)6OH}e(cXiP%I z#T?wbc%7}gCeE5H$Cdmv2&hV7Nm2_LPZh(eaelB#7=_iNC(!`=6Xb7{4DBAK5GXeQ-%AdwFG)hI|9rg5-$!aQqoDCd7M^4G60=|=*(Zo$rDRuxy<9^9vGUj)sDOdp z_LFh{NMybo0nw=ec+fBhCmIit{WyaAk=fWPuo#zQOG&I%W+Yk974{s9@`%C8U|EiKQ~;xMLTKx*Q1{^{ha4%xE;6C~70!f2IkSSw+w{JWb*zgJHM32t6i9p-1RVnlr)^B~0A8?O&}KP|k|SB9u+Fr{gZ z!kS|~n0(9tw-t6#;WbH|o+^pC`Wl$@))0*+YpBc0IS?L^gtWtI=wbha9tQTrxxhg5 zO3;PP)h39)4?~O`2V=)%V${`bv~G$JraaBW$1A4D)-1r?_BiYrtpmZdo~W|Uz{jP- zFe>>Z9o?;m9Uif`vVSCk#|R;2UN}4zmg0a?5>k}oP^+7PD}!Uv1tX$zRY0%k6q# zN6}MyG$0f=0|(-HzY~<${Dp#-8RKDGDD;hjF(hUN8BWuN^Qgt>eIgT$XQh9*Ub_6J zvotDVe;3z_p@qJ_$#0&e`N8~?>qV`Dws+8R9WEB;Pp%i)4!W{~FY!;Vn|}YBFNXZD zTrWDEu9v_5`bXc$)L38tH`hNM%s;Jv{yER`hx-Np$^Ej;bv1o=zkGj5<W`_E|>AmS>{>gizzCnuj>6|fh^7s-aBu<3Je;%~swTRe`Em0yn#PX`G zT5C2k+!Y1f6v?%`^h7)Q)O`>m{dgu*nYV>8^oiiC8KllUGCFFlA@`6c)V`TJ&~bL_ z{Lz`bIZhjx9ZkZ#!eQ^32g&QW?=~;6wg~kj`DxvlfiuIHCzrLET^U2TYR$`-_vtG* z9y4F??5ZT`{=5cWX6XRN(|HJUd}tKs)Z{zN(eqb1Sws4@x{3JH$>VbvpB}rJ%(fz? zfB1He^^l{??YJcFi3?|VIjPg={L)$`E4Ym@C@)|ftCw?BPI&O*ee<>_S*>C;go?Qe zPcnJ!*`=))$Gb7tGu^CxrmSK5p2^}|zi(}oSO1tRBuPxN&0^+*z9O?$%81*~#E}`8 zCCe3l`j&TIN1a?ZO=3nU#W6Pqer43B9p{90UB?Wq*}#!{yO-DfrXrp93FM4c>E1e{ zY&CDk0ejxslM;+^$+lMUSx0&H+f=AKqIm_%Wy}MWZcI~~Hdn=cDKl@Y1$SOAN8Y3v zDpb&IEu*-tkjWalhbbH7$%_>iVb1zb;AXAXVm>^$%#AJ5;;d}T$%7 z%Qty;GXI9x?lbqlPVe&D2crJ!^E4Avjt;MLmL+EPPz zKh=`gkvh6`wu}ZFETv0aHeIZef<@ST*H_TZSG6=gxsrJC^<;HuAz3!n)3CGEGRW!7bI|+I1&xyu3tl+yH`@Ccmr+TP)$aotLVV}YU*{N zip&g_(0h$aQmCz`F!v_nx>nPXeq~hFP(eATYRT?s6~#E!lER5PnjBI^yN1+~(#3M} zy;)5k@*8RW*?M}@T1PiR>M5wAf%Lc4(c}G1lww&$zcxWle-jNcw$L~F^MCO5OAYB{ z{z*gr@mT)<`zF!Oy!m0iEc|r-JwJW^n8V-7;^*I)CqF-a*`2cU`S|yI%C85%ZhSs| zm>|cL`=KQ3~FIA+2n^ln~9ph}vt}yJB z??tb+8e`u^U|*#fG~`?`Y>^$ldV4{8&nTq43&+tzf>5(#sMbCRYSt~3UYmjGQf_c+ z@eNV#bkD?wCEC#XxE{LpeQ^B7 zEU3)L1ta!|y6;uS!0lC->064mU3y?}sxJ)7rEp}m4ldc5B5Cy`YW@-nG#a2%)e_!8 z^JspfKgKpTV8uHJWLa8)SC9_TdCjQmK~QNLibu-!SS-^YR~=W=q&jbSUhIwCr+Pzl zzYaFB^{TXJz~_JisI)#wWv3$WrmzBg$9d8snX!0SGarEqD)E@t6|pw2>4}ssq+iR! z>}v>CKL0}Xih!9zAj%8&lbom}t}Zq~_f#q9KDbXIF-Ixa@GY5-$fv_GVyMuH$IOND z*e9uogFB{UhDCQ6^bkk!i8&D3bcvK>La&4Y6x7w_H}VNIkCj+Ndbxij-U*zkyPX za!78ZE2dwFK+n?}RF}$dgHEod2jLSjO`D5>ijMfW$rz<4dLxw?h6&{cFuU=VBGfBr zX`BT(kLz%Dl@Qy$6_ll7j7c*pkX`NoRlyjD3k^o$gpKGr@CcPo%mVjh8jiewLRU3a zV9iTIhsbrz|0hN;yn9{hEdg6p`3beJC{AplV=u+`MK1 zN29Nl6tkQ9nkOJHQ3DMr9xTTCVp5G2I1O*eabYHXw9^2YoS}75Iq208gPzwK(KLHI zWgVFf#XhkK!7S~f;p%*WD)a_Evjhk6dvMa3`y=-7(mV}my0qRvs^)Tbm< zpo3I7EwoHh$JcdzAu?77oVrqU@AiP!jf=)UAsYnkd_lECdSXjqSA6{ZklJ-m(T%mo zsjsI4+;4OT&&?K>hN(ls-32by6_6}f!m?FynBSm|F@6&fDe{H(=i6Yray^c})xtFK z!7y2`f~ds#5K^#(RG-7NSj8V2MgA~|vqsew7l`!kgBsUYgng`hqwi3Zh$TiED&pmX zcr--%L&_u^aGBayQ<;(mvRa^I1KTZO_1b}hYFJi)FrV55>4X~-t2)7x?Z@u zRuDdyuF&|&^Pv8EBHwNPrw=_{ z=88U1@_3!d+N^UPcwHrofM*7HwRRvJjIvR?ZX!nbiD3TtKo+OZ$)k849%`?`qw&|N ze~-D4z8{Cl@7K~}kDeGKJsrv3#hCKhhU8U8LPw(r8RaIBmU>R>opd1_oI@{*y)bX5 z8x9|x1D8lYq};cF$ng-|ol=9ZzEk0R)eskx!qEI=HFoXJrxV8wQF6>1{!&~_N@O_q z_TQ@nB5zz(V7yzw!nAARX@gYMfDL(^3YLmXDYxWNxSK0PFni9NAt zTqtIz3nTK-6RKP^3tHg>$?pB|RV9~PGp0dXV*_Tq9EHYbN?31fkH;r6P#!q~yvxS) za;h#~NHig`APM$c`y#)HwYi#oux;%}l3p+lSLC12-osx={Yw_|ddHyX*dtof1t=Nr zh&Hnk=utWwGj(jK;KXVwTre8KpN3+fl_K0Q zP1otH&@cp8RiWN+5Y~R4i`}G-WrKUdXsH37^1FHbY`Y)QR@>vo(D7Isu@d@$?6h)5I$HNuVR_XGT9)R7h}pAHUz3T!qp#A7 zE01ZcjxJKTX1J>@g7VUA%H&8;fP^Fl^yrV)WscY|#1G!;;#d~Z4_9W*#A$;mm}shx z_4UJ`D`$cAUOliUPzah5(&Q-U0R`a!=*?u|kz5zJ$46q^of0S;$YaF**CY}bjOAw{ zAlLl@O?>N&>8<7nY+6YhlmcKsN{!Br$j2iizU{L_U3&tgug$@i4dxjBJ_@U}H=y^f z7%YEbkED|kSRC#I%ME%cKlz2OSt=uc-2xP^o{X#hOYp9jARPB6!mo8EjW&(M+yyJ) z*OrJOX&O)%Hv(cV*$Bw%0nWZ@*f7)^`fcN3e$)tOX%Ho-G31cfgs38WoT+HV!X7bL zCo&JY){iJMH5P*0B`D?%Mvq2i92=W~@i}=2_fmoLsS0e_orz10vtj?91C^8@iWc8R z7ZWnkbFCV-yxIa@cX|NFHq7x8Tv|4b(3u z2J-3oxLJLWW{G~F8=+}<`!)yTmwQp{$7{6Trb?J07Q^UBeKmD|HaI)0-x`+^474GO*q}5QlDWL$4)?5D6KG)P6_lWnOP+4^jX= zK*7Jm!L$09 zK#(Mo6eI~EAW3qN(EE{`l`J44Ne~fG5F;W2B0&U1$xYKuG=PEu6xxKM$DFg6b55wJ zU;B>EotZV~z32XQ*UaaFVmH;h>X&w`dUlmM^7lGmUa$gc?Sru6WfL}TVqo2Cp3k~n zi1KUA=oKD`#UJc3uEPqlvvRT0!xe(Br7>~MSXiz!MKcv3Al?}<;af;IVvs)Y`l+7Vb#Rb3+)biC6R+En#Fh=c!^T1 zoiM!F4SNs9V22`?7~2f-=4%<+5~A_YI0ld07-%>@1_In_a%tI!ol#Tp<{1-?yB1+h z-CMfju@Ju7XP{-`5n8m*0(}l)6ta5-hFo(H`++y^36pT`qcS?9v~gWwfQIi$K#ATg z1V>&XW8(lkn^cF#ViveK+Z!eQNjNcQGge*Ffygv7=$Xb~mB%~Ukj{X;_-Wc+pN$b! zfq0yqfpC#z1WSZrtbQ_@7N4e0*F&Ts?SZJ8a#UOqKvEH5laT__2HkM%lNP4eZ=;Jr z?ewO_8;@2i@NAt3WT#tVnnfMpW(U(?UaVW=k4?|Q@os)0%q`iJR49RW4q2EbOqjlX z2`)7yU}u0lDqgsw`}Hf*?O%xEvJWIJvl`1DWMjg92DpCVcyj(2bv+vc;@OW5r8$Ta zwm`zsNVJWqhv@cbR0x`(&HgLBbn6ZwD%@?DvR|?>$ zm*7aYHBO!r#pha2C^g=p(*6Q$bMrud?G)&3v`6lGp8SN1BmLrjO5pN=yc=;ggM(-n z6^!4ejE-kEkj*_#fmXNZ{Y@Rrir$XZ7PIK;$xG`S4bYD( zqV-xLSSufa>rwhp3z~}?2FXw!Kb6i6jv$}Q@pvR{h-PIojPW+bxx{r8*{B4|#|rSB zsR`+X+l)lT7?Sz63QO40h+i88L8M?H_!=p!PQ$aX^+*xOLb6i=B5T4>-&}x4i7pt@ z8;0~OAq4epB=HzN^e!)gRH_v=kDrOn^H0&}&%XGSUy92Dg|slfoMN%Biuo_Oxv_k8Zhjgt?9-c1#STsWm8K>7GMSB|L`%EDxrHJ!8PtZBz4yusXMQMHo z9`pxd_||MVd{~KnccJP_P zC!xzo7$Np;bp3!KO7b^AVhJCU7T!!yhDM|ibn8_x_Q7lf zm}Q}1lsXdca7ca5Vw8=@hNG7z?1r@Pa?VqVv(5#7UK*yax2V_;tPa662a!1baztQ_YDeEWo7yRej z59fa(t)BkeJ^LqB{Yl0DBEsr_Lspgjy{P>^{yMFRp|R;-y_4~a`5&&+{zg{)MwR{h z>$Jz68GXl_2L6UBOXSJ^KfnLD@BeR9*>q=Dk7=%6zCMA0_Fg~EJk{CP$IHXs%XO-~ zFFZVaeH~o=;2Yp&4|jLh0AB}ZFK16L1O_0`$IHjw-Pz9}VA>C8T3>7mZ%g(3Hv9)P zP3RXit&jKfUj$7vF|jf?viggtn_tZTU?@5-NO2`CdVAXrz3iQEQGpCLba2-Q9p`Ql z7Tq^@H5lQs^SJkiZMIubQOLc=G^2IvB)Lictz7Tt2e=$zYsOpMo!qrMW9`CRM7a~q zt!Qf4j^3cQXzuFITe!5xZ|}xY72GjLyX-PFHLL@l2a@Wxe(qq|SME3KR_-#n1-2=| zc3dY(HHOfns@^4&)#;Ur3|If1EY}Ym+@+6p^r{Wr;3h2jgTdXrsdr;$1VxEPa=#x< zA!G9AMz&;gU-oghpVbcA z3A`!bZtk;U@KJQH;1OQ?PEyg$tEt}q$Awy{u-{T zb~fYc+!(HZW0hTNUc)PnR*aTY)R)VoX9S z9J!**HQZ3w>)bh4B>oySO+)xUJLa)4voJIMmpaY9KU0!l%seIO`wh+cO@Gth^f&!Y z|35$&`~F)Yj25P*#{XLI^QRETU(A02!dSBLZ*k*hH5Pv1qSQGsQ{O(ZD_ysb1vJ3>P*fZe!B5+*)$- zV^ZTmHjTJZPJ5=+(Hmltj6RE&w6dsAsD)%h*%Z*rq8M2YrKIydpJ!7`dkqy|ts(wv z4RpGUO|#l+Y1ghMdNYkl(P4F@eYu6I1zEJ~Z5^rJY@o4nwbWN!Ps0z?k%d`3C2nk{ z+9e!%;mD@p`qgw&w3#Z{4b-!YcWg)%DTpxX67P6MH;bOkX`;JJ>&fzK1Id1EqHDEm ziXX-sUm%CqKAUF9)Ki=}o2G7M(Q4rux;dhij_~?ypnL%knYJEirYdz7@sF>iGa(!@=w#8W zJB=hfn?(~B)Kf91p0upk^!f~kjK^1#Ludmfr_|Ew18iC;-aw=3*tBM5Ewv4+r=V9X z+HO-vvhB5We>#&&wQ8ySL^C~4siVG%Mw0GsAhRSEIh?7ag2pO3(^^mMYwO5ysGe?} z;E>d{dJ4#4k;#-Aa=BeckLGe{nq?!+U(2DMp#~aA}1k*wFb%`WYKd!79E>XP1ks1 zE^MeJpIIzgVcS4ABRJG7#FLGldOH1pO(KI;lz5|oawjy>dj%FnE3s&QE0ba#RTIOi znhcgV(Z1ugRJw=FldD?lSK|;{uZoh_u;{?FMoPR~K?+Jh8Sm%Gf)i5$K{@3UH|GCxp$@~vi z<2S7H|87|4k9L2}{g1YP$-<9ge(?P`{zpH4Y(M((WB3g?{W#Yz{$K0* z1)hsvN(H=!ati}($iJQNwP!0 z$E$Q?C>j|n_s~nm2nas5MevjaeCe71jmIVkd2EEruvb*Q!UgR{O_=mJ9J#mFWB!My zWc_dpE*?5ZSqn2^QCbXn^Ffkd?}m2@>u_DQ5hu!~L0zt$-eu{-=)57cm30v!wU$CU z$H6Tw2r3*m92x%y<-~YF)q#bvjUK2N5sS)qv$69_9t@8P;Aqcd8q%ALvOom{o|C{X zMI!{5d%-y6H-|9~B(Jg<=x$ZV{l^Z-`Cd%d1ZN<$tsPg(=b*JD5Zk>oFl?{@ z8K+;;wO9ij-x7qPaT;jydP}cnca!ZsC8S3JZUdRfJfRJSei@Cjw}zQXGq`6bfu&l6 z#-+kgP`yTrZt7yxi1`pYn~Y_bdPy^S5na0^gooT7RIA-x{y^*#FOf9Je(H?+sBDm@WCl8~RcQm+YDIz@qD;*QGv>4#zk{48ZMil1Hvgv7y9gdjH z#L+{B&?`GjuV1gi7lj%+mAVu?OBi@i8V~=zVX!-9hm3_1C~KJlb@ewiGG#y6I@mAZ?e9b&eQPZx zc9)QHz&u5?wjUd)`=6OZsP&FFP3OFwSw6AmE^9;2Q?R2 ze9PTJtrjA9>^B^e8WEWPM**^yj>5OMQ?Pu4H%3>gV#j_d*g8leS|kXm8J}rep*|i` zHWE$8!$4deyC;icUZFG6_VOVjP7sSWt%CgH955GapmtUcdifI37stS>>z3#^dz@@S zSK;Q6DUOu*^5n(@GsiE*?3=?7@pLqfxTj(ueI30_cY>I-4X&?C#^^KSz|UWU<8{-( zS3L@2{Jin*o5e_b*+~(9m|%LGEynhY zg0YV*_3`ZBwNzo)&mE8Ri(4=v{Sj$gv&4nd52?M&1slfe!7}qP?a9i;<@p7OJ^6+B z1gGHP#adc7b1AYU#gTBl0Nb*%z;KViv{~x7h92#oILLt9D};uL-GJhO=M zCKBe4(#6G(i@{eEiN|*`$YIN4+TIt1&K^aq7?6bfY8!ZTwxJ@n5VQDW;Rqurew+z| zah`+&U3BiA4PLaiB5XhzwRS2q80A3wMy)lObwmuEnO7c$COb z#^YRl6zB|6)mlslj&%>LYi7b2k>knS@ZHAMnCGdGN6+^pIpm@gudDEk?wPphn z59HyCha{YMYrIyZ6gNXmaWtk8ab5}A`%YNCZycF)i6FnC7=~A8U|vQC-jZf1FRRTeQ(Xc`M~aMkEIN0y$*9|s-h6Y#f|rxvF~$^H1zym!H-)2?9P%0!FwSm0UYf~) z=~#w~i7{|0vci+xnRv6~51Pq~&t9K3#|ifq1RXYlt8_PcMw_GUmJd#FY9T9{0IkPn zDB@SZI`eO|3y>E|#vJj@|}mgnI@ep+Si!hq7>2s6b!- zWYpa3#o_iuSk&w%Pfip*@5;u64b!0d#RFE}Gf1{)0d`&sLI!WH8=9lAY{@7jy)?p_ zBmB77oQ3*B%V3C6F!&Y-$rF_rJxI937YOy7RY;s}j)od>^j(yIz>DFS=_ZJQWOa0E zza`F!Rt%V>!;&2Zv4>8m)bPW*ViQ!^i{kV3gEZ)BjR#e=Jm1?v5{aw9os))$20eT= zGll)AESjcKNFg2spVb!77gj)I>lD~`Zp7^Opl(kt_2`Vk z=>i|bELMaALm6uiMBwXzR#^0SgRz|tV{)az>Z}F-=|)U?&x@hg&W3MN2b~*;$Dx2t zkikN9o^-?8rUJZMISsduyr%nBpJ~fx6U5b5!slc--d}S@kFXz>zGNVvIR06@hk7#Bb3yZ(A;67<1O7eD~ zt0xavw;rd78h2@}y)))rtHAIFPPpS#h7FI*(XBQgickEJnK197fD}eJO!jZPH znU_w^$AYm_z`Rie<$J+km~5u7H%hoM%?uvFEqD|?8KzrX@Jdw}CZ#bDOUVIySSzhb z8;zC2a$sZDh}kL!Ni5MHQ|{a1is)OaZF@rbpNbH+Kb#I#Ym&{!OtifhMt&>MVd9F< zN=xVx&+p{Dl7YPb5i0BLru&_lbZO*7V38k6cLzh9nG8es+ca*31CoZCFtVlr-Djj= zJUjtIWAw4vI|2?LLU%&dhjNF zy8Mh9lP6+fU^e3YwIMoJ2h37FJTtY$?pk?#C}q>ERBudGH^DazFJx;o|cgINQa~?$avLVE4TmGmd&dg+zob4(oi1}h(HJA29 zPsQ%`cGM(QVQpyu+IYU?kfs|J%5yQFBl*l&bkijoT=1DZ z@44Z{ig#3HT8iYnnOH8kp9J2l#-*dY{LRq_M*03oKl2AE=(bQ~QUZ$ZdEtqGEYxCj zU~qObW|>VukXC4k zLBp&Pt&g3Nt7S`P6eeT3+j?AHxfXJrE_kjy5tnw&Mex_Tn4UZb(PFxA%z4a^opYL= zEiJ(2G6Ss2>cW~m=?FbigFAc2;G?q?Mh#O$u!=688Rn2kz9eQ(D}cZpW%xDiMyB3c zD0}V0M%sW6DMd&tU_*GFCjJP1L-K=K7`FBS@ypneb*zH;d-(}&LePImry{(XYF^}3Gs~~HX9~f_Q5OXyF9^oD+k_|w6#uYk{cAhL2 zjKZTW9QYUCAq(lZWZU0O{)(dbYUB*DRq}8S+75k%MC8lKfGzluZlws|ltVVkVuayv zaVzZBdZB*3FZ$M~AZ`Mi7EY+6V|R<;++>55@7G~d`vOF7_kx=1HwwP21idDH>{C@l z(-c3PO+7*LRovmo<9!l8LfUW9Dj7pn3Z{=|t|NQLiH~!9E3JBovW6S_h}nqoKV-67=a7tvIBE z=rj)Ic#2`uBo7FS24IQvOvJ`rr&SKhXz3V^r3@}jeI*8?%Vy|a_L0(O&A?Tjz6iY0 zLW@HP>aUnXYK#QVh)hDN`fx;rPA1#JFnC;@f7EnRegYJ(nj&lAe)=OJUH8DwT= z5<`R!$tEAk%kmWozY@iIJ$YOYc}RgAQ+yf0p<6GMVRl9X^Y2_FzP|DJbkqpbb~fSc zNnpq!9HV9K(amZjJeApiCQ~n*pDKx-nhvVGu!XA4)uDc@1mF0h@F+19W;V-E={5sK z(Nj>%nFzc5kbZ0iUd z+N^=ItQ9bOm59!?U0C@-6Q{&h!aKGC6<2%cP_hdec{%&FigLtRYGT*+Z7^X=!e>c3 zj(%H!du87#Z$K093&#q*DBQ5tLB_o2bok<2q_joiZCMI@4YrWlreie2s-1>i-b6*; zf?*>mgju}~IMT+(!FqGNEhc|@g3*1vcxh}2u7qU5;*cedYDhwH%M?gR zPe5_<47|8H9NLf8kY#orBIakI(sB|Ew;O=JFA!4MDF_x{jg|p@L@29)ACs{r<|JKf z=%X+9`C%&2ie+D(QLeEE_%@Bg;rAOcAhsBL_D-M$qnuFT(mc2S(_rXQ5zyI_^Df zf{<}4T)(=*$W;>iZpM*FaUl4osT)pVJPvp!Q#vL4JM)>G%F zKulBYKuRc^YUjkE?s5rM#pKh+H}Xh7pN**5A?Ug-g``G#sFX~=#1LWZjmXE1C5LDX z&-R|!?+vjKMeOLSpth&h&^{fH?vy@qOJ-nv9xnz=(!nOS1_r%n(G7MX0&?f$Y?T3G znc_IlbDgUbva#ab1Uyw*3#swP=yH_>j0X~N;5Li6ygY&Bp@m!Dj?uU^R#+iD4fP4J ze=n8!HvHd|%FJ>9Zz+{AF*YfXY??{Kpioo27L6;g5|jik4|yB}ZZo$c_kx3m3#tzAh1wzbIcNZ zojbrCH?ontD2jTXem=!LvSQG#K+DX|)J}!k&y3QC`HPo)`G)U^}X)ZVM5x4SXCwGVC8@t{h2k!g+Lv|tC$8fK0*Qa+w ztGMSDSaVl=UdPpTNbarIo5mfRY1yl7XU6@0(wep!9^y7>PUg?ZaQCa7u&v&h$9*h&-7YM5V(+av z`ZQ|d0q()0`?&`awsQB6Bu3#oAufNsB6q*G0#{&Y1Q|xea>u5-aRbv1aiykJ*)?W& za<8AdVC&+l!&RK%NiI{8xXDL4xgiPLxU!rqJF9P{-2O!}j0NH!d%Zp?(OAtiuIH`M z+;Dpi_vX!(-nqllxz{((Wc*RHx7YL<(a9aM+zo;1+?J9p+(*Z^+x(@{r?2Aw;ZNnZ zv@$d{wEQQGPsXr6e|G)fxPNHLu-};e-}E>AO@Gth^f&!Yf79ReH~qh~rj?g8{FfHB zS{a&}S^n!IRQ{}}_3zC8P}KVCvir}g_J1#8eZlzW3-bS3+$!={i(0=4{inY_&CJ}y z$k_0&CZRvM{~-z0TUJYL!i}^&vywhGG*FatJsAu(kd;1*z8Ke0*Xw%n)#T8T!8!`7 zsU>i0sXB#2Cvz+4URxc#xm`m8$5`}yZymi}(?kWW9MW3MrlBt!QgvZb53`<(uh$TB zSS>Y4v*~a)o0#3TWH7y%x`kL2l3GVkmo?Mo@oXv(swW5c267Imp=_}RvVBuWOXKV5 z)wpK5^pZ*Er?Tm2dKEozYNnY7IdtZNgG2rk8cE8u zh349_Nm8(p-W7Am+oX|7ch}L#u6oMot*6;N)f5@rN}qDrBrH%(4ksFDk7q57Yhlxs zr8VTosi#vD%SrSsn^sS%r}%gEB-T?wkC-iVhgnI15-fT+Z!={-Xe5i5brih3mL?ms zNkF%bwmfQ}llxiJc#uVhCb38YY&xmSp{dWS$<4flB4;pZ>t+^tJ>t;W9u}Eru;^$n zi?p7yNPK54X}_-_xf`|Q_@_FB5k>tAKN5Th5Kq~(7c|q0@-vyzKQ~^nrK^e z3pw)Dkf(SxeQ;pW`V)<0xS@#-bg_x7n6!vdNA|Dkss2no^%yZJMWvCX&(_nv*(^G| zw}$Rrt)Y__nRLgSMHg6&WxXQ%4W$*|h&z1F7=lOtrg#TAo#s>Yf@pm(Qlcy=>}c zHPC&xCR)m>p)1}TDnDLJON}b%!CIaUY-pjt`f57mSVuQ~YU!3?BdvYTA&pWd1;1g_ z!SFiLQ)7~7PYopv)X4vaXqb3W0BghMvCIp(D)_1 zeWx3VrNySRa*gCz$)+=wjikoYyXSG$wBtr2#fGzJ>YX}T6<1HK$7*SNC5x=B8|la> zHYp`k(IfeK5?)|CONa~NWz~<_jqGW84WZ+K(4QccIq z>S>uv9X+zDr)1u`f4a$}r>QKmD`}z{J>J^5vMHIrk%T>(X`T?9WJ>BN*SLYU4d>|) zZyrZZXrlL#)pTJ)9i?qyl3#xvMb*|)FHdfcRCCDvehZyFUPmmRT?tF8CEs=y?Rv(d z*q1yVljhKnYc*YaQbVu%o9V7dBdOkEQOT4_62HQvB`aBEbhMGS7}S%pZ!KNo*_I!b z=bL$U;8tTJojg%P3whfk!B%=w$D+z!Ci&{{# zGziYUE*Kq%H4fikp zAMN~oeLRo+T^@ex%x|6fZ_%0m#Yg^nH`k>9@I2DU%E-jr^q+5LZWa5pJn|Pe&qr?k ziLfhg%f$%oDtepgg!j_3!P@(ZmVa}B>I)wvy5@ubm>ITaMnm_8Awr(a0-xD(WC+Zo z1si2CWW&I$$E%U(Er7BNJ9O873pXV3AEME( zl+nJk5HhBFC`2-jmh{>n#Il79`Yn(m$Uw+KaqJc!ph-Q>NIOuAwkyVno|Onm+h|;r zYlL#$VNx*5b7iW4|1aT0K$bZGelrKT}bcFpjJgK?nbB(trMyl1xlc zUO5FimsjHEI}KFi_#xwaJGgto5T9m?<%(IbnlcGhGaRtAX)IRyjKlRCKJa>@f-4<< z5bn}K&Q&$|-W~z|m9bcESBKufcAWZnl!WAR(SElMeCwVNf4)0jd$nM|Hxo_fvNRw) z2?2(NF!@o`bKDWH9UbAK6a@HAh3=R+V222+bvVkGLwu|olo=L>1BrE*q5Xzr z&uhXWqaOWlOOTp50h_xfVqei3Z17D*`TR#DU!#cl$oKT=`gn+cjKYhO=j3s8FV(e` zpin3iD}%+bMq3!3w}oIECk&z71vvR>4RCFcdU76-z)nMy3`{`7obPlg@D?57zenF@ znm~XzzSo7Flu@dT*a!_2Do0}4yCqaBl|uXu5}3I|63n+=*s(YcQo3FkS>cR{7mQ(d zPYD*%dq`!g9G=G10t_`QIsA&gJaI+kT{g-KXJWW;9oSuoXlJ%y2`^7SbYME%SFJyY%6=z^#hO|OFUrThk4q7y-v@nj8%(H-h12OWJl1)YQ@NwZ% z)`d6Z?yQKsb14)tQXEihwz*?&T?5&6^5%7GI7E|r zu()T4xP6NeHLwvuMuD^Hvj?z`?xO<24y=Id9 zcX7NI)W?Yv0nnNxf$=Tk*m-UQubm8hs*%LYi?boT*9U8*b$EKUn)+?8)3_Cau-yEW zT$B1}dc85?zDMJ-{RZ$mpQpx+qUfy5#55@`ZTKvM-He%Vx|jmjR59e0`d|y`BKPA2 z)Sp#@ZiXnH4mx3mffKHEYr~Q^ubOd7AZluWuZLm~m%fX>^Kz{v{$7xsnn`WLHek|H zJ4E{B!a7;pO}A=Z(VnnMo~<&4+R9dZ-nRrRMRPFgKsS{ipNxRp?btTo43m%8Vf*cP7^xkAkqs#T6n>gvs^Ql>1Z@D#au4^lKGOS~wNc12*8U>oa0J2*P#$JZKp&pht`d z3ft!k(N*SfTJDJ^%~;%6afpuTXkzLTOB89>!>uh1GXlEEr%Mr5I(&3l+Y`1&y72wD z0)`KyVfOTO*z43w)rWMEec#OjQU2(~x5KfhB3fm@)R~JSgH02`ITR5V4 zxjjNSJ~-5GiVFv}(<%9IJPO!?JT(&(nwj9TN(?SImB2!@pSE2(M4n;FaEvX4{nu6+ zzM`34C=(0^wIMLm8Lc4_XewBSHg^qB>~L68COq$T(QqopxVa&a86td^a)bZ8w-j}5 zDL%yppy%K!c;-(<-`)@?ia6oGy&8tdqAHqPRfpSw6R{^}I|dVDknqS1@y9(--XnzP zDdXWC!jqF!S2XY$;%b8n;-9{y`+Y&+^Q*%}dqVBD1gzn(!R*dMque2n}djZYf+;;mDXP{L(-H=)Gl3%F^w1L{n~w0;WrH` zqsBmZkqWjqnbMN8v#2g_3YwEAU??jM%RbG)k@9dFWoe4ww?%leD;K92{`got8m?RP z;V3>Dh(16n#xeAKfgfIWD5Fu%7be_~G<|dk1_leTqeL49Db5HlOMne$9m0kUlD0%T zRy|mS@Z=Io?OF-*SAHZiF%jpTy>PMF2jBOnpbg~O#; zXCt6IqaK-)y|G}`G<0!GaY=A7=K9~JZCf(I(#pm4JM%D3XFk3xt)!ZYi&*M6p{-|SGnmILhva`16)fX%5&a(ki+4W9%Ue2~IO z{>kXNn}d&&S5eT#64GApi-B})2#vLa&s}evFOq`zG)WY<@2694>kwDth(YIO7(I1? z|6CdNroE>y$e=!Q)x*)f34{kn6g3f*`yp$J3*i&_grthJ|C0poxeeNxsSxVjLYF{=O*;_GzVF9aq8gSn|21?T+F?_Wl&Yamzj2Ahm zS~3HN&n|`iv#Ai`bHIx_JNUY=NHzKttw@x|#v@I*kk?LgZbieU-yh>9Mnj=y4Q6*O zA@k6Dk}L{^$Vq+lZ;;0PQ4H*y+5`u42EN8@#n*AcxEz{<^I29%eYcxLkG4|M55$5iVhLfYo>IChMK+?k^xd9fS;aywDBn~hmlb71~q6*k#@ zC9e7q&A&1cT_zlCQHaDi(RN(%j|At6F9aPG5jAo&Hd@J|`b!5LZVd;+_ADuBj6kyT zIIteDv2|P^qB0ox5T$~nB~ua9yb*e}wkS^7h7J)OT-r4s4To|e<-wxsOk-^Rz8WGt zUp!H17aiH557(<6xIULjr|p!n%vBnqb2d^lbg-7sA8ocXv3#W^%ngD1g1sajnF#Y} zGsu|hL0;8}abYl!dKVbrZ1QM)9Tf*pcSn3XHXNG1m+cW_)Pswu~WM!%ft!V_t`k1G7LA5Dd5G$Fl3x^#H(|B z=<%_K$exMNnqmeg^8}~{d7`;}EHb?bv)QZZ@N$1FYXy9-ECC?T^X?q3~j*P_p3np;5b)9q-o>6w; zNNnw{Ky8dEUWI4j!O=9FpQ?<}0=d}cdYSgLwvbOnCH_dU!^YGdFdm+Y8+o1>8XkiX zH5cqzO4v3=0@W)DXb+zNOmH>Adr4iK^^k_x)O@69`9kDOCtY&ZK-SiL$R|}{ z)b_EMAESegejT`IPQt-_o^3fMj2J~HJT0Gs3%--!x8DW~Z7rO>(oDDbW+U|vHw1+i zBWhhXRBN``0Nx1Th0*tJKM347e~*$ zI~iZby27Hr5i@h&Q{abCWXlDY-Ej zJ()cJ^*{`Z5@*o0qc2D>`Xzm;;nKa8b6_JThsC)zaMX2$!{>2`KXaavi|yc?#l#Mz zAvOFAeQVT)tWOcnOsayorVz?sX+tl&7^C0K!~E4pyh&qsPcSjvUrU+6E!K=*vU*o%JLY-vGYMO_&j~8wCRu*tjkO zp(9KnJLM%QTpNY8qeo(mqY(DGW#d5LFyw6vLeT{~Fq^K?!#i^5WUwh-gqQ!!WFt*f z3?I`hP;+xM{F1)X6BT)UHQq!@)7HT?Iu==LH8FEY0u2oz@H|w6EF~3q3zXrI0wL~z zCgjI1MYw|*b?h^Qx>o^mgF}#--i+syR^VotEk<^a2d6I_ImbL8ePRk8jJCmuXNLHE zX#@E*+4E#?9K?*{F*3c1^C{j;A)Tr1kX$0(A7jt zHwVPa#qr(K3TKa$;z8tMm<&cC+;KFb-6znp=6*VEmxqWA)`+s&jX~8EoOqD}c$b2$+CluGVK@u;la#*6hBP20y}zWlI45aZG4|xCP&gN@k)#+^uqlI0ua&9fVjpKY%{n{6ThFOhhakSPY^<_ z)HNzQ@s%D}1fVcN5L$tHn3{cu-jDOb=DqrOk#2@lqEG0?G$$y}EXHGx5X5dshusTF z+<7mF@%iQOO<#=%4j*WJ{1x)m+D}%dN^tk9K*VbeY+h3e^@p=@&My=S-0gSn7}|(2?`1-D$#&|AnPX> zh@yq47*P}z17HRbL=ccD5)=?ICsag53vxf4l0AaYy$Z-S_o& ze-C@CmovsW`>eIsobzGLP1LRmQN5_l{EL311?`g;ArIukX)~@g--vYfV^-P$P9*#; zF6zm|ElcPo`aO_vd(rHJM$o9Rs?YkgvUq#TcdnFweINR#91)dibT%<0N z^n1CYy1z;`ADcVF(KXd+gx}DVb+MEnY zBd%J9woONIb=p+~omKgLRwu5i)>l zBma`y5B2G~)Q|j>tt6WJ@#y9jCj8P5E2GxbEK!`dW=d$Ec`LUUx5C138RJJRkwv}t zGA}!w5WN&MSLBnTlF#_7MqJEKV9b&@0>69F%dI~;@$KZ^xQpU%w1Rsoz38R6pQHEZ z5Kz$iA=voVw}9g&0K+K3MW?k~!In_c;%o zQZ6dj;`u|vgv>!3`BArkOIvMd+Fgg&8a_z+0U4^IMn}_8G~K#}_$eo)Ns|~px@R-o z;;m%&>rPP9a5~ISLVedc`K+sjeo7lcZ{Cqy-(-|OXb?C`i|@Hj(SO~VotY0L{hA}@ zy+`w+K^+6nP{xhkDL3|SmipytJh-1PCqE?NecpqT*XKmp`;1WShNIDTY6pIhtbuWi zf8)Y=vR!mUJdQ=~sWLdjSB>x&gSaq$K zyba?0>xpRX^(3RmO$nH;Oq<$LiJVv~;R>#G`*Ai2-^Oxc?+_|pPG`Z%dhtHuH)Q8IQ`9z%8iKxq$#sy{{ht#D(nt{nlllJ!}v6R0677b5;V|}&$~RBuBdg-_r40=P%S$*wdGvDD~3dB(!*AjV|U#cys5Q955AU%KSNojHySUiG3*XlD=*d- zi(0P>V*A%?>2RSpd%w=))K^oQWz|Udw}o8qZAIgeYS<`r!066uEK*&}I1JkO648DTpLCV64s~U1atN)z&mn$QsCcE^mo6*(xploiN3X;Z>1T-kv}ZDT zy9HBHV)$0{Lp+^_bKESNZe~fmZ~YPzb4@L(2;#FiW#VW zPEzhpMqgtxbC%{vt6Ol@Is^Ak!*Fsjl;mP1JXa0DCAbGeH4DjC=%tshc5==5nb;3p zOt;G!sHoLR#s+1cXY58jYXeQJyv3r`uaZ|YpBUxNyncRC7H(ZBb-p^hJ==+{6IaQy zef46xz=hw>T$Sgmf_YTip612(WN5M@M|`YtoD#>8Ypv*dy;f%YG{L;hVR`ekqXIX| ziUo6+QOg_M2U$MPjja19y5ruHp~>d>5)TjIp{ zW_MnOR*L(nZF1|1Bats#@z+2DiWm2%X5UKb6zt5lrc*h&Y60t`t@x=qipMo0(K^;3 zPK_;j*e{PeFVrx+Fq`2qA7pofGw+Q0^7M!nO5>MP=QfGz_rr*-8qeiVWzuJOQ)>53 zq)uC*`Rx5DpEr-lC`VqvoekRkPmHZi00jxpw&%0)9zg|EkoC1sNx?`x(|TC;-AJ@VwT0&|i^ zEu*ZnkF+a|7v&+|TrnDnh5?+vQwI(`M51(%- zRH&C=SNT~!?OlQPl4zFyxn5@7H)h{FD|DA95a+RwgtcZIbIM@II!~I-G2>HJN2=O* zNzVyd?9gyz zRjz1tr^-B;>%JCr`!$u$Xs>9yE|0Yt&HIHKN~c0}{($N|&rb+`fHWhHI@QC3PO&53kDYgEL8d z;fvbwX)Gv;#`gLax&K#@?AtgPk4>Yn*x8JV@h(*VS1N>%BYU+N5z(DG-$sQbBBDnf1?%i23{B4f>ZMHQGeDsu4RLcD{peH zO7jI#n0k+q4;LfRvKqnRhu&PAc>>p+6%zW&m!Acx)SK>=%`qorXNN*|KF#Bowuw?~ zqt5WE#+-dVnPCT&(Fpq{Sv`Wp!EOMB>O&}V4`J~-CoVUAVm zP~Il$vwW8UIU^s*k9!t;9o(I86DwSHJF)iqU!wK<2rhJ4g>!Spyw1>9g!k) zd2d#iBs2MXJVuW#>BtVLDC)}cigk=UY{Z7jVBGh1=Ws~~4q=(n&?g9&k-5x%Vvg&( zmS}ZM;_8Np}qe0>Zb61Oe8OdO=ak#OsbxzaiV+_%}bov#ZjrSJ|?x-a_Ri8 zC)$<=35yD)%fjY#n>(Gju=f(uX9>y^G?`xWLynDCVbG;C`m651%C|(O*o>z~d_2}e z+j8f*1uBa7`e(1;ua^08LPwQpO4sFEm^(9W=TkR+3Ma$TxYBhj`Uy5XG+V=|&7Bx^ zcnJ&|!_!`~W%8;%GWA;ztXiun^4*PXzmDTj3l~&^?Ra$KiWHaT@OxOj^t~5EvHfH^ zmzh%GZ-LFK)zm-qKgmYg2gK#TVV0{ZjU*FP4u@dEgL7?HZxIQkAF#n6ORJZze`2 z@=$96xArK`aP|qgY-&kK`c&-RJrt!6y*N}pfYe1BnSFmZ4y8?baVZGzqBIUXDwpne zbZKd$;0}+*vqB?Remi19CyjKzyRBqm>#=xMJ91@lzVl1Ft8yx4Jpp>ooCx2A$=*eH zPppu;t0P(6%%5M{jHR8HC3?5K<#u^G(r73Td-Y}aktp*2&6xFjOVaPuNc5cL#2!sy z-$?`Rd>D_;fnB&BjN;2zD{h+PvFt_x4z+DrVByN28#;4XQ-`w$hmd7aB+E8s(eYRi zCZ2}7cqXWHjRfY<9!QjhV~i7F8yDK9i1eqc{vj0x}HEq?=EXN z#KLP~EdOH6mO%{I!9uNw5!T#Ievcp{o>A>3F>8H;5Cd9!_-LBTEL_S?dTkAfk$0`Oyehzd$bX^XDLy)QiUg%w=++DJ5LVx zWLiX>#Q7M|$*B!vd>4t$-0rM+AII6^3Xz*hj4s)W$(w8nX71-plbayhw9!M0HtHJe3DA=ExN6T7>Y}r4QqlcIVEz0W{mN6V0L^ zJZsKN*{$AeJn&JX?ZR;|=#Q%NY4)ru7vncEWLmAlxFJ=Fmh?o;d?QXek<9WxFLkv& z$-Ev&_{SQ#7TbeEpX|Bx^|hSUPm~vh9vGX3kkm%e^9mIIcUzV$DRIU-xerY*m&(!| zPv!K|`O>RKk^cn)SU7Md1FjC`eeYy(e|1KV9*QCHTo`&&JF#%T4@av)8LV%E#|CZM zO*6#W;DprsX>V>;v-vP^hl@JFdnz`rl+xKp4uy*FbJ$>dq(Pc#3Q!y^JWKNVR zCEZ+6@34mcjZE=b-=5MD#>6ztl){uT95Qs`fP!}{UDA@#CXM;2;Ju^N^Qf5b#4Weh z^lm$oW=ZN~tSy(qKiz2;5RItYkbEhI7iR(qN$J2!1s{A9S0H67H)KkoEt`z8Nvif{ zZJ-sEH%H>2&;rUHsvLjyOit?EliH#}2F3R$ynZ)^6N9mC@M7)h%<5PuxtP{|^cH*d7y?E)?GF#ID8nq<%*vt&0t?a&JW526O6{=(6yAfeicJ zAle;2$lRcCn)Qz&>9Z3fZZ<*3?wWjh8mrjvXr|tDCi9sZDGrJ}r?0{EiwZrnT{&LZ zg~;W4#D9D&lYV$$*EN$0g_gH5{Y}(YIa1xWIemA15wq^+Wwm6>nc8ep&Ex4G+l(JZ zF}Tg2$m{8TydLAjpdPC9zZfo|{l-vPWy#=J9iHF)g*AVbaI(9C?>4R#|7m}S&&w2E zr?;oF#{%%xM!nns+l%Am{MDv79(gK#8Y{G~?s&?-sMAp+nBhL#Xl~<(SxaqhuE{6w z%_7{d8S$bf4VOV3d8w&^p5;K!e2y0H0#~-HPT=qWA6(YhGCOevA<+&vT^UAg;A8n{ zsgB=}G6~r-Mpl1HQuHVr?ui%K!QPZi?aK9%T8W%sz?-Ce<|Yni)+>K$;%G=<(OOhI zl1QIwK=qj_*{_j7cuYDi7mg=-Z6^vm!sJ-dA+hY5gY6ARbRX{|H+Tj~y1Ur?+Js~N ze~7!TEw?7TmrJvEitDR`;^>o(UB5Ef)u{m4pGPmX&Gh&qftG8sNpLBo{dP-sfBPZ6 zhxNJIwMxznJukCb-;!N73yDzh=oX>px$-iL^KnNRRyB@1y)1ggKa;OPhh%+LFGX*^ zFVnXimw1QK%=(eUfb-4CUk_jIy%U{-ZD_gli9GLW#%BNR^ej}PQ(Rx3^s16M3kEZ_ z{-eC^k|o~h@!VPOOTw$)#p6vsd~I8CPc@yQ$qN3^VK+|o3GBGkp8){|{CT*e44%D5 z+@B>e$I61KEB8^@X#p2@suO&?2S1v86Ww?#OkO+?!(mIg*1k^KdnoGtm@}F^`|;?mCk;(ys`QbjFnd|&%dBFsAMXg_cUP{KJ1q|>}#l3lhbl927_>XRs`gSIL{tJ2k zWhsxtS1{w#6yB|x%cOqMa%|68sYy&`P?i%Xo|JJYE}V6b<5;NBNaZU2JT-Ubx5AFh z)(w?mSA zrQ%Ye)W)pf)2SHhp4ii7x*4AXgXn2^NmS}9#KE@%FAr{^>e3}yl0SzCG3UF+BRQZ_ zBcBp9h*NH<*!K%5*;OGvGc73fYlm8_4f76elOylIJ2#u|?z6bH`>q72f0bk}FB)BY zAV(tuIUf{@=c_&}srn)%-kHR?M6)CJmeejkB%?CEOV-{Y?D?8ao|_fR{j%vsoH#BgjZM@$zoZ`?>Ow25Q0!#E~(-7cTXqF6FwD|4e-arK2INAE1-+vYfw?ZZj1 z*Cn%eS8_h?;%m-Qh88$u+kt$Hci4`fmKawmt&Qq%-BwT^a#wwa@bhzUuwa= zgX-ii^_I*k7iMd&$F|6UHJ_Fe+w-Qxbel}$=>gm@(I(j5jzfcf!Ebq<47Ana$&o_# zD|@ltI9;x2nvik(ikQ~Fl~LASn0ISTv55iV@5C?DbFhpwq4>^zdN}GautPi_nk3N4 zK$(z;W~}b93eQ<9sadDa{;J(vNv#yM>S=5mvWjhK+r=(mD0daOGEY%kiC=TXe!U7M z{?myvvP84ML4ofFB-ZM)Gv)($x1pWCXTiY zbDl4a(<1n}aSQe@*Wg~V1bx+S(s9s2_BJ%gyTi?SaqOPxyIqr^N@4WSNM>`a0gg-i zV^jW4$`o}zpjkZCcdS|K+8*OimV}OdCKg4zSg=C zULpB^m|Fbj>Hjdb_%E~7f31J5f31J5|F4av|FfyZtAm|y7ykA?ou(D&^gjT2Y^;R@ GAOQfS-O{-L diff --git a/tests/utils/compression_test.py b/tests/utils/compression_test.py new file mode 100644 index 00000000..1a6ec1c5 --- /dev/null +++ b/tests/utils/compression_test.py @@ -0,0 +1,57 @@ +"""The client-side encoder for an MSA carried compressed inside a request payload.""" + +import numpy as np + +from esm.utils.compression import compress_state_dict, decompress_state_dict +from esm.utils.msa import MSA +from esm.utils.structure.input_builder import ( + ProteinInput, + StructurePredictionInput, + deserialize_structure_prediction_input, + serialize_structure_prediction_input, +) + + +def test_a_compressed_msa_round_trips(): + """The decoder returns the state dict; rebuilding stays with the type, so this is + the call a caller actually writes. `deletions` is the a3m insertion counts, and is + lost silently if the encoder serializes only the sequences.""" + msa = MSA.from_state_dict( + {"sequences": ["MNM", "MNQ"], "deletions": [[0.0, 0.0, 0.0], [0.0, 1.0, 0.0]]} + ) + + restored = MSA.from_state_dict(decompress_state_dict(compress_state_dict(msa))) + + assert restored.sequences == msa.sequences + assert restored.deletions is not None + np.testing.assert_array_equal(restored.deletions, msa.deletions) + + +def test_a_chain_msa_is_compressed_on_the_way_out_and_expanded_on_the_way_back(): + """Callers only ever hold the object -- the multimer path compresses per chain on + serialize, so the wire form is the blob and deserialize has to invert it.""" + msa = MSA.from_sequences(["MNM", "MNQ"]) + built = serialize_structure_prediction_input( + StructurePredictionInput( + sequences=[ProteinInput(id="A", sequence="MNM", msa=msa)] + ) + ) + + assert built["sequences"][0]["msa"] == compress_state_dict(msa) + + restored = deserialize_structure_prediction_input(built) + chain = restored.sequences[0] + assert isinstance(chain, ProteinInput) + assert isinstance(chain.msa, MSA) + assert chain.msa.sequences == ["MNM", "MNQ"] + + +def test_the_encoding_is_generic_over_the_state_dict_convention(): + class Custom: + def state_dict(self, json_serializable: bool = False) -> dict: + return {"values": [1, 2, 3], "serializable": json_serializable} + + assert decompress_state_dict(compress_state_dict(Custom())) == { + "values": [1, 2, 3], + "serializable": True, + } diff --git a/tests/utils/msa_test.py b/tests/utils/msa_test.py index 148e3048..bc4a543a 100644 --- a/tests/utils/msa_test.py +++ b/tests/utils/msa_test.py @@ -104,6 +104,19 @@ def test_slicing_carries_deletions(tmp_path): np.testing.assert_array_equal(by_slice, _EXPECTED_DELETIONS[:, 1:]) +def test_an_empty_selection_is_allowed_once_deletions_are_set(tmp_path): + msa = _a3m_msa(tmp_path) + assert msa.deletions is not None + + empty = msa.select_sequences([]) + + assert empty.sequences == [] + assert empty.depth == 0 + # 0 rather than an IndexError off entries[0]; the column subselect below reads it. + assert empty.seqlen == 0 + assert empty.select_positions([0, 1]).sequences == [] + + def test_sliced_deletions_flow_through_featurization(tmp_path): """A per-chain column subselect (as chainbreak splitting does) keeps deletions, so featurizing the sliced MSA yields the sliced counts, not zeros.""" diff --git a/tests/utils/track_round_trip_test.py b/tests/utils/track_round_trip_test.py new file mode 100644 index 00000000..ee17217d --- /dev/null +++ b/tests/utils/track_round_trip_test.py @@ -0,0 +1,60 @@ +"""Decode -> encode round trips must return the tokens they came from.""" + +import pytest +import torch + +from esm.tokenization.sasa_tokenizer import SASADiscretizingTokenizer +from esm.tokenization.ss_tokenizer import SecondaryStructureTokenizer +from esm.utils.decoding import decode_sasa, decode_secondary_structure +from esm.utils.encoding import tokenize_sasa, tokenize_secondary_structure + + +def test_sasa_decode_encode_round_trip(): + tokenizer = SASADiscretizingTokenizer() + # BOS, , a bin, , , top bin, EOS + tokens = torch.tensor([0, 0, 10, 1, 2, 18, 0]) + + values = decode_sasa(tokens, tokenizer) + round_tripped = tokenize_sasa(values, tokenizer, add_special_tokens=True) + + assert torch.equal(round_tripped, tokens) + + +def test_sasa_decode_float_specials(): + tokenizer = SASADiscretizingTokenizer() + + values = decode_sasa(torch.tensor([0, 0, 1, 2, 10, 0]), tokenizer) + + # is None because that is what tokenize_sasa maps back to the mask token; the + # other specials have no midpoint, so they come back as their vocab string. + assert values == [None, "", "", pytest.approx(46.75)] + + +def test_sasa_tokenize_rejects_nan(): + tokenizer = SASADiscretizingTokenizer() + + with pytest.raises(ValueError, match="NaN"): + tokenize_sasa([1.0, float("nan")], tokenizer, add_special_tokens=False) + + +def test_secondary_structure_decode_encode_round_trip(): + tokenizer = SecondaryStructureTokenizer() + # BOS, H, (chainbreak), E, EOS + tokens = torch.tensor([0, 4, 0, 7, 0]) + + decoded = decode_secondary_structure(tokens, tokenizer) + round_tripped = tokenize_secondary_structure( + decoded, tokenizer, add_special_tokens=True + ) + + assert torch.equal(round_tripped, tokens) + + +def test_secondary_structure_decode_is_one_char_per_residue(): + tokenizer = SecondaryStructureTokenizer() + tokens = torch.tensor([0, 4, 0, 7, 0]) + + decoded = decode_secondary_structure(tokens, tokenizer) + + # A literal "" here would measure 5 residues instead of 1. + assert decoded == "H_E" From 4fe4099a924b3b17247559fb3ba3a1087e7fbe94 Mon Sep 17 00:00:00 2001 From: Fausto Milletari Date: Thu, 3 Sep 2026 20:13:38 +0000 Subject: [PATCH 2/2] Re-sync: pick up the base_forge_client_test move Sync from monorepo main (bd0dd1df0d). The only change since the 3.4.1 sync is monorepo PR #15367, which moved `base_forge_client_test.py` into `opensource/tests/sdk/`. It no longer ships inside the `esm` package. Co-Authored-By: Claude Opus 5 (1M context) --- {esm => tests}/sdk/base_forge_client_test.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename {esm => tests}/sdk/base_forge_client_test.py (100%) diff --git a/esm/sdk/base_forge_client_test.py b/tests/sdk/base_forge_client_test.py similarity index 100% rename from esm/sdk/base_forge_client_test.py rename to tests/sdk/base_forge_client_test.py