Chain assets 2

defi/l2/utils.ts

@@ -114,7 +114,10 @@ async function getOsmosisSupplies(tokens: string[], timestamp?: number): Promise
 async function getAptosSupplies(tokens: string[], timestamp?: number): Promise<{ [token: string]: number }> {
   if (timestamp) throw new Error(`timestamp incompatible with Aptos adapter!`);
   const supplies: { [token: string]: number } = {};
-  const rpc = process.env.APTOS_RPC;
+  // Public fullnode is rate-limited but keeps the pipeline working when
+  // APTOS_RPC isn't set (e.g. local dev). Prod should still set APTOS_RPC
+  // to a dedicated endpoint.
+  const rpc = process.env.APTOS_RPC || "https://fullnode.mainnet.aptoslabs.com";
 
   await PromisePool.withConcurrency(1)
     .for(tokens)
@@ -294,34 +297,70 @@ async function getStellarSupplies(tokens: string[], timestamp?: number): Promise
   if (timestamp) throw new Error(`timestamp incompatible with Stellar adapter!`);
   const supplies: { [token: string]: number } = {};
 
+  // ES stores versioned variants of the same Stellar asset (e.g.
+  // "blnd-gdjehtbe...-1" in addition to "blnd-gdjehtbe..."). Both hit the
+  // same Horizon record, and both would otherwise be summed by symbol
+  // downstream — doubling the USDC / USDY / etc. figures for stellar.
+  // Dedupe by (code, issuer) pair, keeping the first-seen rawToken as the
+  // canonical key we store the supply under.
+  const seen = new Map<string, string>();
+  const dedupedTokens: string[] = [];
+  for (const rawToken of tokens) {
+    const upper = rawToken.toUpperCase();
+    let canonicalKey: string;
+    if (isSorobanContractId(upper)) {
+      canonicalKey = upper;
+    } else {
+      const classicKey = stellarSacToClassic[upper] ?? upper;
+      // base (code, issuer); issuer is the trailing 56-char G-prefixed string;
+      // anything after that (e.g. "-1", "-2") is a versioning artifact.
+      const m = classicKey.match(/^(.+-[A-Z2-7]{56})(?:-\d+)?$/);
+      canonicalKey = m ? m[1] : classicKey;
+    }
+    if (seen.has(canonicalKey)) continue;
+    seen.set(canonicalKey, rawToken);
+    dedupedTokens.push(rawToken);
+  }
+
   await PromisePool.withConcurrency(3)
-    .for(tokens)
-    .process(async (token) => {
+    .for(dedupedTokens)
+    .process(async (rawToken) => {
       try {
+        // Upstream (ES whitelist) stores stellar keys lowercased. Stellar asset
+        // issuers and Soroban contract IDs are G/C-prefixed base32 strings that
+        // are case-sensitive on Horizon — re-uppercase before querying.
+        const token = rawToken.toUpperCase();
+
         // Native Soroban contracts: call total_supply() via RPC
         if (isSorobanContractId(token)) {
           const supply = await getSorobanTokenSupply(token);
-          if (supply != null && supply > BigInt(0)) supplies[`stellar:${token}`] = Number(supply);
+          // getSorobanTokenSupply returns number | null; mixing BigInt(0) into
+          // the comparison would throw at runtime, so compare in the number domain.
+          if (supply != null && supply > 0) supplies[`stellar:${rawToken}`] = supply;
           return;
         }
 
         // Resolve Soroban SAC contract IDs to classic "code-issuer" format
         const classicKey = stellarSacToClassic[token] ?? token;
         if (classicKey === "XLM") return; // native asset handled by ownTokens
 
-        // Token format: "{asset_code}-{asset_issuer}" (dash-separated)
+        // Token format: "{asset_code}-{asset_issuer}" (dash-separated).
+        // Horizon is case-insensitive for asset_code but case-sensitive for
+        // asset_issuer.
         const dashIdx = classicKey.lastIndexOf("-");
         if (dashIdx === -1) return;
         const asset_code = classicKey.substring(0, dashIdx);
-        const asset_issuer = classicKey.substring(dashIdx + 1);
+        const asset_issuer = classicKey.substring(dashIdx + 1).toUpperCase();
         const res = await fetch(
           `https://horizon.stellar.org/assets?asset_code=${asset_code}&asset_issuer=${asset_issuer}&limit=1`
         ).then((r) => r.json());
         const record = res?._embedded?.records?.[0];
         if (record?.balances?.authorized != null) {
           // Horizon exposes amount in display units with 7 implicit decimal places.
           // Multiply by 1e7 to align with decimals=7 returned by the price API.
-          supplies[`stellar:${token}`] = Math.round(parseFloat(record.balances.authorized) * 1e7);
+          // Key the supply under the original (lowercase) token so it matches
+          // the keys coins.getPrices returned.
+          supplies[`stellar:${rawToken}`] = Math.round(parseFloat(record.balances.authorized) * 1e7);
         }
       } catch (e) {}
     });

defi/l2/v2/file-cache.ts

@@ -0,0 +1,74 @@
+import fs from "fs";
+import path from "path";
+
+const CACHE_VERSION = "v1.0";
+const CACHE_DIR = process.env.CHAIN_ASSETS_CACHE_DIR || path.join(__dirname, ".chain-assets-cache");
+const VERSIONED_CACHE_DIR = path.join(CACHE_DIR, CACHE_VERSION);
+
+const pathExistsMap: { [key: string]: Promise<void> } = {};
+
+async function ensureDirExists(folder: string): Promise<void> {
+  if (!pathExistsMap[folder]) {
+    pathExistsMap[folder] = (async () => {
+      try {
+        await fs.promises.access(folder);
+      } catch {
+        try {
+          await fs.promises.mkdir(folder, { recursive: true });
+        } catch (e) {
+          console.error("Error creating directory:", (e as any)?.message);
+        }
+      }
+    })();
+  }
+  return pathExistsMap[folder];
+}
+
+async function storeData(subPath: string, data: any): Promise<void> {
+  const filePath = path.join(VERSIONED_CACHE_DIR, subPath);
+  await ensureDirExists(path.dirname(filePath));
+  // Write to a temp file in the same directory and rename, so a crash mid-write
+  // can't leave a truncated/empty cache file in place.
+  const tmpPath = `${filePath}.${process.pid}.${Date.now()}.tmp`;
+  try {
+    await fs.promises.writeFile(tmpPath, JSON.stringify(data));
+    await fs.promises.rename(tmpPath, filePath);
+  } catch (e) {
+    console.error(`Error storing cache ${filePath}:`, (e as any)?.message);
+    fs.promises.unlink(tmpPath).catch(() => {});
+  }
+}
+
+async function readData(subPath: string): Promise<any> {
+  const filePath = path.join(VERSIONED_CACHE_DIR, subPath);
+  try {
+    const raw = await fs.promises.readFile(filePath, "utf8");
+    return JSON.parse(raw);
+  } catch {
+    return null;
+  }
+}
+
+// Strip anything that could escape the cache directory (slashes, dots, control
+// chars) before the chain name is interpolated into a file path. Dashes and
+// underscores stay so existing chain keys like "polygon-zkevm" round-trip.
+function normalizeChain(chain: string): string {
+  const cleaned = (chain ?? "").replace(/[^a-zA-Z0-9_-]/g, "").toLowerCase();
+  return cleaned || "unknown";
+}
+
+export async function storeChainHistory(chain: string, data: any[]): Promise<void> {
+  await storeData(`history/${normalizeChain(chain)}.json`, data);
+}
+
+export async function readChainHistory(chain: string): Promise<any[] | null> {
+  return readData(`history/${normalizeChain(chain)}.json`);
+}
+
+export async function storeAllChainsHistory(data: any[]): Promise<void> {
+  await storeData("history/all.json", data);
+}
+
+export async function readAllChainsHistory(): Promise<any[] | null> {
+  return readData("history/all.json");
+}

defi/l2/v2/index.ts

@@ -21,7 +21,7 @@ import { additional, excluded } from "../adapters/manual";
 import { storeHistoricalToDB } from "./storeToDb";
 import { stablecoins } from "../../src/getProtocols";
 import { metadata as rwaMetadata } from "../../src/rwa/protocols";
-import { verifyChanges } from "../verifyChanges";
+import { verifyChangesV2 } from "./verifyChanges";
 import { fetchTokensList } from "../../src/utils/coinsApi";
 import { getClosestProtocolItem, getLatestProtocolItems, initializeTVLCacheDB } from "../../src/api2/db";
 import { hourlyRawTokensTvl } from "../../src/utils/getLastRecord";
@@ -208,21 +208,26 @@ async function fetchOutgoingAmountsFromDB(timestamp: number): Promise<{
   const sourceChainAmounts: { [chain: Chain]: { [token: string]: BigNumber } } = {};
   const protocolAmounts: { [chain: Chain]: { [token: string]: BigNumber } } = {};
   const destinationChainAmounts: { [chain: Chain]: { [token: string]: BigNumber } } = {};
+
   tvls.map(({ data, id }: any) => {
     if (!ids.includes(id)) return;
-    Object.keys(data).map((chain: string) => {
-      if (excludedTvlKeys.includes(chain)) return;
+    Object.keys(data).map((rawChain: string) => {
+      if (excludedTvlKeys.includes(rawChain)) return;
+      // Some canonical-bridge adapters (xion, noble, echelon_initia, inertia, ...)
+      // report their source chain in display case ("XION", "Noble"), which would
+      // never match our lowercase chain keys when we later look up sourceChainAmounts[chain].
+      const chain = rawChain.toLowerCase();
       if (!sourceChainAmounts[chain]) sourceChainAmounts[chain] = {};
-      Object.keys(data[chain]).map((token: string) => {
+      Object.keys(data[rawChain]).map((token: string) => {
         const key = normalizeKey(token);
         if (!sourceChainAmounts[chain][key]) sourceChainAmounts[chain][key] = zero;
-        sourceChainAmounts[chain][key] = sourceChainAmounts[chain][key].plus(data[chain][token]);
+        sourceChainAmounts[chain][key] = sourceChainAmounts[chain][key].plus(data[rawChain][token]);
 
         if (Object.keys(protocolBridgeIds).includes(id)) {
           const protocolSlug = protocolBridgeIds[id];
           if (!protocolAmounts[protocolSlug]) protocolAmounts[protocolSlug] = {};
           if (!protocolAmounts[protocolSlug][key]) protocolAmounts[protocolSlug][key] = zero;
-          protocolAmounts[protocolSlug][key] = protocolAmounts[protocolSlug][key].plus(data[chain][token]);
+          protocolAmounts[protocolSlug][key] = protocolAmounts[protocolSlug][key].plus(data[rawChain][token]);
           return;
         }
 
@@ -231,7 +236,7 @@ async function fetchOutgoingAmountsFromDB(timestamp: number): Promise<{
         if (!destinationChainAmounts[destinationChain]) destinationChainAmounts[destinationChain] = {};
         if (!destinationChainAmounts[destinationChain][key]) destinationChainAmounts[destinationChain][key] = zero;
         destinationChainAmounts[destinationChain][key] = destinationChainAmounts[destinationChain][key].plus(
-          data[chain][token]
+          data[rawChain][token]
         );
       });
     });
@@ -283,22 +288,45 @@ async function fetchExcludedAmounts(timestamp: number) {
   return excludedAmounts;
 }
 
+// The open endpoints (yields.llama.fi/lsdRates, stablecoins.llama.fi/stablecoins)
+// are paywalled for some hosts. Use the authenticated pro-api path when
+// INTERNAL_API_KEY is set — same base used throughout the rest of the codebase
+// (see src/rwa/index.ts, src/updateSearch.ts, etc.).
+function proApi(path: string, publicFallback: string): string {
+  const key = process.env.INTERNAL_API_KEY;
+  return key ? `https://pro-api.llama.fi/${key}/${path.replace(/^\//, "")}` : publicFallback;
+}
+
 // fetch stablecoin symbols
 async function fetchStablecoinSymbols() {
-  const { peggedAssets } = await cachedFetch({
-    key: "stablecoin-symbols",
-    endpoint: "https://stablecoins.llama.fi/stablecoins",
-  });
-  const symbols = peggedAssets.map((s: any) => s.symbol);
+  let symbols: string[] = [];
+  try {
+    const res: any = await cachedFetch({
+      key: "stablecoin-symbols",
+      endpoint: proApi("stablecoins/stablecoins", "https://stablecoins.llama.fi/stablecoins"),
+    });
+    if (Array.isArray(res?.peggedAssets)) symbols = res.peggedAssets.map((s: any) => s.symbol);
+    else console.warn("fetchStablecoinSymbols: unexpected response shape, falling back to hardcoded list");
+  } catch (e: any) {
+    console.warn("fetchStablecoinSymbols failed, falling back to hardcoded list:", e?.message);
+  }
   const allSymbols = [...new Set([...symbols, ...stablecoins].map((t) => t.toUpperCase()))];
   return allSymbols;
 }
 
 // fetch lst symbols
 async function fetchLstSymbols() {
-  const assets = await cachedFetch({ key: "lst-symbols", endpoint: "https://yields.llama.fi/lsdRates" });
-  const symbols = assets.map((s: any) => s.symbol.toUpperCase());
-  return symbols;
+  try {
+    const assets: any = await cachedFetch({
+      key: "lst-symbols",
+      endpoint: proApi("yields/lsdRates", "https://yields.llama.fi/lsdRates"),
+    });
+    if (Array.isArray(assets)) return assets.map((s: any) => s.symbol?.toUpperCase()).filter(Boolean);
+    console.warn("fetchLstSymbols: non-array response, skipping LST classification");
+  } catch (e: any) {
+    console.warn("fetchLstSymbols failed, skipping LST classification:", e?.message);
+  }
+  return [] as string[];
 }
 
 // fetch rwa symbols
@@ -356,7 +384,7 @@ const newChainAssets = () => ({
 });
 
 // main function
-export async function storeChainAssetsV2(override: boolean = false) {
+export async function storeChainAssetsV2(override: boolean = false, dryRun: boolean = false) {
   if (!process.env.COINS_V4_API_URL) {
     throw new Error("storeChainAssetsV2 requires COINS_V4_API_URL — run with coins v4 API configured");
   }
@@ -369,6 +397,60 @@ export async function storeChainAssetsV2(override: boolean = false) {
   const lstSymbols = await fetchLstSymbols();
   const rwaSymbols = fetchRwaSymbols();
 
+  // Bridge / protocol-bridge adapters (noble, xion, echelon_initia, inertia, ...)
+  // emit tokens keyed by CoinGecko slug ("usd-coin", "xion-2", "celestia", "initia")
+  // or sometimes by bare symbol ("osmo", "axl"). `fetchNativeAndMcaps` only prices
+  // tokens it saw on per-chain native lists, so keys like `coingecko:usd-coin`
+  // are absent from allPrices. Collect everything the bridge-fetch step produced,
+  // pick out whatever allPrices is missing, and resolve it in one batch.
+  const allBridgeKeys = new Set<string>();
+  Object.values(destinationChainAmounts).forEach((m) => Object.keys(m).forEach((k) => allBridgeKeys.add(k)));
+  Object.values(protocolAmounts).forEach((m) => Object.keys(m).forEach((k) => allBridgeKeys.add(k)));
+  const missingBridgeKeys = [...allBridgeKeys].filter((k) => !allPrices[k]);
+  if (missingBridgeKeys.length) {
+    console.log(`[bridge-price-fill] resolving ${missingBridgeKeys.length} missing bridge/protocol keys via coins API`);
+    try {
+      const extra = await coins.getPrices(missingBridgeKeys, timestamp);
+      const resolvedKeys = Object.keys(extra);
+      console.log(`[bridge-price-fill]   resolved ${resolvedKeys.length}/${missingBridgeKeys.length}  (sample:`, resolvedKeys.slice(0, 5), ")");
+      resolvedKeys.forEach((k) => {
+        if (k.startsWith("coingecko:")) (extra[k] as any).decimals = 0; // match native-side convention
+        allPrices[k] = extra[k];
+      });
+      // nativeDataAfterMcaps drops keys that have no mcap entry, so the bridge
+      // fill must include mcaps for the same keys or every newly-priced bridge
+      // amount silently disappears in the next stage.
+      if (resolvedKeys.length) {
+        try {
+          const extraMcaps = await coins.getMcaps(resolvedKeys, timestamp);
+          Object.keys(extraMcaps).forEach((k) => {
+            allMcaps[k] = extraMcaps[k];
+          });
+        } catch (e: any) {
+          console.warn("[bridge-price-fill] mcap fetch failed:", e?.message);
+        }
+      }
+    } catch (e: any) {
+      console.warn("[bridge-price-fill] failed:", e?.message);
+    }
+  }
+
+  // Some canonical-bridge adapters emit tokens keyed by uppercase symbol
+  // ("AXLUSDC", "ATOM"). normalizeKey turns those into "coingecko:axlusdc", which
+  // almost never matches a real coingecko id. Build a reverse symbol → pricing
+  // index from allPrices so those can still be resolved in the destination loop.
+  // We also keep the resolved key alongside the data so the destination loop
+  // can re-key the amount under a key that actually exists in allMcaps; storing
+  // under the unresolved "coingecko:axlusdc" would cause it to be dropped by
+  // the mcap stage downstream.
+  const symbolToPrice: { [symbol: string]: { key: string; data: CoinsApiData } } = {};
+  Object.entries(allPrices).forEach(([priceKey, p]) => {
+    const sym = p?.symbol?.toUpperCase();
+    if (!sym) return;
+    // Prefer the first resolution; symbol collisions are ambiguous by construction.
+    if (!symbolToPrice[sym]) symbolToPrice[sym] = { key: priceKey, data: p };
+  });
+
   // adjust native asset balances by excluded and outgoing amounts
   const nativeDataAfterDeductions: { [chain: Chain]: { [token: string]: BigNumber } } = {};
   allChainKeys.map((chain: Chain) => {
@@ -398,10 +480,28 @@ export async function storeChainAssetsV2(override: boolean = false) {
       const destinationChainAmount = destinationChainAmounts[chain];
       Object.keys(destinationChainAmount).map((token: string) => {
         const key = normalizeKey(token);
-        const coinData = allPrices[key];
+        let coinData = allPrices[key];
+        let outputKey = key;
+        // When the adapter was symbol-keyed, `key` looks like "coingecko:axlusdc" but
+        // there is no such coingecko id. Fall back to a symbol reverse lookup.
+        // Symbol-keyed adapter output is also typically in whole tokens, not base
+        // units, so we must skip the `/ 10^decimals` step for these.
+        let isWholeTokenAmount = false;
+        if ((!coinData || !coinData.price) && key.startsWith("coingecko:")) {
+          const symbol = key.slice("coingecko:".length).toUpperCase();
+          const resolved = symbolToPrice[symbol];
+          if (resolved?.data?.price) {
+            coinData = resolved.data;
+            // Re-key the amount under the resolved key so allMcaps lookups
+            // downstream actually find an entry.
+            outputKey = resolved.key;
+            isWholeTokenAmount = true;
+          }
+        }
         if (!coinData || !coinData.price) return;
-        const usdAmount = destinationChainAmount[token].times(coinData.price).div(BigNumber(10).pow(coinData.decimals));
-        nativeDataAfterDeductions[chain][key] = usdAmount;
+        const divisor = isWholeTokenAmount ? BigNumber(1) : BigNumber(10).pow(coinData.decimals);
+        const usdAmount = destinationChainAmount[token].times(coinData.price).div(divisor);
+        nativeDataAfterDeductions[chain][outputKey] = usdAmount;
       });
     }
   });
@@ -485,8 +585,9 @@ export async function storeChainAssetsV2(override: boolean = false) {
     });
   });
 
-  // create symbol key data
-  const symbolMapPromise = storeR2JSONString("chainAssetsSymbolMap", JSON.stringify(symbolMap));
+  // The symbol-map R2 write is deferred to the post-validation Promise.all
+  // below so we never persist an updated map for a snapshot that
+  // verifyChangesV2 ends up rejecting.
   [rawData, symbolData].map((allData) => {
     Object.keys(allData).map((chain: Chain) => {
       let totalTotal = zero;
@@ -524,10 +625,15 @@ export async function storeChainAssetsV2(override: boolean = false) {
     });
   });
 
-  if (!override) await verifyChanges(symbolData);
+  if (!override) await verifyChangesV2(symbolData);
+
+  if (dryRun) {
+    console.log("[dryRun] skipping prod writes (chainAssetsSymbolMap R2, chainassets2 DB, chainAssets2 R2)");
+    return { rawData, symbolData };
+  }
 
   await Promise.all([
-    symbolMapPromise,
+    storeR2JSONString("chainAssetsSymbolMap", JSON.stringify(symbolMap)),
     storeHistoricalToDB({ timestamp: getCurrentUnixTimestamp(), value: rawData }),
     storeR2JSONString("chainAssets2", JSON.stringify({ timestamp: getCurrentUnixTimestamp(), value: symbolData })),
   ]);