Core: cache website fee estimates per chain (BD50)

Author: gemcoder21

core/apps/api/src/routes/public/fee.rs

@@ -5,8 +5,13 @@ use primitives::ChainFeeEstimates;
 use services::chain::FeeEstimatesClient;
 
 use crate::error::ApiError;
+use crate::request::{ChainParam, Path};
 use crate::response::ApiResponse;
 
 pub async fn get_fee_estimates(State(client): State<Arc<FeeEstimatesClient>>) -> Result<ApiResponse<Vec<ChainFeeEstimates>>, ApiError> {
     Ok(client.get_fee_estimates().await?.into())
 }
+
+pub async fn get_chain_fee_estimates(Path(chain): Path<ChainParam>, State(client): State<Arc<FeeEstimatesClient>>) -> Result<ApiResponse<ChainFeeEstimates>, ApiError> {
+    Ok(client.get_chain_fee_estimates(chain.0).await?.into())
+}

core/apps/api/src/routes/public/mod.rs

@@ -36,5 +36,6 @@ pub fn router() -> Router<AppState> {
         .route("/markets", get(markets::get_markets))
         .route("/rewards/leaderboard", get(markets::get_rewards_leaderboard))
         .route("/chain/fee-estimates", get(fee::get_fee_estimates))
+        .route("/chain/fee-estimates/{chain}", get(fee::get_chain_fee_estimates))
         .merge(webhooks::router())
 }

core/apps/api/testdata/contract.json

@@ -491,6 +491,20 @@
       }
     }
   },
+  "v1_chain_fee_estimates": {
+    "status": 200,
+    "content_type": "application/json",
+    "body": "<volatile>"
+  },
+  "v1_chain_fee_estimates_bad": {
+    "status": 422,
+    "content_type": "application/json",
+    "body": {
+      "error": {
+        "message": "Invalid chain: mars"
+      }
+    }
+  },
   "v1_fee_estimates": {
     "status": 200,
     "content_type": "application/json",

core/apps/api/tests/contract.rs

@@ -358,6 +358,8 @@ fn public_cases() -> Vec<Case> {
         Case::new("v1_markets", Method::GET, "/v1/markets"),
         Case::new("v1_rewards_leaderboard", Method::GET, "/v1/rewards/leaderboard"),
         Case::new("v1_fee_estimates", Method::GET, "/v1/chain/fee-estimates").volatile(),
+        Case::new("v1_chain_fee_estimates", Method::GET, "/v1/chain/fee-estimates/tron").volatile(),
+        Case::new("v1_chain_fee_estimates_bad", Method::GET, "/v1/chain/fee-estimates/mars"),
         Case::new("v1_nft_asset_preview", Method::GET, "/v1/nft/assets/ethereum_0x0000000000000000000000000000000000000001::1/preview").volatile(),
         Case::new("v1_nft_asset_preview_bad", Method::GET, "/v1/nft/assets/not-an-id/preview"),
         Case::new("v1_nft_collection_preview", Method::GET, "/v1/nft/collections/ethereum_0x0000000000000000000000000000000000000001/preview").volatile(),

core/crates/cacher/src/cache.rs

@@ -0,0 +1,65 @@
+use std::{
+    error::Error,
+    future::Future,
+    time::{Duration, SystemTime},
+};
+
+use futures::future::BoxFuture;
+use gem_tracing::warn_with_fields;
+use serde::{Deserialize, Serialize, de::DeserializeOwned};
+
+use crate::{CacheKey, CacherClient};
+
+pub type CacheFuture<'a, T> = BoxFuture<'a, Result<T, Box<dyn Error + Send + Sync>>>;
+
+#[derive(Serialize, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub(crate) struct Cached<T> {
+    pub(crate) value: T,
+    updated_at: SystemTime,
+}
+
+impl<T> Cached<T> {
+    fn is_fresh(&self, duration: Duration) -> bool {
+        self.updated_at.elapsed().is_ok_and(|elapsed| elapsed < duration)
+    }
+}
+
+impl CacherClient {
+    pub(crate) async fn get_or_fetch<T, F>(&self, key: CacheKey<'_>, duration: Duration, fetch: F) -> Result<T, Box<dyn Error + Send + Sync>>
+    where
+        T: DeserializeOwned + Serialize,
+        F: Future<Output = Result<T, Box<dyn Error + Send + Sync>>>,
+    {
+        if let Some(cached) = self.get::<Cached<T>>(key).await?
+            && cached.is_fresh(duration)
+        {
+            return Ok(cached.value);
+        }
+        let cached = Cached {
+            value: fetch.await?,
+            updated_at: SystemTime::now(),
+        };
+        if let Err(error) = self.set(key, &cached).await {
+            warn_with_fields!("cache write failed", error = error.as_ref());
+        }
+        Ok(cached.value)
+    }
+
+    pub(crate) async fn get_or_fetch_with_fallback<T, F>(&self, key: CacheKey<'_>, duration: Duration, fetch: F) -> Result<T, Box<dyn Error + Send + Sync>>
+    where
+        T: DeserializeOwned + Serialize,
+        F: Future<Output = Result<T, Box<dyn Error + Send + Sync>>>,
+    {
+        match self.get_or_fetch(key, duration, fetch).await {
+            Ok(value) => Ok(value),
+            Err(error) => {
+                if let Some(cached) = self.get::<Cached<T>>(key).await? {
+                    warn_with_fields!("cache refresh failed, serving previous value", error = error.as_ref());
+                    return Ok(cached.value);
+                }
+                Err(error)
+            }
+        }
+    }
+}

core/crates/cacher/src/cachers/fee_estimates.rs

@@ -1,39 +1,28 @@
 use std::error::Error;
+use std::time::Duration;
 
 use async_trait::async_trait;
 use primitives::{Chain, ChainFeeEstimates};
 use strum::IntoEnumIterator;
 
-use crate::{CacheKey, CacherClient};
+use crate::cache::Cached;
+use crate::{CacheFuture, CacheKey, CacherClient};
 
 #[async_trait]
 pub trait FeeEstimatesCacher: Send + Sync {
-    async fn fresh_estimates(&self, chain: Chain) -> Result<Option<ChainFeeEstimates>, Box<dyn Error + Send + Sync>>;
-    async fn set_estimates(&self, chain: Chain, estimates: &ChainFeeEstimates) -> Result<(), Box<dyn Error + Send + Sync>>;
+    async fn get_or_fetch_estimates(&self, chain: Chain, duration: Duration, fetch: CacheFuture<'_, ChainFeeEstimates>) -> Result<ChainFeeEstimates, Box<dyn Error + Send + Sync>>;
     async fn all_estimates(&self) -> Result<Vec<ChainFeeEstimates>, Box<dyn Error + Send + Sync>>;
 }
 
 #[async_trait]
 impl FeeEstimatesCacher for CacherClient {
-    async fn fresh_estimates(&self, chain: Chain) -> Result<Option<ChainFeeEstimates>, Box<dyn Error + Send + Sync>> {
-        let (cached, fresh) = futures::try_join!(
-            self.get::<ChainFeeEstimates>(CacheKey::TransactionFeeEstimates(chain.as_ref())),
-            self.get::<()>(CacheKey::TransactionFeeEstimatesFresh(chain.as_ref())),
-        )?;
-        Ok(match (cached, fresh) {
-            (Some(estimates), Some(())) => Some(estimates),
-            (Some(_), None) | (None, _) => None,
-        })
-    }
-
-    async fn set_estimates(&self, chain: Chain, estimates: &ChainFeeEstimates) -> Result<(), Box<dyn Error + Send + Sync>> {
-        self.set(CacheKey::TransactionFeeEstimates(chain.as_ref()), estimates).await?;
-        self.set(CacheKey::TransactionFeeEstimatesFresh(chain.as_ref()), &()).await
+    async fn get_or_fetch_estimates(&self, chain: Chain, duration: Duration, fetch: CacheFuture<'_, ChainFeeEstimates>) -> Result<ChainFeeEstimates, Box<dyn Error + Send + Sync>> {
+        self.get_or_fetch_with_fallback(CacheKey::TransactionFeeEstimates(chain.as_ref()), duration, fetch).await
     }
 
     async fn all_estimates(&self) -> Result<Vec<ChainFeeEstimates>, Box<dyn Error + Send + Sync>> {
         let chains = Chain::iter().collect::<Vec<_>>();
         let keys = chains.iter().map(|chain| CacheKey::TransactionFeeEstimates(chain.as_ref())).collect::<Vec<_>>();
-        self.get_many(&keys).await
+        Ok(self.get_many::<Cached<ChainFeeEstimates>>(&keys).await?.into_iter().map(|cached| cached.value).collect())
     }
 }

core/crates/cacher/src/keys.rs

@@ -3,6 +3,7 @@ use primitives::{SECONDS_PER_DAY, SECONDS_PER_HOUR, SECONDS_PER_MINUTE};
 
 const SECONDS_PER_YEAR: u64 = 365 * SECONDS_PER_DAY;
 
+#[derive(Clone, Copy)]
 pub(crate) enum CacheKey<'a> {
     ReferralIpCheck(&'a str),
 
@@ -56,7 +57,6 @@ pub(crate) enum CacheKey<'a> {
     PendingTransactions(&'a str),
     TransactionCheckSchedule(&'a str),
     TransactionFeeEstimates(&'a str),
-    TransactionFeeEstimatesFresh(&'a str),
 
     ScanSafe(&'a str, &'a str, u64),
 }
@@ -101,8 +101,7 @@ impl CacheKey<'_> {
             Self::PerpetualObserverCheckpoint(chain, address) => format!("perpetual:last_seen:{}:{}", chain, address),
             Self::PendingTransactions(chain) => format!("transactions:pending:{}", chain),
             Self::TransactionCheckSchedule(queue) => format!("transactions:check_schedule:{}", queue),
-            Self::TransactionFeeEstimates(chain) => format!("transactions:fee_estimates:{}", chain),
-            Self::TransactionFeeEstimatesFresh(chain) => format!("transactions:fee_estimates:fresh:{}", chain),
+            Self::TransactionFeeEstimates(chain) => format!("transactions:fee_estimates:v2:{}", chain),
             Self::ScanSafe(scan_type, target, _) => format!("scan:safe:{}:{}", scan_type, target),
         }
     }
@@ -145,7 +144,6 @@ impl CacheKey<'_> {
             Self::PerpetualObserverCheckpoint(_, _) => 30 * SECONDS_PER_DAY,
             Self::PendingTransactions(_) | Self::TransactionCheckSchedule(_) => 30 * SECONDS_PER_DAY,
             Self::TransactionFeeEstimates(_) => 5 * SECONDS_PER_YEAR,
-            Self::TransactionFeeEstimatesFresh(_) => SECONDS_PER_HOUR,
             Self::ScanSafe(_, _, ttl) => *ttl,
         }
     }
@@ -169,6 +167,12 @@ mod tests {
         assert_eq!(key.ttl(), 3600);
     }
 
+    #[test]
+    fn test_fee_estimates_schema_and_retention() {
+        assert_eq!(CacheKey::TransactionFeeEstimates("tron").key(), "transactions:fee_estimates:v2:tron");
+        assert_eq!(CacheKey::TransactionFeeEstimates("tron").ttl(), 5 * 365 * SECONDS_PER_DAY);
+    }
+
     #[test]
     fn test_fiat_quote() {
         let key = CacheKey::FiatQuote(1, 2, "quote");

core/crates/cacher/src/lib.rs

@@ -4,9 +4,11 @@ use redis::{AsyncCommands, Client, aio::ConnectionManager};
 use serde::Serialize;
 use serde::de::DeserializeOwned;
 
+mod cache;
 mod cachers;
 mod error;
 mod keys;
+pub use cache::CacheFuture;
 pub use cachers::*;
 pub use error::*;
 pub(crate) use keys::CacheKey;

core/crates/config_keys/src/config_param_key.rs

@@ -118,6 +118,7 @@ impl RateLimit {
 pub enum ConfigParamKey {
     TransactionsRequestLimit(Chain),
     TransactionsPendingErrorMaxAge(Chain),
+    TransactionsFeeEstimatesCacheDuration(Chain),
     SwapperVaultAddresses(SwapProvider),
     PriceProviderAssetsLimit(PriceProvider),
     PriceProviderAssetsDuration(PriceProvider),
@@ -137,6 +138,7 @@ impl ConfigParamKey {
     pub fn all() -> Vec<Self> {
         let transactions = Chain::all().into_iter().map(Self::TransactionsRequestLimit);
         let pending_transactions = Chain::all().into_iter().map(Self::TransactionsPendingErrorMaxAge);
+        let fee_estimates = Chain::all().into_iter().map(Self::TransactionsFeeEstimatesCacheDuration);
         let swapper = SwapProvider::cross_chain_providers().into_iter().map(Self::SwapperVaultAddresses);
         let assets_limit = PriceProvider::all().into_iter().map(Self::PriceProviderAssetsLimit);
         let assets = PriceProvider::all().into_iter().map(Self::PriceProviderAssetsDuration);
@@ -152,6 +154,7 @@ impl ConfigParamKey {
         let rate_limits = RateLimitKey::iter().flat_map(|key| RateLimitWindow::ALL.into_iter().map(move |window| Self::RateLimit(key, window)));
         transactions
             .chain(pending_transactions)
+            .chain(fee_estimates)
             .chain(swapper)
             .chain(assets_limit)
             .chain(assets)
@@ -172,6 +175,7 @@ impl ConfigParamKey {
         match self {
             Self::TransactionsRequestLimit(chain) => format!("{}.{}", self.as_ref(), chain.as_ref()),
             Self::TransactionsPendingErrorMaxAge(chain) => format!("{}.{}", self.as_ref(), chain.as_ref()),
+            Self::TransactionsFeeEstimatesCacheDuration(chain) => format!("{}.{}", self.as_ref(), chain.as_ref()),
             Self::SwapperVaultAddresses(provider) => format!("{}.{}", self.as_ref(), provider.as_ref()),
             Self::PriceProviderAssetsLimit(provider) => format!("{}.{}", self.as_ref(), provider.as_ref()),
             Self::PriceProviderAssetsDuration(provider) => format!("{}.{}", self.as_ref(), provider.as_ref()),
@@ -192,6 +196,7 @@ impl ConfigParamKey {
         match self {
             Self::TransactionsRequestLimit(_) => "100".to_string(),
             Self::TransactionsPendingErrorMaxAge(_) => "3d".to_string(),
+            Self::TransactionsFeeEstimatesCacheDuration(_) => "1m".to_string(),
             Self::SwapperVaultAddresses(_) => "5m".to_string(),
             Self::PriceProviderAssetsLimit(PriceProvider::TonApi) => "1000".to_string(),
             Self::PriceProviderAssetsLimit(_) => "5000".to_string(),
@@ -233,6 +238,15 @@ mod tests {
         assert_eq!(bitcoin.default_value(), "3d");
     }
 
+    #[test]
+    fn test_transactions_fee_estimates_cache_duration() {
+        let key = ConfigParamKey::TransactionsFeeEstimatesCacheDuration(Chain::Tron);
+
+        assert_eq!(key.key(), "transactionsFeeEstimatesCacheDuration.tron");
+        assert_eq!(key.default_value(), "1m");
+        assert_eq!(ConfigParamKey::all().iter().filter(|key| matches!(key, ConfigParamKey::TransactionsFeeEstimatesCacheDuration(_))).count(), Chain::all().len());
+    }
+
     #[test]
     fn test_scan_type_enable() {
         let key = ConfigParamKey::ScanTypeEnable(ScanType::AddressPoisoning);

core/crates/services/src/backend.rs

@@ -350,7 +350,7 @@ impl Services {
     }
 
     pub fn fee_estimates(&self, assets: AssetsClient, prices: PriceClient, cacher: CacherClient, user_agent: &str) -> FeeEstimatesClient {
-        FeeEstimatesClient::new(self.chain(user_agent), assets, prices, Arc::new(cacher))
+        FeeEstimatesClient::new(self.chain(user_agent), assets, prices, Arc::new(cacher), self.config())
     }
 
     pub fn nodes_status(&self) -> NodesStatusClient {