fix: tx prices linking, dashboard load, initial sync errors

.env.from-dump

@@ -1,2 +1,4 @@
 IGNORE_SEEDING=true
 DONT_EXECUTE_TRIGGERS=true
+FROM_DUMP=true
+SKIP_CACHE_REBUILD=true

.gitignore

@@ -39,3 +39,4 @@ prisma-local/seeds/productionTxs.csv
 paybutton-config.json
 
 dump.sql*
+.forever

Makefile

@@ -37,6 +37,18 @@ dev-from-dump:
 	$(touch_local_env)
 	docker compose -f docker-compose-from-dump.yml --env-file .env --env-file .env.local --env-file .env.from-dump up --build -d
 
+stop-dev-from-dump:
+	docker compose -f docker-compose-from-dump.yml --env-file .env --env-file .env.local --env-file .env.from-dump down
+
+reset-dev-from-dump:
+	make stop-dev-from-dump && make dev-from-dump
+
+reset-dev-from-dump-keep-db:
+	docker compose -f docker-compose-from-dump.yml --env-file .env --env-file .env.local --env-file .env.from-dump up --build -d --force-recreate --no-deps paybutton
+
+nuke-dev-from-dump:
+	docker compose -f docker-compose-from-dump.yml --env-file .env --env-file .env.local --env-file .env.from-dump down -v
+
 logs-dev:
 	docker logs -f paybutton-dev
 

constants/index.ts

@@ -269,8 +269,10 @@ export const XEC_TX_EXPLORER_URL = 'https://explorer.e.cash/tx/'
 export const BCH_TX_EXPLORER_URL = 'https://blockchair.com/bitcoin-cash/transaction/'
 
 export const MAX_MEMPOOL_TXS_TO_PROCESS_AT_A_TIME = 2
+export const MAX_CONFIRMED_TXS_TO_PROCESS_AT_A_TIME = 6
 export const CHRONIK_INITIALIZATION_DELAY = 2000
 export const MEMPOOL_PROCESS_DELAY = 100
+export const CONFIRMED_TX_PROCESS_DELAY = 100
 
 /* WIP RENAME ALL THOSE */
 // When fetching some address transactions, number of transactions to fetch at a time.

docker-compose-from-dump.yml

@@ -42,6 +42,7 @@ services:
       - ./scripts/db/init-supertokens-db.sql:/home/mysql/raw_entrypoint/3-init-supertokens-db.sql
       - ./scripts/db/prisma-shadow-db-fix.sql:/home/mysql/raw_entrypoint/4-prisma-shadow-db-fix.sql
       - ./dump.sql:/home/mysql/raw_entrypoint/5-dump.sql
+      - paybutton-dump-db-data:/var/lib/mysql
 
   users-service:
     container_name: paybutton-users-service
@@ -69,3 +70,6 @@ services:
       - ./redis:/data/redis:z
       - ./scripts:/data/scripts
     command: sh -c "sed -i 's/\\r//g' ./scripts/redis-start.sh && sh ./scripts/redis-start.sh"
+
+volumes:
+  paybutton-dump-db-data:

pages/api/payments/count/index.ts

@@ -1,4 +1,3 @@
-import { CacheGet } from 'redis/index'
 import { fetchUserProfileFromId } from 'services/userService'
 import { setSession } from 'utils/setSession'
 import { getFilteredTransactionCount } from 'services/transactionService'
@@ -28,15 +27,8 @@ export default async (req: any, res: any): Promise<void> => {
     if (typeof req.query.endDate === 'string' && req.query.endDate !== '') {
       endDate = req.query.endDate as string
     }
-    if (((buttonIds !== undefined) && buttonIds.length > 0) ||
-        ((years !== undefined) && years.length > 0) ||
-        (startDate !== undefined && endDate !== undefined && startDate !== '' && endDate !== '')) {
-      const totalCount = await getFilteredTransactionCount(userId, buttonIds, years, timezone, startDate, endDate)
-      res.status(200).json(totalCount)
-    } else {
-      const totalCount = await CacheGet.paymentsCount(userId, timezone)
-      res.status(200).json(totalCount)
-    }
+    const totalCount = await getFilteredTransactionCount(userId, buttonIds, years, timezone, startDate, endDate)
+    res.status(200).json(totalCount)
   } else {
     res.status(405).json({ error: 'Method not allowed' })
   }

pages/dashboard/dashboard.module.css

@@ -250,4 +250,14 @@ body[data-theme='dark'] .show_filters_button img {
   display: flex;
   align-items: center;
   gap: 10px;
+}
+
+.cache_banner {
+  background-color: #fff3cd;
+  color: #856404;
+  padding: 10px 16px;
+  border-radius: 6px;
+  margin-top: 12px;
+  font-size: 14px;
+  font-weight: 500;
 }

pages/dashboard/index.tsx

@@ -140,6 +140,21 @@ export default function Dashboard ({ user }: PaybuttonsProps): React.ReactElemen
     }
   }, [selectedButtonIds])
 
+  useEffect(() => {
+    if (dashboardData?.cacheRebuilding !== true) return
+    const timer = setTimeout(() => {
+      let url = 'api/dashboard'
+      if (selectedButtonIds.length > 0) {
+        url += `?buttonIds=${selectedButtonIds.join(',')}`
+      }
+      fetch(url, { headers: { Timezone: moment.tz.guess() } })
+        .then(async res => await res.json())
+        .then(json => { setDashboardData(json) })
+        .catch(console.error)
+    }, 15000)
+    return () => clearTimeout(timer)
+  }, [dashboardData?.cacheRebuilding, selectedButtonIds])
+
   useEffect(() => {
     setPeriodFromString(dashboardData, activePeriodString)
     if (dashboardData !== undefined) {
@@ -203,6 +218,11 @@ export default function Dashboard ({ user }: PaybuttonsProps): React.ReactElemen
               </div>
             </div>
       )}
+      {dashboardData.cacheRebuilding === true && (
+        <div className={style.cache_banner}>
+          Data still being calculated. Values will update automatically.
+        </div>
+      )}
       <div className={style.number_ctn}>
         <NumberBlock value={'$'.concat(formatQuoteValue(activePeriod.totalRevenue, user.userProfile.preferredCurrencyId)) } text='Revenue' />
         <NumberBlock value={activePeriod.totalPayments} text='Payments' />

prisma-local/clientInstance.ts

@@ -1,22 +1,39 @@
 import { PrismaClient } from '@prisma/client'
 
 let prisma: PrismaClient
+let backgroundPrisma: PrismaClient
 
 interface CustomNodeJsGlobal extends NodeJS.Global {
   prisma: PrismaClient
+  backgroundPrisma: PrismaClient
 }
 declare const global: CustomNodeJsGlobal
 
+const backgroundDbUrl = (process.env.DATABASE_URL ?? '').replace(
+  /connection_limit=\d+/,
+  'connection_limit=3'
+)
+
 if (process.env.NODE_ENV === 'production') {
   prisma = new PrismaClient()
+  backgroundPrisma = new PrismaClient({
+    datasources: { db: { url: backgroundDbUrl } }
+  })
 } else {
   if (global.prisma === undefined) {
     global.prisma = new PrismaClient()
   }
+  if (global.backgroundPrisma === undefined) {
+    global.backgroundPrisma = new PrismaClient({
+      datasources: { db: { url: backgroundDbUrl } }
+    })
+  }
 
   prisma = global.prisma
+  backgroundPrisma = global.backgroundPrisma
 }
 
 export default prisma
+export { backgroundPrisma }
 
 // https://github.com/prisma/prisma/issues/1983#issuecomment-620621213

redis/dashboardCache.ts

@@ -1,5 +1,5 @@
 import { redis } from './clientInstance'
-import { getPaymentStream } from 'redis/paymentCache'
+import { getPaymentStream, isBackgroundRebuildActive } from 'redis/paymentCache'
 import { ChartData, DashboardData, Payment, ButtonData, PaymentDataByButton, ChartColor, PeriodData, ButtonDisplayData } from './types'
 import { Prisma } from '@prisma/client'
 import moment, { DurationInputArg2 } from 'moment'
@@ -379,7 +379,14 @@ export const getUserDashboardData = async function (userId: string, timezone: st
       timezone,
       paybuttonIds
     )
-    await cacheDashboardData(userId, dashboardData)
+
+    const rebuilding = isBackgroundRebuildActive(userId)
+    if (rebuilding) {
+      dashboardData.cacheRebuilding = true
+      await cacheDashboardDataWithTTL(userId, dashboardData, 30)
+    } else {
+      await cacheDashboardData(userId, dashboardData)
+    }
     return dashboardData
   }
   return dashboardData
@@ -393,6 +400,14 @@ export const cacheDashboardData = async (userId: string, dashboardData: Dashboar
   await redis.set(key, JSON.stringify(cachable))
 }
 
+const cacheDashboardDataWithTTL = async (userId: string, dashboardData: DashboardData, ttlSeconds: number): Promise<void> => {
+  const key = getDashboardSummaryKey(userId)
+  const {
+    ...cachable
+  } = dashboardData
+  await redis.set(key, JSON.stringify(cachable), 'EX', ttlSeconds)
+}
+
 export const getCachedDashboardData = async (userId: string): Promise<DashboardData | null> => {
   const key = getDashboardSummaryKey(userId)
   const dashboardString = await redis.get(key)

redis/index.ts

@@ -76,34 +76,34 @@ export const CacheSet = {
 
 type MethodName = 'dashboardData' | 'paymentList' | 'addressBalance' | 'paymentsCount'
 
-interface PendingCalls {
-  [userId: string]: Set<MethodName>
-}
-
 export class CacheGet {
-  private static pendingCalls: PendingCalls = {}
+  private static readonly pendingPromises = new Map<string, Map<MethodName, Promise<any>>>()
 
   private static async executeCall<T>(
     userId: string,
     methodName: MethodName,
     fn: () => Promise<T>
   ): Promise<T> {
-    if (this.pendingCalls[userId] === undefined) {
-      this.pendingCalls[userId] = new Set()
+    let userMap = this.pendingPromises.get(userId)
+    if (userMap === undefined) {
+      userMap = new Map()
+      this.pendingPromises.set(userId, userMap)
     }
 
-    if (this.pendingCalls[userId].has(methodName)) {
-      throw new Error(`Method "${methodName}" is already being executed for user "${userId}".`)
+    const existing = userMap.get(methodName)
+    if (existing !== undefined) {
+      return await existing as T
     }
 
-    this.pendingCalls[userId].add(methodName)
+    const promise = fn()
+    userMap.set(methodName, promise)
 
     try {
-      return await fn()
+      return await promise
     } finally {
-      this.pendingCalls[userId].delete(methodName)
-      if (this.pendingCalls[userId].size === 0) {
-        this.pendingCalls[userId] = undefined as unknown as Set<MethodName>
+      userMap.delete(methodName)
+      if (userMap.size === 0) {
+        this.pendingPromises.delete(userId)
       }
     }
   }