diff --git a/.gitignore b/.gitignore index 9f97c59f3..fd78a1a8a 100644 --- a/.gitignore +++ b/.gitignore @@ -15,6 +15,7 @@ launch_* gen/ cardano-chain-gen/test/testfiles/temp/ /secp256k1/ +monitoring/data/ # Vim *.swp @@ -31,4 +32,4 @@ result* /.vscode # MacOS -.DS_Store \ No newline at end of file +.DS_Store diff --git a/cardano-chain-gen/test/Test/Cardano/Db/Mock/Config.hs b/cardano-chain-gen/test/Test/Cardano/Db/Mock/Config.hs index 5f9e89384..1b4eb262d 100644 --- a/cardano-chain-gen/test/Test/Cardano/Db/Mock/Config.hs +++ b/cardano-chain-gen/test/Test/Cardano/Db/Mock/Config.hs @@ -410,6 +410,9 @@ emptyMetricsSetters = , metricsSetDbSlotHeight = \_ -> pure () , metricsSetDbEpochSyncDuration = \_ -> pure () , metricsSetDbEpochSyncNumber = \_ -> pure () + , metricsSetDbBlocksPerSecond = \_ -> pure () + , metricsSetInsertDuration = \_ -> pure () + , metricsSetCacheHitRate = \_ _ -> pure () } withFullConfig :: diff --git a/cardano-db-sync/src/Cardano/DbSync/Database.hs b/cardano-db-sync/src/Cardano/DbSync/Database.hs index 86d841714..1550d9cff 100644 --- a/cardano-db-sync/src/Cardano/DbSync/Database.hs +++ b/cardano-db-sync/src/Cardano/DbSync/Database.hs @@ -23,6 +23,7 @@ import Cardano.Prelude hiding (atomically) import Cardano.Slotting.Slot (SlotNo (..), WithOrigin (..)) import Control.Concurrent.Class.MonadSTM.Strict import Control.Monad.Extra (whenJust) +import Data.Time.Clock (diffUTCTime, getCurrentTime) import Ouroboros.Network.Block (BlockNo (..), Point (..)) import Ouroboros.Network.Point (blockPointHash, blockPointSlot) @@ -126,7 +127,21 @@ runActions syncEnv actions = do lift $ atomically $ putTMVar resultVar (points, blockNo) dbEvent Continue ys (ys, zs) -> do + -- Record start time and block count for performance metrics + startTime <- liftIO getCurrentTime + let blockCount = length ys + + -- Process blocks ExceptT $ insertListBlocks syncEnv ys + + -- Calculate and record blocks per second + when (blockCount > 0) $ do + endTime <- liftIO getCurrentTime + let duration = realToFrac $ diffUTCTime endTime startTime + when (duration > 0) $ do + let blocksPerSecond = fromIntegral blockCount / duration + liftIO $ setDbBlocksPerSecond (envMetricSetters syncEnv) blocksPerSecond + if null zs then pure Continue else dbEvent Continue zs diff --git a/cardano-db-sync/src/Cardano/DbSync/Default.hs b/cardano-db-sync/src/Cardano/DbSync/Default.hs index 0ac739f7f..2d9b72f15 100644 --- a/cardano-db-sync/src/Cardano/DbSync/Default.hs +++ b/cardano-db-sync/src/Cardano/DbSync/Default.hs @@ -21,6 +21,7 @@ import qualified Data.ByteString.Short as SBS import Data.List (span) import qualified Data.Set as Set import qualified Data.Strict.Maybe as Strict +import Data.Time.Clock (diffUTCTime, getCurrentTime) import Ouroboros.Consensus.Cardano.Block (HardForkBlock (..)) import qualified Ouroboros.Consensus.HardFork.Combinator as Consensus import Ouroboros.Network.Block (blockHash, blockNo, getHeaderFields, headerFieldBlockNo, unBlockNo) @@ -42,6 +43,7 @@ import Cardano.DbSync.Error (SyncNodeError (..), mkSyncNodeCallStack) import Cardano.DbSync.Ledger.State (applyBlockAndSnapshot, defaultApplyResult) import Cardano.DbSync.Ledger.Types (ApplyResult (..)) import Cardano.DbSync.LocalStateQuery +import Cardano.DbSync.Metrics (setInsertDuration) import Cardano.DbSync.Rollback import Cardano.DbSync.Types import Cardano.DbSync.Util @@ -151,6 +153,9 @@ insertBlock :: Bool -> ExceptT SyncNodeError DB.DbM () insertBlock syncEnv cblk applyRes firstAfterRollback tookSnapshot = do + -- Start timing for insert duration metric + startTime <- liftIO getCurrentTime + !epochEvents <- liftIO $ atomically $ generateNewEpochEvents syncEnv (apSlotDetails applyRes) let !applyResult = applyRes {apEvents = sort $ epochEvents <> apEvents applyRes} let !details = apSlotDetails applyResult @@ -206,6 +211,11 @@ insertBlock syncEnv cblk applyRes firstAfterRollback tookSnapshot = do do lift $ DB.deleteConsumedTxOut tracer txOutVariantType (getSafeBlockNoDiff syncEnv) commitOrIndexes withinTwoMin withinHalfHour + + -- Record insert duration metric + endTime <- liftIO getCurrentTime + let duration = realToFrac $ diffUTCTime endTime startTime + liftIO $ setInsertDuration (envMetricSetters syncEnv) duration where tracer = getTrace syncEnv txOutVariantType = getTxOutVariantType syncEnv diff --git a/cardano-db-sync/src/Cardano/DbSync/Era/Cardano/Util.hs b/cardano-db-sync/src/Cardano/DbSync/Era/Cardano/Util.hs index a8f08041d..6e3008d9d 100644 --- a/cardano-db-sync/src/Cardano/DbSync/Era/Cardano/Util.hs +++ b/cardano-db-sync/src/Cardano/DbSync/Era/Cardano/Util.hs @@ -35,6 +35,7 @@ insertEpochSyncTime :: UTCTime -> ExceptT SyncNodeError DB.DbM () insertEpochSyncTime epochNo syncState epochStats endTime = do + currentTime <- liftIO Time.getCurrentTime void . lift $ DB.insertEpochSyncTime @@ -42,6 +43,7 @@ insertEpochSyncTime epochNo syncState epochStats endTime = do { DB.epochSyncTimeNo = unEpochNo epochNo - 1 , DB.epochSyncTimeSeconds = ceiling (realToFrac (Time.diffUTCTime endTime (elsStartTime epochStats)) :: Double) , DB.epochSyncTimeState = syncState + , DB.epochSyncTimeSyncedAt = Just currentTime } initEpochStatistics :: MonadIO m => m (StrictTVar IO EpochStatistics) diff --git a/cardano-db-sync/src/Cardano/DbSync/Era/Universal/Insert/LedgerEvent.hs b/cardano-db-sync/src/Cardano/DbSync/Era/Universal/Insert/LedgerEvent.hs index f67866a31..6b3e40249 100644 --- a/cardano-db-sync/src/Cardano/DbSync/Era/Universal/Insert/LedgerEvent.hs +++ b/cardano-db-sync/src/Cardano/DbSync/Era/Universal/Insert/LedgerEvent.hs @@ -19,7 +19,7 @@ import Cardano.Slotting.Slot (EpochNo (..)) import Cardano.DbSync.Api import Cardano.DbSync.Api.Types (EpochStatistics (..), InsertOptions (..), SyncEnv (..), UnicodeNullSource, formatUnicodeNullSource) -import Cardano.DbSync.Cache.Types (textShowCacheStats) +import Cardano.DbSync.Cache.Types (CacheStatistics (..), textShowCacheStats) import Cardano.DbSync.Era.Cardano.Util (insertEpochSyncTime, resetEpochStatistics) import qualified Cardano.DbSync.Era.Shelley.Generic as Generic import Cardano.DbSync.Era.Universal.Adjust (adjustEpochRewards) @@ -31,7 +31,7 @@ import Cardano.DbSync.Types import Cardano.DbSync.Error (SyncNodeError) import Cardano.DbSync.Ledger.Types -import Cardano.DbSync.Metrics (setDbEpochSyncDuration, setDbEpochSyncNumber) +import Cardano.DbSync.Metrics (setCacheHitRate, setDbEpochSyncDuration, setDbEpochSyncNumber) import Control.Concurrent.Class.MonadSTM.Strict (readTVarIO, writeTVar) import Control.Monad.Extra (whenJust) import qualified Data.Map.Strict as Map @@ -96,6 +96,17 @@ insertNewEpochLedgerEvents syncEnv applyRes currentEpochNo@(EpochNo curEpoch) = liftIO $ setDbEpochSyncDuration metricSetters (epochDurationSeconds (elsStartTime epochStats) currentTime) liftIO $ setDbEpochSyncNumber metricSetters (fromIntegral $ unEpochNo en - 1) + -- Calculate and set cache hit rates + let cacheStats = elsCaches epochStats + hitRate hits queries = if queries == 0 then 0.0 else fromIntegral hits / fromIntegral queries + liftIO $ setCacheHitRate metricSetters CacheStake (hitRate (credsHits cacheStats) (credsQueries cacheStats)) + liftIO $ setCacheHitRate metricSetters CachePools (hitRate (poolsHits cacheStats) (poolsQueries cacheStats)) + liftIO $ setCacheHitRate metricSetters CacheDatum (hitRate (datumHits cacheStats) (datumQueries cacheStats)) + liftIO $ setCacheHitRate metricSetters CacheMultiAssets (hitRate (multiAssetsHits cacheStats) (multiAssetsQueries cacheStats)) + liftIO $ setCacheHitRate metricSetters CachePrevBlock (hitRate (prevBlockHits cacheStats) (prevBlockQueries cacheStats)) + liftIO $ setCacheHitRate metricSetters CacheAddress (hitRate (addressHits cacheStats) (addressQueries cacheStats)) + liftIO $ setCacheHitRate metricSetters CacheTxIds (hitRate (txIdsHits cacheStats) (txIdsQueries cacheStats)) + -- Log comprehensive epoch statistics liftIO . logInfo tracer $ mconcat diff --git a/cardano-db-sync/src/Cardano/DbSync/Metrics.hs b/cardano-db-sync/src/Cardano/DbSync/Metrics.hs index d6a687446..58d8fc482 100644 --- a/cardano-db-sync/src/Cardano/DbSync/Metrics.hs +++ b/cardano-db-sync/src/Cardano/DbSync/Metrics.hs @@ -9,12 +9,15 @@ module Cardano.DbSync.Metrics ( setDbSlotHeight, setDbEpochSyncDuration, setDbEpochSyncNumber, + setDbBlocksPerSecond, + setInsertDuration, + setCacheHitRate, makeMetrics, withMetricSetters, withMetricsServer, ) where -import Cardano.DbSync.Types (MetricSetters (..)) +import Cardano.DbSync.Types (MetricSetters (..), CacheType(..)) import Cardano.Prelude import Cardano.Slotting.Slot (SlotNo (..), WithOrigin (..), fromWithOrigin) import Ouroboros.Network.Block (BlockNo (..)) @@ -41,6 +44,24 @@ data Metrics = Metrics -- ^ The duration of the last epoch sync in seconds. , mDbEpochSyncNumber :: !Gauge -- ^ The number of the last epoch that was synced. + , mDbBlocksPerSecond :: !Gauge + -- ^ The number of blocks being processes per second. + , mInsertDuration :: !Gauge + -- ^ The duration of the last insert operation in seconds. + , mCacheStakeHitRate :: !Gauge + -- ^ Cache hit rate for stake cache. + , mCachePoolsHitRate :: !Gauge + -- ^ Cache hit rate for pools cache. + , mCacheDatumHitRate :: !Gauge + -- ^ Cache hit rate for datum cache. + , mCacheMultiAssetsHitRate :: !Gauge + -- ^ Cache hit rate for multi_assets cache. + , mCachePrevBlockHitRate :: !Gauge + -- ^ Cache hit rate for prev_block cache. + , mCacheAddressHitRate :: !Gauge + -- ^ Cache hit rate for address cache. + , mCacheTxIdsHitRate :: !Gauge + -- ^ Cache hit rate for tx_ids cache. } -- This enables us to be much more flexibile with what we actually measure. @@ -61,8 +82,21 @@ withMetricSetters prometheusPort action = Gauge.set duration $ mDbEpochSyncDuration metrics , metricsSetDbEpochSyncNumber = \epochNo -> Gauge.set (fromIntegral epochNo) $ mDbEpochSyncNumber metrics + , metricsSetDbBlocksPerSecond = \bps -> + Gauge.set bps $ mDbBlocksPerSecond metrics + , metricsSetInsertDuration = \duration -> + Gauge.set duration $ mInsertDuration metrics + , metricsSetCacheHitRate = \cacheName hitRate -> + case cacheName of + CacheStake -> Gauge.set hitRate $ mCacheStakeHitRate metrics + CachePools -> Gauge.set hitRate $ mCachePoolsHitRate metrics + CacheDatum -> Gauge.set hitRate $ mCacheDatumHitRate metrics + CacheMultiAssets -> Gauge.set hitRate $ mCacheMultiAssetsHitRate metrics + CachePrevBlock -> Gauge.set hitRate $ mCachePrevBlockHitRate metrics + CacheAddress -> Gauge.set hitRate $ mCacheAddressHitRate metrics + CacheTxIds -> Gauge.set hitRate $ mCacheTxIdsHitRate metrics } - + withMetricsServer :: Int -> (Metrics -> IO a) -> IO a withMetricsServer port action = do -- Using both `RegistryT` and `bracket` here is overkill. Unfortunately the @@ -83,6 +117,15 @@ makeMetrics = <*> registerGauge "cardano_db_sync_db_slot_height" mempty <*> registerGauge "cardano_db_sync_db_epoch_sync_duration_seconds" mempty <*> registerGauge "cardano_db_sync_db_epoch_sync_number" mempty + <*> registerGauge "cardano_db_sync_blocks_per_second" mempty + <*> registerGauge "cardano_db_sync_insert_duration_seconds" mempty + <*> registerGauge "cardano_db_sync_cache_stake_hit_rate" mempty + <*> registerGauge "cardano_db_sync_cache_pools_hit_rate" mempty + <*> registerGauge "cardano_db_sync_cache_datum_hit_rate" mempty + <*> registerGauge "cardano_db_sync_cache_multi_assets_hit_rate" mempty + <*> registerGauge "cardano_db_sync_cache_prev_block_hit_rate" mempty + <*> registerGauge "cardano_db_sync_cache_address_hit_rate" mempty + <*> registerGauge "cardano_db_sync_cache_tx_ids_hit_rate" mempty setNodeBlockHeight :: MetricSetters -> WithOrigin BlockNo -> IO () setNodeBlockHeight setters woBlkNo = @@ -102,3 +145,12 @@ setDbEpochSyncDuration = metricsSetDbEpochSyncDuration setDbEpochSyncNumber :: MetricSetters -> Word64 -> IO () setDbEpochSyncNumber = metricsSetDbEpochSyncNumber + +setDbBlocksPerSecond :: MetricSetters -> Double -> IO () +setDbBlocksPerSecond = metricsSetDbBlocksPerSecond + +setInsertDuration :: MetricSetters -> Double -> IO () +setInsertDuration = metricsSetInsertDuration + +setCacheHitRate :: MetricSetters -> CacheType -> Double -> IO () +setCacheHitRate = metricsSetCacheHitRate diff --git a/cardano-db-sync/src/Cardano/DbSync/Types.hs b/cardano-db-sync/src/Cardano/DbSync/Types.hs index 9ddef3747..1f2dc3faf 100644 --- a/cardano-db-sync/src/Cardano/DbSync/Types.hs +++ b/cardano-db-sync/src/Cardano/DbSync/Types.hs @@ -24,6 +24,7 @@ module Cardano.DbSync.Types ( SyncState (..), TPraosStandard, MetricSetters (..), + CacheType (..), OffChainPoolWorkQueue (..), OffChainVoteWorkQueue (..), SimplifiedOffChainPoolData (..), @@ -139,11 +140,24 @@ data MetricSetters = MetricSetters , metricsSetDbSlotHeight :: SlotNo -> IO () , metricsSetDbEpochSyncDuration :: Double -> IO () , metricsSetDbEpochSyncNumber :: Word64 -> IO () + , metricsSetDbBlocksPerSecond :: Double -> IO () + , metricsSetInsertDuration :: Double -> IO () + , metricsSetCacheHitRate :: CacheType -> Double -> IO () } data SyncState = SyncLagging | SyncFollowing deriving (Eq, Show) +data CacheType + = CacheStake + | CachePools + | CacheDatum + | CacheMultiAssets + | CachePrevBlock + | CacheAddress + | CacheTxIds + deriving (Eq, Show) + ------------------------------------------------------------------------------------- -- OffChain ------------------------------------------------------------------------------------- diff --git a/cardano-db/src/Cardano/Db/Schema/Core/EpochAndProtocol.hs b/cardano-db/src/Cardano/Db/Schema/Core/EpochAndProtocol.hs index 33a6b7458..fef380a5e 100644 --- a/cardano-db/src/Cardano/Db/Schema/Core/EpochAndProtocol.hs +++ b/cardano-db/src/Cardano/Db/Schema/Core/EpochAndProtocol.hs @@ -324,6 +324,7 @@ data EpochSyncTime = EpochSyncTime { epochSyncTimeNo :: !Word64 -- sqltype=word31type , epochSyncTimeSeconds :: !Word64 -- sqltype=word63type , epochSyncTimeState :: !SyncState -- sqltype=syncstatetype + , epochSyncTimeSyncedAt :: !(Maybe UTCTime) } deriving (Show, Eq, Generic) @@ -338,6 +339,7 @@ epochSyncTimeEncoder = [ epochSyncTimeNo >$< E.param (E.nonNullable $ fromIntegral >$< E.int8) , epochSyncTimeSeconds >$< E.param (E.nonNullable $ fromIntegral >$< E.int8) , epochSyncTimeState >$< E.param (E.nonNullable syncStateEncoder) + , epochSyncTimeSyncedAt >$< E.param (E.nullable E.timestamptz) ] ----------------------------------------------------------------------------------------------------------------------------------- diff --git a/dev-tools/README.md b/dev-tools/README.md deleted file mode 100644 index 4bdec8f24..000000000 --- a/dev-tools/README.md +++ /dev/null @@ -1,110 +0,0 @@ -# Cardano DB Sync - Developer Tools - -Local development tools for monitoring and profiling cardano-db-sync. - -## Overview - -Two complementary tools for understanding cardano-db-sync performance: - -### πŸ” [Monitoring](monitoring/README.md) -Real-time metrics with Prometheus/Grafana -- PostgreSQL metrics (queries, cache, connections) -- System metrics (CPU, memory, disk I/O) -- Time-series visualization - -### 🧠 [Profiling](profiling/README.md) -Memory profiling with ghc-debug -- Interactive heap exploration -- Memory leak detection -- Retainer chain analysis - -## Installation - -### Monitoring - -```bash -# macOS -brew install tmux prometheus postgres_exporter node_exporter grafana - -# Linux (apt) -sudo apt-get install tmux prometheus postgres-exporter prometheus-node-exporter grafana - -# Linux (yum) -sudo yum install tmux prometheus postgres_exporter node_exporter grafana -``` - -### Profiling - -```bash -# Install ghc-debug-brick -git clone https://gitlab.haskell.org/ghc/ghc-debug.git -cd ghc-debug/brick -cabal install ghc-debug-brick - -# Instrument cardano-db-sync (see profiling/README.md for details) -``` - -## Running - -### Start Monitoring - -```bash -cd dev-tools/monitoring -./scripts/start-monitoring.sh -``` - -Access at: -- Prometheus: http://localhost:9090 -- Grafana: http://localhost:3000 (if started separately) - -### Start Profiling - -```bash -cd dev-tools/profiling -./scripts/start-profiling.sh - -# In another terminal (after several hours): -ghc-debug-brick /tmp/cardano-db-sync.ghc-debug -``` - -### Run Both Together - -```bash -# Terminal 1: Monitoring -cd dev-tools/monitoring && ./scripts/start-monitoring.sh - -# Terminal 2: Profiling -cd dev-tools/profiling && ./scripts/start-profiling.sh -``` - -## Directory Structure - -``` -dev-tools/ -β”œβ”€β”€ README.md # This file -β”‚ -β”œβ”€β”€ monitoring/ # Prometheus/Grafana monitoring -β”‚ β”œβ”€β”€ README.md # Full monitoring documentation -β”‚ β”œβ”€β”€ scripts/ -β”‚ β”‚ └── start-monitoring.sh -β”‚ β”œβ”€β”€ config/ -β”‚ β”‚ └── prometheus.yml -β”‚ β”œβ”€β”€ data/ # Prometheus data (gitignored) -β”‚ └── docs/ -β”‚ └── METRICS.md # Available metrics -β”‚ -└── profiling/ # ghc-debug profiling - β”œβ”€β”€ README.md # Full profiling documentation - β”œβ”€β”€ scripts/ - β”‚ β”œβ”€β”€ start-profiling.sh - β”‚ └── analyze-memory.sh - β”œβ”€β”€ snapshots/ # Heap snapshots (gitignored) - β”œβ”€β”€ logs/ # Memory logs (gitignored) - └── reports/ # Analysis reports (commit these) -``` - -## Documentation - -- [Monitoring Setup Guide](monitoring/README.md) -- [Profiling Setup Guide](profiling/README.md) -- [Available Metrics Reference](monitoring/docs/METRICS.md) diff --git a/dev-tools/monitoring/README.md b/dev-tools/monitoring/README.md deleted file mode 100644 index 836962b1c..000000000 --- a/dev-tools/monitoring/README.md +++ /dev/null @@ -1,114 +0,0 @@ -# Cardano DB Sync - Monitoring - -Local development monitoring using Prometheus and Grafana for PostgreSQL and system metrics. - -## Quick Setup - -### 1. Install Tools - -```bash -# macOS -brew install prometheus postgres_exporter node_exporter grafana - -# Linux (apt) -sudo apt-get install prometheus postgres-exporter prometheus-node-exporter grafana - -# Linux (yum) -sudo yum install prometheus postgres_exporter node_exporter grafana -``` - -### 2. Setup PostgreSQL Monitoring User - -```sql --- Connect to your database -psql -U postgres -d cexplorer - --- Create monitoring user -CREATE USER postgres_exporter WITH PASSWORD 'secure_password'; -GRANT pg_monitor TO postgres_exporter; -GRANT CONNECT ON DATABASE cexplorer TO postgres_exporter; -``` - -### 3. Start Monitoring - -```bash -cd dev-tools/monitoring -./scripts/start-monitoring.sh -``` - -This launches a tmux session with Prometheus and exporters. - -### 4. Start Grafana - -```bash -# macOS -brew services start grafana - -# Linux -sudo systemctl start grafana-server -``` - -## Access Dashboards - -- **Prometheus**: http://localhost:9090 -- **Grafana**: http://localhost:3000 (default: admin/admin) - -## Grafana Setup - -1. Open http://localhost:3000 -2. Add Prometheus data source: - - Configuration β†’ Data Sources β†’ Add data source - - Select Prometheus - - URL: `http://localhost:9090` - - Save & Test -3. Import dashboard: - - Dashboards β†’ Import - - Upload `config/grafana-db-sync.json` - - Select Prometheus data source - -## Key Metrics - -- **System**: CPU, memory usage (node_exporter) -- **PostgreSQL**: Connections, transaction rates, cache hit ratio -- **Sync Progress**: Block height, epoch duration - -## Troubleshooting - -**postgres_exporter fails to connect**: -- Verify monitoring user exists and has permissions -- Set `DATA_SOURCE_NAME` with correct credentials in `scripts/start-monitoring.sh` - -**Prometheus shows "DOWN" targets**: -- Check exporters are running: `ps aux | grep exporter` -- Test endpoints: `curl http://localhost:9187/metrics` and `curl http://localhost:9100/metrics` - -**Port already in use**: -```bash -# Kill processes using ports -lsof -ti:9090 | xargs kill # Prometheus -lsof -ti:9100 | xargs kill # node_exporter -lsof -ti:9187 | xargs kill # postgres_exporter -``` - -## Stop Services - -```bash -# Stop monitoring -tmux kill-session -t cardano-monitoring - -# Stop Grafana -brew services stop grafana # macOS -sudo systemctl stop grafana-server # Linux -``` - -## Files - -- `scripts/start-monitoring.sh` - Launch monitoring suite -- `config/prometheus.yml` - Prometheus configuration -- `config/grafana-db-sync.json` - Grafana dashboard -- `docs/METRICS.md` - Complete metrics reference - -## See Also - -- [Prometheus Documentation](https://prometheus.io/docs/) -- [Grafana Documentation](https://grafana.com/docs/) diff --git a/dev-tools/monitoring/config/prometheus.yml b/dev-tools/monitoring/config/prometheus.yml deleted file mode 100644 index 29c7604a8..000000000 --- a/dev-tools/monitoring/config/prometheus.yml +++ /dev/null @@ -1,42 +0,0 @@ -# Prometheus Configuration for Cardano DB Sync Monitoring - -global: - scrape_interval: 15s - evaluation_interval: 15s - external_labels: - monitor: 'cardano-db-sync-dev' - -# Scrape configurations -scrape_configs: - # PostgreSQL metrics via postgres_exporter - - job_name: 'postgres-cexplorer' - static_configs: - - targets: ['localhost:9187'] - labels: - instance: 'cardano-db-sync-db' - environment: 'development' - scrape_interval: 5s - - # System metrics via node_exporter - - job_name: 'node' - static_configs: - - targets: ['localhost:9100'] - labels: - instance: 'cardano-db-sync-host' - environment: 'development' - scrape_interval: 10s - - # Cardano DB Sync application metrics - - job_name: 'cardano-db-sync' - static_configs: - - targets: ['localhost:8080'] - labels: - instance: 'cardano-db-sync' - environment: 'development' - scrape_interval: 10s - # This endpoint will expose: - # - Sync progress metrics - # - Database operation metrics - # - Memory/GC metrics from ghc-debug integration - # - Cache hit/miss rates - # - Transaction processing rates diff --git a/dev-tools/monitoring/docs/METRICS.md b/dev-tools/monitoring/docs/METRICS.md deleted file mode 100644 index 7449bb907..000000000 --- a/dev-tools/monitoring/docs/METRICS.md +++ /dev/null @@ -1,126 +0,0 @@ -# Cardano DB Sync - Available Metrics - -This document describes the metrics exposed by the monitoring suite for cardano-db-sync. - -## Metric Sources - -### 1. PostgreSQL Metrics (postgres_exporter) - -Exposed on port **9187** via `postgres_exporter`. - -#### Database Size Metrics -- `pg_database_size_bytes{datname="cexplorer"}` - Total database size in bytes - -#### Connection Metrics -- `pg_stat_database_numbackends` - Number of active connections -- `pg_stat_database_xact_commit` - Total transactions committed -- `pg_stat_database_xact_rollback` - Total transactions rolled back -- `pg_stat_database_deadlocks` - Number of deadlocks detected - -#### Table Metrics -- `pg_stat_user_tables_seq_scan` - Sequential scans on tables -- `pg_stat_user_tables_idx_scan` - Index scans on tables -- `pg_stat_user_tables_n_tup_ins` - Rows inserted -- `pg_stat_user_tables_n_tup_upd` - Rows updated -- `pg_stat_user_tables_n_tup_del` - Rows deleted -- `pg_stat_user_tables_n_live_tup` - Estimated live rows -- `pg_stat_user_tables_n_dead_tup` - Estimated dead rows - -#### Index Metrics -- `pg_stat_user_indexes_idx_scan` - Index scans performed -- `pg_stat_user_indexes_idx_tup_read` - Index entries returned -- `pg_stat_user_indexes_idx_tup_fetch` - Live rows fetched by index scans - -#### Transaction/WAL Metrics -- `pg_stat_database_blks_read` - Disk blocks read -- `pg_stat_database_blks_hit` - Disk blocks found in cache (buffer hit) -- `pg_stat_database_tup_returned` - Rows returned by queries -- `pg_stat_database_tup_fetched` - Rows fetched by queries -- `pg_stat_database_tup_inserted` - Rows inserted -- `pg_stat_database_tup_updated` - Rows updated -- `pg_stat_database_tup_deleted` - Rows deleted - -#### Replication Metrics (if applicable) -- `pg_stat_replication_lag` - Replication lag in bytes - -### 2. System Metrics (node_exporter) - -Exposed on port **9100** via `node_exporter`. - -#### CPU Metrics -- `node_cpu_seconds_total` - CPU time spent in various modes (user, system, idle, etc.) -- `node_load1`, `node_load5`, `node_load15` - System load averages - -#### Memory Metrics -- `node_memory_MemTotal_bytes` - Total memory -- `node_memory_MemFree_bytes` - Free memory -- `node_memory_MemAvailable_bytes` - Available memory -- `node_memory_Buffers_bytes` - Memory used for buffers -- `node_memory_Cached_bytes` - Memory used for cache -- `node_memory_SwapTotal_bytes` - Total swap space -- `node_memory_SwapFree_bytes` - Free swap space - -#### Disk Metrics -- `node_disk_read_bytes_total` - Total bytes read from disk -- `node_disk_written_bytes_total` - Total bytes written to disk -- `node_disk_read_time_seconds_total` - Time spent reading -- `node_disk_write_time_seconds_total` - Time spent writing -- `node_filesystem_size_bytes` - Filesystem size -- `node_filesystem_avail_bytes` - Filesystem space available -- `node_filesystem_free_bytes` - Filesystem space free - -#### Network Metrics -- `node_network_receive_bytes_total` - Network bytes received -- `node_network_transmit_bytes_total` - Network bytes transmitted -- `node_network_receive_errs_total` - Network receive errors -- `node_network_transmit_errs_total` - Network transmit errors - -### 3. Cardano DB Sync Application Metrics - -Exposed on port **8080** via Prometheus endpoint in cardano-db-sync. - -**Status**: To be implemented when Prometheus endpoint is added to cardano-db-sync. - -#### Planned Sync Metrics -- `dbsync_block_height` - Current synced block height -- `dbsync_slot_number` - Current synced slot number -- `dbsync_epoch_number` - Current synced epoch number -- `dbsync_sync_progress_percent` - Sync progress percentage -- `dbsync_blocks_per_second` - Block processing rate -- `dbsync_tx_per_second` - Transaction processing rate -- `dbsync_rollback_count` - Number of rollbacks performed -- `dbsync_rollback_depth` - Depth of most recent rollback - -#### Planned Database Operation Metrics -- `dbsync_db_insert_duration_seconds` - Histogram of insert operation durations -- `dbsync_db_query_duration_seconds` - Histogram of query operation durations -- `dbsync_db_bulk_insert_size` - Size of bulk insert batches -- `dbsync_grouped_data_flush_duration_seconds` - Time spent flushing grouped data -- `dbsync_grouped_data_size_bytes` - Size of grouped data in memory - -#### Planned Cache Metrics -- `dbsync_cache_hits_total` - Cache hits by cache type -- `dbsync_cache_misses_total` - Cache misses by cache type -- `dbsync_cache_size_entries` - Current cache size in entries -- `dbsync_cache_size_bytes` - Estimated cache size in bytes -- `dbsync_cache_evictions_total` - Number of cache evictions - -#### Planned Memory/GC Metrics (from ghc-debug integration) -- `dbsync_memory_heap_size_bytes` - Current heap size -- `dbsync_memory_live_bytes` - Live data in heap -- `dbsync_memory_gc_count` - Number of GC collections -- `dbsync_memory_gc_cpu_seconds` - CPU time spent in GC -- `dbsync_memory_gc_wall_seconds` - Wall time spent in GC -- `dbsync_memory_max_live_bytes` - Maximum live data observed -- `dbsync_memory_allocated_bytes_total` - Total bytes allocated - -#### Planned Ledger State Metrics -- `dbsync_ledger_state_size_bytes` - Size of ledger state -- `dbsync_ledger_snapshot_duration_seconds` - Time to save ledger snapshot -- `dbsync_ledger_events_processed_total` - Ledger events processed - -## See Also - -- [Prometheus Query Documentation](https://prometheus.io/docs/prometheus/latest/querying/basics/) -- [PostgreSQL Statistics Views](https://www.postgresql.org/docs/current/monitoring-stats.html) -- [Node Exporter Metrics](https://github.com/prometheus/node_exporter#enabled-by-default) diff --git a/dev-tools/monitoring/scripts/start-monitoring.sh b/dev-tools/monitoring/scripts/start-monitoring.sh deleted file mode 100755 index 0adbc5cdb..000000000 --- a/dev-tools/monitoring/scripts/start-monitoring.sh +++ /dev/null @@ -1,197 +0,0 @@ -#!/bin/bash - -set -euo pipefail - -# Cardano DB Sync - Monitoring Suite Launcher -# Starts Prometheus, postgres_exporter, and node_exporter in a tmux session - -SESSION="cardano-monitoring" -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -MONITORING_DIR="$(dirname "$SCRIPT_DIR")" -CONFIG_DIR="$MONITORING_DIR/config" -DATA_DIR="$MONITORING_DIR/data" - -# Colors -RED='\033[0;31m' -GREEN='\033[0;32m' -YELLOW='\033[1;33m' -NC='\033[0m' # No Color - -print_info() { - echo -e "${GREEN}[INFO]${NC} $1" -} - -print_warn() { - echo -e "${YELLOW}[WARN]${NC} $1" -} - -print_error() { - echo -e "${RED}[ERROR]${NC} $1" -} - -# Check dependencies -check_dependencies() { - local missing=() - - command -v tmux >/dev/null 2>&1 || missing+=("tmux") - command -v prometheus >/dev/null 2>&1 || missing+=("prometheus") - command -v postgres_exporter >/dev/null 2>&1 || missing+=("postgres_exporter") - command -v node_exporter >/dev/null 2>&1 || missing+=("node_exporter") - - if [ ${#missing[@]} -gt 0 ]; then - print_error "Missing required dependencies: ${missing[*]}" - print_info "Install with:" - echo " brew install tmux prometheus postgres_exporter node_exporter" - exit 1 - fi -} - -# Check if PostgreSQL is running -check_postgres() { - if ! pg_isready -h 127.0.0.1 -p 5432 >/dev/null 2>&1; then - print_warn "PostgreSQL does not appear to be running on localhost:5432" - print_info "Start PostgreSQL before running monitoring suite" - read -p "Continue anyway? [y/N] " -n 1 -r - echo - [[ ! $REPLY =~ ^[Yy]$ ]] && exit 1 - fi -} - -# Create data directory -ensure_data_dir() { - mkdir -p "$DATA_DIR" - print_info "Data directory: $DATA_DIR" -} - -# Kill existing session if it exists -cleanup_existing() { - if tmux has-session -t "$SESSION" 2>/dev/null; then - print_warn "Killing existing session: $SESSION" - tmux kill-session -t "$SESSION" - - # Kill any orphaned exporters/prometheus - pkill -f postgres_exporter 2>/dev/null || true - pkill -f node_exporter 2>/dev/null || true - pkill -f prometheus 2>/dev/null || true - - sleep 1 - fi -} - -# Database connection settings -DB_HOST="${DB_HOST:-127.0.0.1}" -DB_PORT="${DB_PORT:-5432}" -DB_NAME="${DB_NAME:-cexplorer}" -DB_USER="${DB_USER:-postgres_exporter}" - -# Exporter ports -POSTGRES_EXPORTER_PORT="${POSTGRES_EXPORTER_PORT:-9187}" -NODE_EXPORTER_PORT="${NODE_EXPORTER_PORT:-9100}" -PROMETHEUS_PORT="${PROMETHEUS_PORT:-9090}" - -print_info "Starting Cardano DB Sync Monitoring Suite" -print_info "==========================================" - -check_dependencies -check_postgres -ensure_data_dir -cleanup_existing - -# Create new tmux session -tmux new-session -d -s "$SESSION" - -# Configure layout: 4 panes -# β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”¬β”€β”€β”€β”€β”€β”€β”€β”€β”€β” -# β”‚ 0 β”‚ 2 β”‚ -# β”‚ postgresβ”‚ node β”‚ -# β”‚ exporterβ”‚ exporterβ”‚ -# β”œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€ -# β”‚ 1 β”‚ 3 β”‚ -# β”‚prometheusβ”‚ info β”‚ -# β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”΄β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ - -tmux rename-window -t "$SESSION" "monitoring" -tmux split-window -h -t "$SESSION" -tmux split-window -v -t "$SESSION:0.0" -tmux split-window -v -t "$SESSION:0.2" - -# Pane 0: postgres_exporter -print_info "Starting postgres_exporter on port $POSTGRES_EXPORTER_PORT" -tmux send-keys -t "$SESSION:0.0" "cd '$MONITORING_DIR'" C-m -tmux send-keys -t "$SESSION:0.0" "echo '=== PostgreSQL Exporter ==='" C-m -tmux send-keys -t "$SESSION:0.0" "export DATA_SOURCE_NAME='postgresql://${DB_USER}@${DB_HOST}:${DB_PORT}/${DB_NAME}?sslmode=disable'" C-m -tmux send-keys -t "$SESSION:0.0" "postgres_exporter --web.listen-address=:${POSTGRES_EXPORTER_PORT}" C-m - -# Pane 1: prometheus -print_info "Starting Prometheus on port $PROMETHEUS_PORT" -tmux send-keys -t "$SESSION:0.1" "cd '$MONITORING_DIR'" C-m -tmux send-keys -t "$SESSION:0.1" "echo '=== Prometheus ==='" C-m -tmux send-keys -t "$SESSION:0.1" "prometheus --config.file='$CONFIG_DIR/prometheus.yml' --storage.tsdb.path='$DATA_DIR' --web.listen-address=:${PROMETHEUS_PORT}" C-m - -# Pane 2: node_exporter -print_info "Starting node_exporter on port $NODE_EXPORTER_PORT" -tmux send-keys -t "$SESSION:0.2" "cd '$MONITORING_DIR'" C-m -tmux send-keys -t "$SESSION:0.2" "echo '=== Node Exporter ==='" C-m -tmux send-keys -t "$SESSION:0.2" "node_exporter --web.listen-address=:${NODE_EXPORTER_PORT} --no-collector.thermal" C-m - -# Pane 3: Info pane -tmux send-keys -t "$SESSION:0.3" "cd '$MONITORING_DIR'" C-m -tmux send-keys -t "$SESSION:0.3" "clear" C-m -tmux send-keys -t "$SESSION:0.3" "cat << 'EOF' -╔═══════════════════════════════════════════════════════════════╗ -β•‘ Cardano DB Sync Monitoring Suite β•‘ -β•šβ•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β• - -Monitoring Status: - βœ“ PostgreSQL Exporter : http://localhost:${POSTGRES_EXPORTER_PORT} - βœ“ Node Exporter : http://localhost:${NODE_EXPORTER_PORT} - βœ“ Prometheus : http://localhost:${PROMETHEUS_PORT} - -Quick Links: - - Prometheus UI : http://localhost:${PROMETHEUS_PORT} - - Metrics Explorer : http://localhost:${PROMETHEUS_PORT}/graph - - Targets Status : http://localhost:${PROMETHEUS_PORT}/targets - -Grafana Setup: - 1. Install Grafana: brew install grafana - 2. Start Grafana: brew services start grafana - 3. Access: http://localhost:3000 (admin/admin) - 4. Add Prometheus datasource: http://localhost:${PROMETHEUS_PORT} - 5. Import dashboard from docs/grafana-dashboard.json - -Useful Commands: - - View Prometheus config : cat config/prometheus.yml - - View logs : Check panes above - - Stop monitoring : tmux kill-session -t ${SESSION} - - Detach from session : Ctrl+b, d - - Re-attach to session : tmux attach -t ${SESSION} - -Database Connection: - Host : ${DB_HOST} - Port : ${DB_PORT} - Database : ${DB_NAME} - User : ${DB_USER} - -Documentation: - - See docs/README.md for full setup guide - - See docs/METRICS.md for available metrics - - See docs/grafana-dashboard.json for Grafana template - -EOF -" C-m - -sleep 2 - -print_info "" -print_info "Monitoring suite started successfully!" -print_info "" -print_info " Prometheus UI : http://localhost:${PROMETHEUS_PORT}" -print_info " Targets : http://localhost:${PROMETHEUS_PORT}/targets" -print_info "" -print_info "Attaching to tmux session '$SESSION'..." -print_info "(Press Ctrl+b, d to detach; tmux attach -t $SESSION to reattach)" -print_info "" - -# Attach to session -tmux select-pane -t "$SESSION:0.3" -tmux attach-session -t "$SESSION" diff --git a/dev-tools/profiling/README.md b/dev-tools/profiling/README.md deleted file mode 100644 index c8b04447a..000000000 --- a/dev-tools/profiling/README.md +++ /dev/null @@ -1,94 +0,0 @@ -# Cardano DB Sync - ghc-debug Profiling - -Interactive heap profiling for cardano-db-sync using ghc-debug. - -## Quick Setup - -### 1. Install ghc-debug-brick - -```bash -cabal install ghc-debug-brick -``` - -This automatically installs all ghc-debug dependencies. - -### 2. Configure ghc-debug (first time only) - -The ghc-debug dependencies are configured in `cabal.project.local` (not committed to git). - -If you don't have them yet, add to your `cabal.project.local`: - -```cabal -source-repository-package - type: git - location: https://gitlab.haskell.org/ghc/ghc-debug.git - tag: 1b0f36fab86e9baa9734c88dcc1dbe17d10d8c93 - subdir: stub - -source-repository-package - type: git - location: https://gitlab.haskell.org/ghc/ghc-debug.git - tag: 1b0f36fab86e9baa9734c88dcc1dbe17d10d8c93 - subdir: common -``` - -**Note**: The ghc-debug commit above is from master branch (0.7.0.0). If you have GHC compatibility issues, you may need to adjust this commit. - -### 3. Build cardano-db-sync with profiling - -```bash -# From repository root -cabal build --flag enable-ghc-debug exe:cardano-db-sync - -# Or use the build script -./dev-tools/profiling/scripts/build-profiling.sh -``` - -### 4. Run cardano-db-sync - -Start cardano-db-sync as you normally would. The `enable-ghc-debug` flag automatically enables ghc-debug support: - -```bash -# Using the start script (launches in tmux) -./dev-tools/profiling/scripts/start-profiling.sh - -# Or run manually with your usual command -PGPASSFILE=config/pgpass-mainnet cabal run cardano-db-sync -- \ - --config config/mainnet-config.yaml \ - --socket-path /path/to/node.socket \ - --state-dir ledger-state/mainnet \ - --schema-dir schema/ -``` - -**Note**: When running with `enable-ghc-debug`, cardano-db-sync will print: -``` -Starting ghc-debug on socket: ~/.local/share/ghc-debug/debuggee/sockets/-cardano-db-sync -``` - -### 5. Connect with ghc-debug-brick - -In a new terminal, while cardano-db-sync is running: - -```bash -ghc-debug-brick -``` - -## Files - -- `scripts/build-profiling.sh` - Build with profiling enabled -- `scripts/start-profiling.sh` - Launch cardano-db-sync in tmux with monitoring - -## Troubleshooting - -**ghc-debug-brick shows "0 found"**: -1. Run `ghc-debug-brick` without arguments -2. Press `Tab` to switch to process view -3. Check socket exists: `ls ~/.local/share/ghc-debug/debuggee/sockets/` -4. Verify cardano-db-sync is running: `ps aux | grep cardano-db-sync` - -**MacOS Do NOT set `GHC_DEBUG_SOCKET` environment variable** - it can break auto-discovery. - -## See Also - -- [ghc-debug Documentation](https://gitlab.haskell.org/ghc/ghc-debug) -- [ouroboros-consensus PR #1731](https://github.com/IntersectMBO/ouroboros-consensus/pull/1731) - Similar approach diff --git a/dev-tools/profiling/scripts/build-profiling.sh b/dev-tools/profiling/scripts/build-profiling.sh deleted file mode 100755 index 991793a04..000000000 --- a/dev-tools/profiling/scripts/build-profiling.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/bin/bash -# build-profiling.sh -# Build cardano-db-sync with ghc-debug profiling support enabled - -set -e - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -PROFILING_DIR="$(dirname "$SCRIPT_DIR")" -DEV_TOOLS_DIR="$(dirname "$PROFILING_DIR")" -CARDANO_DB_SYNC_DIR="$(dirname "$DEV_TOOLS_DIR")" - -cd "$CARDANO_DB_SYNC_DIR" - -echo "Building cardano-db-sync with profiling support..." -echo "" - -cabal build --flag enable-ghc-debug exe:cardano-db-sync - -DBSYNC_BIN=$(cabal list-bin exe:cardano-db-sync --flag enable-ghc-debug) - -echo "" -echo "Build complete: $DBSYNC_BIN" -echo "" diff --git a/dev-tools/profiling/scripts/start-profiling.sh b/dev-tools/profiling/scripts/start-profiling.sh deleted file mode 100755 index 63d718706..000000000 --- a/dev-tools/profiling/scripts/start-profiling.sh +++ /dev/null @@ -1,123 +0,0 @@ -#!/bin/bash -# start-profiling.sh -# Run cardano-db-sync with ghc-debug profiling enabled -# Based on ouroboros-consensus team's approach - -set -e - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -PROFILING_DIR="$(dirname "$SCRIPT_DIR")" -DEV_TOOLS_DIR="$(dirname "$PROFILING_DIR")" -CARDANO_DB_SYNC_DIR="$(dirname "$DEV_TOOLS_DIR")" - -HOMEIOG=${HOMEIOG:-$HOME/Code/IOG} -CARDANO_NODE_DIR="${CARDANO_NODE_DIR:-$HOMEIOG/cardano-node}" -TESTNET_DIR="${TESTNET_DIR:-$HOMEIOG/testnet}" - -# Detect OS for pgpass file -if [[ "$OSTYPE" == "darwin"* ]]; then - PGPASS_FILE="${PGPASS_FILE:-$CARDANO_DB_SYNC_DIR/config/pgpass-mainnet-macos}" -else - PGPASS_FILE="${PGPASS_FILE:-$CARDANO_DB_SYNC_DIR/config/pgpass-mainnet}" -fi - -# Profiling configuration -SNAPSHOT_DIR="$PROFILING_DIR/snapshots" -LOG_DIR="$PROFILING_DIR/logs" -MEMORY_LOG="$LOG_DIR/memory-$(date +%Y%m%d-%H%M%S).csv" -# Note: Don't set GHC_DEBUG_SOCKET - let ghc-debug-stub use default XDG location -# Socket will be at: ~/.local/share/ghc-debug/debuggee/sockets/-cardano-db-sync - -# Create profiling directories -mkdir -p "$SNAPSHOT_DIR" -mkdir -p "$LOG_DIR" - -# Build cardano-db-sync with profiling support -echo "Building profiling-enabled binary..." -"$SCRIPT_DIR/build-profiling.sh" -echo "" - -# Find the profiling-enabled binary (similar to run-everything-tmux.sh) -dbsync="$(find "$CARDANO_DB_SYNC_DIR"/ -name cardano-db-sync -type f | head -1)" - -if [ -z "$dbsync" ]; then - echo "ERROR: Could not find cardano-db-sync binary" - echo "Build may have failed. Check output above." - exit 1 -fi - -echo "======================================" -echo "cardano-db-sync Profiling Setup" -echo "======================================" -echo "DB Sync binary: $dbsync" -echo "Memory log: $MEMORY_LOG" -echo "Snapshot dir: $SNAPSHOT_DIR" -echo "Socket location: ~/.local/share/ghc-debug/debuggee/sockets/-cardano-db-sync" -echo "======================================" -echo "" - -# Check if ghc-debug-brick is installed -if ! command -v ghc-debug-brick &> /dev/null; then - echo "WARNING: ghc-debug-brick not found in PATH" - echo "To install:" - echo " git clone https://gitlab.haskell.org/ghc/ghc-debug.git" - echo " cd ghc-debug/brick" - echo " cabal install ghc-debug-brick" - echo "" -fi - -# Note: Sockets are auto-managed by ghc-debug in XDG directory - -# Set up tmux session -session="DB-SYNC-PROFILE" - -# Kill existing session if it exists -if tmux has-session -t $session 2>/dev/null; then - echo "Killing existing tmux session: $session" - tmux kill-session -t $session - killall cardano-node 2>/dev/null || true - pkill -f cardano-db-sync 2>/dev/null || true - sleep 2 -fi - -echo "Creating tmux session: $session" -tmux new-session -d -s $session - -# Rename the window -tmux rename-window -t $session "profiling" - -# Split into 3 panes: -# 0: cardano-node -# 1: cardano-db-sync -# 2: monitoring -tmux split-window -h -t $session -tmux split-window -v -t $session:0.1 - -# Pane 0: Cardano Node -echo "Setting up cardano-node (pane 0)..." -tmux send-keys -t $session:0.0 "cd $CARDANO_NODE_DIR/" 'C-m' -tmux send-keys -t $session:0.0 "echo 'Starting cardano-node...'" 'C-m' -tmux send-keys -t $session:0.0 "cardano-node run --config $TESTNET_DIR/config.json --database-path $TESTNET_DIR/db/ --socket-path $TESTNET_DIR/db/node.socket --host-addr 0.0.0.0 --port 1337 --topology $TESTNET_DIR/topology.json" 'C-m' - -# Wait for node to start -sleep 5 - -# Pane 1: Cardano DB-Sync with profiling -echo "Setting up cardano-db-sync with profiling (pane 1)..." -tmux send-keys -t $session:0.1 "cd $CARDANO_DB_SYNC_DIR/" 'C-m' -tmux send-keys -t $session:0.1 "export PGPASSFILE=$PGPASS_FILE" 'C-m' -sleep 2 -tmux send-keys -t $session:0.1 "PGPASSFILE=$PGPASS_FILE $dbsync --config $TESTNET_DIR/db-sync-config.json --socket-path $TESTNET_DIR/db/node.socket --state-dir $TESTNET_DIR/ledger-state --schema-dir $CARDANO_DB_SYNC_DIR/schema/" 'C-m' - -# Pane 2: Monitoring and instructions -echo "Setting up monitoring pane (pane 2)..." -tmux send-keys -t $session:0.2 "cd $CARDANO_DB_SYNC_DIR/" 'C-m' - -# Start memory monitoring -tmux send-keys -t $session:0.2 "$SCRIPT_DIR/monitor-dbsync-memory.sh $MEMORY_LOG 300 2>&1 | tee -a $LOG_DIR/monitor.log" 'C-m' - -# Set pane sizes -tmux resize-pane -t $session:0.0 -x 50% -tmux resize-pane -t $session:0.1 -x 50% - -tmux -CC attach-session -t $session diff --git a/monitoring/README.md b/monitoring/README.md new file mode 100644 index 000000000..16c4d9ee7 --- /dev/null +++ b/monitoring/README.md @@ -0,0 +1,77 @@ +# Cardano DB Sync - Monitoring + +Monitor cardano-db-sync performance with Telegraf, Prometheus, and Grafana. + +## Quick Start + +### 1. Install Dependencies + +**macOS:** +```bash +brew install zellij telegraf prometheus grafana +``` + +**Linux (Ubuntu/Debian):** +```bash +sudo apt-get update +sudo apt-get install zellij telegraf prometheus grafana +``` + +**Linux (Fedora/RHEL/CentOS):** +```bash +sudo yum install zellij telegraf prometheus grafana +``` + +Note: You may need to add package repositories first. See [Telegraf](https://docs.influxdata.com/telegraf/latest/install/), [Grafana](https://grafana.com/docs/grafana/latest/setup-grafana/installation/) docs for details. + +### 2. Configure PostgreSQL + +Enable `pg_stat_statements` in `postgresql.conf`: +```ini +shared_preload_libraries = 'pg_stat_statements' +``` + +Restart PostgreSQL and run setup: +```bash +brew services restart postgresql@14 # macOS +sudo systemctl restart postgresql # Linux + +psql -U postgres -d cexplorer -f monitoring/scripts/grant-telegraf-permissions.sql +``` + +### 3. Start Monitoring + +```bash +./scripts/run-everything-zellij.sh +``` + +This starts cardano-node, cardano-db-sync, Telegraf, and Prometheus in a Zellij session. + +### 4. Setup Grafana + +Open http://localhost:3000 (admin/admin) and run: +```bash +./monitoring/scripts/setup-grafana.sh +``` + +Or manually add data sources (Prometheus at `:9090`, PostgreSQL at `:5432`) and import `monitoring/grafana/dashboards/cardano-db-sync-complete.json`. + +## Troubleshooting + +Run the verification script: +```bash +./monitoring/scripts/verify-monitoring.sh +``` + +**Common fixes:** +- Telegraf can't connect: Check `telegraf_monitor` user exists and `pg_hba.conf` allows local connections +- Prometheus targets down: Verify services are running with `ps aux | grep -E 'telegraf|prometheus|cardano-db-sync'` +- Port conflicts: `lsof -ti:9090 | xargs kill` (or :9273, :8080, :3000) + +**Access:** +- Grafana: http://localhost:3000 +- Prometheus: http://localhost:9090 +- Metrics: http://localhost:9273/metrics (Telegraf), http://localhost:8080 (db-sync) + +**Resources:** +- [Telegraf](https://docs.influxdata.com/telegraf/latest/) | [Prometheus](https://prometheus.io/docs/) | [Grafana](https://grafana.com/docs/) diff --git a/dev-tools/monitoring/config/grafana-db-sync.json b/monitoring/config/grafana-db-sync.json similarity index 100% rename from dev-tools/monitoring/config/grafana-db-sync.json rename to monitoring/config/grafana-db-sync.json diff --git a/monitoring/config/prometheus-rules.yml b/monitoring/config/prometheus-rules.yml new file mode 100644 index 000000000..d14efaa5b --- /dev/null +++ b/monitoring/config/prometheus-rules.yml @@ -0,0 +1,23 @@ +groups: + - name: cardano_epoch_tracking + interval: 30s + rules: + # Record epoch duration whenever epoch number changes + # This creates a counter that increments with each epoch's duration + - record: cardano_db_sync_epoch_duration_history + expr: | + cardano_db_sync_db_epoch_sync_duration_seconds + * on() group_right() + (cardano_db_sync_db_epoch_sync_number != bool 0) + labels: + epoch: "{{cardano_db_sync_db_epoch_sync_number}}" + + # Track epoch completion rate (epochs per hour) + - record: cardano_db_sync_epochs_per_hour + expr: | + rate(cardano_db_sync_db_epoch_sync_number[1h]) * 3600 + + # Average epoch duration over last hour + - record: cardano_db_sync_avg_epoch_duration_1h + expr: | + avg_over_time(cardano_db_sync_db_epoch_sync_duration_seconds[1h]) diff --git a/monitoring/config/prometheus.yml b/monitoring/config/prometheus.yml new file mode 100644 index 000000000..8e373b3c3 --- /dev/null +++ b/monitoring/config/prometheus.yml @@ -0,0 +1,40 @@ +# Prometheus Configuration for Cardano DB Sync Monitoring + +global: + scrape_interval: 15s + evaluation_interval: 15s + external_labels: + monitor: 'cardano-db-sync-dev' + +# Scrape configurations +scrape_configs: + # Cardano DB Sync application metrics (from Haskell) + - job_name: 'cardano-db-sync' + static_configs: + - targets: ['localhost:8080'] + labels: + instance: 'cardano-db-sync' + environment: 'development' + component: 'application' + scrape_interval: 10s + metrics_path: '/' # Note: metrics are served at root path, not /metrics + # This endpoint exposes: + # - Sync progress metrics (block height, slot height, epoch) + # - Database operation metrics (transaction duration, insert performance) + # - Queue metrics (depth, processing time) + # - Rollback metrics + + # Telegraf - System + PostgreSQL + Process metrics (replaces postgres_exporter + node_exporter) + - job_name: 'telegraf' + static_configs: + - targets: ['localhost:9273'] + labels: + instance: 'macos-dev' + environment: 'development' + component: 'telegraf' + scrape_interval: 10s + # This endpoint exposes: + # - macOS system metrics (CPU, RAM, disk I/O, network) + # - PostgreSQL database metrics (connections, transactions, cache hit ratio) + # - Process-specific metrics (cardano-db-sync and postgres processes) + # - Custom cardano database queries (block count, tx count, table sizes) diff --git a/monitoring/config/telegraf.conf b/monitoring/config/telegraf.conf new file mode 100644 index 000000000..78e50955e --- /dev/null +++ b/monitoring/config/telegraf.conf @@ -0,0 +1,288 @@ +# Telegraf Configuration for Cardano DB Sync Monitoring +# This configuration collects system, PostgreSQL, and process metrics +# and exposes them in Prometheus format at :9273/metrics + +############################################################################### +# GLOBAL AGENT CONFIG # +############################################################################### + +[agent] + ## Default data collection interval + interval = "10s" + + ## Rounds collection interval to 'interval' + round_interval = true + + ## Telegraf will send metrics to outputs in batches + metric_batch_size = 1000 + + ## Maximum number of unwritten metrics per output + metric_buffer_limit = 10000 + + ## Flush interval for all outputs + flush_interval = "10s" + + ## Override hostname (auto-detected if not specified) + # hostname = "cardano-db-sync-dev" + + ## Log only error level messages + quiet = false + logfile = "/tmp/telegraf.log" + +############################################################################### +# OUTPUT PLUGINS # +############################################################################### + +# Expose metrics in Prometheus format +[[outputs.prometheus_client]] + ## Address to listen on + listen = ":9273" + + ## Metric endpoint path + path = "/metrics" + + ## Expiration interval for metrics that haven't been updated + expiration_interval = "60s" + + ## Export metric collection timestamp + export_timestamp = true + + ## Metric type override + # This ensures histograms and summaries are properly exported + collectors_exclude = ["gocollector", "process"] + +############################################################################### +# SYSTEM METRICS (macOS) # +############################################################################### + +# CPU usage metrics +[[inputs.cpu]] + ## Whether to report per-cpu stats or not + percpu = true + + ## Whether to report total system cpu stats or not + totalcpu = true + + ## If true, collect raw CPU time metrics + collect_cpu_time = false + + ## If true, compute and report the sum of all non-idle CPU states + report_active = true + + [inputs.cpu.tags] + source = "telegraf" + component = "system" + +# Memory usage metrics +[[inputs.mem]] + [inputs.mem.tags] + source = "telegraf" + component = "system" + +# Disk usage metrics +[[inputs.disk]] + ## Ignore mount points by filesystem type + ignore_fs = ["tmpfs", "devtmpfs", "devfs", "iso9660", "overlay", "aufs", "squashfs"] + + [inputs.disk.tags] + source = "telegraf" + component = "system" + +# Disk I/O metrics +[[inputs.diskio]] + ## Devices to collect stats for + ## If empty, stats for all devices are collected + # devices = ["disk0", "disk1"] + + [inputs.diskio.tags] + source = "telegraf" + component = "system" + +# Network interface metrics +[[inputs.net]] + ## By default, telegraf gathers stats from any up interface (excluding loopback) + ## On macOS, en0 is typically WiFi, en1 might be Ethernet + interfaces = ["en*"] + + [inputs.net.tags] + source = "telegraf" + component = "system" + +# Network protocol statistics +[[inputs.netstat]] + [inputs.netstat.tags] + source = "telegraf" + component = "system" + +# System-wide statistics +[[inputs.system]] + ## Collect additional metrics + # Uncomment to collect more metrics (may require elevated permissions) + # fielddrop = ["uptime_format"] + + [inputs.system.tags] + source = "telegraf" + component = "system" + +# Process statistics (overall) +[[inputs.processes]] + ## Collect per-process stats + # use_sudo = false + + [inputs.processes.tags] + source = "telegraf" + component = "system" + +############################################################################### +# CARDANO PROCESS METRICS # +############################################################################### + +# Monitor cardano-db-sync process specifically +# Using 'exe' matcher to avoid duplicate counting and matching unwanted processes +[[inputs.procstat]] + ## Match by executable name (most precise) + exe = "cardano-db-sync" + + ## Name for this process in metrics + process_name = "cardano-db-sync" + + ## Fields to collect + ## Available: cpu_time, cpu_time_guest, cpu_time_guest_nice, cpu_time_idle, + ## cpu_time_iowait, cpu_time_irq, cpu_time_nice, cpu_time_softirq, + ## cpu_time_steal, cpu_time_system, cpu_time_user, cpu_usage, + ## memory_data, memory_locked, memory_rss, memory_stack, memory_swap, + ## memory_vms, num_fds, num_threads, pid + fieldinclude = [ + "cpu_usage", + "memory_rss", + "memory_vms", + "num_threads", + "num_fds" + ] + + [inputs.procstat.tags] + service = "cardano-db-sync" + component = "application" + source = "telegraf" + +# Monitor cardano-node process (separate from db-sync) +[[inputs.procstat]] + ## Match by executable name + exe = "cardano-node" + + ## Name for this process in metrics + process_name = "cardano-node" + + ## Fields to collect + fieldinclude = [ + "cpu_usage", + "memory_rss", + "memory_vms", + "num_threads", + "num_fds" + ] + + [inputs.procstat.tags] + service = "cardano-node" + component = "node" + source = "telegraf" + +############################################################################### +# POSTGRESQL METRICS # +############################################################################### + +# PostgreSQL standard metrics +[[inputs.postgresql]] + ## PostgreSQL connection string + ## Connection without password (using local trust auth) + address = "host=localhost user=telegraf_monitor dbname=postgres sslmode=disable" + + ## A list of databases to explicitly track. If not specified, metrics for all databases are gathered + databases = ["cexplorer"] + + ## Whether to use prepared statements when connecting to the database + ## This can help with performance on databases with many tables + prepared_statements = true + + [inputs.postgresql.tags] + service = "postgresql" + database = "cexplorer" + source = "telegraf" + +# PostgreSQL extended metrics with custom queries +[[inputs.postgresql_extensible]] + address = "host=localhost user=telegraf_monitor dbname=cexplorer sslmode=disable" + + ## Custom query: Block and transaction counts + [[inputs.postgresql_extensible.query]] + sqlquery = """SELECT COUNT(*) as block_count, COALESCE(MAX(block_no), 0) as max_block_no, COALESCE(MIN(block_no), 0) as min_block_no FROM block""" + min_version = 901 + withdbname = false + tagvalue = "" + measurement = "cardano_db_blocks" + + ## Custom query: Transaction count + [[inputs.postgresql_extensible.query]] + sqlquery = """SELECT COUNT(*) as tx_count FROM tx""" + min_version = 901 + withdbname = false + tagvalue = "" + measurement = "cardano_db_transactions" + + ## Custom query: Cache hit ratio + [[inputs.postgresql_extensible.query]] + sqlquery = """SELECT CASE WHEN (blks_hit + blks_read) = 0 THEN 100.0 ELSE ROUND(100.0 * blks_hit::numeric / (blks_hit + blks_read)::numeric, 2) END as cache_hit_ratio, blks_hit, blks_read FROM pg_stat_database WHERE datname = 'cexplorer'""" + min_version = 901 + withdbname = false + tagvalue = "" + measurement = "cardano_db_cache" + + ## Custom query: Database size + [[inputs.postgresql_extensible.query]] + sqlquery = """SELECT pg_database_size('cexplorer') as db_size_bytes""" + min_version = 901 + withdbname = false + tagvalue = "" + measurement = "cardano_db_size" + + ## Custom query: Top 5 largest tables + [[inputs.postgresql_extensible.query]] + sqlquery = """SELECT tablename as table_name, pg_total_relation_size('public.' || tablename) as table_size_bytes, n_live_tup as live_tuples, n_dead_tup as dead_tuples FROM pg_stat_user_tables WHERE schemaname = 'public' ORDER BY pg_total_relation_size('public.' || tablename) DESC LIMIT 5""" + min_version = 901 + withdbname = false + tagvalue = "table_name" + measurement = "cardano_db_table_sizes" + + ## Custom query: Current epoch number from epoch_sync_time + [[inputs.postgresql_extensible.query]] + sqlquery = """SELECT COALESCE(MAX(no), 0) as epoch_no FROM epoch_sync_time""" + min_version = 901 + withdbname = false + tagvalue = "" + measurement = "cardano_db_current_epoch" + + ## Custom query: pg_stat_statements - Top queries by total time + ## Note: This requires pg_stat_statements extension to be loaded + ## Run: CREATE EXTENSION IF NOT EXISTS pg_stat_statements; + [[inputs.postgresql_extensible.query]] + sqlquery = """SELECT CASE WHEN query LIKE 'INSERT INTO tx_out%' THEN 'INSERT_tx_out' WHEN query LIKE 'INSERT INTO tx_in%' THEN 'INSERT_tx_in' WHEN query LIKE 'INSERT INTO tx %' OR query LIKE 'INSERT INTO tx(%' THEN 'INSERT_tx' WHEN query LIKE 'INSERT INTO block%' THEN 'INSERT_block' WHEN query LIKE 'INSERT INTO ma_tx_out%' THEN 'INSERT_ma_tx_out' WHEN query LIKE 'INSERT INTO ma_tx_mint%' THEN 'INSERT_ma_tx_mint' WHEN query LIKE 'INSERT INTO tx_metadata%' THEN 'INSERT_tx_metadata' WHEN query LIKE 'INSERT INTO stake_address%' THEN 'INSERT_stake_address' WHEN query LIKE 'INSERT INTO pool_hash%' THEN 'INSERT_pool_hash' WHEN query LIKE 'INSERT INTO datum%' THEN 'INSERT_datum' WHEN query LIKE 'INSERT INTO redeemer%' THEN 'INSERT_redeemer' WHEN query LIKE 'INSERT INTO epoch_stake%' THEN 'INSERT_epoch_stake' WHEN query LIKE 'INSERT INTO reward%' THEN 'INSERT_reward' WHEN query LIKE 'INSERT INTO slot_leader%' THEN 'INSERT_slot_leader' WHEN query LIKE 'INSERT INTO multi_asset%' THEN 'INSERT_multi_asset' WHEN query LIKE 'INSERT INTO extra_key_witness%' THEN 'INSERT_extra_key_without' WHEN query LIKE 'INSERT INTO collateral_tx_in%' THEN 'INSERT_collateral_tx_in' WHEN query LIKE 'INSERT INTO collateral_tx_out%' THEN 'INSERT_collateral_tx_out' WHEN query LIKE 'INSERT INTO withdrawal%' THEN 'INSERT_withdrawal' WHEN query LIKE 'INSERT INTO reference_tx_in%' THEN 'INSERT_reference_tx_in' WHEN query LIKE 'INSERT INTO delegation%' THEN 'INSERT_delegation' WHEN query LIKE 'INSERT INTO stake_registration%' THEN 'INSERT_stake_registration' WHEN query LIKE 'INSERT INTO stake_deregistration%' THEN 'INSERT_stake_deregistration' WHEN query LIKE 'INSERT INTO script%' THEN 'INSERT_script' WHEN query LIKE 'INSERT INTO pool_stat%' THEN 'INSERT_pool_stat' WHEN query LIKE 'INSERT INTO pool_metadata_ref%' THEN 'INSERT_pool_metadata_ref' WHEN query LIKE 'INSERT INTO pool_update%' THEN 'INSERT_pool_update' WHEN query LIKE 'INSERT INTO pool_relay%' THEN 'INSERT_pool_relay' WHEN query LIKE 'INSERT INTO pool_owner%' THEN 'INSERT_pool_owner' WHEN query LIKE 'INSERT INTO treasury%' THEN 'INSERT_treasury' WHEN query LIKE 'INSERT INTO reserve%' THEN 'INSERT_reserve' WHEN query LIKE 'INSERT INTO epoch %' THEN 'INSERT_epoch' WHEN query LIKE 'INSERT INTO epoch(%' THEN 'INSERT_epoch' WHEN query LIKE 'INSERT INTO cost_model%' THEN 'INSERT_cost_model' WHEN query LIKE 'INSERT INTO epoch_sync_time%' THEN 'INSERT_epoch_sync_time' WHEN query LIKE 'INSERT INTO off_chain_pool%' THEN 'INSERT_off_chain_pool' WHEN query LIKE 'SELECT id FROM tx WHERE hash%' THEN 'SELECT_tx_by_hash' WHEN query LIKE 'SELECT id FROM script WHERE hash%' THEN 'SELECT_script_by_hash' WHEN query LIKE 'SELECT id FROM redeemer_data WHERE hash%' THEN 'SELECT_redeemer_data_by_hash' WHEN query LIKE 'SELECT%tx_out%' THEN 'SELECT_tx_out' WHEN query LIKE 'SELECT%tx_in%' THEN 'SELECT_tx_in' WHEN query LIKE 'SELECT%stake_address%' THEN 'SELECT_stake_address' WHEN query LIKE 'SELECT%pool_hash%' THEN 'SELECT_pool_hash' WHEN query LIKE 'SELECT%datum%' THEN 'SELECT_datum' WHEN query LIKE 'SELECT%multi_asset%' THEN 'SELECT_multi_asset' WHEN query LIKE 'SELECT%block%' THEN 'SELECT_block' ELSE 'OTHER' END as query_type, SUM(calls)::float as total_calls, SUM(total_exec_time)::float as total_time_ms, AVG(mean_exec_time)::float as avg_mean_time_ms FROM pg_stat_statements WHERE (query LIKE 'INSERT INTO%' OR query LIKE 'SELECT%') AND query NOT LIKE '%pg_stat_statements%' AND query NOT LIKE '%pg_catalog%' AND query NOT LIKE '%pg_stat_database%' AND query NOT LIKE '%pg_stat_bgwriter%' AND query NOT LIKE 'SELECT COUNT(%)%tx_count%' AND query NOT LIKE 'SELECT pg_database_size%' AND query NOT LIKE 'SELECT CASE WHEN%blks_hit%' AND query NOT LIKE 'SELECT setting%version%' AND query NOT LIKE 'SELECT COUNT(%)FROM epoch_stake%' AND query NOT LIKE 'SELECT COUNT(%)FROM reward%' AND query NOT LIKE 'SELECT migrate()%' AND query NOT LIKE 'SELECT id%voting_anchor%' GROUP BY query_type""" + min_version = 901 + withdbname = false + tagvalue = "query_type" + measurement = "pg_stat_statements_queries" + +# Monitor PostgreSQL process +[[inputs.procstat]] + ## Pattern to match process name (matches postgres server processes) + pattern = "postgres.*-D" + + ## Name for this process in metrics + process_name = "postgres" + + ## User to filter by (optional) + # user = "postgres" + + [inputs.procstat.tags] + service = "postgresql" + component = "database" + source = "telegraf" diff --git a/monitoring/grafana/dashboards/cardano-db-sync-complete.json b/monitoring/grafana/dashboards/cardano-db-sync-complete.json new file mode 100644 index 000000000..0ece46e91 --- /dev/null +++ b/monitoring/grafana/dashboards/cardano-db-sync-complete.json @@ -0,0 +1,1151 @@ +{ + "annotations": { + "list": [] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": null, + "links": [], + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 1, + "panels": [], + "title": "Cardano DB-Sync Overview", + "type": "row" + }, + { + "datasource": { + "type": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + } + }, + "gridPos": { + "h": 4, + "w": 12, + "x": 0, + "y": 1 + }, + "id": 2, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "values": false, + "calcs": ["lastNotNull"], + "fields": "" + }, + "textMode": "auto" + }, + "pluginVersion": "12.2.1", + "targets": [ + { + "datasource": { + "type": "prometheus" + }, + "expr": "cardano_db_sync_db_block_height", + "refId": "A" + } + ], + "title": "DB Block Height", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 5 + }, + { + "color": "red", + "value": 10 + } + ] + }, + "unit": "short" + } + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 12, + "y": 1 + }, + "id": 3, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "values": false, + "calcs": ["lastNotNull"], + "fields": "" + }, + "textMode": "auto" + }, + "pluginVersion": "12.2.1", + "targets": [ + { + "datasource": { + "type": "prometheus" + }, + "expr": "cardano_db_sync_db_queue_length", + "refId": "A" + } + ], + "title": "Queue Depth", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + }, + "unit": "short" + } + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 18, + "y": 1 + }, + "id": 4, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "values": false, + "calcs": ["lastNotNull"], + "fields": "" + }, + "textMode": "auto" + }, + "pluginVersion": "12.2.1", + "targets": [ + { + "datasource": { + "type": "prometheus" + }, + "expr": "cardano_db_current_epoch_epoch_no", + "refId": "A" + } + ], + "title": "Current Epoch", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus" + }, + "description": "Time taken to insert a block's data into the database. Lower is better. High values indicate database write bottlenecks.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Duration", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "tooltip": false, + "viz": false, + "legend": false + }, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 0.08 + }, + { + "color": "red", + "value": 0.15 + } + ] + }, + "unit": "s" + } + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 13 + }, + "id": 5, + "options": { + "legend": { + "calcs": ["mean", "lastNotNull", "max"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.2.1", + "targets": [ + { + "datasource": { + "type": "prometheus" + }, + "expr": "cardano_db_sync_insert_duration_seconds", + "legendFormat": "Insert Duration", + "refId": "A" + } + ], + "title": "Insert Duration (Bottleneck)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "CPU %", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "tooltip": false, + "viz": false, + "legend": false + }, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 50 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percent" + } + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 5 + }, + "id": 111, + "options": { + "legend": { + "calcs": ["lastNotNull"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.2.1", + "targets": [ + { + "datasource": { + "type": "prometheus" + }, + "expr": "max without(status) (procstat_cpu_usage{service=\"cardano-db-sync\",exe=\"cardano-db-sync\",source=\"telegraf\"})", + "legendFormat": "CPU Usage", + "refId": "A" + } + ], + "title": "DB-Sync CPU Usage", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Memory", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "tooltip": false, + "viz": false, + "legend": false + }, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 2147483648 + }, + { + "color": "red", + "value": 4294967296 + } + ] + }, + "unit": "bytes" + } + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 5 + }, + "id": 112, + "options": { + "legend": { + "calcs": ["lastNotNull"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.2.1", + "targets": [ + { + "datasource": { + "type": "prometheus" + }, + "expr": "max without(status) (procstat_memory_rss{service=\"cardano-db-sync\",exe=\"cardano-db-sync\",source=\"telegraf\"})", + "legendFormat": "RSS Memory", + "refId": "A" + } + ], + "title": "DB-Sync Memory Usage", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus" + }, + "description": "Blocks processed per second by cardano-db-sync. Higher is better. Shows throughput of block ingestion from the node.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "tooltip": false, + "viz": false, + "legend": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "ops" + } + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 13 + }, + "id": 6, + "options": { + "legend": { + "calcs": ["mean", "lastNotNull", "max"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.2.1", + "targets": [ + { + "datasource": { + "type": "prometheus" + }, + "expr": "cardano_db_sync_blocks_per_second", + "legendFormat": "Blocks/Second", + "refId": "A" + } + ], + "title": "Block Processing Rate", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 21 + }, + "id": 100, + "panels": [], + "title": "Epoch Performance", + "type": "row" + }, + { + "datasource": { + "type": "postgres", + "uid": "postgres-cardanodb" + }, + "description": "Epoch sync duration from database with accurate historical timestamps. Shows epoch number in tooltip. Each bar represents one epoch completion.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Duration", + "axisPlacement": "auto", + "barAlignment": -1, + "drawStyle": "bars", + "fillOpacity": 100, + "gradientMode": "scheme", + "hideFrom": { + "tooltip": false, + "viz": false, + "legend": false + }, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "area" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 600 + }, + { + "color": "orange", + "value": 1200 + }, + { + "color": "red", + "value": 1800 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 22 + }, + "id": 110, + "options": { + "legend": { + "calcs": ["lastNotNull"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.2.1", + "targets": [ + { + "datasource": { + "type": "postgres", + "uid": "postgres-cardanodb" + }, + "editorMode": "code", + "format": "time_series", + "rawQuery": true, + "rawSql": "SELECT \n est.synced_at - (est.seconds || ' seconds')::interval as time,\n est.seconds as value,\n LPAD(est.no::text, 3, '0') as metric\nFROM epoch_sync_time est\nWHERE est.synced_at IS NOT NULL\n AND est.synced_at BETWEEN $__timeFrom() AND $__timeTo()\nORDER BY time;", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + } + ], + "title": "Epoch Duration History", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 31 + }, + "id": 8, + "panels": [], + "title": "Cache Hit Rates", + "type": "row" + }, + { + "datasource": { + "type": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "tooltip": false, + "viz": false, + "legend": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "max": 1, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "percentunit" + } + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 32 + }, + "id": 9, + "options": { + "legend": { + "calcs": ["mean", "lastNotNull"], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.2.1", + "targets": [ + { + "datasource": { + "type": "prometheus" + }, + "expr": "cardano_db_sync_cache_stake_hit_rate", + "legendFormat": "Stake Cache", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus" + }, + "expr": "cardano_db_sync_cache_pools_hit_rate", + "legendFormat": "Pools Cache", + "refId": "B" + }, + { + "datasource": { + "type": "prometheus" + }, + "expr": "cardano_db_sync_cache_datum_hit_rate", + "legendFormat": "Datum Cache", + "refId": "C" + }, + { + "datasource": { + "type": "prometheus" + }, + "expr": "cardano_db_sync_cache_multi_assets_hit_rate", + "legendFormat": "Multi Assets Cache", + "refId": "D" + }, + { + "datasource": { + "type": "prometheus" + }, + "expr": "cardano_db_sync_cache_prev_block_hit_rate", + "legendFormat": "Prev Block Cache", + "refId": "E" + }, + { + "datasource": { + "type": "prometheus" + }, + "expr": "cardano_db_sync_cache_address_hit_rate", + "legendFormat": "Address Cache", + "refId": "F" + }, + { + "datasource": { + "type": "prometheus" + }, + "expr": "cardano_db_sync_cache_tx_ids_hit_rate", + "legendFormat": "TX IDs Cache", + "refId": "G" + } + ], + "title": "Cache Hit Rates (Application Level)", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 40 + }, + "id": 200, + "panels": [], + "title": "Query Performance Bottlenecks", + "type": "row" + }, + { + "datasource": { + "type": "prometheus" + }, + "description": "Total execution time by query type. Higher values indicate bottlenecks.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Total Time (ms)", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 30, + "gradientMode": "hue", + "hideFrom": { + "tooltip": false, + "viz": false, + "legend": false + }, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 1000 + }, + { + "color": "red", + "value": 5000 + } + ] + }, + "unit": "ms" + } + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 41 + }, + "id": 201, + "options": { + "legend": { + "calcs": ["mean", "lastNotNull", "max"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Mean", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "12.2.1", + "targets": [ + { + "datasource": { + "type": "prometheus" + }, + "expr": "rate(pg_stat_statements_queries_total_time_ms[5m])", + "legendFormat": "{{query_type}}", + "refId": "A" + } + ], + "title": "Query Execution Time by Type (Rate)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus" + }, + "description": "Average execution time per query type. Shows which queries are slowest.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Avg Time (ms)", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "tooltip": false, + "viz": false, + "legend": false + }, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 50 + }, + { + "color": "red", + "value": 100 + } + ] + }, + "unit": "ms" + } + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 41 + }, + "id": 202, + "options": { + "legend": { + "calcs": ["mean", "lastNotNull", "max"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Mean", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "12.2.1", + "targets": [ + { + "datasource": { + "type": "prometheus" + }, + "expr": "pg_stat_statements_queries_avg_mean_time_ms", + "legendFormat": "{{query_type}}", + "refId": "A" + } + ], + "title": "Average Query Execution Time", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus" + }, + "description": "Number of calls per query type over time. Shows query frequency.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Calls/sec", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "tooltip": false, + "viz": false, + "legend": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "cps" + } + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 50 + }, + "id": 203, + "options": { + "legend": { + "calcs": ["mean", "lastNotNull"], + "displayMode": "table", + "placement": "right", + "showLegend": true, + "sortBy": "Mean", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "12.2.1", + "targets": [ + { + "datasource": { + "type": "prometheus" + }, + "expr": "rate(pg_stat_statements_queries_total_calls[5m])", + "legendFormat": "{{query_type}}", + "refId": "A" + } + ], + "title": "Query Call Rate by Type", + "type": "timeseries" + } + ], + "schemaVersion": 39, + "tags": ["cardano", "db-sync", "performance"], + "templating": { + "list": [] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": {}, + "timezone": "browser", + "title": "Cardano DB-Sync - Complete Performance Dashboard", + "uid": "cardano-db-sync-complete", + "version": 1 +} diff --git a/monitoring/grafana/datasources/prometheus.yml b/monitoring/grafana/datasources/prometheus.yml new file mode 100644 index 000000000..cc1b32549 --- /dev/null +++ b/monitoring/grafana/datasources/prometheus.yml @@ -0,0 +1,13 @@ +apiVersion: 1 + +datasources: + - name: Prometheus + type: prometheus + access: proxy + url: http://localhost:9090 + isDefault: true + editable: true + jsonData: + timeInterval: 10s + queryTimeout: 60s + httpMethod: POST diff --git a/monitoring/scripts/grant-telegraf-permissions.sql b/monitoring/scripts/grant-telegraf-permissions.sql new file mode 100644 index 000000000..4d402f708 --- /dev/null +++ b/monitoring/scripts/grant-telegraf-permissions.sql @@ -0,0 +1,47 @@ +-- Setup telegraf_monitor user with proper permissions +-- Run this as a database superuser or the owner of cexplorer database + +-- Create user if it doesn't exist +DO $$ +BEGIN + IF NOT EXISTS (SELECT FROM pg_catalog.pg_roles WHERE rolname = 'telegraf_monitor') THEN + CREATE USER telegraf_monitor; + RAISE NOTICE 'Created user: telegraf_monitor'; + ELSE + RAISE NOTICE 'User telegraf_monitor already exists'; + END IF; +END +$$; + +-- Enable pg_stat_statements extension for query performance tracking +-- This requires shared_preload_libraries='pg_stat_statements' in postgresql.conf +-- If you get an error, add to postgresql.conf and restart PostgreSQL +CREATE EXTENSION IF NOT EXISTS pg_stat_statements; + +-- Grant pg_monitor role (gives access to pg_stat_* views) +GRANT pg_monitor TO telegraf_monitor; + +-- Grant access to pg_stat_statements view +GRANT SELECT ON pg_stat_statements TO telegraf_monitor; + +-- Grant access to all tables in public schema (cardano-db-sync tables) +GRANT SELECT ON ALL TABLES IN SCHEMA public TO telegraf_monitor; + +-- Grant access to all sequences (needed for some table metadata queries) +GRANT SELECT ON ALL SEQUENCES IN SCHEMA public TO telegraf_monitor; + +-- Make the grants permanent for future tables +ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT SELECT ON TABLES TO telegraf_monitor; +ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT SELECT ON SEQUENCES TO telegraf_monitor; + +-- Verify permissions +\echo 'Setup complete!' +\echo 'Checking sample permissions for telegraf_monitor...' +SELECT + schemaname, + tablename, + has_table_privilege('telegraf_monitor', schemaname||'.'||tablename, 'SELECT') as can_select +FROM pg_tables +WHERE schemaname = 'public' + AND tablename IN ('block', 'tx', 'epoch_sync_time', 'tx_out', 'tx_in') +ORDER BY tablename; diff --git a/monitoring/scripts/setup-grafana.sh b/monitoring/scripts/setup-grafana.sh new file mode 100755 index 000000000..6d4d3846b --- /dev/null +++ b/monitoring/scripts/setup-grafana.sh @@ -0,0 +1,85 @@ +#!/bin/bash + +# Setup Grafana datasources and dashboards for Cardano DB-Sync monitoring +# Requires Grafana to be running on localhost:3000 + +GRAFANA_URL="http://localhost:3000" +GRAFANA_USER="admin" +GRAFANA_PASS="pass" +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +MONITORING_DIR="$(dirname "$SCRIPT_DIR")" + +echo "=== Setting up Grafana for Cardano DB-Sync Monitoring ===" +echo "" + +# Check if Grafana is running +echo "Checking if Grafana is accessible at $GRAFANA_URL..." +if ! curl -s "$GRAFANA_URL/api/health" > /dev/null 2>&1; then + echo "ERROR: Grafana is not accessible at $GRAFANA_URL" + echo "Please make sure Grafana is running." + exit 1 +fi +echo "βœ“ Grafana is running" +echo "" + +# Create Prometheus datasource +echo "Setting up Prometheus datasource..." +curl -s -X POST \ + -H "Content-Type: application/json" \ + -u "$GRAFANA_USER:$GRAFANA_PASS" \ + -d '{ + "name": "Prometheus", + "type": "prometheus", + "url": "http://localhost:9090", + "access": "proxy", + "isDefault": true, + "jsonData": { + "timeInterval": "10s", + "queryTimeout": "60s", + "httpMethod": "POST" + } + }' \ + "$GRAFANA_URL/api/datasources" > /dev/null 2>&1 \ +&& echo "βœ“ Prometheus datasource created (or already exists)" \ +|| echo "βœ“ Prometheus datasource already exists (ignoring error)" +echo "" + +# Import dashboard +echo "Importing Cardano DB-Sync Complete dashboard..." +DASHBOARD_JSON="$MONITORING_DIR/grafana/dashboards/cardano-db-sync-complete.json" + +if [ ! -f "$DASHBOARD_JSON" ]; then + echo "ERROR: Dashboard file not found: $DASHBOARD_JSON" + exit 1 +fi + +# Wrap dashboard JSON in the required format +DASHBOARD_PAYLOAD=$(jq '{dashboard: ., overwrite: true, message: "Imported via setup script"}' < "$DASHBOARD_JSON") + +curl -s -X POST \ + -H "Content-Type: application/json" \ + -u "$GRAFANA_USER:$GRAFANA_PASS" \ + -d "$DASHBOARD_PAYLOAD" \ + "$GRAFANA_URL/api/dashboards/db" | jq -r '.status, .url' | while read -r line; do + if [ "$line" = "success" ]; then + echo "βœ“ Dashboard imported successfully" + elif [[ "$line" == /d/* ]]; then + echo "βœ“ Dashboard URL: $GRAFANA_URL$line" + fi +done + +echo "" +echo "=== Setup Complete ===" +echo "" +echo "Access your dashboard at:" +echo " $GRAFANA_URL (navigate to Dashboards to find the imported dashboard)" +echo "" +echo "Default credentials:" +echo " Username: admin" +echo " Password: pass" +echo "" +echo "Make sure the following services are running:" +echo " - Prometheus on http://localhost:9090" +echo " - Telegraf collecting metrics" +echo " - cardano-db-sync with metrics enabled" +echo "" diff --git a/monitoring/scripts/verify-monitoring.sh b/monitoring/scripts/verify-monitoring.sh new file mode 100755 index 000000000..174b9d8d8 --- /dev/null +++ b/monitoring/scripts/verify-monitoring.sh @@ -0,0 +1,136 @@ +#!/bin/bash +# Verify monitoring setup for Cardano DB Sync + +echo "=== Cardano DB Sync Monitoring Verification ===" +echo "" + +# Colors +GREEN='\033[0;32m' +RED='\033[0;31m' +YELLOW='\033[1;33m' +NC='\033[0m' # No Color + +# Check PostgreSQL connection +echo "1. Checking PostgreSQL connection..." +if psql -U postgres -d cexplorer -c "SELECT 1" > /dev/null 2>&1; then + echo -e "${GREEN}βœ“${NC} PostgreSQL is accessible" +else + echo -e "${RED}βœ—${NC} Cannot connect to PostgreSQL" + exit 1 +fi + +# Check pg_stat_statements extension +echo "" +echo "2. Checking pg_stat_statements extension..." +PG_STAT=$(psql -U postgres -d cexplorer -tAc "SELECT COUNT(*) FROM pg_extension WHERE extname = 'pg_stat_statements'") +if [ "$PG_STAT" = "1" ]; then + echo -e "${GREEN}βœ“${NC} pg_stat_statements extension is enabled" + + # Check if it's collecting data + STMT_COUNT=$(psql -U postgres -d cexplorer -tAc "SELECT COUNT(*) FROM pg_stat_statements" 2>/dev/null) + if [ -n "$STMT_COUNT" ] && [ "$STMT_COUNT" -gt "0" ]; then + echo -e "${GREEN}βœ“${NC} pg_stat_statements is collecting data ($STMT_COUNT queries tracked)" + else + echo -e "${YELLOW}⚠${NC} pg_stat_statements is enabled but not collecting data yet" + echo " Run some queries or wait for cardano-db-sync to execute queries" + fi +else + echo -e "${RED}βœ—${NC} pg_stat_statements extension is NOT enabled" + echo " Run: psql -U postgres -d cexplorer -c 'CREATE EXTENSION pg_stat_statements;'" + echo " Note: Requires 'shared_preload_libraries = pg_stat_statements' in postgresql.conf" +fi + +# Check telegraf_monitor user +echo "" +echo "3. Checking telegraf_monitor user..." +TELEGRAF_USER=$(psql -U postgres -d cexplorer -tAc "SELECT COUNT(*) FROM pg_roles WHERE rolname = 'telegraf_monitor'") +if [ "$TELEGRAF_USER" = "1" ]; then + echo -e "${GREEN}βœ“${NC} telegraf_monitor user exists" + + # Check permissions + HAS_PERMS=$(psql -U postgres -d cexplorer -tAc "SELECT has_table_privilege('telegraf_monitor', 'block', 'SELECT')") + if [ "$HAS_PERMS" = "t" ]; then + echo -e "${GREEN}βœ“${NC} telegraf_monitor has SELECT permissions on tables" + else + echo -e "${RED}βœ—${NC} telegraf_monitor missing SELECT permissions" + fi +else + echo -e "${RED}βœ—${NC} telegraf_monitor user does NOT exist" + echo " Run: psql -U postgres -d cexplorer -f monitoring/scripts/grant-telegraf-permissions.sql" +fi + +# Check if Telegraf is running +echo "" +echo "4. Checking if Telegraf is running..." +if pgrep -x "telegraf" > /dev/null; then + echo -e "${GREEN}βœ“${NC} Telegraf is running" + + # Check if metrics endpoint is accessible + if curl -s http://localhost:9273/metrics > /dev/null 2>&1; then + echo -e "${GREEN}βœ“${NC} Telegraf metrics endpoint accessible (http://localhost:9273/metrics)" + + # Check for pg_stat_statements metrics + if curl -s http://localhost:9273/metrics | grep -q "pg_stat_statements_queries"; then + echo -e "${GREEN}βœ“${NC} Query-level metrics are being collected" + else + echo -e "${YELLOW}⚠${NC} Query-level metrics not found in Telegraf output" + echo " This is normal if pg_stat_statements has no data yet" + fi + else + echo -e "${RED}βœ—${NC} Telegraf metrics endpoint not accessible" + fi +else + echo -e "${RED}βœ—${NC} Telegraf is NOT running" + echo " Start with: telegraf --config monitoring/config/telegraf.conf" +fi + +# Check if Prometheus is running +echo "" +echo "5. Checking if Prometheus is running..." +if pgrep -x "prometheus" > /dev/null; then + echo -e "${GREEN}βœ“${NC} Prometheus is running" + + # Check if Prometheus can scrape Telegraf + if curl -s http://localhost:9090/-/healthy > /dev/null 2>&1; then + echo -e "${GREEN}βœ“${NC} Prometheus web UI accessible (http://localhost:9090)" + else + echo -e "${RED}βœ—${NC} Prometheus web UI not accessible" + fi +else + echo -e "${RED}βœ—${NC} Prometheus is NOT running" +fi + +# Check if Grafana is running +echo "" +echo "6. Checking if Grafana is running..." +if curl -s http://localhost:3000/api/health > /dev/null 2>&1; then + echo -e "${GREEN}βœ“${NC} Grafana is running (http://localhost:3000)" +else + echo -e "${YELLOW}⚠${NC} Grafana is not accessible at http://localhost:3000" + echo " Start with: brew services start grafana (macOS) or sudo systemctl start grafana-server (Linux)" +fi + +# Check cardano-db-sync metrics +echo "" +echo "7. Checking cardano-db-sync metrics..." +if pgrep -f "cardano-db-sync" > /dev/null; then + echo -e "${GREEN}βœ“${NC} cardano-db-sync is running" + + if curl -s http://localhost:8080 > /dev/null 2>&1; then + echo -e "${GREEN}βœ“${NC} cardano-db-sync metrics endpoint accessible" + else + echo -e "${YELLOW}⚠${NC} cardano-db-sync metrics endpoint not accessible" + fi +else + echo -e "${YELLOW}⚠${NC} cardano-db-sync is not running" +fi + +echo "" +echo "=== Verification Complete ===" +echo "" +echo "Next steps:" +echo "1. If pg_stat_statements is missing, edit postgresql.conf and add:" +echo " shared_preload_libraries = 'pg_stat_statements'" +echo "2. Restart PostgreSQL: brew services restart postgresql@14" +echo "3. Run: psql -U postgres -d cexplorer -f monitoring/scripts/grant-telegraf-permissions.sql" +echo "4. Import dashboard: monitoring/grafana/dashboards/cardano-db-sync-complete.json" diff --git a/schema/migration-2-0046-20260316.sql b/schema/migration-2-0046-20260316.sql new file mode 100644 index 000000000..8f41066a2 --- /dev/null +++ b/schema/migration-2-0046-20260316.sql @@ -0,0 +1,20 @@ +-- Add synced_at timestamp to epoch_sync_time table +-- This tracks WHEN db-sync processed the epoch (not blockchain time) + +CREATE FUNCTION migrate() RETURNS void AS $$ +DECLARE + next_version int ; +BEGIN + SELECT stage_two + 1 INTO next_version FROM schema_version ; + IF next_version = 46 THEN + EXECUTE 'ALTER TABLE "epoch_sync_time" ADD COLUMN "synced_at" TIMESTAMP WITH TIME ZONE NULL' ; + + UPDATE schema_version SET stage_two = next_version ; + RAISE NOTICE 'DB has been migrated to stage_two version %', next_version ; + END IF ; +END ; +$$ LANGUAGE plpgsql ; + +SELECT migrate() ; + +DROP FUNCTION migrate() ; diff --git a/scripts/run-everything-zellij.sh b/scripts/run-everything-zellij.sh new file mode 100755 index 000000000..a2945daf5 --- /dev/null +++ b/scripts/run-everything-zellij.sh @@ -0,0 +1,87 @@ +#!/bin/bash + +set -e # Exit on error + +# Determine project root directory (parent of scripts/) +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +CARDANO_DB_SYNC_DIR="$(cd "$SCRIPT_DIR/.." && pwd)" + +# Set default paths (can be overridden by environment variables) +HOMEIOG="${HOMEIOG:-$HOME/Code/IOG}" +CARDANO_NODE_DIR="${CARDANO_NODE_DIR:-$HOMEIOG/cardano-node}" +TESTNET_DIR="${TESTNET_DIR:-$HOMEIOG/testnet}" + +# Verify required directories exist +if [ ! -d "$CARDANO_NODE_DIR" ]; then + echo "ERROR: cardano-node directory not found at: $CARDANO_NODE_DIR" + echo "Set CARDANO_NODE_DIR environment variable or update HOMEIOG path" + exit 1 +fi + +if [ ! -d "$TESTNET_DIR" ]; then + echo "ERROR: testnet directory not found at: $TESTNET_DIR" + echo "Set TESTNET_DIR environment variable or update HOMEIOG path" + exit 1 +fi + +# Find cardano-db-sync binary +dbsync="$(find "$CARDANO_DB_SYNC_DIR"/ -name cardano-db-sync -type f | head -1)" + +if [ -z "$dbsync" ]; then + echo "ERROR: cardano-db-sync binary not found in: $CARDANO_DB_SYNC_DIR" + echo "Build the project first with: cabal build cardano-db-sync" + exit 1 +fi + +echo "Using cardano-db-sync binary: $dbsync" + +# Kill any previous instances +echo "Cleaning up previous instances..." +pkill -f cardano-node || true +pkill -f cardano-db-sync || true +pkill -f prometheus || true +pkill -f telegraf || true +sleep 1 + +# Setup database permissions for Telegraf +echo "Setting up database permissions for Telegraf..." +psql -d cexplorer -f "$CARDANO_DB_SYNC_DIR/monitoring/scripts/grant-telegraf-permissions.sql" > /dev/null 2>&1 || echo "Warning: Could not grant permissions (database may not be ready)" +sleep 1 + +echo "Cleanup complete. Starting services..." + +# Generate the layout file with actual paths +# Layout: +# β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”¬β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” +# β”‚ cardano-nodeβ”‚ cardano-db- β”‚ +# β”‚ β”‚ sync β”‚ +# β”œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€ +# β”‚ telegraf β”‚ prometheus β”‚ +# β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”΄β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ +zellij --layout <(cat <