diff --git a/.github/workflows/db-backup.yml b/.github/workflows/db-backup.yml new file mode 100644 index 0000000..b746260 --- /dev/null +++ b/.github/workflows/db-backup.yml @@ -0,0 +1,69 @@ +name: Nightly DB backup + +# Logical dumps of the per-network Postgres (Neon/direct URL). This repo is +# public, so Actions artifacts are world-readable — only age ciphertext is +# uploaded. See docs/backup-restore.md. +on: + schedule: + # 03:00 UTC — one hour before the load test (04:00), so dump I/O and the + # 5k-RPS flood do not overlap. + - cron: '0 3 * * *' + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: db-backup + cancel-in-progress: false + +jobs: + dump: + name: pg_dump (${{ matrix.network }}) + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + network: [mainnet, testnet] + env: + DATABASE_URL_MAINNET: ${{ secrets.DATABASE_URL_MAINNET }} + DATABASE_URL_TESTNET: ${{ secrets.DATABASE_URL_TESTNET }} + AGE_RECIPIENT: ${{ secrets.AGE_RECIPIENT }} + steps: + - uses: actions/checkout@v4 + + - name: Install postgresql-client and age + run: sudo apt-get update && sudo apt-get install -y postgresql-client age + + - name: Dump ${{ matrix.network }} + env: + NETWORK: ${{ matrix.network }} + run: | + set -euo pipefail + if [ "$NETWORK" = "mainnet" ]; then + DATABASE_URL="${DATABASE_URL_MAINNET:-}" + else + DATABASE_URL="${DATABASE_URL_TESTNET:-}" + fi + if [ -z "$DATABASE_URL" ]; then + echo "SKIP ${NETWORK}: secret not configured" + exit 0 + fi + if [ -z "${AGE_RECIPIENT:-}" ]; then + echo "AGE_RECIPIENT is required; refusing an unencrypted dump" + exit 1 + fi + DATE_UTC=$(date -u +%Y%m%d) + OUT="lens-${NETWORK}-${DATE_UTC}.dump" + pg_dump --format=custom --no-owner --no-acl --dbname="$DATABASE_URL" --file="$OUT" + gzip -n "$OUT" + age -r "$AGE_RECIPIENT" -o "${OUT}.gz.age" "${OUT}.gz" + rm -f "$OUT" "${OUT}.gz" + + - uses: actions/upload-artifact@v4 + if: success() + with: + name: lens-${{ matrix.network }}-${{ github.run_id }} + path: lens-${{ matrix.network }}-*.dump.gz.age + retention-days: 14 + if-no-files-found: ignore diff --git a/docs/DUAL_NETWORK.md b/docs/DUAL_NETWORK.md index 9ef7632..6bf9da0 100644 --- a/docs/DUAL_NETWORK.md +++ b/docs/DUAL_NETWORK.md @@ -43,7 +43,7 @@ Dependencies: **#113 → #115 → #116 → #117**; #114 before the network selec | [#116](../../issues/116) | Per-network Soroswap/Reflector/Aquarius/token-list | #113 | | [#117](../../issues/117) | Launch ingesters per network | #113–#116 | | [#118](../../issues/118) | Network selector on routes + per-request x402 | #114 | -| [#119](../../issues/119) | Nightly `pg_dump` backup + restore runbook | — | +| [#119](../../issues/119) | Nightly `pg_dump` backup + [restore runbook](backup-restore.md) | — | | [#120](../../issues/120) | Mainnet deploy guide | the rest | ## Ops (Render + UptimeRobot + external Postgres) @@ -53,5 +53,6 @@ Dependencies: **#113 → #115 → #116 → #117**; #114 before the network selec mainnet always-on, testnet on-demand / a second account / a paid instance. - **Database:** use Neon or another managed Postgres. **Do not** use Render's free Postgres — it is **deleted after 90 days**. One DB per network. -- **Durability fallback:** nightly `pg_dump` (#119) for fast restore; Lens is an - aggregator, so the DB is also re-derivable by re-ingesting from chain. +- **Durability fallback:** nightly `pg_dump` (#119) for fast restore; see + [backup-restore.md](backup-restore.md). Lens is an aggregator, so the DB is + also re-derivable by re-ingesting from chain. diff --git a/docs/backup-restore.md b/docs/backup-restore.md new file mode 100644 index 0000000..9f9359c --- /dev/null +++ b/docs/backup-restore.md @@ -0,0 +1,100 @@ +# Backup and restore + +A dump of this database contains live credentials — `webhooks.secret` (HMAC +signing keys) and `api_keys` rows — and must never land anywhere +world-readable. GitHub Actions artifacts on this public repo are +world-readable, so the workflow only uploads an `age`-encrypted file. + +Nightly `pg_dump` of each network's Postgres. Lens stores price history in +**one database per network** (no `network` column); restoring a mainnet dump +into the testnet instance would mix pairs. See [DUAL_NETWORK.md](DUAL_NETWORK.md). + +Neon also has branching and point-in-time restore. Use that for short-window +accidents. The encrypted dump is the copy that survives a project delete or +a missed PITR window. + +The workflow is [`.github/workflows/db-backup.yml`](../.github/workflows/db-backup.yml): +cron `0 3 * * *` UTC plus `workflow_dispatch`. Encrypted artifacts are kept +**14 days**. + +## Secrets + +Add these on the GitHub repo (Settings → Secrets and variables → Actions). +Use the **direct** connection string, not the pooler: + +| Secret | What | +|--------|------| +| `DATABASE_URL_MAINNET` | mainnet Postgres (direct URI) | +| `DATABASE_URL_TESTNET` | testnet Postgres (direct URI) | +| `AGE_RECIPIENT` | `age` public key (`age1…`) | + +On Neon that means the host **without** `-pooler`, port **5432** (not 6543), +and `sslmode=require`. Prisma's pooled `DATABASE_URL` is the runtime URL; +`pg_dump` / `pg_restore` need the same class of URI as `DIRECT_DATABASE_URL`. + +Generate the keypair on a machine that is not the runner: + +```bash +age-keygen -o lens-backup.agekey +# public key line: age1… +``` + +Put only the public key in `AGE_RECIPIENT`. The identity file stays offline +(password manager / disk not in this repo). GitHub never sees it. + +A missing database URL skips that network (`SKIP : secret not +configured`). A missing `AGE_RECIPIENT` fails the job — it will not upload +plaintext. + +Do not put these URIs, the identity file, or a plaintext dump in the repo, +`.env.example`, or workflow YAML. + +## Download a dump + +1. Actions → **Nightly DB backup** → the run you want. +2. Artifacts: `lens-mainnet-` / `lens-testnet-`. +3. File inside: `lens--YYYYMMDD.dump.gz.age`. + +`workflow_dispatch` is the way to take a dump on demand (and to verify the +job once secrets are in place). + +## Restore into a fresh instance + +Do not `pg_restore` onto the live URL. Spin up an empty database, restore +there, then swap the deployment env. + +1. Create a new Neon project (or local Postgres). Match the dump's major + version; if `pg_restore` complains, install a client ≥ the server + (`postgresql-client-16` / `17` as needed). +2. Copy the **direct** URI for the new instance (`sslmode=require` on Neon). +3. Decrypt with the offline identity, then restore **one network per + instance**: + +```bash +age -d -i lens-backup.agekey \ + -o lens-mainnet-YYYYMMDD.dump.gz \ + lens-mainnet-YYYYMMDD.dump.gz.age + +gzip -dc lens-mainnet-YYYYMMDD.dump.gz \ + | pg_restore --no-owner --no-acl --dbname="$NEW_DIRECT_URL" +``` + +`pg_restore -l` on the gunzipped file lists the TOC if you want to inspect +before loading. Delete the decrypted `.dump.gz` when you are done. + +4. Point the matching Lens deployment at the new instance: + - `DATABASE_URL` — pooled URL is fine for the app + - `DIRECT_DATABASE_URL` — direct URL (Prisma migrations / `db push`) +5. Check `/status`, then one `/prices/history` (or GraphQL equivalent) for a + watched pair. +6. Rollback is swapping those two env vars back to the previous instance. + +Repeat independently for testnet with the testnet artifact and testnet +deployment. Never restore a mainnet dump into testnet, or the reverse. + +## Client notes + +Compose uses Postgres 15. Managed Neon may be 16/17. `pg_dump` / `pg_restore` +must be the same major as the server or newer. The workflow installs Ubuntu's +`postgresql-client` and `age`; bump the client package if dumps start failing +with a version error. diff --git a/src/__tests__/db-backup.workflow.test.ts b/src/__tests__/db-backup.workflow.test.ts new file mode 100644 index 0000000..e2ef418 --- /dev/null +++ b/src/__tests__/db-backup.workflow.test.ts @@ -0,0 +1,90 @@ +import { describe, it, expect } from 'vitest' +import { readFileSync } from 'node:fs' +import { resolve } from 'node:path' +import YAML from 'js-yaml' + +const root = resolve(__dirname, '../..') +const workflowPath = resolve(root, '.github/workflows/db-backup.yml') +const runbookPath = resolve(root, 'docs/backup-restore.md') + +function read(path: string): string { + return readFileSync(path, 'utf8') +} + +describe('nightly pg_dump workflow', () => { + it('exists and is valid YAML', () => { + const raw = read(workflowPath) + expect(raw.length).toBeGreaterThan(0) + expect(() => YAML.load(raw)).not.toThrow() + }) + + it('runs on a nightly schedule and workflow_dispatch only', () => { + const raw = read(workflowPath) + const doc = YAML.load(raw) as { + on?: { + schedule?: { cron: string }[] + workflow_dispatch?: unknown + pull_request?: unknown + push?: unknown + } + } + const crons = (doc.on?.schedule ?? []).map((s) => s.cron) + expect(crons).toContain('0 3 * * *') + expect(doc.on?.workflow_dispatch).toBeDefined() + expect(doc.on?.pull_request).toBeUndefined() + expect(doc.on?.push).toBeUndefined() + }) + + it('covers both networks via secrets and keeps artifacts 14 days', () => { + const raw = read(workflowPath) + expect(raw).toMatch(/secrets\.DATABASE_URL_MAINNET/) + expect(raw).toMatch(/secrets\.DATABASE_URL_TESTNET/) + expect(raw).toMatch(/mainnet/) + expect(raw).toMatch(/testnet/) + expect(raw).toMatch(/retention-days:\s*14/) + expect(raw).not.toMatch(/postgres:\/\/[^\s]+:[^\s]+@/) + }) + + it('dumps custom format, gzips, and does not echo the URL', () => { + const raw = read(workflowPath) + expect(raw).toMatch(/pg_dump/) + expect(raw).toMatch(/--format=custom/) + expect(raw).toMatch(/\bgzip\b/) + expect(raw).toMatch(/--no-owner/) + expect(raw).toMatch(/--no-acl/) + expect(raw).not.toMatch(/echo\s+"?\$\{?DATABASE_URL/) + expect(raw).not.toMatch(/set\s+-x/) + }) + + it('age-encrypts before upload and refuses a missing recipient', () => { + const raw = read(workflowPath) + expect(raw).toMatch(/secrets\.AGE_RECIPIENT/) + expect(raw).toMatch(/\bage -r\b/) + expect(raw).toMatch(/\.dump\.gz\.age/) + expect(raw).toMatch(/AGE_RECIPIENT is required|refusing.*unencrypted/i) + expect(raw).toMatch(/path:\s*lens-\$\{\{\s*matrix\.network\s*\}\}-\*\.dump\.gz\.age/) + expect(raw).not.toMatch(/path:\s*lens-\$\{\{\s*matrix\.network\s*\}\}-\*\.dump\.gz\s*$/m) + }) +}) + +describe('backup restore runbook', () => { + it('documents pg_restore into a fresh instance and URL swap for both networks', () => { + const raw = read(runbookPath) + expect(raw).toMatch(/pg_restore/) + expect(raw).toMatch(/Neon/i) + expect(raw).toMatch(/DATABASE_URL/) + expect(raw).toMatch(/DIRECT_DATABASE_URL/) + expect(raw).toMatch(/mainnet/i) + expect(raw).toMatch(/testnet/i) + expect(raw).toMatch(/pooler|direct/i) + expect(raw).toMatch(/fresh/i) + }) + + it('says dumps hold credentials and must be decrypted off the public artifact', () => { + const raw = read(runbookPath) + expect(raw).toMatch(/webhooks?\.secret|HMAC/i) + expect(raw).toMatch(/world-readable|public/i) + expect(raw).toMatch(/age -d/) + expect(raw).toMatch(/AGE_RECIPIENT/) + }) +})