diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile new file mode 100644 index 000000000..40604623b --- /dev/null +++ b/.devcontainer/Dockerfile @@ -0,0 +1,11 @@ +# Dev container image for sync-engine +FROM mcr.microsoft.com/devcontainers/javascript-node:24 + +# Enable corepack so pnpm is available at the version specified in package.json +RUN corepack enable + +# Install CLI tools useful for development and service health checks +RUN apt-get update && apt-get install -y --no-install-recommends \ + postgresql-client \ + netcat-openbsd \ + && rm -rf /var/lib/apt/lists/* diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json new file mode 100644 index 000000000..d66ca0435 --- /dev/null +++ b/.devcontainer/devcontainer.json @@ -0,0 +1,56 @@ +{ + "name": "Sync Engine", + "dockerComposeFile": "docker-compose.yml", + "service": "app", + "workspaceFolder": "/workspaces/sync-engine", + + "postCreateCommand": "corepack enable && pnpm install && pnpm build", + "postStartCommand": "echo 'Dev container ready. Services: postgres, stripe-mock, temporal.'", + + "forwardPorts": [55432, 12111, 12112, 7233, 8080, 3000, 4010, 4020, 5173], + "portsAttributes": { + "55432": { "label": "Postgres", "onAutoForward": "silent" }, + "12111": { "label": "Stripe Mock (HTTP)", "onAutoForward": "silent" }, + "12112": { "label": "Stripe Mock (HTTPS)", "onAutoForward": "silent" }, + "7233": { "label": "Temporal gRPC", "onAutoForward": "silent" }, + "8080": { "label": "Temporal UI", "onAutoForward": "openBrowser" }, + "3000": { "label": "Engine API", "onAutoForward": "notify" }, + "4010": { "label": "Engine API (Docker)", "onAutoForward": "silent" }, + "4020": { "label": "Service API", "onAutoForward": "notify" }, + "5173": { "label": "Dashboard", "onAutoForward": "openBrowser" } + }, + + "customizations": { + "vscode": { + "extensions": [ + "dbaeumer.vscode-eslint", + "esbenp.prettier-vscode", + "vitest.explorer", + "ckolkman.vscode-postgres", + "ms-azuretools.vscode-docker", + "ms-vscode.vscode-typescript-next" + ], + "settings": { + "editor.defaultFormatter": "esbenp.prettier-vscode", + "editor.formatOnSave": true, + "editor.codeActionsOnSave": { + "source.fixAll.eslint": "explicit" + }, + "typescript.tsdk": "node_modules/typescript/lib", + "typescript.enablePromptUseWorkspaceTsdk": true, + "vitest.commandLine": "pnpm exec vitest", + "terminal.integrated.defaultProfile.linux": "bash", + "files.eol": "\n", + "files.trimTrailingWhitespace": true, + "files.insertFinalNewline": true, + "search.exclude": { + "**/node_modules": true, + "**/dist": true, + "**/.tsbuildinfo": true + } + } + } + }, + + "remoteUser": "node" +} diff --git a/.devcontainer/docker-compose.yml b/.devcontainer/docker-compose.yml new file mode 100644 index 000000000..c8490d0b0 --- /dev/null +++ b/.devcontainer/docker-compose.yml @@ -0,0 +1,41 @@ +# Dev container compose overlay — imports the repo's compose.yml for +# infrastructure services and adds the development container alongside them. +# All services share the same Docker network, so the app container can +# reach postgres, stripe-mock, and temporal by service name. + +include: + - path: ../compose.yml + +services: + app: + build: + context: . + dockerfile: Dockerfile + volumes: + # Mount the workspace source code + - ..:/workspaces/sync-engine:cached + # Persist pnpm store across rebuilds + - pnpm-store:/home/node/.local/share/pnpm/store + # Keep the container running for VS Code to attach + command: sleep infinity + environment: + # Database — uses Docker service hostname, internal port + DATABASE_URL: postgres://postgres:postgres@postgres:5432/postgres?sslmode=disable&search_path=stripe + POSTGRES_URL: postgres://postgres:postgres@postgres:5432/postgres + # Stripe mock — uses Docker service hostname + STRIPE_MOCK_URL: http://stripe-mock:12111 + STRIPE_API_KEY: sk_test_fake123 + # Temporal — uses Docker service hostname + TEMPORAL_ADDRESS: temporal:7233 + # Node + NODE_ENV: development + depends_on: + postgres: + condition: service_healthy + stripe-mock: + condition: service_healthy + temporal: + condition: service_healthy + +volumes: + pnpm-store: diff --git a/.devcontainer/test-devcontainer.sh b/.devcontainer/test-devcontainer.sh new file mode 100755 index 000000000..b30334d0b --- /dev/null +++ b/.devcontainer/test-devcontainer.sh @@ -0,0 +1,73 @@ +#!/usr/bin/env bash +# Validation script for the dev container setup. +# Run inside the dev container to verify all services are reachable +# and the development toolchain is functional. +# +# Usage: bash .devcontainer/test-devcontainer.sh + +set -euo pipefail + +PASS=0 +FAIL=0 + +check() { + local label="$1" + shift + if "$@" >/dev/null 2>&1; then + echo " PASS $label" + ((PASS++)) + else + echo " FAIL $label" + ((FAIL++)) + fi +} + +echo "=== Dev Container Validation ===" +echo "" + +# --- Toolchain --- +echo "Toolchain:" +check "Node.js >= 24" node -e "assert(parseInt(process.versions.node) >= 24)" +check "pnpm available" pnpm --version +check "TypeScript available" pnpm exec tsc --version +check "corepack enabled" corepack --version + +echo "" + +# --- Service connectivity --- +echo "Services:" +check "Postgres (postgres:5432)" pg_isready -h postgres -p 5432 -U postgres +check "stripe-mock (stripe-mock:12111)" nc -z stripe-mock 12111 +check "Temporal gRPC (temporal:7233)" nc -z temporal 7233 + +echo "" + +# --- Database --- +echo "Database:" +check "Postgres connection" psql "$DATABASE_URL" -c "SELECT 1" +check "stripe schema" psql "$POSTGRES_URL" -c "CREATE SCHEMA IF NOT EXISTS stripe" + +echo "" + +# --- Stripe mock --- +echo "Stripe Mock:" +check "GET /v1/customers" curl -sf -H "Authorization: Bearer sk_test_fake123" http://stripe-mock:12111/v1/customers + +echo "" + +# --- Build artifacts --- +echo "Build:" +check "node_modules exists" test -d node_modules +check "Build output exists" test -d packages/protocol/dist + +echo "" + +# --- Environment variables --- +echo "Environment:" +check "DATABASE_URL set" test -n "${DATABASE_URL:-}" +check "STRIPE_MOCK_URL set" test -n "${STRIPE_MOCK_URL:-}" +check "TEMPORAL_ADDRESS set" test -n "${TEMPORAL_ADDRESS:-}" + +echo "" +echo "=== Results: $PASS passed, $FAIL failed ===" +[ "$FAIL" -eq 0 ] || exit 1 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5d3c36578..71fe52dc4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -326,7 +326,7 @@ jobs: # --------------------------------------------------------------------------- e2e_docker: name: E2E Docker - needs: build + needs: [build, publish_npm] runs-on: ubuntu-24.04-arm steps: @@ -381,6 +381,11 @@ jobs: STRIPE_API_KEY: ${{ secrets.STRIPE_API_KEY }} ENGINE_IMAGE: 'ghcr.io/${{ github.repository }}:${{ github.sha }}-arm64' + - name: Set up Node + uses: actions/setup-node@v6 + with: + node-version-file: ./.nvmrc + - name: Publish test run: | if [ -z "${STRIPE_NPM_REGISTRY:-}" ]; then @@ -390,6 +395,7 @@ jobs: bash e2e/publish.test.sh env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + SKIP_PUBLISH: '1' # --------------------------------------------------------------------------- # E2E Stripe — Stripe API + Temporal integration tests (runs on every push/PR) diff --git a/e2e/devcontainer.test.ts b/e2e/devcontainer.test.ts new file mode 100644 index 000000000..08c081e52 --- /dev/null +++ b/e2e/devcontainer.test.ts @@ -0,0 +1,90 @@ +import { readFileSync, existsSync } from 'node:fs' +import { join } from 'node:path' +import { describe, it, expect } from 'vitest' + +const ROOT = join(import.meta.dirname, '..') +const DEVCONTAINER_DIR = join(ROOT, '.devcontainer') + +describe('devcontainer configuration', () => { + it('devcontainer.json is valid JSON with required fields', () => { + const path = join(DEVCONTAINER_DIR, 'devcontainer.json') + expect(existsSync(path), '.devcontainer/devcontainer.json must exist').toBe(true) + + const config = JSON.parse(readFileSync(path, 'utf-8')) + + expect(config.name).toBeDefined() + expect(config.dockerComposeFile).toBe('docker-compose.yml') + expect(config.service).toBe('app') + expect(config.workspaceFolder).toBeDefined() + expect(config.postCreateCommand).toContain('pnpm install') + expect(config.postCreateCommand).toContain('pnpm build') + expect(config.remoteUser).toBe('node') + }) + + it('devcontainer.json forwards required ports', () => { + const config = JSON.parse(readFileSync(join(DEVCONTAINER_DIR, 'devcontainer.json'), 'utf-8')) + + const ports = config.forwardPorts as number[] + expect(ports).toContain(55432) // Postgres + expect(ports).toContain(12111) // stripe-mock HTTP + expect(ports).toContain(7233) // Temporal gRPC + }) + + it('devcontainer.json includes required VS Code extensions', () => { + const config = JSON.parse(readFileSync(join(DEVCONTAINER_DIR, 'devcontainer.json'), 'utf-8')) + + const extensions = config.customizations?.vscode?.extensions as string[] + expect(extensions).toContain('dbaeumer.vscode-eslint') + expect(extensions).toContain('esbenp.prettier-vscode') + expect(extensions).toContain('vitest.explorer') + }) + + it('docker-compose.yml exists and references compose.yml', () => { + const path = join(DEVCONTAINER_DIR, 'docker-compose.yml') + expect(existsSync(path), '.devcontainer/docker-compose.yml must exist').toBe(true) + + const content = readFileSync(path, 'utf-8') + expect(content).toContain('compose.yml') + expect(content).toContain('DATABASE_URL') + expect(content).toContain('STRIPE_MOCK_URL') + expect(content).toContain('TEMPORAL_ADDRESS') + }) + + it('docker-compose.yml app service depends on infrastructure', () => { + const content = readFileSync(join(DEVCONTAINER_DIR, 'docker-compose.yml'), 'utf-8') + + expect(content).toContain('postgres') + expect(content).toContain('stripe-mock') + expect(content).toContain('temporal') + expect(content).toContain('service_healthy') + }) + + it('Dockerfile exists and sets up Node toolchain', () => { + const path = join(DEVCONTAINER_DIR, 'Dockerfile') + expect(existsSync(path), '.devcontainer/Dockerfile must exist').toBe(true) + + const content = readFileSync(path, 'utf-8') + expect(content).toContain('javascript-node:24') + expect(content).toContain('corepack enable') + }) + + it('test-devcontainer.sh exists and is executable-ready', () => { + const path = join(DEVCONTAINER_DIR, 'test-devcontainer.sh') + expect(existsSync(path), '.devcontainer/test-devcontainer.sh must exist').toBe(true) + + const content = readFileSync(path, 'utf-8') + expect(content).toContain('#!/usr/bin/env bash') + expect(content).toContain('set -euo pipefail') + expect(content).toContain('PASS') + expect(content).toContain('FAIL') + }) + + it('environment variables use Docker service hostnames, not localhost', () => { + const content = readFileSync(join(DEVCONTAINER_DIR, 'docker-compose.yml'), 'utf-8') + + // Inside the container, services are reached by hostname, not localhost + expect(content).toMatch(/DATABASE_URL.*@postgres:/) + expect(content).toMatch(/STRIPE_MOCK_URL.*stripe-mock:/) + expect(content).toMatch(/TEMPORAL_ADDRESS.*temporal:/) + }) +}) diff --git a/e2e/publish.test.sh b/e2e/publish.test.sh index 6add67f32..c30a33ce3 100755 --- a/e2e/publish.test.sh +++ b/e2e/publish.test.sh @@ -10,9 +10,12 @@ # # Prerequisites: # - Registry running and STRIPE_NPM_REGISTRY set -# - All packages built (pnpm build) +# - Unless SKIP_PUBLISH=1: packages built (pnpm build) and pnpm available for Step 2 # - For GitHub Packages: GITHUB_TOKEN set (CI provides this automatically) # +# Environment: +# - SKIP_PUBLISH=1 — skip Step 2 (use when packages were just published by CI, e.g. publish_npm job) +# # Usage: # bash e2e/publish.test.sh # @@ -36,6 +39,11 @@ echo "Registry: $REGISTRY" echo "Temp dir: $TMPDIR_BASE" echo "" +ENGINE_VERSION="$(node -p "require(\"${REPO_ROOT}/apps/engine/package.json\").version")" +ENGINE_SPEC="@stripe/sync-engine@${ENGINE_VERSION}" +echo "Engine package spec: $ENGINE_SPEC" +echo "" + # --------------------------------------------------------------------------- # Step 1: Auth for GitHub Packages (Verdaccio is anonymous) # --------------------------------------------------------------------------- @@ -57,21 +65,25 @@ echo "" # --------------------------------------------------------------------------- # Step 2: Publish all workspace packages # --------------------------------------------------------------------------- -echo "--- Step 2: Publishing packages ---" - -pnpm -r --filter '!./e2e' publish \ - --registry "$REGISTRY" \ - --access public \ - --no-git-checks \ - 2>&1 || true -# publish returns non-zero if some packages already exist — that's fine +if [ "${SKIP_PUBLISH:-}" = "1" ]; then + echo "--- Step 2: Skipped (SKIP_PUBLISH=1 — registry already has this commit's packages) ---" +else + echo "--- Step 2: Publishing packages ---" + + pnpm -r --filter '!./e2e' publish \ + --registry "$REGISTRY" \ + --access public \ + --no-git-checks \ + 2>&1 || true + # publish returns non-zero if some packages already exist — that's fine +fi echo "" # --------------------------------------------------------------------------- # Step 3: Smoke test — npx from clean directory # --------------------------------------------------------------------------- -echo "--- Step 3: npx @stripe/sync-engine --version ---" +echo "--- Step 3: npx $ENGINE_SPEC --version ---" cd "$TMPDIR_BASE" mkdir test-npx && cd test-npx @@ -84,11 +96,11 @@ if [[ "$REGISTRY" == *"npm.pkg.github.com"* ]]; then fi echo " .npmrc: $(cat .npmrc | grep -v authToken)" -if VERSION_OUTPUT=$(npx --yes @stripe/sync-engine --version 2>&1); then +if VERSION_OUTPUT=$(npx --yes "$ENGINE_SPEC" --version 2>&1); then echo " Version: $VERSION_OUTPUT" echo " PASS: --version returned output" else - echo " FAIL: npx @stripe/sync-engine --version exited with $?" + echo " FAIL: npx $ENGINE_SPEC --version exited with $?" echo " Output: $VERSION_OUTPUT" exit 1 fi @@ -97,13 +109,13 @@ echo "" # --------------------------------------------------------------------------- # Step 4: npx @stripe/sync-engine --help # --------------------------------------------------------------------------- -echo "--- Step 4: npx @stripe/sync-engine --help ---" +echo "--- Step 4: npx $ENGINE_SPEC --help ---" -if npx --yes @stripe/sync-engine --help > /dev/null 2>&1; then +if npx --yes "$ENGINE_SPEC" --help > /dev/null 2>&1; then echo " PASS: --help exits 0" else echo " FAIL: --help exited with $?" - npx --yes @stripe/sync-engine --help 2>&1 || true + npx --yes "$ENGINE_SPEC" --help 2>&1 || true exit 1 fi echo "" @@ -111,11 +123,11 @@ echo "" # --------------------------------------------------------------------------- # Step 5: npx @stripe/sync-engine check (connector loading) # --------------------------------------------------------------------------- -echo "--- Step 5: npx @stripe/sync-engine check (connector loading) ---" +echo "--- Step 5: npx $ENGINE_SPEC check (connector loading) ---" PARAMS='{"source":{"type":"stripe","stripe":{"api_key":"sk_test_fake"}},"destination":{"type":"postgres","postgres":{"connection_string":"postgresql://fake:fake@localhost:5432/fake"}}}' -CHECK_OUTPUT=$(npx --yes @stripe/sync-engine check --params "$PARAMS" 2>&1 || true) +CHECK_OUTPUT=$(npx --yes "$ENGINE_SPEC" check --params "$PARAMS" 2>&1 || true) # check will fail (bad credentials) but should NOT fail on "not found" (connector loading) if echo "$CHECK_OUTPUT" | grep -qi "not found"; then