diff --git a/.github/workflows/upgrade-integration-test.yml b/.github/workflows/upgrade-integration-test.yml new file mode 100644 index 0000000000..f73b7276e6 --- /dev/null +++ b/.github/workflows/upgrade-integration-test.yml @@ -0,0 +1,437 @@ +# Upgrade Integration Test +# +# Tests that Dokploy can upgrade from version A to version B while keeping +# user projects (Postgres, MongoDB, two web apps, one static site) alive. +# +# Generates upgrade pairs: each stable tag in [floor_version, target_version) +# is paired with target_version. If target_version is empty, the highest tag +# available on Docker Hub is used as the target. +# +# Only triggered manually to avoid burning Actions minutes. + +name: Upgrade Integration Test + +on: + workflow_dispatch: + inputs: + floor_version: + description: 'Oldest version tag to include in pairs (e.g. v0.29.4)' + required: false + default: 'v0.29.4' + target_version: + description: 'Target version to upgrade to (version B). Leave empty to use the highest available.' + required: false + default: '' + +env: + DOKPLOY_IMAGE: dokploy/dokploy + DOKPLOY_SERVICE: dokploy + DOKPLOY_PORT: 3000 + +# ────────────────────────────────────────────────────────────────────────────── +jobs: + + # ── 1. Build the matrix ───────────────────────────────────────────────────── + build-matrix: + name: Build upgrade pair matrix + runs-on: ubuntu-latest + outputs: + pairs: ${{ steps.pairs.outputs.pairs }} + + steps: + - name: Generate pairs + id: pairs + env: + FLOOR: ${{ inputs.floor_version }} + TARGET: ${{ inputs.target_version }} + run: | + set -euo pipefail + + # semver "a >= b" comparison helper (vX.Y.Z, strips leading v) + ge() { + [ "$(printf '%s\n%s\n' "${1#v}" "${2#v}" | sort -V | tail -n1)" = "${1#v}" ] + } + + # Fetch all semver tags from Docker Hub (dokploy/dokploy) + ALL_TAGS=$(curl -fsSL \ + "https://hub.docker.com/v2/repositories/dokploy/dokploy/tags?page_size=100" | \ + jq -r '.results[].name' | \ + grep -E '^v[0-9]+\.[0-9]+\.[0-9]+$' | \ + sort -V) + + echo "All tags found:" + echo "$ALL_TAGS" + + # Target version (version B): explicit input, else highest available. + if [ -n "$TARGET" ]; then + TO="$TARGET" + echo "Using user-supplied target version: $TO" + else + TO=$(echo "$ALL_TAGS" | tail -n1) + echo "No target supplied; using highest available: $TO" + fi + + # Floor semver components + FLOOR_CLEAN="${FLOOR#v}" + IFS='.' read -r F_MAJ F_MIN F_PAT <<< "$FLOOR_CLEAN" + + # Each tag in [FLOOR, TO) → TO + PAIRS='[]' + while read -r TAG; do + [ -z "$TAG" ] && continue + # skip tags above-or-equal to the target (incl. the target itself) + ge "$TAG" "$TO" && continue + TAG_CLEAN="${TAG#v}" + IFS='.' read -r T_MAJ T_MIN T_PAT <<< "$TAG_CLEAN" + if [ "$T_MAJ" -gt "$F_MAJ" ] || \ + { [ "$T_MAJ" -eq "$F_MAJ" ] && [ "$T_MIN" -gt "$F_MIN" ]; } || \ + { [ "$T_MAJ" -eq "$F_MAJ" ] && [ "$T_MIN" -eq "$F_MIN" ] && [ "$T_PAT" -ge "$F_PAT" ]; }; then + PAIRS=$(echo "$PAIRS" | jq -c \ + --arg f "$TAG" --arg t "$TO" \ + '. + [{"from":$f,"to":$t}]') + fi + done <<< "$ALL_TAGS" + + COUNT=$(echo "$PAIRS" | jq 'length') + echo "Total pairs: $COUNT" + echo "$PAIRS" | jq -r '.[] | " \(.from) → \(.to)"' + + echo "pairs=$PAIRS" >> "$GITHUB_OUTPUT" + + + # ── 2. Run one upgrade test per pair ──────────────────────────────────────── + upgrade-test: + name: "${{ matrix.pair.from }} → ${{ matrix.pair.to }}" + needs: build-matrix + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + pair: ${{ fromJSON(needs.build-matrix.outputs.pairs) }} + + steps: + + # ── Environment setup ────────────────────────────────────────────────── + - name: Free disk space + run: | + sudo rm -rf \ + /usr/share/dotnet /opt/ghc /usr/local/share/boost \ + "$AGENT_TOOLSDIRECTORY" /usr/local/lib/android \ + /usr/local/share/chromium /opt/hostedtoolcache + docker system prune -af --volumes + df -h + + # ── Install Dokploy directly at VERSION A ────────────────────────────── + # install.sh: + # • requires root (run via sudo bash) + # • respects DOKPLOY_VERSION → installs that tag directly + # (so we don't need a separate "downgrade" step that would risk + # running B's migrations on A's expected schema) + # • respects ADVERTISE_ADDR → skip the external IP lookup + # • initializes Docker Swarm + dokploy-network itself + - name: Install Dokploy at VERSION A (${{ matrix.pair.from }}) + run: | + curl -fsSL https://dokploy.com/install.sh -o /tmp/install.sh + chmod +x /tmp/install.sh + sudo -E env \ + DOKPLOY_VERSION="${{ matrix.pair.from }}" \ + ADVERTISE_ADDR="127.0.0.1" \ + bash /tmp/install.sh + + - name: Wait for dokploy service to converge on ${{ matrix.pair.from }} + run: | + echo "Waiting for 'dokploy' Swarm service to reach 1/1..." + timeout 240 bash -c ' + until docker service ls --filter name=dokploy \ + --format "{{.Name}} {{.Replicas}}" \ + | grep "^dokploy " | grep -q " 1/1"; do + sleep 4 + done + ' + docker service ls + echo "✅ Service running on ${{ matrix.pair.from }}" + + - name: Wait for Dokploy API to accept requests + run: | + timeout 180 bash -c ' + until curl -sf -o /dev/null \ + "http://localhost:${{ env.DOKPLOY_PORT }}"; do + sleep 3 + done + ' + echo "✅ Dokploy API is up" + + # ── Bootstrap admin user ─────────────────────────────────────────────── + - name: Register first admin user + id: auth + run: | + COOKIE_JAR="$RUNNER_TEMP/dokploy-cookies.txt" + : > "$COOKIE_JAR" + chmod 600 "$COOKIE_JAR" + echo "cookie_jar=$COOKIE_JAR" >> "$GITHUB_OUTPUT" + + # better-auth: sign-up/email (allowed only before any owner exists) + set -x + curl -sS -i -X POST \ + "http://localhost:${{ env.DOKPLOY_PORT }}/api/auth/sign-up/email" \ + -H "Content-Type: application/json" \ + -c "$COOKIE_JAR" -b "$COOKIE_JAR" \ + -d '{"name":"CI Admin","email":"ci@dokploy.test","password":"CiTest1234!"}' \ + | tee /tmp/signup.out + set +x + + # Verify we got a session cookie + if ! grep -qE '(better-auth|auth)\.session' "$COOKIE_JAR"; then + echo "⚠️ No session cookie matched expected name; jar contents:" + cat "$COOKIE_JAR" + fi + + # ── Create test resources ────────────────────────────────────────────── + - name: Create project, databases and applications + id: create + env: + BASE: "http://localhost:${{ env.DOKPLOY_PORT }}" + COOKIE: ${{ steps.auth.outputs.cookie_jar }} + run: | + set -euo pipefail + + # --- tRPC POST helper --- + trpc_mut() { + curl -sf -X POST "$BASE/api/trpc/$1" \ + -H "Content-Type: application/json" \ + -b "$COOKIE" -c "$COOKIE" \ + -d "{\"json\":$2}" + } + + # --- Project --- + PROJECT=$(trpc_mut project.create \ + '{"name":"ci-upgrade-test","description":"CI upgrade integration test"}') + echo "project.create → $PROJECT" + + # project.create in v0.29+ returns nested {project:{projectId}, environment:{environmentId}} + PROJECT_ID=$(echo "$PROJECT" | jq -r \ + '.result.data.json.project.projectId // .result.data.json.projectId // empty') + ENV_ID=$(echo "$PROJECT" | jq -r \ + '.result.data.json.environment.environmentId // empty') + + if [ -z "$PROJECT_ID" ] || [ "$PROJECT_ID" = "null" ]; then + echo "❌ Could not extract projectId from project.create response:" + echo "$PROJECT" | jq . + exit 1 + fi + if [ -z "$ENV_ID" ] || [ "$ENV_ID" = "null" ]; then + echo "❌ Could not extract environmentId — this version may not support environments." + echo " project.create response: $PROJECT" + exit 1 + fi + + echo "project_id=$PROJECT_ID" >> "$GITHUB_OUTPUT" + echo "env_id=$ENV_ID" >> "$GITHUB_OUTPUT" + echo "Detected PROJECT_ID=$PROJECT_ID ENV_ID=$ENV_ID" + + # --- PostgreSQL 15 --- + PG=$(trpc_mut postgres.create \ + "{\"name\":\"ci-pg\",\"appName\":\"ci-pg-db\",\ + \"databaseName\":\"cidb\",\"databaseUser\":\"ciuser\",\ + \"databasePassword\":\"CiPg1pass\",\ + \"dockerImage\":\"postgres:15\",\"environmentId\":\"$ENV_ID\"}") + echo "postgres.create → $PG" + PG_ID=$(echo "$PG" | jq -r '.result.data.json.postgresId') + # deploy (not start) — deploy creates the Swarm service; start only + # scales an already-deployed service and 500s on a fresh resource. + trpc_mut postgres.deploy "{\"postgresId\":\"$PG_ID\"}" > /dev/null + echo "pg_id=$PG_ID" >> "$GITHUB_OUTPUT" + + # --- MongoDB 7.0 --- + MG=$(trpc_mut mongo.create \ + "{\"name\":\"ci-mongo\",\"appName\":\"ci-mongo-db\",\ + \"databaseName\":\"cidb\",\"databaseUser\":\"ciuser\",\ + \"databasePassword\":\"CiMg1pass\",\ + \"dockerImage\":\"mongo:7.0\",\"environmentId\":\"$ENV_ID\"}") + echo "mongo.create → $MG" + MG_ID=$(echo "$MG" | jq -r '.result.data.json.mongoId') + trpc_mut mongo.deploy "{\"mongoId\":\"$MG_ID\"}" > /dev/null + echo "mg_id=$MG_ID" >> "$GITHUB_OUTPUT" + + # --- Docker-image application helper --- + make_app() { + local DISP_NAME=$1 APP_NAME=$2 IMAGE=$3 + APP=$(trpc_mut application.create \ + "{\"name\":\"$DISP_NAME\",\"appName\":\"$APP_NAME\",\"environmentId\":\"$ENV_ID\"}") + APP_ID=$(echo "$APP" | jq -r '.result.data.json.applicationId') + trpc_mut application.saveDockerProvider \ + "{\"applicationId\":\"$APP_ID\",\"dockerImage\":\"$IMAGE\",\ + \"username\":\"\",\"password\":\"\",\"registryUrl\":\"\"}" \ + > /dev/null + trpc_mut application.deploy "{\"applicationId\":\"$APP_ID\"}" \ + > /dev/null + echo "$APP_ID" + } + + # Static site (nginx) + APP_STATIC=$(make_app "ci-static" "ci-static-app" "nginx:alpine") + echo "app_static_id=$APP_STATIC" >> "$GITHUB_OUTPUT" + + # Node.js hello-world (echo server image — small, no args needed) + APP_NODE=$(make_app "ci-node" "ci-node-app" "ealen/echo-server:latest") + echo "app_node_id=$APP_NODE" >> "$GITHUB_OUTPUT" + + # Go HTTP server (traefik/whoami is a tiny Go binary, port 80) + APP_GO=$(make_app "ci-go" "ci-go-app" "traefik/whoami:latest") + echo "app_go_id=$APP_GO" >> "$GITHUB_OUTPUT" + + echo "✅ All resources created" + + # ── Pre-upgrade health check ─────────────────────────────────────────── + - name: Wait for all services → 'done' (pre-upgrade) + env: + BASE: "http://localhost:${{ env.DOKPLOY_PORT }}" + COOKIE: ${{ steps.auth.outputs.cookie_jar }} + PG_ID: ${{ steps.create.outputs.pg_id }} + MG_ID: ${{ steps.create.outputs.mg_id }} + APP_STATIC_ID: ${{ steps.create.outputs.app_static_id }} + APP_NODE_ID: ${{ steps.create.outputs.app_node_id }} + APP_GO_ID: ${{ steps.create.outputs.app_go_id }} + run: | + wait_done() { + local NAME=$1 ENDPOINT=$2 ID_KEY=$3 ID=$4 STATUS_KEY=$5 + echo "Waiting for $NAME to reach 'done'..." + timeout 360 bash -c " + until [ \"\$(curl -sf -G '$BASE/api/trpc/$ENDPOINT' \ + --data-urlencode 'input={\"json\":{\"$ID_KEY\":\"$ID\"}}' \ + -b '$COOKIE' | \ + jq -r '.result.data.json.$STATUS_KEY // \"unknown\"')\" \ + = 'done' ]; do + sleep 5 + done + " + echo "✅ $NAME is done" + } + + wait_done postgres postgres.one postgresId "$PG_ID" applicationStatus + wait_done mongo mongo.one mongoId "$MG_ID" applicationStatus + wait_done static application.one applicationId "$APP_STATIC_ID" applicationStatus + wait_done node-app application.one applicationId "$APP_NODE_ID" applicationStatus + wait_done go-app application.one applicationId "$APP_GO_ID" applicationStatus + + - name: Assert Docker Swarm services healthy (pre-upgrade) + run: | + echo "=== docker service ls ===" + docker service ls + + FAIL=$(docker service ls --format '{{.Name}} {{.Replicas}}' | \ + grep -E '^ci-' | grep -v ' 1/1' || true) + if [ -n "$FAIL" ]; then + echo "❌ User services not healthy before upgrade:" + echo "$FAIL" + exit 1 + fi + echo "✅ All user services healthy before upgrade" + + # ── Upgrade ──────────────────────────────────────────────────────────── + - name: Upgrade Dokploy to VERSION B (${{ matrix.pair.to }}) + run: | + docker service update \ + --image "${{ env.DOKPLOY_IMAGE }}:${{ matrix.pair.to }}" \ + --force \ + "${{ env.DOKPLOY_SERVICE }}" + + echo "Waiting for service to converge on ${{ matrix.pair.to }}..." + timeout 240 bash -c ' + until ! docker service inspect dokploy \ + --format "{{.UpdateStatus.State}}" 2>/dev/null \ + | grep -q "^updating$"; do + sleep 4 + done + until docker service ls --filter name=dokploy \ + --format "{{.Name}} {{.Replicas}}" \ + | grep "^dokploy " | grep -q " 1/1"; do + sleep 4 + done + ' + echo "✅ Service running on ${{ matrix.pair.to }}" + + - name: Wait for Dokploy API to respond post-upgrade + run: | + timeout 180 bash -c ' + until curl -sf -o /dev/null \ + "http://localhost:${{ env.DOKPLOY_PORT }}"; do + sleep 3 + done + ' + echo "✅ Dokploy API is up after upgrade" + + # ── Post-upgrade health check ────────────────────────────────────────── + - name: Verify all services still healthy (post-upgrade) + env: + BASE: "http://localhost:${{ env.DOKPLOY_PORT }}" + COOKIE: ${{ steps.auth.outputs.cookie_jar }} + PG_ID: ${{ steps.create.outputs.pg_id }} + MG_ID: ${{ steps.create.outputs.mg_id }} + APP_STATIC_ID: ${{ steps.create.outputs.app_static_id }} + APP_NODE_ID: ${{ steps.create.outputs.app_node_id }} + APP_GO_ID: ${{ steps.create.outputs.app_go_id }} + run: | + check_status() { + local NAME=$1 ENDPOINT=$2 ID_KEY=$3 ID=$4 STATUS_KEY=$5 + STATUS=$(curl -sf -G "$BASE/api/trpc/$ENDPOINT" \ + --data-urlencode "input={\"json\":{\"$ID_KEY\":\"$ID\"}}" \ + -b "$COOKIE" | \ + jq -r ".result.data.json.$STATUS_KEY // \"unknown\"") + if [ "$STATUS" != "done" ]; then + echo "❌ $NAME status after upgrade: $STATUS" + return 1 + fi + echo "✅ $NAME: $STATUS" + } + + check_status postgres postgres.one postgresId "$PG_ID" applicationStatus + check_status mongo mongo.one mongoId "$MG_ID" applicationStatus + check_status static application.one applicationId "$APP_STATIC_ID" applicationStatus + check_status node-app application.one applicationId "$APP_NODE_ID" applicationStatus + check_status go-app application.one applicationId "$APP_GO_ID" applicationStatus + + echo "=== docker service ls (post-upgrade) ===" + docker service ls + + FAIL=$(docker service ls --format '{{.Name}} {{.Replicas}}' | \ + grep -E '^ci-' | grep -v ' 1/1' || true) + if [ -n "$FAIL" ]; then + echo "❌ User services not healthy after upgrade:" + echo "$FAIL" + exit 1 + fi + echo "✅ All services healthy after upgrade to ${{ matrix.pair.to }}" + + # ── Diagnostics on failure ───────────────────────────────────────────── + - name: Dump state on failure + if: failure() + run: | + echo "=== docker service ls ===" && docker service ls || true + echo "=== dokploy service tasks ===" && \ + docker service ps "${{ env.DOKPLOY_SERVICE }}" --no-trunc || true + echo "=== dokploy logs (last 200) ===" && \ + docker service logs "${{ env.DOKPLOY_SERVICE }}" --tail 200 2>&1 || true + echo "=== install.sh tail ===" && \ + tail -100 /tmp/install.sh 2>&1 || true + echo "=== signup response ===" && \ + cat /tmp/signup.out 2>&1 || true + echo "=== disk usage ===" && df -h + + # ── Job summary ──────────────────────────────────────────────────────── + - name: Write job summary + if: always() + run: | + STATUS="${{ job.status }}" + ICON="✅"; [ "$STATUS" != "success" ] && ICON="❌" + cat >> "$GITHUB_STEP_SUMMARY" <