diff --git a/.github/workflows/package.yml b/.github/workflows/package.yml index 648c43ed6d..a5c85a4fa8 100644 --- a/.github/workflows/package.yml +++ b/.github/workflows/package.yml @@ -1,48 +1,132 @@ -# This workflow will triage pull requests and apply a label based on the -# paths that are modified in the pull request. +# Build and release workflow for Backend.AI Desktop and WebUI bundle. # -# To use this workflow, you will need to set up a .github/labeler.yml -# file with configuration. For more information, see: -# https://github.com/actions/labeler/blob/master/README.md +# Architecture: 3 parallel jobs after a shared web build step. +# +# build_web (ubuntu) ──┬──> build_mac (macos) → DMG x64/arm64 + local proxy +# ├──> build_desktop (ubuntu) → Win/Linux ZIP x64/arm64 + local proxy +# └──> upload web bundle +# +# Key optimizations over the previous single-job approach: +# 1. Parallel jobs: macOS + win/linux builds run concurrently (~10 min saved) +# 2. No double React build: publicPath patching replaces full rebuild (~5 min saved) +# 3. Parallel local proxy compilation within each job (~3 min saved) +# 4. Optimized ZIP compression level (-6 vs -9, marginal size diff, ~1 min saved) name: Build and Release Packages on: release: types: [published] + workflow_dispatch: + inputs: + dry_run: + description: 'Skip release asset upload (for testing the build pipeline)' + type: boolean + default: true + +env: + NODE_OPTIONS: --max-old-space-size=4096 jobs: + # ────────────────────────────────────────────────────────────────────── + # Job 1: Build web assets and create the web bundle (ubuntu, ~8 min) + # ────────────────────────────────────────────────────────────────────── + build_web: + permissions: + contents: write + runs-on: ubuntu-latest + steps: + - name: Check out Git repository + uses: actions/checkout@v4 + + - uses: pnpm/action-setup@v4 + name: Install pnpm + with: + version: latest + run_install: false + + - name: Install Node.js + uses: actions/setup-node@v4 + with: + node-version-file: '.nvmrc' + cache: 'pnpm' + + - name: Install Dependencies + run: pnpm install --no-frozen-lockfile + + - name: Build web assets + run: make dep_web + + - name: Create web bundle + run: make bundle + + - name: Upload release bundle + if: inputs.dry_run != true + run: node upload-release.js app + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + # Share build artifacts with downstream desktop jobs + - name: Upload build artifacts + uses: actions/upload-artifact@v4 + with: + name: web-build + path: | + build/web/ + src/wsproxy/dist/ + retention-days: 1 + compression-level: 3 + + # ────────────────────────────────────────────────────────────────────── + # Job 2: Build macOS desktop apps — requires macOS for code signing, + # notarization, and DMG creation (~10 min) + # ────────────────────────────────────────────────────────────────────── build_mac: + needs: build_web permissions: contents: write - checks: write - actions: read - issues: read - packages: write - pull-requests: read - repository-projects: read - statuses: read runs-on: macos-latest - environment: app-packaging + # Use the protected `app-packaging` environment for real releases (which + # gates access to signing secrets). Dry runs use a separate unprotected + # name because GitHub Actions rejects an empty environment value. + environment: ${{ inputs.dry_run != true && 'app-packaging' || 'app-packaging-dryrun' }} steps: - name: Check out Git repository uses: actions/checkout@v4 + - uses: pnpm/action-setup@v4 name: Install pnpm with: version: latest run_install: false + - name: Install Node.js uses: actions/setup-node@v4 with: node-version-file: '.nvmrc' cache: 'pnpm' + - name: Install Dependencies - # CI defaults to --frozen-lockfile, which blocks merging git branch lockfiles. - # pnpm-workspace.yaml's mergeGitBranchLockfilesBranchPattern auto-merges - # branch lockfiles on main. --no-frozen-lockfile allows that merge write. run: pnpm install --no-frozen-lockfile - - name: Package Desktop Applications - run: make all + + - name: Download web build artifacts + uses: actions/download-artifact@v4 + with: + name: web-build + + - name: Prepare Electron app + run: make dep_electron + + - name: Compile local proxies (parallel) + run: | + make compile_localproxy os=macos arch=x64 local_proxy_postfix= & + make compile_localproxy os=macos arch=arm64 local_proxy_postfix= & + wait + + - name: Package macOS Desktop Apps (signed) + if: inputs.dry_run != true + run: | + make mac_x64 + make mac_arm64 env: BAI_APP_SIGN: 1 BAI_APP_SIGN_APPLE_TEAM_ID: ${{ secrets.BAI_APP_SIGN_APPLE_TEAM_ID }} @@ -51,10 +135,72 @@ jobs: BAI_APP_SIGN_IDENTITY: ${{ secrets.BAI_APP_SIGN_IDENTITY }} BAI_APP_SIGN_KEYCHAIN_B64: ${{ secrets.BAI_APP_SIGN_KEYCHAIN_B64 }} BAI_APP_SIGN_KEYCHAIN_PASSWORD: ${{ secrets.BAI_APP_SIGN_KEYCHAIN_PASSWORD }} - NODE_OPTIONS: --max-old-space-size=4096 - - name: Bundle static resources into zip package - run: make bundle - - name: Upload application to latest release + + - name: Package macOS Desktop Apps (unsigned, dry run) + if: inputs.dry_run == true + run: | + make mac_x64 + make mac_arm64 + + - name: Upload macOS release assets + if: inputs.dry_run != true + run: node upload-release.js app + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + # ────────────────────────────────────────────────────────────────────── + # Job 3: Build Windows + Linux desktop apps (ubuntu, ~8 min) + # No code signing needed — can run on cheaper/faster ubuntu runners. + # ────────────────────────────────────────────────────────────────────── + build_desktop: + needs: build_web + permissions: + contents: write + runs-on: ubuntu-latest + steps: + - name: Check out Git repository + uses: actions/checkout@v4 + + - uses: pnpm/action-setup@v4 + name: Install pnpm + with: + version: latest + run_install: false + + - name: Install Node.js + uses: actions/setup-node@v4 + with: + node-version-file: '.nvmrc' + cache: 'pnpm' + + - name: Install Dependencies + run: pnpm install --no-frozen-lockfile + + - name: Download web build artifacts + uses: actions/download-artifact@v4 + with: + name: web-build + + - name: Prepare Electron app + run: make dep_electron + + - name: Compile local proxies (parallel) + run: | + make compile_localproxy os=win arch=x64 local_proxy_postfix=.exe & + make compile_localproxy os=win arch=arm64 local_proxy_postfix=.exe & + make compile_localproxy os=linux arch=x64 local_proxy_postfix= & + make compile_localproxy os=linux arch=arm64 local_proxy_postfix= & + wait + + - name: Package Windows & Linux Desktop Apps + run: | + make win_x64 + make win_arm64 + make linux_x64 + make linux_arm64 + + - name: Upload release assets + if: inputs.dry_run != true run: node upload-release.js app env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/Makefile b/Makefile index d6734402c4..46314ea990 100644 --- a/Makefile +++ b/Makefile @@ -61,6 +61,7 @@ compile_client_node_ts: clean_client_node_ts compile_wsproxy: compile_client_node_ts @pnpm -w exec webpack-cli --config src/wsproxy/webpack.config.js all: dep + @make compile_all_localproxy @make mac_x64 @make mac_arm64 @make win_x64 @@ -68,28 +69,58 @@ all: dep @make linux_x64 @make linux_arm64 @make bundle -dep: +# Build all local proxy binaries in parallel (saves ~3-4 min vs sequential) +compile_all_localproxy: + @printf "$(GREEN)Compiling all local proxy binaries in parallel...$(NC)\n" + @make compile_localproxy os=macos arch=x64 local_proxy_postfix= & \ + make compile_localproxy os=macos arch=arm64 local_proxy_postfix= & \ + make compile_localproxy os=win arch=x64 local_proxy_postfix=.exe & \ + make compile_localproxy os=win arch=arm64 local_proxy_postfix=.exe & \ + make compile_localproxy os=linux arch=x64 local_proxy_postfix= & \ + make compile_localproxy os=linux arch=arm64 local_proxy_postfix= & \ + wait + @printf "$(YELLOW)All local proxy binaries compiled$(NC)\n" +# Build only the web bundle (no Electron setup). Used by CI web job. +# Also ensures src/wsproxy/dist/wsproxy.js exists, since dep_electron copies it. +dep_web: @if [ ! -f "./config.toml" ]; then \ cp config.toml.sample config.toml; \ fi @mkdir -p ./app - @if [ ! -d "./build/web/" ] || ! grep -q 'es6://static/js/main' react/build/index.html; then \ + @if [ ! -d "./build/web/" ]; then \ make compile; \ + fi + @if [ ! -f "./src/wsproxy/dist/wsproxy.js" ]; then \ make compile_wsproxy; \ - rm -rf build/electron-app; \ - mkdir -p build/electron-app; \ - cp -r electron-app/* build/electron-app/;\ - cp electron-app/.npmrc build/electron-app/;\ - pnpm i --prefix ./build/electron-app --ignore-workspace;\ + fi +# Prepare the Electron app directory. Requires dep_web to have run first. +# Uses publicPath patching instead of a full second React build (~4-8 min savings). +# +# Idempotent: skips when `build/electron-app/app/index.html` already carries +# the patched `es6://static/js/main` marker. This mirrors the original +# Makefile's skip semantics so downstream targets that re-declare `dep` as a +# prerequisite (e.g. `mac_x64`, `win_x64`) do not repeatedly re-copy the web +# bundle. Set `FORCE_DEP_ELECTRON=1` to force a re-sync. +dep_electron: dep_web + @if [ -f "./build/electron-app/app/index.html" ] && grep -q 'es6://static/js/main' ./build/electron-app/app/index.html && [ "$(FORCE_DEP_ELECTRON)" != "1" ]; then \ + printf "$(YELLOW)Electron app already prepared, skipping$(NC)\n"; \ + else \ + if [ ! -d "./build/electron-app" ]; then \ + mkdir -p build/electron-app; \ + cp -r electron-app/* build/electron-app/; \ + cp electron-app/.npmrc build/electron-app/; \ + pnpm i --prefix ./build/electron-app --ignore-workspace; \ + fi; \ + rm -rf build/electron-app/app build/electron-app/resources build/electron-app/manifest; \ cp -Rp build/web build/electron-app/app; \ cp -Rp build/web/resources build/electron-app; \ cp -Rp build/web/manifest build/electron-app; \ - BUILD_TARGET=electron pnpm run build:react-only; \ - cp -Rp react/build/* build/electron-app/app/; \ + node scripts/patch-electron-publicpath.js build/electron-app/app; \ mkdir -p ./build/electron-app/app/wsproxy; \ cp ./src/wsproxy/dist/wsproxy.js ./build/electron-app/app/wsproxy/wsproxy.js; \ cp ./preload.js ./build/electron-app/preload.js; \ fi +dep: dep_electron web: @if [ ! -d "./build/web/" ];then \ make compile; \ @@ -119,17 +150,32 @@ endif # BAI_APP_SIGN_KEYCHAIN_PASSWORD echo Keychain ${KEYCHAIN_NAME} created for build endif # BAI_APP_SIGN_KEYCHAIN_B64 endif # BAI_APP_SIGN_KEYCHAIN +# Concurrency-safe: each (os, arch) build uses a unique staging directory so +# multiple invocations can run in parallel without overwriting each other's +# intermediate file (`backend.ai-local-proxy[.exe]`) packed into the ZIP. +# +# Idempotent: skips rebuild when the output ZIP already exists, so `make all` +# can pre-build everything via `compile_all_localproxy` and downstream targets +# (`mac_x64`, `win_x64`, ...) can reuse the cached artifact without +# re-compiling. Set `FORCE_COMPILE_LOCALPROXY=1` to force a rebuild. compile_localproxy: - @rm -rf ./app/backend.ai-local-proxy-$(BUILD_VERSION)-$(os)-$(arch)$(local_proxy_postfix) - @pnpm exec pkg ./src/wsproxy/local_proxy.js --targets node18-$(os)-$(arch) --output ./app/backend.ai-local-proxy-$(BUILD_VERSION)-$(os)-$(arch)$(local_proxy_postfix) --compress Brotli - @rm -rf ./app/backend.ai-local-proxy$(local_proxy_postfix); cp ./app/backend.ai-local-proxy-$(BUILD_VERSION)-$(os)-$(arch)$(local_proxy_postfix) ./app/backend.ai-local-proxy$(local_proxy_postfix) - @cd app; zip -r -9 ./backend.ai-local-proxy-$(BUILD_VERSION)-$(os)-$(arch).zip "./backend.ai-local-proxy$(local_proxy_postfix)" - @rm -rf ./app/backend.ai-local-proxy$(local_proxy_postfix) + @if [ -f "./app/backend.ai-local-proxy-$(BUILD_VERSION)-$(os)-$(arch).zip" ] && [ "$(FORCE_COMPILE_LOCALPROXY)" != "1" ]; then \ + printf "$(YELLOW)local-proxy $(os)-$(arch) already built, skipping$(NC)\n"; \ + else \ + rm -rf ./app/backend.ai-local-proxy-$(BUILD_VERSION)-$(os)-$(arch)$(local_proxy_postfix); \ + pnpm exec pkg ./src/wsproxy/local_proxy.js --targets node18-$(os)-$(arch) --output ./app/backend.ai-local-proxy-$(BUILD_VERSION)-$(os)-$(arch)$(local_proxy_postfix) --compress Brotli; \ + rm -rf ./app/_lp-stage-$(os)-$(arch); \ + mkdir -p ./app/_lp-stage-$(os)-$(arch); \ + cp ./app/backend.ai-local-proxy-$(BUILD_VERSION)-$(os)-$(arch)$(local_proxy_postfix) ./app/_lp-stage-$(os)-$(arch)/backend.ai-local-proxy$(local_proxy_postfix); \ + rm -f ./app/backend.ai-local-proxy-$(BUILD_VERSION)-$(os)-$(arch).zip; \ + (cd ./app/_lp-stage-$(os)-$(arch); zip -r -6 ../backend.ai-local-proxy-$(BUILD_VERSION)-$(os)-$(arch).zip "./backend.ai-local-proxy$(local_proxy_postfix)"); \ + rm -rf ./app/_lp-stage-$(os)-$(arch); \ + fi package_zip: @printf "$(GREEN)Packaging as ZIP archive...$(NC)" @cp ./configs/$(site).toml ./build/electron-app/app/config.toml @node ./app-packager.js $(os) $(arch) - @cd app; zip -r -9 ./backend.ai-desktop-$(os)-$(arch)-$(BUILD_DATE).zip "./Backend.AI Desktop-$(os_api)-$(arch)" + @cd app; zip -r -6 ./backend.ai-desktop-$(os)-$(arch)-$(BUILD_DATE).zip "./Backend.AI Desktop-$(os_api)-$(arch)" ifeq ($(site),main) @mv ./app/backend.ai-desktop-$(os)-$(arch)-$(BUILD_DATE).zip ./app/backend.ai-desktop-$(BUILD_VERSION)-$(os)-$(arch).zip else @@ -152,9 +198,10 @@ else @mv ./app/backend.ai-desktop-$(arch)-$(BUILD_DATE).dmg ./app/backend.ai-desktop-$(BUILD_VERSION)-$(site)-$(os)-$(arch).dmg endif @printf "$(YELLOW)Finished$(NC)\n" -bundle: dep +bundle: dep_web @printf "$(GREEN)Bundling...$(NC)" - @cd build/web; zip -r -9 ../../app/backend.ai-webui-bundle-$(BUILD_DATE).zip . > /dev/null + @mkdir -p ./app + @cd build/web; zip -r -6 ../../app/backend.ai-webui-bundle-$(BUILD_DATE).zip . > /dev/null @mv ./app/backend.ai-webui-bundle-$(BUILD_DATE).zip ./app/backend.ai-webui-bundle-$(BUILD_VERSION).zip @printf "$(YELLOW)Finished$(NC)\n" mac: dep diff --git a/react/package.json b/react/package.json index d4948529bf..3eec05e6b4 100644 --- a/react/package.json +++ b/react/package.json @@ -88,7 +88,7 @@ "scripts": { "start": "NODE_OPTIONS='--max-old-space-size=4096' craco start", "build": "pnpm run build:only && cp -r ./build/* ../build/web/", - "build:only": "pnpm run relay && craco build", + "build:only": "NODE_OPTIONS='--max-old-space-size=4096' pnpm run relay && NODE_OPTIONS='--max-old-space-size=4096' craco build", "test": "NODE_OPTIONS='$NODE_OPTIONS --no-deprecation --experimental-vm-modules' jest", "eject": "react-scripts eject", "relay": "relay-compiler", diff --git a/scripts/patch-electron-publicpath.js b/scripts/patch-electron-publicpath.js new file mode 100755 index 0000000000..4eff942a99 --- /dev/null +++ b/scripts/patch-electron-publicpath.js @@ -0,0 +1,135 @@ +#!/usr/bin/env node + +/** + * Patch the web build output to use Electron's 'es6://' publicPath. + * + * When building for Electron, webpack's output.publicPath must be 'es6://' + * instead of the default '/'. Previously this required a full second React + * build (~4-8 min). This script achieves the same result by patching the + * already-built files, saving significant CI time. + * + * Usage: node scripts/patch-electron-publicpath.js + * + * The script patches: + * 1. index.html — static asset references (/static/... → es6://static/...) + * 2. asset-manifest.json — all asset paths + * 3. JS bundles — webpack runtime's publicPath assignment (e.g. n.p="/") + * 4. CSS files — url() references to /static/ assets + * 5. Service worker (sw.js) — precache manifest URLs + */ + +const fs = require('fs'); +const path = require('path'); + +const buildDir = process.argv[2]; + +if (!buildDir) { + console.error('Usage: node scripts/patch-electron-publicpath.js '); + process.exit(1); +} + +if (!fs.existsSync(buildDir)) { + console.error(`Build directory not found: ${buildDir}`); + process.exit(1); +} + +const WEB_PUBLIC_PATH = '/'; +const ELECTRON_PUBLIC_PATH = 'es6://'; + +let patchedCount = 0; + +/** + * Replace all occurrences of web publicPath with electron publicPath in a file. + */ +function patchFile(filePath, replacements) { + if (!fs.existsSync(filePath)) return false; + + let content = fs.readFileSync(filePath, 'utf-8'); + const original = content; + + for (const [search, replace] of replacements) { + content = content.split(search).join(replace); + } + + if (content !== original) { + fs.writeFileSync(filePath, content, 'utf-8'); + patchedCount++; + console.log(` Patched: ${path.relative(buildDir, filePath)}`); + return true; + } + return false; +} + +console.log(`Patching publicPath: "${WEB_PUBLIC_PATH}" → "${ELECTRON_PUBLIC_PATH}"`); +console.log(`Build directory: ${buildDir}\n`); + +// 1. Patch index.html +patchFile(path.join(buildDir, 'index.html'), [ + // Script/link tags: src="/static/... → src="es6://static/... + ['="/static/', `="${ELECTRON_PUBLIC_PATH}static/`], + // Href references + ['href="/static/', `href="${ELECTRON_PUBLIC_PATH}static/`], + // Manifest and other root-relative references + ['="/manifest', `="${ELECTRON_PUBLIC_PATH}manifest`], +]); + +// 2. Patch asset-manifest.json +patchFile(path.join(buildDir, 'asset-manifest.json'), [ + [`"/static/`, `"${ELECTRON_PUBLIC_PATH}static/`], +]); + +// 3. Patch JS bundles (webpack runtime publicPath + chunk references) +const jsDir = path.join(buildDir, 'static', 'js'); +if (fs.existsSync(jsDir)) { + const jsFiles = fs.readdirSync(jsDir).filter((f) => f.endsWith('.js')); + for (const file of jsFiles) { + patchFile(path.join(jsDir, file), [ + // Webpack runtime: various minified forms of __webpack_require__.p = "/" + // Common patterns from webpack 5 + terser: + ['.p="/"', `.p="${ELECTRON_PUBLIC_PATH}"`], + [".p='/'", `.p='${ELECTRON_PUBLIC_PATH}'`], + // Static references to /static/ in chunk loading code + ['"/static/', `"${ELECTRON_PUBLIC_PATH}static/`], + ]); + } +} + +// 4. Patch CSS files (url() references) +const cssDir = path.join(buildDir, 'static', 'css'); +if (fs.existsSync(cssDir)) { + const cssFiles = fs.readdirSync(cssDir).filter((f) => f.endsWith('.css')); + for (const file of cssFiles) { + patchFile(path.join(cssDir, file), [ + ['url(/static/', `url(${ELECTRON_PUBLIC_PATH}static/`], + ]); + } +} + +// 5. Patch service worker if present +patchFile(path.join(buildDir, 'sw.js'), [ + ['"/static/', `"${ELECTRON_PUBLIC_PATH}static/`], + ['url:"/static/', `url:"${ELECTRON_PUBLIC_PATH}static/`], +]); + +// 6. Patch workbox precache manifest if present +const swFiles = fs.readdirSync(buildDir).filter((f) => /^workbox-.*\.js$/.test(f)); +for (const file of swFiles) { + patchFile(path.join(buildDir, file), [ + ['"/static/', `"${ELECTRON_PUBLIC_PATH}static/`], + ]); +} + +console.log(`\nDone. Patched ${patchedCount} file(s).`); + +// Verify the patch worked by checking index.html +const indexPath = path.join(buildDir, 'index.html'); +if (fs.existsSync(indexPath)) { + const indexContent = fs.readFileSync(indexPath, 'utf-8'); + if (indexContent.includes('es6://static/js/main')) { + console.log('✓ Verification passed: index.html contains es6://static/js/main'); + } else { + console.error('✗ Verification failed: index.html does not contain es6://static/js/main'); + console.error(' The Electron build may not work correctly.'); + process.exit(1); + } +} diff --git a/upload-release.js b/upload-release.js index f8c371e689..5ad2ada0a3 100644 --- a/upload-release.js +++ b/upload-release.js @@ -63,10 +63,14 @@ const main = async () => { const tag = process.env.RELEASE_TAG || (await getLatestTag()) const releaseId = await getReleaseIdFromTag(tag) + // Fetch the upload URL once — it's constant per release. Calling + // getUploadURL() per asset wastes API quota and risks rate limiting. + const uploadUrl = await getUploadURL(releaseId) - for (const filename of DMGs) { + // Upload files concurrently (up to 4 at a time) for faster release publishing + const CONCURRENCY = 4 + const uploadFile = async (filename) => { console.log(`Uploading file ${filename} to https://github.com/${OWNER}/${REPOSITORY}/releases/${tag}`) - const uploadUrl = await getUploadURL(releaseId) const buf = await fs.promises.readFile(path.join(folder, filename)) try { await octokit.request({ @@ -78,16 +82,22 @@ const main = async () => { data: buf, name: filename, }) - console.log('completed uploading file') + console.log(`completed uploading file: ${filename}`) } catch (e) { - if (e.response.data) { - console.error('error while uploading file:', e.response.data.errors) + if (e.response && e.response.data) { + console.error(`error while uploading file ${filename}:`, e.response.data.errors) } else { - console.error('unknown error while uploading file:') + console.error(`unknown error while uploading file ${filename}:`) console.error(e) } } } + + // Process uploads in batches to avoid overwhelming the API + for (let i = 0; i < DMGs.length; i += CONCURRENCY) { + const batch = DMGs.slice(i, i + CONCURRENCY) + await Promise.all(batch.map(uploadFile)) + } } main()