diff --git a/app/src/remote_server/ssh_transport.rs b/app/src/remote_server/ssh_transport.rs index fcce50d0af..e3c2022aea 100644 --- a/app/src/remote_server/ssh_transport.rs +++ b/app/src/remote_server/ssh_transport.rs @@ -212,17 +212,59 @@ impl RemoteTransport for SshTransport { if output.status.code() == Some(remote_server::setup::NO_HTTP_CLIENT_EXIT_CODE) => { - log::info!("Remote server has no curl/wget, falling back to SCP upload"); + log::info!("Remote has no curl/wget, falling back to SCP upload"); scp_install_fallback(&socket_path) .await .map_err(Error::Other) } + Ok(output) + if output.status.code() + == Some(remote_server::setup::DOWNLOAD_FAILED_EXIT_CODE) => + { + log::info!( + "Remote download failed (both HTTP clients tried), \ + falling back to SCP upload" + ); + scp_install_fallback(&socket_path) + .await + .map_err(Error::Other) + } + Ok(output) + if output.status.code() == Some(remote_server::setup::NO_TAR_EXIT_CODE) => + { + log::info!("Remote has no tar, falling back to direct binary upload"); + scp_install_binary_direct(&socket_path) + .await + .map_err(Error::Other) + } Ok(output) => { let exit_code = output.status.code().unwrap_or(-1); let stderr = String::from_utf8_lossy(&output.stderr).to_string(); + // Do not attempt fallback for host-level failures that no + // alternate download strategy can fix. + if remote_server::setup::is_non_retryable_host_error(&stderr) { + return Err(Error::ScriptFailed { exit_code, stderr }); + } + // SSH exit 255 means the connection itself is dead — + // no point attempting an SCP fallback. + if exit_code == 255 { + return Err(Error::ScriptFailed { exit_code, stderr }); + } Err(Error::ScriptFailed { exit_code, stderr }) } - Err(SshCommandError::TimedOut { .. }) => Err(Error::TimedOut), + // Timeout: the install script did not complete in time. + // If the timeout is likely from the download phase, the SCP + // fallback may succeed. Host-filesystem timeouts are rare; + // if the SCP fallback also times out, the error propagates. + Err(SshCommandError::TimedOut { .. }) => { + log::info!("Install script timed out, attempting SCP fallback"); + scp_install_fallback(&socket_path) + .await + .map_err(|fallback_err| { + log::warn!("SCP fallback also failed after timeout: {fallback_err:#}"); + Error::TimedOut + }) + } Err(e) => Err(Error::Other(e.into())), } }) @@ -315,6 +357,114 @@ impl RemoteTransport for SshTransport { /// the remote via SCP, then re-invokes the install script with the /// staging path baked in so the shared extraction tail runs. async fn scp_install_fallback(socket_path: &Path) -> anyhow::Result<()> { + let (_platform, tmp_dir) = download_tarball_locally(socket_path).await?; + let temp_client_tarball_path = tmp_dir.path().join("oz.tar.gz"); + + let remote_tarball_path = format!( + "{}/oz-upload.tar.gz", + remote_server::setup::remote_server_dir() + ); + let timeout = remote_server::setup::SCP_INSTALL_TIMEOUT; + + // Upload to the remote via SCP. + log::info!("Uploading tarball to remote at {remote_tarball_path}"); + remote_server::ssh::scp_upload( + socket_path, + &temp_client_tarball_path, + &remote_tarball_path, + timeout, + ) + .await?; + + // Run the install script with the staging path baked in. + // The script's `staging_tarball_path` variable is non-empty, so it + // skips the download and extracts from the uploaded tarball. + log::info!("Running extraction via install script with tarball at {remote_tarball_path}"); + + let script = remote_server::setup::install_script(Some(&remote_tarball_path)); + + let output = remote_server::ssh::run_ssh_script(socket_path, &script, timeout).await?; + if output.status.success() { + Ok(()) + } else { + let code = output.status.code().unwrap_or(-1); + let stderr = String::from_utf8_lossy(&output.stderr); + Err(anyhow::anyhow!( + "Extraction script failed (exit {code}): {stderr}" + )) + } +} + +/// Direct binary upload fallback: downloads the tarball locally, extracts +/// it locally, and uploads only the resolved binary via SCP. This avoids +/// requiring `tar` on the remote host. +/// +/// The remote-side steps are: +/// 1. `mkdir -p ` (ensures the directory exists) +/// 2. SCP the binary to a staging path +/// 3. `chmod +x && mv` to the final install path +async fn scp_install_binary_direct(socket_path: &Path) -> anyhow::Result<()> { + let (_platform, tmp_dir) = download_tarball_locally(socket_path).await?; + let temp_client_tarball_path = tmp_dir.path().join("oz.tar.gz"); + let timeout = remote_server::setup::SCP_INSTALL_TIMEOUT; + + // Extract locally using the client machine's tar. + log::info!("Extracting tarball locally for direct binary upload"); + let extract_dir = tmp_dir.path().join("extracted"); + std::fs::create_dir_all(&extract_dir) + .map_err(|e| anyhow::anyhow!("Failed to create local extraction dir: {e}"))?; + + let tar_output = command::r#async::Command::new("tar") + .arg("-xzf") + .arg(&temp_client_tarball_path) + .arg("-C") + .arg(&extract_dir) + .kill_on_drop(true) + .output() + .await + .map_err(|e| anyhow::anyhow!("Failed to spawn local tar: {e}"))?; + if !tar_output.status.success() { + let stderr = String::from_utf8_lossy(&tar_output.stderr); + return Err(anyhow::anyhow!( + "Local tar extraction failed (exit {:?}): {stderr}", + tar_output.status.code() + )); + } + + // Find the binary in the extraction directory. + let binary_path = find_oz_binary_in_dir(&extract_dir)?; + + // Upload the binary directly to a staging path on the remote. + let remote_binary = remote_server::setup::remote_server_binary(); + let remote_staging = format!("{remote_binary}.staging"); + + log::info!("Uploading binary directly to remote at {remote_staging}"); + remote_server::ssh::scp_upload(socket_path, &binary_path, &remote_staging, timeout).await?; + + // chmod +x and move to final location on the remote. + let finalize_cmd = + format!("chmod +x '{remote_staging}' && mv '{remote_staging}' '{remote_binary}'"); + log::info!("Finalizing remote binary: {finalize_cmd}"); + let output = remote_server::ssh::run_ssh_command(socket_path, &finalize_cmd, timeout).await?; + if output.status.success() { + Ok(()) + } else { + let code = output.status.code().unwrap_or(-1); + let stderr = String::from_utf8_lossy(&output.stderr); + Err(anyhow::anyhow!( + "Remote finalize failed (exit {code}): {stderr}" + )) + } +} + +/// Downloads the remote-server tarball to a local temp directory. +/// Returns the detected remote platform and the temp directory handle. +/// +/// Shared by [`scp_install_fallback`] and [`scp_install_binary_direct`] +/// to avoid duplicating the platform-detection + local-download logic. +async fn download_tarball_locally( + socket_path: &Path, +) -> anyhow::Result<(remote_server::setup::RemotePlatform, tempfile::TempDir)> { use std::process::Stdio; // Detect the remote platform so we can construct the correct download URL. @@ -326,22 +476,13 @@ async fn scp_install_fallback(socket_path: &Path) -> anyhow::Result<()> { .map_err(|e| anyhow::anyhow!("SCP fallback: {e:#}"))?; let url = remote_server::setup::download_tarball_url(&platform); - let remote_tarball_path = format!( - "{}/oz-upload.tar.gz", - remote_server::setup::remote_server_dir() - ); - let timeout = remote_server::setup::SCP_INSTALL_TIMEOUT; - // 1. Download the tarball locally into a temp directory. let tmp_dir = tempfile::tempdir().map_err(|e| anyhow::anyhow!("Failed to create local temp dir: {e}"))?; let temp_client_tarball_path = tmp_dir.path().join("oz.tar.gz"); log::info!("Downloading tarball locally from {url}"); let output = command::r#async::Command::new("curl") - // -f: fail silently on HTTP errors (non-zero exit instead of HTML error page) - // -S: show errors even when -f is used - // -L: follow redirects (the CDN may 302 to a regional edge) .arg("-fSL") .arg("--connect-timeout") .arg("15") @@ -362,33 +503,25 @@ async fn scp_install_fallback(socket_path: &Path) -> anyhow::Result<()> { )); } - // 2. Upload to the remote via SCP. - log::info!("Uploading tarball to remote at {remote_tarball_path}"); - remote_server::ssh::scp_upload( - socket_path, - &temp_client_tarball_path, - &remote_tarball_path, - timeout, - ) - .await?; - - // 3. Run the install script with the staging path baked in. - // The script's `staging_tarball_path` variable is non-empty, so it - // skips the download and extracts from the uploaded tarball. - log::info!("Running extraction via install script with tarball at {remote_tarball_path}"); - - let script = remote_server::setup::install_script(Some(&remote_tarball_path)); + Ok((platform, tmp_dir)) +} - let output = remote_server::ssh::run_ssh_script(socket_path, &script, timeout).await?; - if output.status.success() { - Ok(()) - } else { - let code = output.status.code().unwrap_or(-1); - let stderr = String::from_utf8_lossy(&output.stderr); - Err(anyhow::anyhow!( - "Extraction script failed (exit {code}): {stderr}" - )) +/// Walks `dir` for the first file whose name starts with `oz` and is not +/// a `.tar.gz`, matching the install script's `find` invocation. +fn find_oz_binary_in_dir(dir: &Path) -> anyhow::Result { + for entry in walkdir::WalkDir::new(dir) + .into_iter() + .filter_map(|e| e.ok()) + { + if !entry.file_type().is_file() { + continue; + } + let name = entry.file_name().to_string_lossy(); + if name.starts_with("oz") && !name.ends_with(".tar.gz") { + return Ok(entry.into_path()); + } } + Err(anyhow::anyhow!("no binary found in extracted tarball")) } #[cfg(test)] diff --git a/crates/remote_server/src/install_remote_server.sh b/crates/remote_server/src/install_remote_server.sh index 4e8eae8241..4a31cd7bd5 100644 --- a/crates/remote_server/src/install_remote_server.sh +++ b/crates/remote_server/src/install_remote_server.sh @@ -9,6 +9,8 @@ # {version_query} — e.g. &version=v0.2026... (empty when no release tag) # {version_suffix} — e.g. -v0.2026... (empty when no release tag) # {no_http_client_exit_code} — exit code when neither curl nor wget is available +# {download_failed_exit_code} — exit code when both curl and wget fail to download +# {no_tar_exit_code} — exit code when tar is not available # {staging_tarball_path} — path to a pre-uploaded tarball (SCP fallback; empty normally) set -e @@ -64,17 +66,50 @@ if [ -n "$staging_tarball_path" ]; then esac mv "$staging_tarball_path" "$tmpdir/oz.tar.gz" else - # Normal path: download via curl or wget. + # Normal path: download via curl or wget, with retry using the + # alternate client if the primary fails. url="{download_base_url}?package=tar&os=$os_name&arch=$arch_name&channel={channel}{version_query}" - if command -v curl >/dev/null 2>&1; then - curl -fSL "$url" -o "$tmpdir/oz.tar.gz" - elif command -v wget >/dev/null 2>&1; then - wget -q -O "$tmpdir/oz.tar.gz" "$url" - else + has_curl=false + has_wget=false + command -v curl >/dev/null 2>&1 && has_curl=true + command -v wget >/dev/null 2>&1 && has_wget=true + + if [ "$has_curl" = false ] && [ "$has_wget" = false ]; then echo "error: neither curl nor wget is available" >&2 exit {no_http_client_exit_code} fi + + download_ok=false + download_err="" + + # Try primary client, then retry with alternate on failure. + if [ "$has_curl" = true ]; then + if curl -fSL --connect-timeout 15 --retry 1 "$url" -o "$tmpdir/oz.tar.gz" 2>/dev/null; then + download_ok=true + else + download_err="curl failed (exit $?)" + fi + fi + + if [ "$download_ok" = false ] && [ "$has_wget" = true ]; then + if wget -q --timeout=15 -O "$tmpdir/oz.tar.gz" "$url" 2>/dev/null; then + download_ok=true + else + download_err="${download_err:+$download_err; }wget failed (exit $?)" + fi + fi + + if [ "$download_ok" = false ]; then + echo "error: remote download failed: $download_err" >&2 + exit {download_failed_exit_code} + fi +fi + +# Verify tar is available before attempting extraction. +if ! command -v tar >/dev/null 2>&1; then + echo "error: tar is not available" >&2 + exit {no_tar_exit_code} fi tar -xzf "$tmpdir/oz.tar.gz" -C "$tmpdir" diff --git a/crates/remote_server/src/setup.rs b/crates/remote_server/src/setup.rs index fdd254ea46..ba814dec93 100644 --- a/crates/remote_server/src/setup.rs +++ b/crates/remote_server/src/setup.rs @@ -427,6 +427,11 @@ pub fn install_script(staging_tarball_path: Option<&str>) -> String { "{no_http_client_exit_code}", &NO_HTTP_CLIENT_EXIT_CODE.to_string(), ) + .replace( + "{download_failed_exit_code}", + &DOWNLOAD_FAILED_EXIT_CODE.to_string(), + ) + .replace("{no_tar_exit_code}", &NO_TAR_EXIT_CODE.to_string()) .replace("{staging_tarball_path}", staging_tarball_path.unwrap_or("")) } @@ -487,6 +492,17 @@ pub fn download_tarball_url(platform: &RemotePlatform) -> String { /// trigger the SCP upload fallback. pub const NO_HTTP_CLIENT_EXIT_CODE: i32 = 3; +/// Exit code the install script uses when both curl and wget are present +/// but both failed to download the tarball (DNS failure, TLS error, +/// HTTP 403/502, timeout, partial download, etc.). The Rust side matches +/// on this to trigger the SCP upload fallback. +pub const DOWNLOAD_FAILED_EXIT_CODE: i32 = 4; + +/// Exit code the install script uses when `tar` is not available on the +/// remote host. The Rust side matches on this to trigger the direct +/// binary upload fallback (extract locally, upload only the binary). +pub const NO_TAR_EXIT_CODE: i32 = 5; + /// Timeout for the binary existence check. pub const CHECK_TIMEOUT: Duration = Duration::from_secs(10); @@ -499,6 +515,23 @@ pub const INSTALL_TIMEOUT: Duration = Duration::from_secs(60); /// the remote host's direct internet connection. pub const SCP_INSTALL_TIMEOUT: Duration = Duration::from_secs(120); +/// Returns `true` if the install script stderr indicates a host-level +/// condition that should **not** be retried via SCP fallback. These are +/// "true" failures (permission denied, no disk space, read-only +/// filesystem, quota exceeded) that no amount of alternate download +/// strategy can fix. +pub fn is_non_retryable_host_error(stderr: &str) -> bool { + let lower = stderr.to_ascii_lowercase(); + // Each pattern corresponds to a POSIX errno string commonly emitted + // by mkdir, chmod, mv, tar, or the shell on real remote hosts. + lower.contains("permission denied") + || lower.contains("read-only file system") + || lower.contains("no space left on device") + || lower.contains("disk quota exceeded") + || lower.contains("cannot create directory") + || lower.contains("operation not permitted") +} + #[cfg(test)] #[path = "setup_tests.rs"] mod tests; diff --git a/crates/remote_server/src/setup_tests.rs b/crates/remote_server/src/setup_tests.rs index 3fea5927a3..7219ae18fd 100644 --- a/crates/remote_server/src/setup_tests.rs +++ b/crates/remote_server/src/setup_tests.rs @@ -315,3 +315,92 @@ fn parse_preinstall_missing_status_falls_open() { assert_eq!(result.status, PreinstallStatus::Unknown); assert!(result.is_supported()); } + +#[test] +fn exit_code_constants_are_distinct() { + // Guard: the install script exit codes must be distinct so the Rust + // side routes to the correct fallback. + let codes = [ + NO_HTTP_CLIENT_EXIT_CODE, + DOWNLOAD_FAILED_EXIT_CODE, + NO_TAR_EXIT_CODE, + ]; + for (i, a) in codes.iter().enumerate() { + for b in &codes[i + 1..] { + assert_ne!(a, b, "exit code collision: {a} == {b}"); + } + // None of them should collide with the unsupported-arch exit (2) + // or generic failure (1). + assert_ne!(*a, 1); + assert_ne!(*a, 2); + } +} + +#[test] +fn install_script_substitutes_new_exit_codes() { + let script = install_script(None); + // The new exit code placeholders must be resolved in the generated + // script — no raw `{download_failed_exit_code}` or `{no_tar_exit_code}` + // should remain. + assert!( + !script.contains("{download_failed_exit_code}"), + "placeholder not substituted" + ); + assert!( + !script.contains("{no_tar_exit_code}"), + "placeholder not substituted" + ); + // The actual numeric values should appear. + assert!(script.contains(&format!("exit {DOWNLOAD_FAILED_EXIT_CODE}"))); + assert!(script.contains(&format!("exit {NO_TAR_EXIT_CODE}"))); +} + +#[test] +fn is_non_retryable_detects_permission_denied() { + assert!(is_non_retryable_host_error( + "mkdir: cannot create directory '/root/.warp': Permission denied" + )); +} + +#[test] +fn is_non_retryable_detects_no_space() { + assert!(is_non_retryable_host_error( + "tar: oz: Cannot write: No space left on device" + )); +} + +#[test] +fn is_non_retryable_detects_read_only_fs() { + assert!(is_non_retryable_host_error( + "mv: cannot move 'oz': Read-only file system" + )); +} + +#[test] +fn is_non_retryable_detects_quota() { + assert!(is_non_retryable_host_error( + "write failed: Disk quota exceeded" + )); +} + +#[test] +fn is_non_retryable_ignores_download_errors() { + // Network/download errors should NOT be classified as non-retryable, + // because the SCP fallback may succeed. + assert!(!is_non_retryable_host_error( + "curl: (6) Could not resolve host: app.warp.dev" + )); + assert!(!is_non_retryable_host_error( + "wget: unable to resolve host address" + )); + assert!(!is_non_retryable_host_error( + "curl: (60) SSL certificate problem" + )); +} + +#[test] +fn is_non_retryable_detects_operation_not_permitted() { + assert!(is_non_retryable_host_error( + "chmod: changing permissions of 'oz': Operation not permitted" + )); +} diff --git a/script/remote-install-harness/.gitignore b/script/remote-install-harness/.gitignore new file mode 100644 index 0000000000..cd91ba6e77 --- /dev/null +++ b/script/remote-install-harness/.gitignore @@ -0,0 +1,4 @@ +# Runtime artifacts generated by run_harness.sh +.fake_server/ +.harness_* +results/ diff --git a/script/remote-install-harness/profiles/00_baseline_normal.sh b/script/remote-install-harness/profiles/00_baseline_normal.sh new file mode 100644 index 0000000000..829e38966c --- /dev/null +++ b/script/remote-install-harness/profiles/00_baseline_normal.sh @@ -0,0 +1,13 @@ +# Profile: baseline normal install — everything works. +# This profile verifies the harness itself: a clean user with all tools +# and a working download server should complete the install successfully. +PROFILE_USER="harness_baseline" +PROFILE_EXPECTED="Successful install; exit 0" + +profile_setup() { + : # No special setup needed +} + +profile_teardown() { + rm -rf "/home/$PROFILE_USER/.warp-test" 2>/dev/null || true +} diff --git a/script/remote-install-harness/profiles/01_missing_curl.sh b/script/remote-install-harness/profiles/01_missing_curl.sh new file mode 100644 index 0000000000..a6b972e109 --- /dev/null +++ b/script/remote-install-harness/profiles/01_missing_curl.sh @@ -0,0 +1,37 @@ +# Profile: missing curl — only wget is available on the remote host. +# The install script should fall back to wget and succeed. +PROFILE_USER="harness_nocurl" +PROFILE_EXPECTED="Script falls back to wget; should succeed if wget is present" + +profile_setup() { + local home="/home/$PROFILE_USER" + # Hide curl by creating a restricted PATH wrapper + mkdir -p "$home/.harness_bin" + # Symlink everything from /usr/bin except curl + for bin in /usr/bin/*; do + local name + name="$(basename "$bin")" + [ "$name" = "curl" ] && continue + ln -sf "$bin" "$home/.harness_bin/$name" 2>/dev/null || true + done + # Also link from /bin and /usr/sbin + for bin in /bin/* /usr/sbin/*; do + [ -f "$bin" ] || continue + local name + name="$(basename "$bin")" + [ "$name" = "curl" ] && continue + ln -sf "$bin" "$home/.harness_bin/$name" 2>/dev/null || true + done + chown -R "$PROFILE_USER:$PROFILE_USER" "$home/.harness_bin" +} + +profile_run() { + local script="$1" + # Run with restricted PATH so curl is not found + local home="/home/$PROFILE_USER" + ssh_run_script "$PROFILE_USER" "export PATH=$home/.harness_bin; $script" +} + +profile_teardown() { + rm -rf "/home/$PROFILE_USER/.harness_bin" +} diff --git a/script/remote-install-harness/profiles/02_missing_wget.sh b/script/remote-install-harness/profiles/02_missing_wget.sh new file mode 100644 index 0000000000..a0c71bd6f5 --- /dev/null +++ b/script/remote-install-harness/profiles/02_missing_wget.sh @@ -0,0 +1,33 @@ +# Profile: missing wget — only curl is available on the remote host. +# The install script should use curl and succeed. +PROFILE_USER="harness_nowget" +PROFILE_EXPECTED="Script uses curl; should succeed" + +profile_setup() { + local home="/home/$PROFILE_USER" + mkdir -p "$home/.harness_bin" + for bin in /usr/bin/*; do + local name + name="$(basename "$bin")" + [ "$name" = "wget" ] && continue + ln -sf "$bin" "$home/.harness_bin/$name" 2>/dev/null || true + done + for bin in /bin/* /usr/sbin/*; do + [ -f "$bin" ] || continue + local name + name="$(basename "$bin")" + [ "$name" = "wget" ] && continue + ln -sf "$bin" "$home/.harness_bin/$name" 2>/dev/null || true + done + chown -R "$PROFILE_USER:$PROFILE_USER" "$home/.harness_bin" +} + +profile_run() { + local script="$1" + local home="/home/$PROFILE_USER" + ssh_run_script "$PROFILE_USER" "export PATH=$home/.harness_bin; $script" +} + +profile_teardown() { + rm -rf "/home/$PROFILE_USER/.harness_bin" +} diff --git a/script/remote-install-harness/profiles/03_no_http_client.sh b/script/remote-install-harness/profiles/03_no_http_client.sh new file mode 100644 index 0000000000..c12f0d9d16 --- /dev/null +++ b/script/remote-install-harness/profiles/03_no_http_client.sh @@ -0,0 +1,35 @@ +# Profile: neither curl nor wget available on the remote host. +# The install script should exit with NO_HTTP_CLIENT_EXIT_CODE (3). +PROFILE_USER="harness_nohttp" +PROFILE_EXPECTED="Exit code 3 (no HTTP client); Rust side triggers SCP fallback" + +profile_setup() { + local home="/home/$PROFILE_USER" + mkdir -p "$home/.harness_bin" + for bin in /usr/bin/*; do + local name + name="$(basename "$bin")" + [ "$name" = "curl" ] && continue + [ "$name" = "wget" ] && continue + ln -sf "$bin" "$home/.harness_bin/$name" 2>/dev/null || true + done + for bin in /bin/* /usr/sbin/*; do + [ -f "$bin" ] || continue + local name + name="$(basename "$bin")" + [ "$name" = "curl" ] && continue + [ "$name" = "wget" ] && continue + ln -sf "$bin" "$home/.harness_bin/$name" 2>/dev/null || true + done + chown -R "$PROFILE_USER:$PROFILE_USER" "$home/.harness_bin" +} + +profile_run() { + local script="$1" + local home="/home/$PROFILE_USER" + ssh_run_script "$PROFILE_USER" "export PATH=$home/.harness_bin; $script" +} + +profile_teardown() { + rm -rf "/home/$PROFILE_USER/.harness_bin" +} diff --git a/script/remote-install-harness/profiles/04_missing_tar.sh b/script/remote-install-harness/profiles/04_missing_tar.sh new file mode 100644 index 0000000000..2c4e60adf8 --- /dev/null +++ b/script/remote-install-harness/profiles/04_missing_tar.sh @@ -0,0 +1,33 @@ +# Profile: tar is not available on the remote host. +# The install script should fail at the tar extraction step. +PROFILE_USER="harness_notar" +PROFILE_EXPECTED="Failure at tar extraction; 'tar: command not found' or similar" + +profile_setup() { + local home="/home/$PROFILE_USER" + mkdir -p "$home/.harness_bin" + for bin in /usr/bin/*; do + local name + name="$(basename "$bin")" + [ "$name" = "tar" ] && continue + ln -sf "$bin" "$home/.harness_bin/$name" 2>/dev/null || true + done + for bin in /bin/* /usr/sbin/*; do + [ -f "$bin" ] || continue + local name + name="$(basename "$bin")" + [ "$name" = "tar" ] && continue + ln -sf "$bin" "$home/.harness_bin/$name" 2>/dev/null || true + done + chown -R "$PROFILE_USER:$PROFILE_USER" "$home/.harness_bin" +} + +profile_run() { + local script="$1" + local home="/home/$PROFILE_USER" + ssh_run_script "$PROFILE_USER" "export PATH=$home/.harness_bin; $script" +} + +profile_teardown() { + rm -rf "/home/$PROFILE_USER/.harness_bin" +} diff --git a/script/remote-install-harness/profiles/05_unsupported_arch.sh b/script/remote-install-harness/profiles/05_unsupported_arch.sh new file mode 100644 index 0000000000..8d87ad2a31 --- /dev/null +++ b/script/remote-install-harness/profiles/05_unsupported_arch.sh @@ -0,0 +1,48 @@ +# Profile: unsupported architecture via a mocked uname. +# The install script should exit 2 with "unsupported arch: mips64". +PROFILE_USER="harness_badarch" +PROFILE_EXPECTED="Exit 2; 'unsupported arch: mips64'" + +profile_setup() { + local home="/home/$PROFILE_USER" + mkdir -p "$home/.harness_bin" + + # Create fake uname that returns mips64 + cat > "$home/.harness_bin/uname" <<'FEOF' +#!/bin/sh +# Fake uname: return mips64 for -m, Linux for -s +case "$*" in + *-m*) echo "mips64" ;; + *-s*) echo "Linux" ;; + *-sm*|*-ms*) echo "Linux mips64" ;; + *) echo "Linux mips64" ;; +esac +FEOF + chmod +x "$home/.harness_bin/uname" + + # Symlink other binaries + for bin in /usr/bin/*; do + local name + name="$(basename "$bin")" + [ "$name" = "uname" ] && continue + ln -sf "$bin" "$home/.harness_bin/$name" 2>/dev/null || true + done + for bin in /bin/*; do + [ -f "$bin" ] || continue + local name + name="$(basename "$bin")" + [ "$name" = "uname" ] && continue + ln -sf "$bin" "$home/.harness_bin/$name" 2>/dev/null || true + done + chown -R "$PROFILE_USER:$PROFILE_USER" "$home/.harness_bin" +} + +profile_run() { + local script="$1" + local home="/home/$PROFILE_USER" + ssh_run_script "$PROFILE_USER" "export PATH=$home/.harness_bin; $script" +} + +profile_teardown() { + rm -rf "/home/$PROFILE_USER/.harness_bin" +} diff --git a/script/remote-install-harness/profiles/06_dns_failure.sh b/script/remote-install-harness/profiles/06_dns_failure.sh new file mode 100644 index 0000000000..18dff1c2c7 --- /dev/null +++ b/script/remote-install-harness/profiles/06_dns_failure.sh @@ -0,0 +1,20 @@ +# Profile: DNS resolution failure. +# Override the download URL to a non-existent domain so curl/wget fails DNS. +PROFILE_USER="harness_dns" +PROFILE_EXPECTED="curl/wget DNS failure; 'Could not resolve host' or similar" + +profile_setup() { + : # No special setup needed — we override the URL in the script +} + +profile_run() { + local script="$1" + # Replace the download URL with a non-resolvable host + local modified_script + modified_script=$(echo "$script" | sed 's|http://127.0.0.1:[0-9]*/download/cli|http://nonexistent.invalid.warp.test/download/cli|g') + ssh_run_script "$PROFILE_USER" "$modified_script" +} + +profile_teardown() { + : +} diff --git a/script/remote-install-harness/profiles/07_connection_refused.sh b/script/remote-install-harness/profiles/07_connection_refused.sh new file mode 100644 index 0000000000..6a81552104 --- /dev/null +++ b/script/remote-install-harness/profiles/07_connection_refused.sh @@ -0,0 +1,20 @@ +# Profile: connection refused — nothing listening on the download port. +# curl/wget should fail with "Connection refused". +PROFILE_USER="harness_refused" +PROFILE_EXPECTED="curl/wget connection refused; exit non-zero" + +profile_setup() { + : +} + +profile_run() { + local script="$1" + # Point at a port where nothing is listening + local modified_script + modified_script=$(echo "$script" | sed 's|http://127.0.0.1:[0-9]*/download/cli|http://127.0.0.1:19999/download/cli|g') + ssh_run_script "$PROFILE_USER" "$modified_script" +} + +profile_teardown() { + : +} diff --git a/script/remote-install-harness/profiles/08_http_403.sh b/script/remote-install-harness/profiles/08_http_403.sh new file mode 100644 index 0000000000..5a2715f7fa --- /dev/null +++ b/script/remote-install-harness/profiles/08_http_403.sh @@ -0,0 +1,13 @@ +# Profile: HTTP 403 Forbidden from the download server. +# curl -f should fail with exit 22; wget with a non-zero exit. +PROFILE_USER="harness_403" +PROFILE_EXPECTED="HTTP 403; curl exit 22 or wget error" +PROFILE_SERVER_BEHAVIOR="403" + +profile_setup() { + : +} + +profile_teardown() { + : +} diff --git a/script/remote-install-harness/profiles/09_http_502.sh b/script/remote-install-harness/profiles/09_http_502.sh new file mode 100644 index 0000000000..ea9e30d2e2 --- /dev/null +++ b/script/remote-install-harness/profiles/09_http_502.sh @@ -0,0 +1,13 @@ +# Profile: HTTP 502 Bad Gateway from the download server. +# curl -f should fail with exit 22; wget with a non-zero exit. +PROFILE_USER="harness_502" +PROFILE_EXPECTED="HTTP 502; curl exit 22 or wget error" +PROFILE_SERVER_BEHAVIOR="502" + +profile_setup() { + : +} + +profile_teardown() { + : +} diff --git a/script/remote-install-harness/profiles/10_partial_download.sh b/script/remote-install-harness/profiles/10_partial_download.sh new file mode 100644 index 0000000000..41cdf3a57d --- /dev/null +++ b/script/remote-install-harness/profiles/10_partial_download.sh @@ -0,0 +1,13 @@ +# Profile: partial download — server sends truncated tarball. +# tar extraction should fail because the file is corrupted/incomplete. +PROFILE_USER="harness_partial" +PROFILE_EXPECTED="tar extraction failure on corrupt/truncated tarball" +PROFILE_SERVER_BEHAVIOR="partial" + +profile_setup() { + : +} + +profile_teardown() { + : +} diff --git a/script/remote-install-harness/profiles/11_readonly_home.sh b/script/remote-install-harness/profiles/11_readonly_home.sh new file mode 100644 index 0000000000..52c022f5ec --- /dev/null +++ b/script/remote-install-harness/profiles/11_readonly_home.sh @@ -0,0 +1,20 @@ +# Profile: read-only home directory. +# mkdir -p for the install dir should fail. +PROFILE_USER="harness_rohome" +PROFILE_EXPECTED="mkdir failure; 'Read-only file system' or 'Permission denied'" + +profile_setup() { + local home="/home/$PROFILE_USER" + # Create the .warp-test dir first, then make it read-only + mkdir -p "$home/.warp-test" + chown "$PROFILE_USER:$PROFILE_USER" "$home/.warp-test" + chmod 555 "$home/.warp-test" + # Also make home read-only to prevent mkdir -p from creating .warp-test/remote-server + # We keep .ssh writable for auth +} + +profile_teardown() { + local home="/home/$PROFILE_USER" + chmod 755 "$home/.warp-test" 2>/dev/null || true + rm -rf "$home/.warp-test" 2>/dev/null || true +} diff --git a/script/remote-install-harness/profiles/12_permission_denied_home.sh b/script/remote-install-harness/profiles/12_permission_denied_home.sh new file mode 100644 index 0000000000..9409796676 --- /dev/null +++ b/script/remote-install-harness/profiles/12_permission_denied_home.sh @@ -0,0 +1,19 @@ +# Profile: permission denied on install directory. +# The install dir parent is owned by root and not writable by the user. +PROFILE_USER="harness_noperm" +PROFILE_EXPECTED="mkdir -p fails with 'Permission denied'" + +profile_setup() { + local home="/home/$PROFILE_USER" + # Create .warp-test owned by root, not writable by user + mkdir -p "$home/.warp-test" + chown root:root "$home/.warp-test" + chmod 755 "$home/.warp-test" + # The user can read but not write, so mkdir -p .warp-test/remote-server fails +} + +profile_teardown() { + local home="/home/$PROFILE_USER" + chown "$PROFILE_USER:$PROFILE_USER" "$home/.warp-test" 2>/dev/null || true + rm -rf "$home/.warp-test" 2>/dev/null || true +} diff --git a/script/remote-install-harness/profiles/13_no_space.sh b/script/remote-install-harness/profiles/13_no_space.sh new file mode 100644 index 0000000000..540857b5c4 --- /dev/null +++ b/script/remote-install-harness/profiles/13_no_space.sh @@ -0,0 +1,21 @@ +# Profile: no space left on device / disk quota exceeded. +# Simulated by mounting a tiny tmpfs for the install directory. +PROFILE_USER="harness_nospace" +PROFILE_EXPECTED="'No space left on device' during download or extraction" + +profile_setup() { + local home="/home/$PROFILE_USER" + # Create the install dir and mount a tiny (64K) tmpfs on it + mkdir -p "$home/.warp-test/remote-server" + chown -R "$PROFILE_USER:$PROFILE_USER" "$home/.warp-test" + # Use a 1-block tmpfs so even mktemp will trigger ENOSPC. + # Fill it with a single file to exhaust the remaining space. + mount -t tmpfs -o size=4k,uid=$(id -u "$PROFILE_USER"),gid=$(id -g "$PROFILE_USER") tmpfs "$home/.warp-test/remote-server" + dd if=/dev/zero of="$home/.warp-test/remote-server/.filler" bs=4096 count=1 2>/dev/null || true +} + +profile_teardown() { + local home="/home/$PROFILE_USER" + umount "$home/.warp-test/remote-server" 2>/dev/null || true + rm -rf "$home/.warp-test" 2>/dev/null || true +} diff --git a/script/remote-install-harness/profiles/14_readonly_filesystem.sh b/script/remote-install-harness/profiles/14_readonly_filesystem.sh new file mode 100644 index 0000000000..8e4425bcda --- /dev/null +++ b/script/remote-install-harness/profiles/14_readonly_filesystem.sh @@ -0,0 +1,17 @@ +# Profile: read-only filesystem mount for the install directory. +# Simulated by mounting a read-only tmpfs. +PROFILE_USER="harness_rofs" +PROFILE_EXPECTED="'Read-only file system' during mkdir or write" + +profile_setup() { + local home="/home/$PROFILE_USER" + mkdir -p "$home/.warp-test/remote-server" + chown -R "$PROFILE_USER:$PROFILE_USER" "$home/.warp-test" + mount -t tmpfs -o ro,uid=$(id -u "$PROFILE_USER"),gid=$(id -g "$PROFILE_USER") tmpfs "$home/.warp-test/remote-server" +} + +profile_teardown() { + local home="/home/$PROFILE_USER" + umount "$home/.warp-test/remote-server" 2>/dev/null || true + rm -rf "$home/.warp-test" 2>/dev/null || true +} diff --git a/script/remote-install-harness/profiles/15_tar_ownership_failure.sh b/script/remote-install-harness/profiles/15_tar_ownership_failure.sh new file mode 100644 index 0000000000..05629f2386 --- /dev/null +++ b/script/remote-install-harness/profiles/15_tar_ownership_failure.sh @@ -0,0 +1,54 @@ +# Profile: tar extraction fails due to ownership/permission issues. +# The staging temp directory is created but not writable by the user. +PROFILE_USER="harness_tarown" +PROFILE_EXPECTED="tar extraction failure; 'Cannot open' or 'Permission denied'" + +profile_setup() { + local home="/home/$PROFILE_USER" + # Pre-create the install dir so mkdir -p succeeds, but make the + # .install.XXXXXX temp dir creation succeed, then immediately revoke + # write permission. We intercept mktemp to create a non-writable dir. + mkdir -p "$home/.harness_bin" + + # Create a wrapper mktemp that creates a dir but makes it non-writable + cat > "$home/.harness_bin/mktemp" <<'FEOF' +#!/bin/bash +# Create a temp dir that's not writable by the caller +real_mktemp=/usr/bin/mktemp +result=$($real_mktemp "$@") +if [ -d "$result" ]; then + chmod 555 "$result" +fi +echo "$result" +FEOF + chmod +x "$home/.harness_bin/mktemp" + + # Symlink all other binaries + for bin in /usr/bin/*; do + local name + name="$(basename "$bin")" + [ "$name" = "mktemp" ] && continue + ln -sf "$bin" "$home/.harness_bin/$name" 2>/dev/null || true + done + for bin in /bin/*; do + [ -f "$bin" ] || continue + local name + name="$(basename "$bin")" + [ "$name" = "mktemp" ] && continue + ln -sf "$bin" "$home/.harness_bin/$name" 2>/dev/null || true + done + chown -R "$PROFILE_USER:$PROFILE_USER" "$home/.harness_bin" +} + +profile_run() { + local script="$1" + local home="/home/$PROFILE_USER" + ssh_run_script "$PROFILE_USER" "export PATH=$home/.harness_bin:\$PATH; $script" +} + +profile_teardown() { + local home="/home/$PROFILE_USER" + # Fix permissions for cleanup + find "$home/.warp-test" -type d -exec chmod 755 {} \; 2>/dev/null || true + rm -rf "$home/.harness_bin" "$home/.warp-test" 2>/dev/null || true +} diff --git a/script/remote-install-harness/profiles/16_startup_file_permission.sh b/script/remote-install-harness/profiles/16_startup_file_permission.sh new file mode 100644 index 0000000000..012d6e5272 --- /dev/null +++ b/script/remote-install-harness/profiles/16_startup_file_permission.sh @@ -0,0 +1,25 @@ +# Profile: .bashrc is not readable (permission denied on startup file). +# SSH login may still work but bash startup will emit errors. +# The install script itself should still run since it's piped via `bash -s`. +PROFILE_USER="harness_rcperm" +PROFILE_EXPECTED="Bash startup error from unreadable .bashrc; install may still succeed" + +profile_setup() { + local home="/home/$PROFILE_USER" + # Create a .bashrc that the user cannot read + echo 'echo "bashrc loaded"' > "$home/.bashrc" + chown root:root "$home/.bashrc" + chmod 000 "$home/.bashrc" + # Also create .profile with the same issue + echo 'echo "profile loaded"' > "$home/.profile" + chown root:root "$home/.profile" + chmod 000 "$home/.profile" +} + +profile_teardown() { + local home="/home/$PROFILE_USER" + chmod 644 "$home/.bashrc" 2>/dev/null || true + chown "$PROFILE_USER:$PROFILE_USER" "$home/.bashrc" 2>/dev/null || true + chmod 644 "$home/.profile" 2>/dev/null || true + chown "$PROFILE_USER:$PROFILE_USER" "$home/.profile" 2>/dev/null || true +} diff --git a/script/remote-install-harness/profiles/17_forced_disconnect.sh b/script/remote-install-harness/profiles/17_forced_disconnect.sh new file mode 100644 index 0000000000..5192713dd1 --- /dev/null +++ b/script/remote-install-harness/profiles/17_forced_disconnect.sh @@ -0,0 +1,43 @@ +# Profile: forced SSH disconnect (exit 255). +# Simulate a host that kills the SSH session mid-install. +PROFILE_USER="harness_disconnect" +PROFILE_EXPECTED="SSH connection killed mid-script; exit 255 from ssh" + +profile_setup() { + local home="/home/$PROFILE_USER" + # Create a wrapper bash that kills its parent ssh after a brief delay + mkdir -p "$home/.harness_bin" + + cat > "$home/.harness_bin/force_disconnect.sh" <<'FEOF' +#!/bin/bash +# This script reads from stdin (the install script) but kills the +# SSH connection after 1 second to simulate a forced disconnect. +sleep 1 +# Kill the parent process (sshd child handling this session) +kill -9 $PPID 2>/dev/null +exit 255 +FEOF + chmod +x "$home/.harness_bin/force_disconnect.sh" + chown -R "$PROFILE_USER:$PROFILE_USER" "$home/.harness_bin" +} + +profile_run() { + local script="$1" + local home="/home/$PROFILE_USER" + # Instead of running `bash -s`, run the disconnect script + ssh -p "$SSH_PORT" \ + -i "$CLIENT_KEY" \ + -o StrictHostKeyChecking=no \ + -o UserKnownHostsFile=/dev/null \ + -o ConnectTimeout=5 \ + -o BatchMode=yes \ + -o LogLevel=ERROR \ + -o ServerAliveInterval=1 \ + -o ServerAliveCountMax=2 \ + "$PROFILE_USER@127.0.0.1" \ + "bash $home/.harness_bin/force_disconnect.sh" <<< "$script" +} + +profile_teardown() { + rm -rf "/home/$PROFILE_USER/.harness_bin" +} diff --git a/script/remote-install-harness/profiles/18_missing_bash.sh b/script/remote-install-harness/profiles/18_missing_bash.sh new file mode 100644 index 0000000000..ee40280adc --- /dev/null +++ b/script/remote-install-harness/profiles/18_missing_bash.sh @@ -0,0 +1,44 @@ +# Profile: bash not available (shell is /bin/sh only). +# The install script is sent via `bash -s` so this simulates +# what happens when bash doesn't exist. +PROFILE_USER="harness_nobash" +PROFILE_EXPECTED="'bash: command not found' or connection failure if shell is not bash" + +profile_setup() { + local home="/home/$PROFILE_USER" + mkdir -p "$home/.harness_bin" + # Symlink everything except bash + for bin in /usr/bin/*; do + local name + name="$(basename "$bin")" + [ "$name" = "bash" ] && continue + ln -sf "$bin" "$home/.harness_bin/$name" 2>/dev/null || true + done + for bin in /bin/*; do + [ -f "$bin" ] || continue + local name + name="$(basename "$bin")" + [ "$name" = "bash" ] && continue + ln -sf "$bin" "$home/.harness_bin/$name" 2>/dev/null || true + done + chown -R "$PROFILE_USER:$PROFILE_USER" "$home/.harness_bin" +} + +profile_run() { + local script="$1" + local home="/home/$PROFILE_USER" + # Run with PATH that doesn't include bash + ssh -p "$SSH_PORT" \ + -i "$CLIENT_KEY" \ + -o StrictHostKeyChecking=no \ + -o UserKnownHostsFile=/dev/null \ + -o ConnectTimeout=10 \ + -o BatchMode=yes \ + -o LogLevel=ERROR \ + "$PROFILE_USER@127.0.0.1" \ + "export PATH=$home/.harness_bin; bash -s" <<< "$script" +} + +profile_teardown() { + rm -rf "/home/$PROFILE_USER/.harness_bin" +} diff --git a/script/remote-install-harness/profiles/19_tls_ca_failure.sh b/script/remote-install-harness/profiles/19_tls_ca_failure.sh new file mode 100644 index 0000000000..f18cc8f8fe --- /dev/null +++ b/script/remote-install-harness/profiles/19_tls_ca_failure.sh @@ -0,0 +1,28 @@ +# Profile: TLS/CA certificate verification failure. +# Point curl at an HTTPS URL but with an invalid CA bundle so +# certificate verification fails. +PROFILE_USER="harness_tls" +PROFILE_EXPECTED="curl/wget TLS error; 'certificate verify failed' or similar" + +profile_setup() { + local home="/home/$PROFILE_USER" + # Create an empty CA bundle so TLS verification always fails + mkdir -p "$home/.harness_ssl" + touch "$home/.harness_ssl/empty_ca.crt" + chown -R "$PROFILE_USER:$PROFILE_USER" "$home/.harness_ssl" +} + +profile_run() { + local script="$1" + local home="/home/$PROFILE_USER" + # Replace the download URL with an HTTPS endpoint (use a real domain + # that exists but will fail CA verification with our empty bundle) + local modified_script + modified_script=$(echo "$script" | sed 's|http://127.0.0.1:[0-9]*/download/cli|https://app.warp.dev/download/cli|g') + # Force curl to use our empty CA bundle + ssh_run_script "$PROFILE_USER" "export CURL_CA_BUNDLE=$home/.harness_ssl/empty_ca.crt; export SSL_CERT_FILE=$home/.harness_ssl/empty_ca.crt; $modified_script" +} + +profile_teardown() { + rm -rf "/home/$PROFILE_USER/.harness_ssl" +} diff --git a/script/remote-install-harness/profiles/20_download_write_failure.sh b/script/remote-install-harness/profiles/20_download_write_failure.sh new file mode 100644 index 0000000000..78fe83f2eb --- /dev/null +++ b/script/remote-install-harness/profiles/20_download_write_failure.sh @@ -0,0 +1,59 @@ +# Profile: download write failure — the staging temp dir exists but curl +# cannot write the output file (simulate disk/permission issue mid-download). +# We use a mktemp wrapper that creates a dir owned by root. +PROFILE_USER="harness_writefail" +PROFILE_EXPECTED="curl/wget cannot write output file; 'Permission denied' or similar" + +profile_setup() { + local home="/home/$PROFILE_USER" + mkdir -p "$home/.harness_bin" + + # Wrapper mktemp: create the dir but make it owned by root so the + # user can't write into it (but the dir itself exists so `set -e` + # in the install script won't trap on mktemp itself). + cat > "$home/.harness_bin/mktemp" <<'FEOF' +#!/bin/bash +real_mktemp=/usr/bin/mktemp +result=$($real_mktemp "$@") +if [ -d "$result" ]; then + # Transfer ownership to root so the test user can't write + chown root:root "$result" 2>/dev/null || true + chmod 555 "$result" 2>/dev/null || true +fi +echo "$result" +FEOF + chmod +x "$home/.harness_bin/mktemp" + # suid so the chown call inside works (fallback: test will still + # demonstrate the scenario because the dir will be non-writable) + + for bin in /usr/bin/*; do + local name + name="$(basename "$bin")" + [ "$name" = "mktemp" ] && continue + ln -sf "$bin" "$home/.harness_bin/$name" 2>/dev/null || true + done + for bin in /bin/*; do + [ -f "$bin" ] || continue + local name + name="$(basename "$bin")" + [ "$name" = "mktemp" ] && continue + ln -sf "$bin" "$home/.harness_bin/$name" 2>/dev/null || true + done + chown -R "$PROFILE_USER:$PROFILE_USER" "$home/.harness_bin" + # The mktemp wrapper itself needs to run chown as root + chown root:root "$home/.harness_bin/mktemp" + chmod 4755 "$home/.harness_bin/mktemp" +} + +profile_run() { + local script="$1" + local home="/home/$PROFILE_USER" + ssh_run_script "$PROFILE_USER" "export PATH=$home/.harness_bin:\$PATH; $script" +} + +profile_teardown() { + local home="/home/$PROFILE_USER" + find "$home/.warp-test" -type d -exec chmod 755 {} \; 2>/dev/null || true + find "$home/.warp-test" -exec chown "$PROFILE_USER:$PROFILE_USER" {} \; 2>/dev/null || true + rm -rf "$home/.harness_bin" "$home/.warp-test" 2>/dev/null || true +} diff --git a/script/remote-install-harness/run_harness.sh b/script/remote-install-harness/run_harness.sh new file mode 100755 index 0000000000..eb44aa3bdd --- /dev/null +++ b/script/remote-install-harness/run_harness.sh @@ -0,0 +1,427 @@ +#!/usr/bin/env bash +# Remote server install failure harness. +# +# Builds local OpenSSH host profiles that reproduce each CSV failure family, +# runs the Warp install script against each, and captures output. +# +# Usage: +# ./run_harness.sh [--profile PROFILE_NAME] [--install-script PATH] +# +# Without --profile, runs all profiles. The default install script is +# the one checked in at crates/remote_server/src/install_remote_server.sh, +# with placeholders substituted for local testing. +set -o pipefail + +HARNESS_DIR="$(cd "$(dirname "$0")" && pwd)" +REPO_ROOT="$(cd "$HARNESS_DIR/../.." && pwd)" +RESULTS_DIR="$HARNESS_DIR/results" +PROFILES_DIR="$HARNESS_DIR/profiles" +SSH_PORT=2222 +SSHD_CONFIG="$HARNESS_DIR/.harness_sshd_config" +HOST_KEY="$HARNESS_DIR/.harness_host_key" +CLIENT_KEY="$HARNESS_DIR/.harness_client_key" +SSHD_PID_FILE="$HARNESS_DIR/.harness_sshd.pid" +FAKE_DOWNLOAD_PORT=18443 +FAKE_DOWNLOAD_PID_FILE="$HARNESS_DIR/.harness_httpd.pid" + +# Colors +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +CYAN='\033[0;36m' +NC='\033[0m' + +log_info() { echo -e "${CYAN}[harness]${NC} $*"; } +log_ok() { echo -e "${GREEN}[ OK ]${NC} $*"; } +log_fail() { echo -e "${RED}[ FAIL ]${NC} $*"; } +log_warn() { echo -e "${YELLOW}[ WARN ]${NC} $*"; } + +# --------------------------------------------------------------------------- +# Cleanup +# --------------------------------------------------------------------------- +cleanup() { + log_info "Cleaning up..." + if [ -f "$SSHD_PID_FILE" ]; then + kill "$(cat "$SSHD_PID_FILE")" 2>/dev/null || true + rm -f "$SSHD_PID_FILE" + fi + if [ -f "$FAKE_DOWNLOAD_PID_FILE" ]; then + kill "$(cat "$FAKE_DOWNLOAD_PID_FILE")" 2>/dev/null || true + rm -f "$FAKE_DOWNLOAD_PID_FILE" + fi +} +trap cleanup EXIT + +# --------------------------------------------------------------------------- +# Parse args +# --------------------------------------------------------------------------- +SINGLE_PROFILE="" +INSTALL_SCRIPT_PATH="" +while [[ $# -gt 0 ]]; do + case "$1" in + --profile) SINGLE_PROFILE="$2"; shift 2 ;; + --install-script) INSTALL_SCRIPT_PATH="$2"; shift 2 ;; + *) echo "Unknown arg: $1" >&2; exit 1 ;; + esac +done + +# --------------------------------------------------------------------------- +# Prepare install script with substituted placeholders +# --------------------------------------------------------------------------- +prepare_install_script() { + local template="$REPO_ROOT/crates/remote_server/src/install_remote_server.sh" + if [ -n "$INSTALL_SCRIPT_PATH" ]; then + template="$INSTALL_SCRIPT_PATH" + fi + local install_dir="~/.warp-test/remote-server" + local binary_name="oz-test" + local channel="dev" + local download_base_url="http://127.0.0.1:${FAKE_DOWNLOAD_PORT}/download/cli" + + sed \ + -e "s|{download_base_url}|${download_base_url}|g" \ + -e "s|{channel}|${channel}|g" \ + -e "s|{install_dir}|${install_dir}|g" \ + -e "s|{binary_name}|${binary_name}|g" \ + -e "s|{version_query}||g" \ + -e "s|{version_suffix}||g" \ + -e "s|{no_http_client_exit_code}|3|g" \ + -e "s|{download_failed_exit_code}|4|g" \ + -e "s|{no_tar_exit_code}|5|g" \ + -e "s|{staging_tarball_path}||g" \ + "$template" +} + +# --------------------------------------------------------------------------- +# SSH infrastructure +# --------------------------------------------------------------------------- +setup_ssh_keys() { + if [ ! -f "$HOST_KEY" ]; then + ssh-keygen -t ed25519 -f "$HOST_KEY" -N "" -q + fi + if [ ! -f "$CLIENT_KEY" ]; then + ssh-keygen -t ed25519 -f "$CLIENT_KEY" -N "" -q + fi +} + +start_sshd() { + log_info "Starting sshd on port $SSH_PORT..." + mkdir -p /run/sshd # Required for privilege separation + + cat > "$SSHD_CONFIG" < "$tarball_dir/oz-test" + echo 'echo "fake oz binary running"' >> "$tarball_dir/oz-test" + chmod +x "$tarball_dir/oz-test" + tar -czf "$server_dir/download/oz.tar.gz" -C "$tarball_dir" oz-test + rm -rf "$tarball_dir" + + # Start Python HTTP server + cat > "$server_dir/server.py" <<'PYEOF' +import http.server +import os +import sys +import json +import time +import threading + +PORT = int(sys.argv[1]) +TARBALL_PATH = os.path.join(os.path.dirname(__file__), "download", "oz.tar.gz") +# Profile-specific behaviors loaded from env or config +BEHAVIOR = os.environ.get("FAKE_SERVER_BEHAVIOR", "normal") + +class Handler(http.server.BaseHTTPRequestHandler): + def log_message(self, format, *args): + pass # Suppress logging + + def do_GET(self): + if "/download/cli" in self.path: + if BEHAVIOR == "403": + self.send_error(403, "Forbidden") + return + elif BEHAVIOR == "502": + self.send_error(502, "Bad Gateway") + return + elif BEHAVIOR == "partial": + # Send partial content then close + with open(TARBALL_PATH, "rb") as f: + data = f.read() + self.send_response(200) + self.send_header("Content-Length", str(len(data))) + self.send_header("Content-Type", "application/octet-stream") + self.end_headers() + # Send only first 10 bytes + self.wfile.write(data[:10]) + self.wfile.flush() + return + elif BEHAVIOR == "slow": + # Simulate timeout with very slow delivery + with open(TARBALL_PATH, "rb") as f: + data = f.read() + self.send_response(200) + self.send_header("Content-Length", str(len(data))) + self.send_header("Content-Type", "application/octet-stream") + self.end_headers() + for byte in data: + self.wfile.write(bytes([byte])) + self.wfile.flush() + time.sleep(2) # Very slow + return + elif BEHAVIOR == "refuse": + # Immediately close connection + self.connection.close() + return + else: + # Normal: serve the tarball + with open(TARBALL_PATH, "rb") as f: + data = f.read() + self.send_response(200) + self.send_header("Content-Length", str(len(data))) + self.send_header("Content-Type", "application/octet-stream") + self.end_headers() + self.wfile.write(data) + else: + self.send_error(404, "Not Found") + +server = http.server.HTTPServer(("0.0.0.0", PORT), Handler) +print(f"Fake server on port {PORT}, behavior={BEHAVIOR}", flush=True) +server.serve_forever() +PYEOF + + FAKE_SERVER_BEHAVIOR="${FAKE_SERVER_BEHAVIOR:-normal}" \ + python3 "$server_dir/server.py" "$FAKE_DOWNLOAD_PORT" & + echo $! > "$FAKE_DOWNLOAD_PID_FILE" + sleep 0.5 + log_ok "Fake download server started on port $FAKE_DOWNLOAD_PORT (behavior=${FAKE_SERVER_BEHAVIOR:-normal})" +} + +restart_fake_server_with_behavior() { + local behavior="$1" + if [ -f "$FAKE_DOWNLOAD_PID_FILE" ]; then + kill "$(cat "$FAKE_DOWNLOAD_PID_FILE")" 2>/dev/null || true + sleep 0.3 + fi + FAKE_SERVER_BEHAVIOR="$behavior" start_fake_download_server +} + +# --------------------------------------------------------------------------- +# User account management +# --------------------------------------------------------------------------- +create_test_user() { + local username="$1" + local home="/home/$username" + if id "$username" &>/dev/null; then + return 0 # Already exists + fi + useradd -m -s /bin/bash "$username" 2>/dev/null || true + # Unlock the account for SSH pubkey auth (useradd creates locked accounts) + passwd -u "$username" 2>/dev/null || usermod -p '*' "$username" 2>/dev/null || true + mkdir -p "$home/.ssh" + cp "${CLIENT_KEY}.pub" "$home/.ssh/authorized_keys" + chmod 700 "$home/.ssh" + chmod 600 "$home/.ssh/authorized_keys" + chown -R "$username:$username" "$home/.ssh" +} + +# --------------------------------------------------------------------------- +# SSH helper: run a command as a test user via SSH +# --------------------------------------------------------------------------- +ssh_run() { + local username="$1" + shift + ssh -p "$SSH_PORT" \ + -i "$CLIENT_KEY" \ + -o StrictHostKeyChecking=no \ + -o UserKnownHostsFile=/dev/null \ + -o ConnectTimeout=10 \ + -o BatchMode=yes \ + -o LogLevel=ERROR \ + "$username@127.0.0.1" \ + "$@" +} + +ssh_run_script() { + local username="$1" + local script="$2" + ssh -p "$SSH_PORT" \ + -i "$CLIENT_KEY" \ + -o StrictHostKeyChecking=no \ + -o UserKnownHostsFile=/dev/null \ + -o ConnectTimeout=10 \ + -o BatchMode=yes \ + -o LogLevel=ERROR \ + "$username@127.0.0.1" \ + "bash -s" <<< "$script" +} + +# --------------------------------------------------------------------------- +# Profile runner +# --------------------------------------------------------------------------- +run_profile() { + local profile_name="$1" + local profile_script="$PROFILES_DIR/${profile_name}.sh" + + if [ ! -f "$profile_script" ]; then + log_warn "Profile script not found: $profile_script" + return 1 + fi + + log_info "═══════════════════════════════════════════════════════════" + log_info "Running profile: $profile_name" + log_info "═══════════════════════════════════════════════════════════" + + local result_file="$RESULTS_DIR/${profile_name}.log" + mkdir -p "$RESULTS_DIR" + + # Source the profile to get setup/teardown/expected behavior + # Each profile defines: + # profile_setup() — prepare the user environment + # profile_teardown() — cleanup after test + # PROFILE_USER — username for this profile + # PROFILE_EXPECTED — expected outcome description + # PROFILE_SERVER_BEHAVIOR — fake server behavior (optional) + unset -f profile_setup profile_teardown profile_extra_setup profile_run 2>/dev/null + unset PROFILE_USER PROFILE_EXPECTED PROFILE_SERVER_BEHAVIOR 2>/dev/null + + source "$profile_script" + + # Create user if needed + if [ -n "$PROFILE_USER" ]; then + create_test_user "$PROFILE_USER" + fi + + # Set up server behavior if specified + if [ -n "$PROFILE_SERVER_BEHAVIOR" ]; then + restart_fake_server_with_behavior "$PROFILE_SERVER_BEHAVIOR" + else + restart_fake_server_with_behavior "normal" + fi + + # Run profile setup + if declare -f profile_setup >/dev/null 2>&1; then + profile_setup + fi + + # Prepare and run the install script + local install_script + install_script="$(prepare_install_script)" + + { + echo "===== Profile: $profile_name =====" + echo "Expected: $PROFILE_EXPECTED" + echo "User: $PROFILE_USER" + echo "Server behavior: ${PROFILE_SERVER_BEHAVIOR:-normal}" + echo "Timestamp: $(date -u +%Y-%m-%dT%H:%M:%SZ)" + echo "===== Install Script Output =====" + } > "$result_file" + + local exit_code=0 + # Some profiles need custom invocation + if declare -f profile_run >/dev/null 2>&1; then + profile_run "$install_script" >> "$result_file" 2>&1 + exit_code=$? + else + ssh_run_script "$PROFILE_USER" "$install_script" >> "$result_file" 2>&1 + exit_code=$? + fi + + { + echo "" + echo "===== Exit Code: $exit_code =====" + } >> "$result_file" + + # Run teardown + if declare -f profile_teardown >/dev/null 2>&1; then + profile_teardown + fi + + # Report + if [ $exit_code -ne 0 ]; then + log_ok "Profile '$profile_name' failed as expected (exit $exit_code)" + echo "RESULT: EXPECTED_FAILURE (exit $exit_code)" >> "$result_file" + else + log_warn "Profile '$profile_name' unexpectedly succeeded (exit 0)" + echo "RESULT: UNEXPECTED_SUCCESS (exit 0)" >> "$result_file" + fi + + echo "" + cat "$result_file" + echo "" +} + +# --------------------------------------------------------------------------- +# Main +# --------------------------------------------------------------------------- +main() { + log_info "Remote Install Failure Harness" + log_info "Repo root: $REPO_ROOT" + + setup_ssh_keys + start_sshd + start_fake_download_server + + mkdir -p "$RESULTS_DIR" + + if [ -n "$SINGLE_PROFILE" ]; then + run_profile "$SINGLE_PROFILE" + else + # Run all profiles in alphabetical order + for profile_script in "$PROFILES_DIR"/*.sh; do + [ -f "$profile_script" ] || continue + local profile_name + profile_name="$(basename "$profile_script" .sh)" + run_profile "$profile_name" + done + fi + + # Summary + echo "" + log_info "═══════════════════════════════════════════════════════════" + log_info "Summary" + log_info "═══════════════════════════════════════════════════════════" + for result in "$RESULTS_DIR"/*.log; do + [ -f "$result" ] || continue + local name result_line + name="$(basename "$result" .log)" + result_line="$(grep '^RESULT:' "$result" || echo 'RESULT: NO_RESULT')" + echo " $name: $result_line" + done +} + +main