diff --git a/docs/architecture/index.md b/docs/architecture/index.md index c7d90ebf6..06f9c747e 100644 --- a/docs/architecture/index.md +++ b/docs/architecture/index.md @@ -176,7 +176,9 @@ Core responsibilities: - Attach an egress sidecar when `networkPolicy` is requested and Docker networking is compatible. - Create Docker-backed persistent snapshots as local images and restore sandboxes from those snapshot images. -Docker pause/resume uses container-level pause/resume. Docker snapshots are exposed through the public snapshot API. +Docker pause/resume uses container-level pause/resume. For sandboxes with an egress sidecar, +pause freezes the sandbox container before the sidecar, while resume unfreezes the sidecar +before the sandbox container. Docker snapshots are exposed through the public snapshot API. ### 4.2 Kubernetes Runtime @@ -339,7 +341,7 @@ Create request with networkPolicy ```text Pause / resume -> lifecycle server delegates to runtime provider - -> Docker pauses/resumes the container + -> Docker pauses/resumes the sandbox container and its egress sidecar when present -> BatchSandbox uses rootfs snapshot commit/recreate for supported single-replica sandboxes Public snapshot API diff --git a/server/opensandbox_server/services/docker/docker_service.py b/server/opensandbox_server/services/docker/docker_service.py index ab359e2bd..cba44a780 100644 --- a/server/opensandbox_server/services/docker/docker_service.py +++ b/server/opensandbox_server/services/docker/docker_service.py @@ -290,6 +290,12 @@ def _get_container_by_sandbox_id(self, sandbox_id: str): return containers[0] + def _get_egress_sidecars(self, sandbox_id: str) -> list[Any]: + """Return all egress sidecars associated with a sandbox.""" + return self.docker_client.containers.list( + all=True, filters={"label": f"{EGRESS_SIDECAR_LABEL}={sandbox_id}"} + ) + def _schedule_expiration( self, sandbox_id: str, @@ -1111,6 +1117,20 @@ def pause_sandbox(self, sandbox_id: str) -> None: }, ) + labels = container.attrs.get("Config", {}).get("Labels") or {} + egress_expected = bool(labels.get(SANDBOX_EGRESS_AUTH_TOKEN_METADATA_KEY)) + try: + with self._docker_operation("query egress sidecar", sandbox_id): + sidecars = self._get_egress_sidecars(sandbox_id) + except DockerException as exc: + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail={ + "code": SandboxErrorCodes.SANDBOX_PAUSE_FAILED, + "message": f"Failed to query egress sidecar: {str(exc)}", + }, + ) from exc + try: with self._docker_operation("pause sandbox container", sandbox_id): container.pause() @@ -1123,6 +1143,65 @@ def pause_sandbox(self, sandbox_id: str) -> None: }, ) from exc + if egress_expected and not sidecars: + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail={ + "code": SandboxErrorCodes.SANDBOX_PAUSE_FAILED, + "message": ( + "Sandbox container was paused, but the expected egress sidecar was not found." + ), + }, + ) + + paused_sidecars: list[Any] = [] + try: + for sidecar in sidecars: + sidecar_state = sidecar.attrs.get("State", {}) + if sidecar_state.get("Paused", False): + continue + if not sidecar_state.get("Running", False): + raise DockerException( + f"Egress sidecar {sidecar.id} is not in a running state." + ) + with self._docker_operation("pause egress sidecar", sandbox_id): + sidecar.pause() + paused_sidecars.append(sidecar) + except DockerException as exc: + rollback_errors: list[str] = [] + for paused_sidecar in reversed(paused_sidecars): + try: + with self._docker_operation("rollback egress sidecar pause", sandbox_id): + paused_sidecar.unpause() + except DockerException as rollback_exc: + logger.warning( + "sandbox=%s | failed to rollback egress sidecar pause: %s", + sandbox_id, + rollback_exc, + ) + rollback_errors.append(str(rollback_exc)) + try: + with self._docker_operation("rollback sandbox pause", sandbox_id): + container.unpause() + except DockerException as rollback_exc: + logger.warning( + "sandbox=%s | failed to rollback sandbox pause: %s", + sandbox_id, + rollback_exc, + ) + rollback_errors.append(str(rollback_exc)) + + message = f"Failed to pause egress sidecar: {str(exc)}" + if rollback_errors: + message += f"; rollback failed: {'; '.join(rollback_errors)}" + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail={ + "code": SandboxErrorCodes.SANDBOX_PAUSE_FAILED, + "message": message, + }, + ) from exc + def resume_sandbox(self, sandbox_id: str) -> None: """ Resume a paused sandbox using Docker. @@ -1144,15 +1223,70 @@ def resume_sandbox(self, sandbox_id: str) -> None: }, ) + labels = container.attrs.get("Config", {}).get("Labels") or {} + egress_expected = bool(labels.get(SANDBOX_EGRESS_AUTH_TOKEN_METADATA_KEY)) + try: + with self._docker_operation("query egress sidecar", sandbox_id): + sidecars = self._get_egress_sidecars(sandbox_id) + except DockerException as exc: + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail={ + "code": SandboxErrorCodes.SANDBOX_RESUME_FAILED, + "message": f"Failed to query egress sidecar: {str(exc)}", + }, + ) from exc + + if egress_expected and not sidecars: + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail={ + "code": SandboxErrorCodes.SANDBOX_RESUME_FAILED, + "message": "The expected egress sidecar was not found; sandbox remains paused.", + }, + ) + + resumed_sidecars: list[Any] = [] + resuming_main = False try: + for sidecar in sidecars: + sidecar_state = sidecar.attrs.get("State", {}) + if not sidecar_state.get("Paused", False): + if sidecar_state.get("Running", False): + continue + raise DockerException( + f"Egress sidecar {sidecar.id} is not in a paused state." + ) + with self._docker_operation("resume egress sidecar", sandbox_id): + sidecar.unpause() + resumed_sidecars.append(sidecar) + + resuming_main = True with self._docker_operation("resume sandbox container", sandbox_id): container.unpause() except DockerException as exc: + rollback_errors: list[str] = [] + for resumed_sidecar in reversed(resumed_sidecars): + try: + with self._docker_operation("rollback egress sidecar resume", sandbox_id): + resumed_sidecar.pause() + except DockerException as rollback_exc: + logger.warning( + "sandbox=%s | failed to rollback egress sidecar resume: %s", + sandbox_id, + rollback_exc, + ) + rollback_errors.append(str(rollback_exc)) + + failed_component = "sandbox container" if resuming_main else "egress sidecar" + message = f"Failed to resume {failed_component}: {str(exc)}" + if rollback_errors: + message += f"; rollback failed: {'; '.join(rollback_errors)}" raise HTTPException( status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail={ "code": SandboxErrorCodes.SANDBOX_RESUME_FAILED, - "message": f"Failed to resume sandbox container: {str(exc)}", + "message": message, }, ) from exc diff --git a/server/tests/test_docker_service.py b/server/tests/test_docker_service.py index 38ae9361a..cf62eee9a 100644 --- a/server/tests/test_docker_service.py +++ b/server/tests/test_docker_service.py @@ -1371,6 +1371,239 @@ def host_cfg_side_effect(**kwargs): assert "3389/udp" in sidecar_kwargs["ports"] assert "8006" in sidecar_kwargs["ports"] + +def _lifecycle_container( + container_id: str, + *, + running: bool, + paused: bool, + egress_expected: bool = False, +) -> MagicMock: + container = MagicMock() + container.id = container_id + labels = {} + if egress_expected: + labels[SANDBOX_EGRESS_AUTH_TOKEN_METADATA_KEY] = "egress-token" + container.attrs = { + "Config": {"Labels": labels}, + "State": {"Running": running, "Paused": paused}, + } + return container + + +def test_pause_sandbox_pauses_main_before_egress_sidecar(): + service = DockerSandboxService(config=_app_config()) + main = _lifecycle_container("main-id", running=True, paused=False, egress_expected=True) + sidecar = _lifecycle_container("sidecar-id", running=True, paused=False) + events: list[str] = [] + main.pause.side_effect = lambda: events.append("main.pause") + sidecar.pause.side_effect = lambda: events.append("sidecar.pause") + + with ( + patch.object(service, "_get_container_by_sandbox_id", return_value=main), + patch.object(service, "_get_egress_sidecars", return_value=[sidecar]), + ): + service.pause_sandbox("sandbox-id") + + assert events == ["main.pause", "sidecar.pause"] + + +def test_resume_sandbox_resumes_egress_sidecar_before_main(): + service = DockerSandboxService(config=_app_config()) + main = _lifecycle_container("main-id", running=True, paused=True, egress_expected=True) + sidecar = _lifecycle_container("sidecar-id", running=True, paused=True) + events: list[str] = [] + sidecar.unpause.side_effect = lambda: events.append("sidecar.unpause") + main.unpause.side_effect = lambda: events.append("main.unpause") + + with ( + patch.object(service, "_get_container_by_sandbox_id", return_value=main), + patch.object(service, "_get_egress_sidecars", return_value=[sidecar]), + ): + service.resume_sandbox("sandbox-id") + + assert events == ["sidecar.unpause", "main.unpause"] + + +def test_pause_sandbox_without_egress_sidecar_preserves_main_only_behavior(): + service = DockerSandboxService(config=_app_config()) + main = _lifecycle_container("main-id", running=True, paused=False) + + with ( + patch.object(service, "_get_container_by_sandbox_id", return_value=main), + patch.object(service, "_get_egress_sidecars", return_value=[]), + ): + service.pause_sandbox("sandbox-id") + + main.pause.assert_called_once_with() + + +def test_resume_sandbox_without_egress_sidecar_preserves_main_only_behavior(): + service = DockerSandboxService(config=_app_config()) + main = _lifecycle_container("main-id", running=True, paused=True) + + with ( + patch.object(service, "_get_container_by_sandbox_id", return_value=main), + patch.object(service, "_get_egress_sidecars", return_value=[]), + ): + service.resume_sandbox("sandbox-id") + + main.unpause.assert_called_once_with() + + +def test_pause_sandbox_pauses_main_when_expected_egress_sidecar_is_missing(): + service = DockerSandboxService(config=_app_config()) + main = _lifecycle_container("main-id", running=True, paused=False, egress_expected=True) + + with ( + patch.object(service, "_get_container_by_sandbox_id", return_value=main), + patch.object(service, "_get_egress_sidecars", return_value=[]), + pytest.raises(HTTPException) as exc_info, + ): + service.pause_sandbox("sandbox-id") + + assert exc_info.value.status_code == status.HTTP_500_INTERNAL_SERVER_ERROR + assert exc_info.value.detail["code"] == SandboxErrorCodes.SANDBOX_PAUSE_FAILED + assert "expected egress sidecar was not found" in exc_info.value.detail["message"] + main.pause.assert_called_once_with() + main.unpause.assert_not_called() + + +def test_resume_sandbox_fails_closed_when_expected_egress_sidecar_is_missing(): + service = DockerSandboxService(config=_app_config()) + main = _lifecycle_container("main-id", running=True, paused=True, egress_expected=True) + + with ( + patch.object(service, "_get_container_by_sandbox_id", return_value=main), + patch.object(service, "_get_egress_sidecars", return_value=[]), + pytest.raises(HTTPException) as exc_info, + ): + service.resume_sandbox("sandbox-id") + + assert exc_info.value.status_code == status.HTTP_500_INTERNAL_SERVER_ERROR + assert exc_info.value.detail["code"] == SandboxErrorCodes.SANDBOX_RESUME_FAILED + assert "expected egress sidecar was not found" in exc_info.value.detail["message"] + main.unpause.assert_not_called() + + +def test_pause_sandbox_rolls_back_main_when_egress_sidecar_pause_fails(): + service = DockerSandboxService(config=_app_config()) + main = _lifecycle_container("main-id", running=True, paused=False, egress_expected=True) + sidecar = _lifecycle_container("sidecar-id", running=True, paused=False) + events: list[str] = [] + main.pause.side_effect = lambda: events.append("main.pause") + main.unpause.side_effect = lambda: events.append("main.unpause") + + def fail_sidecar_pause() -> None: + events.append("sidecar.pause") + raise DockerException("sidecar pause failed") + + sidecar.pause.side_effect = fail_sidecar_pause + + with ( + patch.object(service, "_get_container_by_sandbox_id", return_value=main), + patch.object(service, "_get_egress_sidecars", return_value=[sidecar]), + pytest.raises(HTTPException) as exc_info, + ): + service.pause_sandbox("sandbox-id") + + assert exc_info.value.detail["code"] == SandboxErrorCodes.SANDBOX_PAUSE_FAILED + assert events == ["main.pause", "sidecar.pause", "main.unpause"] + + +def test_resume_sandbox_rolls_back_sidecar_when_main_resume_fails(): + service = DockerSandboxService(config=_app_config()) + main = _lifecycle_container("main-id", running=True, paused=True, egress_expected=True) + sidecar = _lifecycle_container("sidecar-id", running=True, paused=True) + events: list[str] = [] + sidecar.unpause.side_effect = lambda: events.append("sidecar.unpause") + sidecar.pause.side_effect = lambda: events.append("sidecar.pause") + + def fail_main_resume() -> None: + events.append("main.unpause") + raise DockerException("main resume failed") + + main.unpause.side_effect = fail_main_resume + + with ( + patch.object(service, "_get_container_by_sandbox_id", return_value=main), + patch.object(service, "_get_egress_sidecars", return_value=[sidecar]), + pytest.raises(HTTPException) as exc_info, + ): + service.resume_sandbox("sandbox-id") + + assert exc_info.value.detail["code"] == SandboxErrorCodes.SANDBOX_RESUME_FAILED + assert events == ["sidecar.unpause", "main.unpause", "sidecar.pause"] + + +def test_pause_sandbox_does_not_mutate_when_egress_query_fails(): + service = DockerSandboxService(config=_app_config()) + main = _lifecycle_container("main-id", running=True, paused=False, egress_expected=True) + + with ( + patch.object(service, "_get_container_by_sandbox_id", return_value=main), + patch.object( + service, + "_get_egress_sidecars", + side_effect=DockerException("sidecar query failed"), + ), + pytest.raises(HTTPException) as exc_info, + ): + service.pause_sandbox("sandbox-id") + + assert exc_info.value.detail["code"] == SandboxErrorCodes.SANDBOX_PAUSE_FAILED + main.pause.assert_not_called() + + +def test_resume_sandbox_does_not_mutate_when_egress_query_fails(): + service = DockerSandboxService(config=_app_config()) + main = _lifecycle_container("main-id", running=True, paused=True, egress_expected=True) + + with ( + patch.object(service, "_get_container_by_sandbox_id", return_value=main), + patch.object( + service, + "_get_egress_sidecars", + side_effect=DockerException("sidecar query failed"), + ), + pytest.raises(HTTPException) as exc_info, + ): + service.resume_sandbox("sandbox-id") + + assert exc_info.value.detail["code"] == SandboxErrorCodes.SANDBOX_RESUME_FAILED + main.unpause.assert_not_called() + + +def test_pause_sandbox_skips_egress_sidecar_that_is_already_paused(): + service = DockerSandboxService(config=_app_config()) + main = _lifecycle_container("main-id", running=True, paused=False, egress_expected=True) + sidecar = _lifecycle_container("sidecar-id", running=True, paused=True) + + with ( + patch.object(service, "_get_container_by_sandbox_id", return_value=main), + patch.object(service, "_get_egress_sidecars", return_value=[sidecar]), + ): + service.pause_sandbox("sandbox-id") + + main.pause.assert_called_once_with() + sidecar.pause.assert_not_called() + + +def test_resume_sandbox_skips_egress_sidecar_that_is_already_running(): + service = DockerSandboxService(config=_app_config()) + main = _lifecycle_container("main-id", running=True, paused=True, egress_expected=True) + sidecar = _lifecycle_container("sidecar-id", running=True, paused=False) + + with ( + patch.object(service, "_get_container_by_sandbox_id", return_value=main), + patch.object(service, "_get_egress_sidecars", return_value=[sidecar]), + ): + service.resume_sandbox("sandbox-id") + + sidecar.unpause.assert_not_called() + main.unpause.assert_called_once_with() + + def test_expire_cleans_sidecar(): service = DockerSandboxService(config=_app_config()) mock_container = MagicMock()