Skip to content

fix(server): pause egress sidecar with sandbox #1441

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
luppyw wants to merge 1 commit into
base: main
Choose a base branch
from
Open

fix(server): pause egress sidecar with sandbox #1441

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 4 additions & 2 deletions docs/architecture/index.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -176,7 +176,9 @@ Core responsibilities:
- Attach an egress sidecar when `networkPolicy` is requested and Docker networking is compatible.
- Create Docker-backed persistent snapshots as local images and restore sandboxes from those snapshot images.

Docker pause/resume uses container-level pause/resume. Docker snapshots are exposed through the public snapshot API.
Docker pause/resume uses container-level pause/resume. For sandboxes with an egress sidecar,
pause freezes the sandbox container before the sidecar, while resume unfreezes the sidecar
before the sandbox container. Docker snapshots are exposed through the public snapshot API.

### 4.2 Kubernetes Runtime

Expand Down Expand Up @@ -339,7 +341,7 @@ Create request with networkPolicy
```text
Pause / resume
-> lifecycle server delegates to runtime provider
-> Docker pauses/resumes the container
-> Docker pauses/resumes the sandbox container and its egress sidecar when present
-> BatchSandbox uses rootfs snapshot commit/recreate for supported single-replica sandboxes

Public snapshot API
Expand Down
136 changes: 135 additions & 1 deletion server/opensandbox_server/services/docker/docker_service.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -290,6 +290,12 @@ def _get_container_by_sandbox_id(self, sandbox_id: str):

return containers[0]

def _get_egress_sidecars(self, sandbox_id: str) -> list[Any]:
"""Return all egress sidecars associated with a sandbox."""
return self.docker_client.containers.list(
all=True, filters={"label": f"{EGRESS_SIDECAR_LABEL}={sandbox_id}"}
)

def _schedule_expiration(
self,
sandbox_id: str,
Expand Down Expand Up @@ -1111,6 +1117,20 @@ def pause_sandbox(self, sandbox_id: str) -> None:
},
)

labels = container.attrs.get("Config", {}).get("Labels") or {}
egress_expected = bool(labels.get(SANDBOX_EGRESS_AUTH_TOKEN_METADATA_KEY))
try:
with self._docker_operation("query egress sidecar", sandbox_id):
sidecars = self._get_egress_sidecars(sandbox_id)
except DockerException as exc:
raise HTTPException(
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
detail={
"code": SandboxErrorCodes.SANDBOX_PAUSE_FAILED,
"message": f"Failed to query egress sidecar: {str(exc)}",
},
) from exc

try:
with self._docker_operation("pause sandbox container", sandbox_id):
container.pause()
Expand All @@ -1123,6 +1143,65 @@ def pause_sandbox(self, sandbox_id: str) -> None:
},
) from exc

if egress_expected and not sidecars:
raise HTTPException(
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
detail={
"code": SandboxErrorCodes.SANDBOX_PAUSE_FAILED,
"message": (
"Sandbox container was paused, but the expected egress sidecar was not found."
),
},
)

paused_sidecars: list[Any] = []
try:
for sidecar in sidecars:
sidecar_state = sidecar.attrs.get("State", {})
if sidecar_state.get("Paused", False):
continue
if not sidecar_state.get("Running", False):
raise DockerException(
f"Egress sidecar {sidecar.id} is not in a running state."
)
with self._docker_operation("pause egress sidecar", sandbox_id):
sidecar.pause()
paused_sidecars.append(sidecar)
except DockerException as exc:
rollback_errors: list[str] = []
for paused_sidecar in reversed(paused_sidecars):
try:
with self._docker_operation("rollback egress sidecar pause", sandbox_id):
paused_sidecar.unpause()
except DockerException as rollback_exc:
logger.warning(
"sandbox=%s | failed to rollback egress sidecar pause: %s",
sandbox_id,
rollback_exc,
)
rollback_errors.append(str(rollback_exc))
try:
with self._docker_operation("rollback sandbox pause", sandbox_id):
container.unpause()
except DockerException as rollback_exc:
logger.warning(
"sandbox=%s | failed to rollback sandbox pause: %s",
sandbox_id,
rollback_exc,
)
rollback_errors.append(str(rollback_exc))

message = f"Failed to pause egress sidecar: {str(exc)}"
if rollback_errors:
message += f"; rollback failed: {'; '.join(rollback_errors)}"
raise HTTPException(
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
detail={
"code": SandboxErrorCodes.SANDBOX_PAUSE_FAILED,
"message": message,
},
) from exc

def resume_sandbox(self, sandbox_id: str) -> None:
"""
Resume a paused sandbox using Docker.
Expand All @@ -1144,15 +1223,70 @@ def resume_sandbox(self, sandbox_id: str) -> None:
},
)

labels = container.attrs.get("Config", {}).get("Labels") or {}
egress_expected = bool(labels.get(SANDBOX_EGRESS_AUTH_TOKEN_METADATA_KEY))
try:
with self._docker_operation("query egress sidecar", sandbox_id):
sidecars = self._get_egress_sidecars(sandbox_id)
except DockerException as exc:
raise HTTPException(
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
detail={
"code": SandboxErrorCodes.SANDBOX_RESUME_FAILED,
"message": f"Failed to query egress sidecar: {str(exc)}",
},
) from exc

if egress_expected and not sidecars:
raise HTTPException(
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
detail={
"code": SandboxErrorCodes.SANDBOX_RESUME_FAILED,
"message": "The expected egress sidecar was not found; sandbox remains paused.",
},
)

resumed_sidecars: list[Any] = []
resuming_main = False
try:
for sidecar in sidecars:
sidecar_state = sidecar.attrs.get("State", {})
if not sidecar_state.get("Paused", False):
if sidecar_state.get("Running", False):
continue
raise DockerException(
f"Egress sidecar {sidecar.id} is not in a paused state."
)
with self._docker_operation("resume egress sidecar", sandbox_id):
sidecar.unpause()
resumed_sidecars.append(sidecar)

resuming_main = True
with self._docker_operation("resume sandbox container", sandbox_id):
container.unpause()
except DockerException as exc:
rollback_errors: list[str] = []
for resumed_sidecar in reversed(resumed_sidecars):
try:
with self._docker_operation("rollback egress sidecar resume", sandbox_id):
resumed_sidecar.pause()
except DockerException as rollback_exc:
logger.warning(
"sandbox=%s | failed to rollback egress sidecar resume: %s",
sandbox_id,
rollback_exc,
)
rollback_errors.append(str(rollback_exc))

failed_component = "sandbox container" if resuming_main else "egress sidecar"
message = f"Failed to resume {failed_component}: {str(exc)}"
if rollback_errors:
message += f"; rollback failed: {'; '.join(rollback_errors)}"
raise HTTPException(
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
detail={
"code": SandboxErrorCodes.SANDBOX_RESUME_FAILED,
"message": f"Failed to resume sandbox container: {str(exc)}",
"message": message,
},
) from exc

Expand Down
Loading
Loading