diff --git a/configs/basic-single-node/k8s.yaml b/configs/basic-single-node/k8s.yaml index f7e2810f6..ed3d5d501 100644 --- a/configs/basic-single-node/k8s.yaml +++ b/configs/basic-single-node/k8s.yaml @@ -1,12 +1,16 @@ +# memory is in bytes, not a "32Gi" quantity string: the published bundle is +# parsed straight into the backend's PhysicalResources (`memory: int`), and +# only the SDK path converts suffixes — a string passes local `compute-config +# create -f` and 422s on launch. 8Gi = 8589934592, 32Gi = 34359738368. head_node: required_resources: CPU: 4 - memory: 8Gi + memory: 8589934592 resources: CPU: 0 worker_nodes: - name: cpu_worker required_resources: CPU: 8 - memory: 32Gi + memory: 34359738368 max_nodes: 1 diff --git a/configs/getting-started/k8s.yaml b/configs/getting-started/k8s.yaml index d7e7aaa8b..b422dcacd 100644 --- a/configs/getting-started/k8s.yaml +++ b/configs/getting-started/k8s.yaml @@ -1,12 +1,16 @@ +# memory is in bytes, not a "32Gi" quantity string: the published bundle is +# parsed straight into the backend's PhysicalResources (`memory: int`), and +# only the SDK path converts suffixes — a string passes local `compute-config +# create -f` and 422s on launch. 8Gi = 8589934592, 32Gi = 34359738368. head_node: required_resources: CPU: 4 - memory: 8Gi + memory: 8589934592 resources: CPU: 0 worker_nodes: - name: cpu_worker required_resources: CPU: 8 - memory: 32Gi + memory: 34359738368 max_nodes: 2 diff --git a/configs/pytorch-fsdp/k8s.yaml b/configs/pytorch-fsdp/k8s.yaml index 1b5649d18..965ce8d5c 100644 --- a/configs/pytorch-fsdp/k8s.yaml +++ b/configs/pytorch-fsdp/k8s.yaml @@ -1,14 +1,18 @@ +# memory is in bytes, not a "16Gi" quantity string: the published bundle is +# parsed straight into the backend's PhysicalResources (`memory: int`), and +# only the SDK path converts suffixes — a string passes local `compute-config +# create -f` and 422s on launch. 8Gi = 8589934592, 16Gi = 17179869184. head_node: required_resources: CPU: 4 - memory: 8Gi + memory: 8589934592 resources: CPU: 0 worker_nodes: - name: gpu_worker required_resources: CPU: 4 - memory: 16Gi + memory: 17179869184 GPU: 1 required_labels: ray.io/accelerator-type: T4 diff --git a/scripts/hooks/validate-build-yaml.py b/scripts/hooks/validate-build-yaml.py index ed3a93d15..2d709d410 100755 --- a/scripts/hooks/validate-build-yaml.py +++ b/scripts/hooks/validate-build-yaml.py @@ -463,6 +463,22 @@ def check_k8s_configs_declarative(entries: list[Entry]) -> list[str]: f"read by the launch-time GPU validation — use " f"`required_labels: {{ray.io/accelerator-type: ...}}`" ) + # memory is bytes, never a Kubernetes quantity string. `rayapp + # build` copies required_resources verbatim into the published + # bundle, and the console clone path parses that bundle into the + # backend's PhysicalResources, whose `memory` is an int — "8Gi" + # there is a 422. Only the SDK (`compute-config create -f`) + # converts suffixes, so a string survives local testing and fails + # at launch; nothing else in the pipeline catches it. + mem = rr.get("memory") + if mem is not None and (isinstance(mem, bool) or not isinstance(mem, int)): + errors.append( + f"{path}: {loc}.required_resources.memory must be an " + f"integer number of bytes, not {mem!r} — the published " + f"bundle goes to the backend unconverted " + f"(8Gi = 8589934592, 16Gi = 17179869184, " + f"32Gi = 34359738368)" + ) # Mirror the backend's check_gpu_accelerator_consistency: GPU # count and accelerator-type label come together (TPU labels are # paired with TPU fields backend-side instead).