diff --git a/.claude/skills/template/references/conventions.md b/.claude/skills/template/references/conventions.md index 7cc34659f..feee459be 100644 --- a/.claude/skills/template/references/conventions.md +++ b/.claude/skills/template/references/conventions.md @@ -15,7 +15,7 @@ templates/ ├── BUILD.yaml # Template definitions ├── templates// # Content (code, notebooks, Dockerfiles) ├── tests// # Test scripts -├── configs// # Compute configs (aws.yaml / gce.yaml) +├── configs// # Compute configs (aws.yaml / gce.yaml [/ k8s.yaml]) └── .claude/skills/template/schemas/ # BUILD.yaml + compute-config schemas ``` @@ -24,7 +24,7 @@ templates/ Run the `check-build-yaml` hook first (see **Validate locally**) — it authoritatively covers BUILD.yaml schema, referenced paths, naming, and the compute-config schema check. Fix what it reports, then verify: - **BUILD.yaml entry** matches `../schemas/build-yaml-schema.yaml`. `owner_team` is required (`ray-serve` | `ray-data` | `llm` | `ray-train` | `general`) — deduce per that schema file's rule. Under `cluster_env:`, use either `cluster_env.image_uri` OR `cluster_env.byod`, never both. Image taxonomy: SKILL.md "Image URI cases". -- **Compute configs** present at `configs//aws.yaml` and `configs//gce.yaml`. Schema: `../schemas/compute-config-schema.yaml`. +- **Compute configs** present at `configs//aws.yaml` and `configs//gce.yaml`; optional `k8s.yaml` alongside them (declarative `required_resources`, K8s-stack clouds). Schema: `../schemas/compute-config-schema.yaml`. - **Test** present at `tests//tests.sh`. - **Dependencies pinned.** Declare template deps in `templates//requirements.txt`, the BYOD `Dockerfile` (`RUN pip install "pkg==x.y.z"`), or notebook `!pip install` — with exact versions. (The repo-root `dependencies/` directory is Ray base-image constraint management, not per-template deps.) - **README** — author content in `README.ipynb`; `README.md` is its `jupyter nbconvert --to markdown` render. Never hand-edit `README.md`; regenerate it (`check-readme` enforces a byte-exact match). diff --git a/.claude/skills/template/schemas/build-yaml-schema.yaml b/.claude/skills/template/schemas/build-yaml-schema.yaml index b9bf1d90a..4c21d3e64 100644 --- a/.claude/skills/template/schemas/build-yaml-schema.yaml +++ b/.claude/skills/template/schemas/build-yaml-schema.yaml @@ -36,6 +36,10 @@ compute_config: AWS: configs//aws.yaml GCP: configs//gce.yaml + # OPTIONAL. One declarative config for all Kubernetes-stack clouds + # (AKS/EKS/GKE): required_resources only, never instance_type — resolved + # server-side into free pods, no registered instance types needed. + K8S: configs//k8s.yaml # REQUIRED for entries under templates/ — active templates are tested. Entries # under archive/ are exempt (test-free — e.g. fast event iteration or a retired @@ -88,8 +92,10 @@ # --- Cross-field rules (enforced by scripts/hooks/validate-build-yaml.py) --- # # - tests_path basename must equal name → tests// -# - compute_config GCP and AWS must share the same configs// +# - compute_config paths (GCP, AWS, and K8S when present) must share the same configs// # - that dir's basename must equal name, OR be `basic-single-node` +# - K8S configs are declarative: every node sets required_resources, never instance_type; +# no auto_select_worker_config; GPU nodes pair the count with required_labels ray.io/accelerator-type # - For byod on the Anyscale GCP registry, docker_image must be exactly: # us-docker.pkg.dev/anyscale-workspace-templates/workspace-templates/: # (image basename == name, tag == byod.ray_version). push-custom-image-to-gcp.sh produces this form. diff --git a/.claude/skills/template/schemas/compute-config-schema.yaml b/.claude/skills/template/schemas/compute-config-schema.yaml index 7c3f75c5c..0efab3ba2 100644 --- a/.claude/skills/template/schemas/compute-config-schema.yaml +++ b/.claude/skills/template/schemas/compute-config-schema.yaml @@ -1,7 +1,7 @@ # Compute Config Schema (ComputeConfig — user-facing API format) # -# These YAML files live under configs//{aws,gce}.yaml and use the -# user-facing ComputeConfig schema: +# These YAML files live under configs//{aws,gce,k8s}.yaml and use +# the user-facing ComputeConfig schema: # https://docs.anyscale.com/reference/compute-config-api#computeconfig # # IMPORTANT: Omit fields that match their default value. Only include what you @@ -74,6 +74,26 @@ auto_select_worker_config: true # Default: false. Omit if false. When tru # max_nodes: 4 # market_type: PREFER_SPOT # spot, fall back to on-demand # +# k8s.yaml (Kubernetes-stack clouds — AKS/EKS/GKE): declarative shapes ONLY, +# per https://docs.anyscale.com/configuration/compute/declarative. Every node +# sets required_resources, never instance_type (validator-enforced); the +# backend resolves the requirements into free pods, so nothing needs to be +# registered on the cloud. auto_select_worker_config is NOT compatible with +# declarative configs — declare explicit worker_nodes. GPU nodes pair the GPU +# count with `required_labels: {ray.io/accelerator-type: ...}` — the label, +# not a required_resources `accelerator` key (launch validation only reads +# the label). Accelerator values must exist in the target clouds' GPU pools — +# Azure has T4/A10/A100/H100, NOT L4/A10G: +# head_node: +# required_resources: {CPU: 4, memory: 8Gi} +# resources: {CPU: 0} +# worker_nodes: +# - name: gpu_worker +# required_resources: {CPU: 4, memory: 16Gi, GPU: 1} +# required_labels: {ray.io/accelerator-type: T4} +# min_nodes: 2 +# max_nodes: 2 +# # advanced_instance_config (cloud-provider passthrough; the key is provider-neutral, the body is not): # # AWS: {BlockDeviceMappings: [{DeviceName: /dev/sda1, Ebs: {VolumeSize: 1000, DeleteOnTermination: true}}]} # # GCP: {instance_properties: {labels: {as-feature-multi-zone: 'true'}}} diff --git a/.claude/skills/template/workflows/create-template.md b/.claude/skills/template/workflows/create-template.md index 6e56dfa8f..fdc4c9d9f 100644 --- a/.claude/skills/template/workflows/create-template.md +++ b/.claude/skills/template/workflows/create-template.md @@ -42,7 +42,7 @@ That returns the ComputeConfig shape `configs/` uses directly (full fields + pat - drop auto-detected node `resources` (with workers present, the head is unschedulable by default); keep explicit overrides like `CPU: 0` - keep `max_nodes` explicit on every worker group -Write `configs//aws.yaml` and `gce.yaml` by instance family. +Write `configs//aws.yaml` and `gce.yaml` by instance family; optionally `k8s.yaml` (declarative `required_resources`, no instance types — see the schema file's k8s.yaml pattern). **Fallback — guided Q&A.** No tested workspace → walk the user through those same fields. diff --git a/BUILD.yaml b/BUILD.yaml index 1d36522e6..a5fee3ce0 100644 --- a/BUILD.yaml +++ b/BUILD.yaml @@ -23,6 +23,7 @@ compute_config: GCP: configs/basic-single-node/gce.yaml AWS: configs/basic-single-node/aws.yaml + K8S: configs/basic-single-node/k8s.yaml test: tests_path: tests/workspace-intro/ command: bash tests.sh @@ -338,6 +339,7 @@ compute_config: AWS: configs/pytorch-fsdp/aws.yaml GCP: configs/pytorch-fsdp/gce.yaml + K8S: configs/pytorch-fsdp/k8s.yaml test: tests_path: tests/pytorch-fsdp/ command: bash tests.sh @@ -484,6 +486,7 @@ compute_config: GCP: configs/getting-started/gce.yaml AWS: configs/getting-started/aws.yaml + K8S: configs/getting-started/k8s.yaml test: tests_path: tests/getting-started/ command: bash tests.sh diff --git a/configs/basic-single-node/k8s.yaml b/configs/basic-single-node/k8s.yaml new file mode 100644 index 000000000..f7e2810f6 --- /dev/null +++ b/configs/basic-single-node/k8s.yaml @@ -0,0 +1,12 @@ +head_node: + required_resources: + CPU: 4 + memory: 8Gi + resources: + CPU: 0 +worker_nodes: +- name: cpu_worker + required_resources: + CPU: 8 + memory: 32Gi + max_nodes: 1 diff --git a/configs/getting-started/k8s.yaml b/configs/getting-started/k8s.yaml new file mode 100644 index 000000000..d7e7aaa8b --- /dev/null +++ b/configs/getting-started/k8s.yaml @@ -0,0 +1,12 @@ +head_node: + required_resources: + CPU: 4 + memory: 8Gi + resources: + CPU: 0 +worker_nodes: +- name: cpu_worker + required_resources: + CPU: 8 + memory: 32Gi + max_nodes: 2 diff --git a/configs/pytorch-fsdp/k8s.yaml b/configs/pytorch-fsdp/k8s.yaml new file mode 100644 index 000000000..1b5649d18 --- /dev/null +++ b/configs/pytorch-fsdp/k8s.yaml @@ -0,0 +1,16 @@ +head_node: + required_resources: + CPU: 4 + memory: 8Gi + resources: + CPU: 0 +worker_nodes: +- name: gpu_worker + required_resources: + CPU: 4 + memory: 16Gi + GPU: 1 + required_labels: + ray.io/accelerator-type: T4 + min_nodes: 2 + max_nodes: 2 diff --git a/scripts/hooks/validate-build-yaml.py b/scripts/hooks/validate-build-yaml.py index cb5aaabce..ed3a93d15 100755 --- a/scripts/hooks/validate-build-yaml.py +++ b/scripts/hooks/validate-build-yaml.py @@ -74,6 +74,21 @@ class ComputeConfig(Strict): # archived ones (which co-locate their compute config under the template). GCP: str = Field(pattern=r"^(?:configs|archive)/.+/gce\.yaml$") AWS: str = Field(pattern=r"^(?:configs|archive)/.+/aws\.yaml$") + # Kubernetes-stack clouds (AKS/EKS/GKE) — one config covers all of them. + # Declarative shapes only (required_resources → free pods, no registered + # instance types; enforced by check_k8s_configs_declarative). Optional + # while templates migrate. + K8S: Optional[str] = Field( + default=None, pattern=r"^(?:configs|archive)/.+/k8s\.yaml$" + ) + + +def cloud_paths(cc: ComputeConfig): + """(cloud, path) pairs for an entry's compute configs; K8S only when set.""" + for cloud in ("GCP", "AWS", "K8S"): + path = getattr(cc, cloud) + if path is not None: + yield cloud, path class Test(Strict): @@ -229,24 +244,25 @@ def check_filesystem_and_uniqueness(entries: list[Entry]) -> list[str]: f"equal name {e.name!r} (expected tests/{e.name}/)" ) - # GCP and AWS configs must live in the same directory under configs/. + # All clouds' configs must live in the same directory under configs/. # Catches one-cloud-customized-but-not-the-other mistakes. - gcp_parent = Path(e.compute_config.GCP).parent - aws_parent = Path(e.compute_config.AWS).parent - if gcp_parent != aws_parent: + parents = {c: Path(p).parent for c, p in cloud_paths(e.compute_config)} + same_dir = len(set(parents.values())) == 1 + if not same_dir: + got = ", ".join(f"{c}: {p}" for c, p in parents.items()) errors.append( - f"{e.name}.compute_config: GCP and AWS configs must live in " - f"the same `configs//` directory; got {gcp_parent} and " - f"{aws_parent}" + f"{e.name}.compute_config: all clouds' configs must live in " + f"the same `configs//` directory; got {got}" ) # Custom compute config dirs must be named after the entry (the # `name` field is the source of truth across templates/, tests/, # and configs/). Shared `configs/basic-single-node/` is exempt. + gcp_parent = parents["GCP"] cfg_dir_basename = gcp_parent.name if ( gcp_parent.parts and gcp_parent.parts[0] == "configs" - and gcp_parent == aws_parent + and same_dir and cfg_dir_basename != "basic-single-node" and cfg_dir_basename != e.name ): @@ -258,8 +274,7 @@ def check_filesystem_and_uniqueness(entries: list[Entry]) -> list[str]: if not (REPO_ROOT / e.dir).is_dir(): errors.append(f"{e.name}: dir not found: {e.dir}") - for cloud in ("GCP", "AWS"): - path = getattr(e.compute_config, cloud) + for cloud, path in cloud_paths(e.compute_config): if not (REPO_ROOT / path).is_file(): errors.append(f"{e.name}.compute_config.{cloud}: not found: {path}") if e.test is not None: @@ -275,6 +290,7 @@ def check_filesystem_and_uniqueness(entries: list[Entry]) -> list[str]: BASIC_CONFIGS = ( "configs/basic-single-node/aws.yaml", "configs/basic-single-node/gce.yaml", + "configs/basic-single-node/k8s.yaml", ) @@ -290,8 +306,7 @@ def check_redundant_compute_configs(entries: list[Entry]) -> list[str]: basics[p] = full.read_bytes() for e in entries: - for cloud in ("GCP", "AWS"): - path = getattr(e.compute_config, cloud) + for cloud, path in cloud_paths(e.compute_config): if path in BASIC_CONFIGS: continue full = REPO_ROOT / path @@ -299,6 +314,10 @@ def check_redundant_compute_configs(entries: list[Entry]) -> list[str]: continue content = full.read_bytes() for basic_path, basic_bytes in basics.items(): + # Same cloud only (aws vs aws, k8s vs k8s): a cross-cloud + # byte-match is never "reference the shared config instead". + if Path(basic_path).name != Path(path).name: + continue if content == basic_bytes: warnings.append( f"{e.name}.compute_config.{cloud}: {path} is byte-identical " @@ -322,8 +341,8 @@ def check_compute_configs(entries: list[Entry]) -> list[str]: errors: list[str] = [] paths: set[str] = set() for e in entries: - paths.add(e.compute_config.GCP) - paths.add(e.compute_config.AWS) + for _, path in cloud_paths(e.compute_config): + paths.add(path) for path in sorted(paths): full = REPO_ROOT / path if not full.is_file(): @@ -359,8 +378,8 @@ def check_head_nodes(entries: list[Entry]) -> list[str]: for e in entries: if e.dir.startswith("archive/"): continue # archived templates are retired/unmaintained — exempt, like the test requirement - paths.add(e.compute_config.GCP) - paths.add(e.compute_config.AWS) + for _, path in cloud_paths(e.compute_config): + paths.add(path) for path in sorted(paths): full = REPO_ROOT / path if not full.is_file(): @@ -384,6 +403,92 @@ def check_head_nodes(entries: list[Entry]) -> list[str]: return errors +# ------------------------------------------ K8S configs must be declarative + +def check_k8s_configs_declarative(entries: list[Entry]) -> list[str]: + """Every node in a K8S config must be shaped with `required_resources`, + never `instance_type` — the backend resolves resource requirements into + free pods, while named instance types are per-cluster registrations on + K8s; naming one would silently reintroduce the registration dependency + the declarative form exists to avoid. Declarative constraints (per + https://docs.anyscale.com/configuration/compute/declarative): no + auto_select_worker_config, and GPU nodes carry the accelerator type in + `required_labels` (the launch-time GPU validation only reads the + ray.io/accelerator-type label, not `required_resources.accelerator`).""" + errors: list[str] = [] + paths: set[str] = set() + for e in entries: + if e.compute_config.K8S is not None: + paths.add(e.compute_config.K8S) + for path in sorted(paths): + full = REPO_ROOT / path + if not full.is_file(): + continue # already reported by check_filesystem_and_uniqueness + data = yaml.safe_load(full.read_text()) or {} + if not isinstance(data, dict): + continue # malformed file — reported by check_compute_configs + if data.get("auto_select_worker_config"): + errors.append( + f"{path}: auto_select_worker_config is not supported with " + f"declarative compute configs — declare explicit worker_nodes " + f"with `required_resources`" + ) + nodes = [("head_node", data.get("head_node") or {})] + for i, worker in enumerate(data.get("worker_nodes") or []): + nodes.append((f"worker_nodes[{i}]", worker or {})) + for loc, node in nodes: + if not isinstance(node, dict): + continue # malformed node — reported by check_compute_configs + if node.get("instance_type") is not None: + errors.append( + f"{path}: {loc}.instance_type is not allowed in a K8S " + f"config — declare the shape with `required_resources` " + f"instead" + ) + continue + rr = node.get("required_resources") + if not rr: + # None, absent, or {} — the SDK rejects empty + # required_resources at deploy time (needs CPU>0 or memory>0). + errors.append( + f"{path}: {loc} needs a non-empty `required_resources` — " + f"K8S configs are declarative; specify CPU/memory" + ) + continue + if not isinstance(rr, dict): + continue # malformed — reported by check_compute_configs + if rr.get("accelerator"): + errors.append( + f"{path}: {loc}.required_resources.accelerator is not " + f"read by the launch-time GPU validation — use " + f"`required_labels: {{ray.io/accelerator-type: ...}}`" + ) + # Mirror the backend's check_gpu_accelerator_consistency: GPU + # count and accelerator-type label come together (TPU labels are + # paired with TPU fields backend-side instead). + gpu = rr.get("GPU") or 0 + accel = None + for label_key in ("required_labels", "labels"): + lbls = node.get(label_key) + if isinstance(lbls, dict) and lbls.get("ray.io/accelerator-type"): + accel = lbls["ray.io/accelerator-type"] + break + if accel and str(accel).upper().startswith("TPU"): + continue + if bool(gpu) != bool(accel): + if gpu: + errors.append( + f"{path}: {loc} sets GPU but no accelerator type — " + f"add `required_labels: {{ray.io/accelerator-type: ...}}`" + ) + else: + errors.append( + f"{path}: {loc} sets ray.io/accelerator-type but no " + f"GPU count in `required_resources`" + ) + return errors + + # ----------------------------------------------- GCP image naming + access GCP_BYOD_REGISTRY = ( @@ -493,6 +598,7 @@ def main() -> int: errors = check_filesystem_and_uniqueness(entries) errors.extend(check_compute_configs(entries)) errors.extend(check_head_nodes(entries)) + errors.extend(check_k8s_configs_declarative(entries)) errors.extend(check_gcp_byod_images(entries, network=not args.no_network)) if errors: