|
| 1 | +#!/usr/bin/env python3 |
| 2 | +"""Test all CloudFormation templates: validate, deploy, verify, delete. |
| 3 | +
|
| 4 | +Usage: python3 test-cfn.py [--parallel N] [--skip-deploy] [--region REGION] |
| 5 | +""" |
| 6 | + |
| 7 | +import argparse |
| 8 | +import boto3 |
| 9 | +import json |
| 10 | +import os |
| 11 | +import sys |
| 12 | +import time |
| 13 | +from concurrent.futures import ThreadPoolExecutor, as_completed |
| 14 | +from dataclasses import dataclass, field |
| 15 | +from pathlib import Path |
| 16 | + |
| 17 | +PREREQ_STACKS = { |
| 18 | + "tutorial-prereqs-bucket": "cfn/prereq-bucket.yaml", |
| 19 | + "tutorial-prereqs-vpc-public": "cfn/prereq-vpc-public.yaml", |
| 20 | + "tutorial-prereqs-vpc-private": "cfn/prereq-vpc-private.yaml", |
| 21 | +} |
| 22 | + |
| 23 | +@dataclass |
| 24 | +class TestResult: |
| 25 | + template: str |
| 26 | + validate: str = "SKIP" |
| 27 | + deploy: str = "SKIP" |
| 28 | + delete: str = "SKIP" |
| 29 | + duration: float = 0 |
| 30 | + error: str = "" |
| 31 | + stack_name: str = "" |
| 32 | + |
| 33 | +def find_templates(repo_root): |
| 34 | + """Find all cfn-*.yaml templates in tutorial directories.""" |
| 35 | + templates = {} |
| 36 | + for p in sorted(Path(repo_root, "tuts").glob("*/cfn-*.yaml")): |
| 37 | + tut = p.parent.name |
| 38 | + templates[tut] = str(p) |
| 39 | + return templates |
| 40 | + |
| 41 | +def detect_prereqs(template_path): |
| 42 | + """Check which prerequisite stacks a template needs.""" |
| 43 | + content = Path(template_path).read_text() |
| 44 | + needed = [] |
| 45 | + if "prereqs-bucket" in content or "prereq-bucket" in content: |
| 46 | + needed.append("tutorial-prereqs-bucket") |
| 47 | + if "prereqs-vpc-public" in content or "prereq-vpc-public" in content: |
| 48 | + needed.append("tutorial-prereqs-vpc-public") |
| 49 | + if "prereqs-vpc-private" in content or "prereq-vpc-private" in content: |
| 50 | + needed.append("tutorial-prereqs-vpc-private") |
| 51 | + return needed |
| 52 | + |
| 53 | +def needs_iam(template_path): |
| 54 | + content = Path(template_path).read_text() |
| 55 | + if "RoleName" in content or "PolicyName" in content: |
| 56 | + return "CAPABILITY_NAMED_IAM" |
| 57 | + if "AWS::IAM::" in content: |
| 58 | + return "CAPABILITY_IAM" |
| 59 | + return None |
| 60 | + |
| 61 | +def validate_template(cfn, template_path): |
| 62 | + body = Path(template_path).read_text() |
| 63 | + cfn.validate_template(TemplateBody=body) |
| 64 | + |
| 65 | +def deploy_stack(cfn, stack_name, template_path, timeout=600): |
| 66 | + body = Path(template_path).read_text() |
| 67 | + caps = [] |
| 68 | + cap = needs_iam(template_path) |
| 69 | + if cap: |
| 70 | + caps = [cap] |
| 71 | + |
| 72 | + try: |
| 73 | + cfn.create_stack( |
| 74 | + StackName=stack_name, |
| 75 | + TemplateBody=body, |
| 76 | + Capabilities=caps, |
| 77 | + Tags=[{"Key": "test-run", "Value": "cfn-test"}], |
| 78 | + TimeoutInMinutes=10, |
| 79 | + OnFailure="DELETE", |
| 80 | + ) |
| 81 | + except cfn.exceptions.AlreadyExistsException: |
| 82 | + cfn.delete_stack(StackName=stack_name) |
| 83 | + waiter = cfn.get_waiter("stack_delete_complete") |
| 84 | + waiter.wait(StackName=stack_name, WaiterConfig={"Delay": 10, "MaxAttempts": 60}) |
| 85 | + cfn.create_stack( |
| 86 | + StackName=stack_name, |
| 87 | + TemplateBody=body, |
| 88 | + Capabilities=caps, |
| 89 | + Tags=[{"Key": "test-run", "Value": "cfn-test"}], |
| 90 | + TimeoutInMinutes=10, |
| 91 | + OnFailure="DELETE", |
| 92 | + ) |
| 93 | + |
| 94 | + waiter = cfn.get_waiter("stack_create_complete") |
| 95 | + waiter.wait(StackName=stack_name, WaiterConfig={"Delay": 15, "MaxAttempts": int(timeout / 15)}) |
| 96 | + |
| 97 | +def delete_stack(cfn, stack_name, timeout=300): |
| 98 | + cfn.delete_stack(StackName=stack_name) |
| 99 | + waiter = cfn.get_waiter("stack_delete_complete") |
| 100 | + waiter.wait(StackName=stack_name, WaiterConfig={"Delay": 10, "MaxAttempts": int(timeout / 10)}) |
| 101 | + |
| 102 | +def ensure_prereqs(cfn, repo_root, needed_stacks): |
| 103 | + """Deploy prerequisite stacks if they don't exist. Returns set of failed prereqs.""" |
| 104 | + failed = set() |
| 105 | + for stack_name in needed_stacks: |
| 106 | + try: |
| 107 | + resp = cfn.describe_stacks(StackName=stack_name) |
| 108 | + status = resp["Stacks"][0]["StackStatus"] |
| 109 | + if status in ("CREATE_COMPLETE", "UPDATE_COMPLETE"): |
| 110 | + print(f" Prereq {stack_name}: exists ({status})") |
| 111 | + continue |
| 112 | + elif "ROLLBACK" in status or "FAILED" in status: |
| 113 | + print(f" Prereq {stack_name}: cleaning up failed stack...") |
| 114 | + cfn.delete_stack(StackName=stack_name) |
| 115 | + cfn.get_waiter("stack_delete_complete").wait( |
| 116 | + StackName=stack_name, WaiterConfig={"Delay": 10, "MaxAttempts": 30}) |
| 117 | + except cfn.exceptions.ClientError: |
| 118 | + pass |
| 119 | + |
| 120 | + template_file = PREREQ_STACKS.get(stack_name) |
| 121 | + if not template_file: |
| 122 | + failed.add(stack_name) |
| 123 | + continue |
| 124 | + |
| 125 | + template_path = os.path.join(repo_root, template_file) |
| 126 | + print(f" Prereq {stack_name}: deploying...") |
| 127 | + try: |
| 128 | + caps = [] |
| 129 | + if needs_iam(template_path): |
| 130 | + caps = [needs_iam(template_path)] |
| 131 | + body = Path(template_path).read_text() |
| 132 | + cfn.create_stack( |
| 133 | + StackName=stack_name, TemplateBody=body, Capabilities=caps, |
| 134 | + Tags=[{"Key": "test-run", "Value": "cfn-test"}], |
| 135 | + TimeoutInMinutes=10, OnFailure="DELETE", |
| 136 | + ) |
| 137 | + waiter = cfn.get_waiter("stack_create_complete") |
| 138 | + waiter.wait(StackName=stack_name, WaiterConfig={"Delay": 15, "MaxAttempts": 40}) |
| 139 | + print(f" Prereq {stack_name}: ready") |
| 140 | + except Exception as e: |
| 141 | + print(f" Prereq {stack_name}: FAILED ({e})") |
| 142 | + failed.add(stack_name) |
| 143 | + try: |
| 144 | + cfn.delete_stack(StackName=stack_name) |
| 145 | + except Exception: |
| 146 | + pass |
| 147 | + return failed |
| 148 | + |
| 149 | +def test_template(cfn, tut_name, template_path, skip_deploy): |
| 150 | + """Test a single template: validate, deploy, delete.""" |
| 151 | + result = TestResult(template=tut_name) |
| 152 | + stack_name = f"cfn-test-{tut_name[:40]}" |
| 153 | + result.stack_name = stack_name |
| 154 | + start = time.time() |
| 155 | + |
| 156 | + # Validate |
| 157 | + try: |
| 158 | + validate_template(cfn, template_path) |
| 159 | + result.validate = "PASS" |
| 160 | + except Exception as e: |
| 161 | + result.validate = "FAIL" |
| 162 | + result.error = str(e)[:200] |
| 163 | + result.duration = time.time() - start |
| 164 | + return result |
| 165 | + |
| 166 | + if skip_deploy: |
| 167 | + result.duration = time.time() - start |
| 168 | + return result |
| 169 | + |
| 170 | + # Deploy |
| 171 | + try: |
| 172 | + deploy_stack(cfn, stack_name, template_path) |
| 173 | + result.deploy = "PASS" |
| 174 | + except Exception as e: |
| 175 | + result.deploy = "FAIL" |
| 176 | + result.error = str(e)[:200] |
| 177 | + result.duration = time.time() - start |
| 178 | + # Try cleanup |
| 179 | + try: |
| 180 | + delete_stack(cfn, stack_name) |
| 181 | + except Exception: |
| 182 | + pass |
| 183 | + return result |
| 184 | + |
| 185 | + # Delete |
| 186 | + try: |
| 187 | + delete_stack(cfn, stack_name) |
| 188 | + result.delete = "PASS" |
| 189 | + except Exception as e: |
| 190 | + result.delete = "FAIL" |
| 191 | + result.error = f"Delete failed: {str(e)[:150]}" |
| 192 | + |
| 193 | + result.duration = time.time() - start |
| 194 | + return result |
| 195 | + |
| 196 | +def main(): |
| 197 | + parser = argparse.ArgumentParser(description="Test CloudFormation templates") |
| 198 | + parser.add_argument("--parallel", type=int, default=3, help="Max parallel deployments") |
| 199 | + parser.add_argument("--skip-deploy", action="store_true", help="Validate only, don't deploy") |
| 200 | + parser.add_argument("--region", default="us-east-1") |
| 201 | + parser.add_argument("--repo", default=".", help="Repo root directory") |
| 202 | + args = parser.parse_args() |
| 203 | + |
| 204 | + repo_root = os.path.abspath(args.repo) |
| 205 | + cfn = boto3.client("cloudformation", region_name=args.region) |
| 206 | + |
| 207 | + # Find templates |
| 208 | + templates = find_templates(repo_root) |
| 209 | + print(f"Found {len(templates)} templates") |
| 210 | + |
| 211 | + if not args.skip_deploy: |
| 212 | + # Collect all needed prereqs |
| 213 | + all_prereqs = set() |
| 214 | + template_prereqs = {} |
| 215 | + for tut, path in templates.items(): |
| 216 | + prereqs = detect_prereqs(path) |
| 217 | + template_prereqs[tut] = prereqs |
| 218 | + all_prereqs.update(prereqs) |
| 219 | + |
| 220 | + failed_prereqs = set() |
| 221 | + if all_prereqs: |
| 222 | + print(f"\nDeploying prerequisites: {', '.join(sorted(all_prereqs))}") |
| 223 | + failed_prereqs = ensure_prereqs(cfn, repo_root, sorted(all_prereqs)) |
| 224 | + if failed_prereqs: |
| 225 | + print(f"\nFailed prereqs: {', '.join(failed_prereqs)}") |
| 226 | + |
| 227 | + # Test templates in parallel |
| 228 | + print(f"\nTesting {len(templates)} templates (parallel={args.parallel})...\n") |
| 229 | + results = [] |
| 230 | + |
| 231 | + with ThreadPoolExecutor(max_workers=args.parallel) as pool: |
| 232 | + futures = {} |
| 233 | + for tut, path in templates.items(): |
| 234 | + # Skip if prereqs failed |
| 235 | + if not args.skip_deploy: |
| 236 | + missing = set(template_prereqs.get(tut, [])) & failed_prereqs |
| 237 | + if missing: |
| 238 | + r = TestResult(template=tut, validate="PASS", deploy="SKIP", error=f"Prereq failed: {', '.join(missing)}") |
| 239 | + results.append(r) |
| 240 | + print(f" ⊘ {tut}: skipped (prereq failed)") |
| 241 | + continue |
| 242 | + # Each thread gets its own client |
| 243 | + thread_cfn = boto3.client("cloudformation", region_name=args.region) |
| 244 | + future = pool.submit(test_template, thread_cfn, tut, path, args.skip_deploy) |
| 245 | + futures[future] = tut |
| 246 | + |
| 247 | + for future in as_completed(futures): |
| 248 | + tut = futures[future] |
| 249 | + result = future.result() |
| 250 | + results.append(result) |
| 251 | + status = "✓" if result.deploy in ("PASS", "SKIP") and result.validate == "PASS" else "✗" |
| 252 | + print(f" {status} {result.template}: validate={result.validate} deploy={result.deploy} delete={result.delete} ({result.duration:.0f}s)") |
| 253 | + if result.error: |
| 254 | + print(f" Error: {result.error}") |
| 255 | + |
| 256 | + # Report |
| 257 | + results.sort(key=lambda r: r.template) |
| 258 | + passed = sum(1 for r in results if r.validate == "PASS" and r.deploy in ("PASS", "SKIP")) |
| 259 | + failed = len(results) - passed |
| 260 | + |
| 261 | + print(f"\n{'='*70}") |
| 262 | + print(f"RESULTS: {passed} passed, {failed} failed, {len(results)} total") |
| 263 | + print(f"{'='*70}") |
| 264 | + print(f"{'Template':<45} {'Validate':<10} {'Deploy':<10} {'Delete':<10} {'Time':<8}") |
| 265 | + print(f"{'-'*45} {'-'*10} {'-'*10} {'-'*10} {'-'*8}") |
| 266 | + for r in results: |
| 267 | + print(f"{r.template:<45} {r.validate:<10} {r.deploy:<10} {r.delete:<10} {r.duration:<8.0f}s") |
| 268 | + |
| 269 | + # Cleanup prereqs if all tests passed |
| 270 | + if not args.skip_deploy and failed == 0: |
| 271 | + print(f"\nAll tests passed. Prerequisite stacks left running for reuse.") |
| 272 | + print(f"To delete: python3 {sys.argv[0]} --cleanup-prereqs") |
| 273 | + |
| 274 | + return 1 if failed > 0 else 0 |
| 275 | + |
| 276 | +if __name__ == "__main__": |
| 277 | + sys.exit(main()) |
0 commit comments