DSPX 2655 gemini simplified workflow

otdf-local/pyproject.toml

@@ -7,6 +7,7 @@ requires-python = ">=3.11"
 dependencies = [
     "httpx>=0.27.0",
     "pydantic-settings>=2.2.0",
+    "pyyaml>=6.0.3",
     "rich>=13.7.0",
     "ruamel.yaml>=0.18.0",
     "typer>=0.12.0",

otdf-local/src/otdf_local/ci.py

@@ -0,0 +1,223 @@
+"""CI-specific commands for otdf-local.
+
+These commands adapt the local environment management for GitHub Actions CI,
+where the platform is already started by an external action and we only need
+to start KAS instances as background processes.
+"""
+
+from __future__ import annotations
+
+import os
+import sys
+from pathlib import Path
+from typing import Annotated
+
+import typer
+
+from otdf_local.config.ports import Ports
+from otdf_local.config.settings import Settings
+from otdf_local.health.waits import WaitTimeoutError, wait_for_health
+from otdf_local.services import get_kas_manager
+from otdf_local.utils.console import (
+    print_error,
+    print_info,
+    print_success,
+    print_warning,
+)
+from otdf_local.utils.yaml import load_yaml, save_yaml, set_nested
+
+ci_app = typer.Typer(
+    name="ci",
+    help="CI-specific commands for GitHub Actions workflows.",
+    no_args_is_help=True,
+)
+
+
+def _emit_github_output(key: str, value: str) -> None:
+    """Write a key=value pair to $GITHUB_OUTPUT if available, else print to stdout."""
+    github_output = os.environ.get("GITHUB_OUTPUT")
+    if github_output:
+        with open(github_output, "a") as f:
+            f.write(f"{key}={value}\n")
+    else:
+        # Fallback for local testing
+        print(f"{key}={value}", file=sys.stdout)
+
+
+def _prepare_kas_template(
+    settings: Settings, root_key: str | None, ec_tdf_enabled: bool
+) -> None:
+    """Ensure the KAS template config has the right root key and EC TDF settings.
+
+    In CI, the platform config may have a root_key that differs from what
+    we want for additional KAS instances. This updates the platform config
+    in-place so that KASService._generate_config reads the correct root_key.
+    """
+    if root_key:
+        config = load_yaml(settings.platform_config)
+        set_nested(config, "services.kas.root_key", root_key)
+        if ec_tdf_enabled:
+            set_nested(config, "services.kas.preview.ec_tdf_enabled", True)
+        save_yaml(settings.platform_config, config)
+
+
+@ci_app.command("start-kas")
+def start_kas(
+    platform_dir: Annotated[
+        Path,
+        typer.Option(
+            "--platform-dir",
+            help="Path to the platform checkout (must contain opentdf-kas-mode.yaml)",
+            envvar="OTDF_LOCAL_PLATFORM_DIR",
+        ),
+    ],
+    root_key: Annotated[
+        str | None,
+        typer.Option(
+            "--root-key",
+            help="Root key for KAS instances (overrides platform config value)",
+            envvar="OT_ROOT_KEY",
+        ),
+    ] = None,
+    ec_tdf_enabled: Annotated[
+        bool,
+        typer.Option(
+            "--ec-tdf-enabled/--no-ec-tdf",
+            help="Enable EC TDF support",
+        ),
+    ] = True,
+    key_management: Annotated[
+        bool,
+        typer.Option(
+            "--key-management/--no-key-management",
+            help="Enable key management on km1/km2 instances",
+        ),
+    ] = False,
+    log_type: Annotated[
+        str,
+        typer.Option(
+            "--log-type",
+            help="Log format type (json, text)",
+        ),
+    ] = "json",
+    health_timeout: Annotated[
+        int,
+        typer.Option(
+            "--health-timeout",
+            help="Seconds to wait for each KAS instance to become healthy",
+        ),
+    ] = 60,
+    instances: Annotated[
+        str | None,
+        typer.Option(
+            "--instances",
+            help="Comma-separated KAS instance names (default: all)",
+        ),
+    ] = None,
+) -> None:
+    """Start KAS instances for CI and emit GitHub Actions outputs.
+
+    Expects the platform to already be running (started by start-up-with-containers).
+    Starts all 6 KAS instances (alpha, beta, gamma, delta, km1, km2) as background
+    processes, waits for each to pass health checks, and emits log file paths as
+    GitHub Actions step outputs.
+
+    Output keys (written to $GITHUB_OUTPUT):
+      kas-alpha-log-file, kas-beta-log-file, kas-gamma-log-file,
+      kas-delta-log-file, kas-km1-log-file, kas-km2-log-file
+    """
+    platform_dir = platform_dir.resolve()
+    if not platform_dir.is_dir():
+        print_error(f"Platform directory does not exist: {platform_dir}")
+        raise typer.Exit(1)
+
+    # Check for required template files
+    kas_template = platform_dir / "opentdf-kas-mode.yaml"
+    platform_config = platform_dir / "opentdf-dev.yaml"
+    if not kas_template.exists():
+        # Fall back to opentdf.yaml if opentdf-kas-mode.yaml doesn't exist
+        kas_template_alt = platform_dir / "opentdf.yaml"
+        if kas_template_alt.exists():
+            print_info(
+                f"Using {kas_template_alt} as KAS template (opentdf-kas-mode.yaml not found)"
+            )
+        else:
+            print_error(
+                f"Neither opentdf-kas-mode.yaml nor opentdf.yaml found in {platform_dir}"
+            )
+            raise typer.Exit(1)
+
+    if not platform_config.exists():
+        # Try opentdf.yaml as fallback
+        platform_config_alt = platform_dir / "opentdf.yaml"
+        if platform_config_alt.exists():
+            platform_config = platform_config_alt
+
+    # Build settings with CI-specific overrides
+    # We use a fresh xtest_root derived from this package's location
+    settings = Settings(
+        platform_dir=platform_dir,
+    )
+    settings.ensure_directories()
+
+    # Update root key in platform config if provided
+    if root_key:
+        _prepare_kas_template(settings, root_key, ec_tdf_enabled)
+
+    # Determine which instances to start
+    if instances:
+        kas_names = [n.strip() for n in instances.split(",")]
+        for name in kas_names:
+            if name not in Ports.all_kas_names():
+                print_error(f"Unknown KAS instance: {name}")
+                raise typer.Exit(1)
+    else:
+        kas_names = Ports.all_kas_names()
+
+    # Start KAS instances
+    print_info(f"Starting KAS instances: {', '.join(kas_names)}...")
+    kas_manager = get_kas_manager(settings)
+
+    failed = []
+    for name in kas_names:
+        kas = kas_manager.get(name)
+        if kas is None:
+            print_error(f"KAS instance {name} not found in manager")
+            failed.append(name)
+            continue
+        if not kas.start():
+            print_error(f"Failed to start KAS {name}")
+            failed.append(name)
+
+    if failed:
+        print_error(f"Failed to start: {', '.join(failed)}")
+        raise typer.Exit(1)
+
+    # Wait for health
+    print_info("Waiting for KAS health checks...")
+    unhealthy = []
+    for name in kas_names:
+        port = Ports.get_kas_port(name)
+        try:
+            wait_for_health(
+                f"http://localhost:{port}/healthz",
+                timeout=health_timeout,
+                service_name=f"KAS {name}",
+            )
+        except WaitTimeoutError as e:
+            print_warning(str(e))
+            unhealthy.append(name)
+
+    if unhealthy:
+        print_error(f"KAS instances failed health check: {', '.join(unhealthy)}")
+        raise typer.Exit(1)
+
+    print_success(f"All {len(kas_names)} KAS instances are healthy")
+
+    # Emit outputs
+    for name in kas_names:
+        log_path = settings.get_kas_log_path(name)
+        output_key = f"kas-{name}-log-file"
+        _emit_github_output(output_key, str(log_path))
+
+    print_success("CI KAS startup complete")

otdf-local/src/otdf_local/cli.py

@@ -11,6 +11,8 @@
 from rich.live import Live
 
 from otdf_local import __version__
+from otdf_local.ci import ci_app
+from otdf_local.suite.cli import suite_app
 from otdf_local.config.ports import Ports
 from otdf_local.config.settings import get_settings
 from otdf_local.health.waits import WaitTimeoutError, wait_for_health, wait_for_port
@@ -43,6 +45,9 @@
     pretty_exceptions_enable=sys.stderr.isatty(),
 )
 
+app.add_typer(ci_app, name="ci")
+app.add_typer(suite_app, name="suite")
+
 
 def _show_provision_error(result: ProvisionResult, target: str) -> None:
     """Display provisioning error with stderr details."""

otdf-local/src/otdf_local/config/settings.py

@@ -91,9 +91,18 @@ class Settings(BaseSettings):
 
     # Directory paths - computed from xtest_root
     xtest_root: Path = Field(default_factory=_find_xtest_root)
-    platform_dir: Path = Field(
-        default_factory=lambda: _find_platform_dir(_find_xtest_root())
-    )
+    _platform_dir: Path | None = None
+
+    @property
+    def platform_dir(self) -> Path:
+        """Platform directory path."""
+        if self._platform_dir:
+            return self._platform_dir
+        return _find_platform_dir(self.xtest_root)
+
+    @platform_dir.setter
+    def platform_dir(self, value: Path) -> None:
+        self._platform_dir = value
 
     @property
     def logs_dir(self) -> Path:

otdf-local/src/otdf_local/suite/cli.py

@@ -0,0 +1,113 @@
+"""CLI commands for X-Test suite management."""
+
+from __future__ import annotations
+
+import json
+import os
+from pathlib import Path
+from typing import Annotated, Optional
+
+import typer
+import yaml
+
+from otdf_local.config.settings import get_settings
+from otdf_local.suite.models import (
+    PlatformVersion,
+    SDKVersion,
+    SuiteConfig,
+    TestJob,
+)
+from otdf_local.utils.console import console, print_error, print_info
+
+
+suite_app = typer.Typer(
+    name="suite",
+    help="X-Test suite orchestration commands.",
+    no_args_is_help=True,
+)
+
+
+@suite_app.command("generate-shard")
+def generate_shard(
+    platform_ref: Annotated[str, typer.Option("--platform-ref", help="Platform ref to test")],
+    platform_sha: Annotated[Optional[str], typer.Option("--platform-sha", help="Platform SHA to test")] = None,
+    sdk: Annotated[str, typer.Option("--sdk", help="SDK to focus on (go, java, js)")],
+    go_ref: Annotated[str, typer.Option("--go-ref")] = "main",
+    go_sha: Annotated[Optional[str], typer.Option("--go-sha")] = None,
+    java_ref: Annotated[str, typer.Option("--java-ref")] = "main",
+    java_sha: Annotated[Optional[str], typer.Option("--java-sha")] = None,
+    js_ref: Annotated[str, typer.Option("--js-ref")] = "main",
+    js_sha: Annotated[Optional[str], typer.Option("--js-sha")] = None,
+    output: Annotated[Optional[Path], typer.Option("--output", "-o", help="Output YAML file")] = None,
+) -> None:
+    """Generate a self-contained SuiteConfig YAML for a specific matrix shard."""
+    platform = PlatformVersion(tag=platform_ref, sha=platform_sha)
+
+    sdks = {
+        "go": [SDKVersion(tag=go_ref, sha=go_sha)],
+        "java": [SDKVersion(tag=java_ref, sha=java_sha)],
+        "js": [SDKVersion(tag=js_ref, sha=js_sha)],
+    }
+
+    # Define jobs - match the standard jobs in xtest.yml
+    jobs = [
+        TestJob(
+            name=f"standard-{sdk}",
+            pytest_args=["-ra", "-v", "test_tdfs.py", "test_policytypes.py"],
+            focus_sdk=sdk
+        ),
+        TestJob(
+            name=f"legacy-{sdk}",
+            pytest_args=["-ra", "-v", "test_legacy.py"],
+            focus_sdk=sdk
+        ),
+        TestJob(
+            name=f"abac-{sdk}",
+            pytest_args=["-ra", "-v", "test_abac.py"],
+            focus_sdk=sdk,
+            requires_kas=True
+        ),
+    ]
+
+    config = SuiteConfig(
+        platforms=[platform],
+        sdks=sdks,
+        jobs=jobs
+    )
+
+    yaml_data = yaml.dump(config.model_dump(), sort_keys=False)
+
+    if output:
+        output.write_text(yaml_data)
+        print_info(f"Generated shard config at {output}")
+    else:
+        # Print to stdout (wrapped in markdown for GHA summary)
+        summary = f"<details><summary><b>shard.yaml</b> (for reproduction)</summary>\n\n```yaml\n{yaml_data}```\n</details>"
+        console.print(summary)
+
+        # Also write to GITHUB_STEP_SUMMARY if it exists
+        github_summary = os.environ.get("GITHUB_STEP_SUMMARY")
+        if github_summary:
+            with open(github_summary, "a") as f:
+                f.write(f"\n{summary}\n")
+
+
+@suite_app.command("run")
+def run_suite(
+    config_path: Annotated[Path, typer.Argument(help="Path to SuiteConfig YAML")],
+) -> None:
+    """Run an X-Test suite from a configuration file."""
+    if not config_path.exists():
+        print_error(f"Config file not found: {config_path}")
+        raise typer.Exit(1)
+
+    with open(config_path) as f:
+        data = yaml.safe_load(f)
+        config = SuiteConfig.model_validate(data)
+
+    from otdf_local.suite.runner import SuiteRunner
+    runner = SuiteRunner(config, get_settings())
+
+    success = runner.run()
+    if not success:
+        raise typer.Exit(1)