feat(compare): add offline result comparison

CHANGELOG.md

@@ -7,6 +7,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+### Added
+- `skill-up compare <old-result.json> <new-result.json>` compares offline
+  evaluation results, including case transitions and optional regression and
+  total-token growth gates for CI.
+
 ## [0.9.0] - 2026-08-12
 
 ### Fixed

README.md

@@ -235,6 +235,7 @@ skill-up import ./evals/evals.json --output ./evals
 | `skill-up validate [path]`           | Validate `eval.yaml` and case files         |
 | `skill-up list-cases [path]`         | List all cases referenced by the config     |
 | `skill-up report <result.json>`      | Generate reports from a previous run        |
+| `skill-up compare <old> <new>`        | Compare two offline evaluation results      |
 | `skill-up import <evals.json>`       | Import Anthropic `evals.json` to YAML cases |
 | `skill-up debug judge <input.json>`  | Debug judge module with a JSON input        |
 | `skill-up debug report <input.json>` | Debug report module with a JSON input       |

README.zh.md

@@ -154,6 +154,7 @@ Windows 的安装方式与已知限制请参阅
 | `skill-up validate [path]`           | 校验 `eval.yaml` 和用例文件                |
 | `skill-up list-cases [path]`         | 列出配置引用的所有用例                     |
 | `skill-up report <result.json>`      | 从已有结果生成报告                         |
+| `skill-up compare <old> <new>`        | 对比两个离线评测结果                       |
 | `skill-up import <evals.json>`       | 将 Anthropic `evals.json` 导入为 YAML 用例 |
 | `skill-up debug judge <input.json>`  | 使用 JSON 输入调试 judge 模块              |
 | `skill-up debug report <input.json>` | 使用 JSON 输入调试 report 模块             |

internal/cli/compare.go

@@ -0,0 +1,134 @@
+package cli
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"math"
+	"os"
+	"strings"
+
+	"github.com/spf13/cobra"
+
+	comparepkg "github.com/alibaba/skill-up/internal/compare"
+	"github.com/alibaba/skill-up/internal/judge"
+	"github.com/alibaba/skill-up/internal/report"
+)
+
+var compareCmd = &cobra.Command{
+	Use:   "compare <old-result.json> <new-result.json>",
+	Short: "Compare two offline evaluation results",
+	Args:  cobra.ExactArgs(2),
+	RunE:  runCompare,
+}
+
+func init() {
+	compareCmd.Flags().String("format", "text", "Output format: text, json")
+	compareCmd.Flags().Bool("fail-on-regression", false, "Fail when any case regresses")
+	compareCmd.Flags().Float64("max-token-increase-percent", 0, "Maximum allowed total token increase percentage")
+}
+
+func runCompare(cmd *cobra.Command, args []string) error {
+	format, err := cmd.Flags().GetString("format")
+	if err != nil {
+		return fmt.Errorf("get format flag: %w", err)
+	}
+	if format != "text" && format != jsonFormat {
+		return fmt.Errorf("unsupported compare format %q; supported formats: text, json", format)
+	}
+
+	oldInput, err := loadCompareInput("old", args[0])
+	if err != nil {
+		return err
+	}
+	newInput, err := loadCompareInput("new", args[1])
+	if err != nil {
+		return err
+	}
+
+	failOnRegression, err := cmd.Flags().GetBool("fail-on-regression")
+	if err != nil {
+		return fmt.Errorf("get fail-on-regression flag: %w", err)
+	}
+	options := comparepkg.Options{FailOnRegression: failOnRegression}
+	maxTokenIncreasePercent, err := maxTokenIncreasePercent(cmd)
+	if err != nil {
+		return err
+	}
+	if maxTokenIncreasePercent != nil {
+		options.MaxTokenIncreasePercent = maxTokenIncreasePercent
+	}
+
+	result := comparepkg.Compare(oldInput, newInput, options)
+	if format == jsonFormat {
+		encoder := json.NewEncoder(cmd.OutOrStdout())
+		if err := encoder.Encode(result); err != nil {
+			return fmt.Errorf("write JSON comparison: %w", err)
+		}
+	} else if _, err := fmt.Fprint(cmd.OutOrStdout(), comparepkg.RenderText(result)); err != nil {
+		return fmt.Errorf("write text comparison: %w", err)
+	}
+
+	if !result.Gates.Passed {
+		return fmt.Errorf("comparison gates failed: %v", result.Gates.Failures)
+	}
+	return nil
+}
+
+func maxTokenIncreasePercent(cmd *cobra.Command) (*float64, error) {
+	if !cmd.Flags().Changed("max-token-increase-percent") {
+		return nil, nil
+	}
+	value, err := cmd.Flags().GetFloat64("max-token-increase-percent")
+	if err != nil {
+		return nil, fmt.Errorf("get max-token-increase-percent flag: %w", err)
+	}
+	if value < 0 || math.IsNaN(value) || math.IsInf(value, 0) {
+		return nil, errors.New("max-token-increase-percent must be a finite non-negative number")
+	}
+	return &value, nil
+}
+
+func loadCompareInput(role, path string) (report.Input, error) {
+	data, err := os.ReadFile(path)
+	if err != nil {
+		return report.Input{}, fmt.Errorf("read %s result %q: %w", role, path, err)
+	}
+
+	var input report.Input
+	if err := json.Unmarshal(data, &input); err != nil {
+		return report.Input{}, fmt.Errorf("parse %s result %q: %w", role, path, err)
+	}
+	if err := validateCompareInput(input); err != nil {
+		return report.Input{}, fmt.Errorf("validate %s result %q: %w", role, path, err)
+	}
+	return input, nil
+}
+
+func validateCompareInput(input report.Input) error {
+	if input.SkillName == "" || input.SchemaVersion == "" || input.EngineName == "" || input.StartTime.IsZero() || input.EndTime.IsZero() {
+		return errors.New("missing required result metadata")
+	}
+	primary := input.PrimaryCaseResults()
+	if len(primary) == 0 {
+		return errors.New("no primary case results")
+	}
+	for _, result := range input.CaseResults {
+		if strings.TrimSpace(result.CaseID) == "" {
+			return errors.New("case result has an empty case ID")
+		}
+		if !isCompareStatus(result.Status) {
+			return fmt.Errorf("case %q has invalid status %q", result.CaseID, result.Status)
+		}
+	}
+	return nil
+}
+
+func isCompareStatus(status judge.Status) bool {
+	switch status {
+	case judge.StatusPass, judge.StatusFail, judge.StatusSkip, judge.StatusError:
+		return true
+	default:
+		return false
+	}
+}