diff --git a/cmd/integration/main.go b/cmd/integration/main.go index c830d5b9..c47af37e 100644 --- a/cmd/integration/main.go +++ b/cmd/integration/main.go @@ -102,6 +102,9 @@ func parseFlags(cfg *config.Config) error { doNotCompareError := flag.Bool("E", false, "compare error code only, ignore error message") flag.BoolVar(doNotCompareError, "do-not-compare-error", false, "compare error code only, ignore error message") + maxFailures := flag.Int("M", cfg.MaxFailures, "stop after this many failures, 0 = unlimited") + flag.IntVar(maxFailures, "max-failures", cfg.MaxFailures, "stop after this many failures, 0 = unlimited") + reportFile := flag.String("R", "", "write CSV summary report to file") flag.StringVar(reportFile, "report-file", "", "write CSV summary report to file") @@ -137,6 +140,7 @@ func parseFlags(cfg *config.Config) error { cfg.WithoutCompareResults = *withoutCompare cfg.DoNotCompareError = *doNotCompareError cfg.TestsOnLatestBlock = *testOnLatest + cfg.MaxFailures = *maxFailures cfg.ReportFile = *reportFile cfg.CpuProfile = *cpuProfile cfg.MemProfile = *memProfile @@ -223,6 +227,7 @@ func usage() { fmt.Println(" -w, --waiting-time wait time after test execution in milliseconds") fmt.Println(" -S, --serial all tests run in serial way [default: parallel]") fmt.Println(" -L, --tests-on-latest-block runs only test on latest block") + fmt.Println(" -M, --max-failures stop after n failures, 0 = unlimited [default: 100]") fmt.Println(" -R, --report-file write summary report to file (.csv or .txt)") fmt.Println(" --cpuprofile write cpu profile to file") fmt.Println(" --memprofile write memory profile to file") diff --git a/integration/README.md b/integration/README.md index 87970103..55587f1e 100644 --- a/integration/README.md +++ b/integration/README.md @@ -53,6 +53,7 @@ Options: -w, --waiting-time wait time after test execution in milliseconds -S, --serial all tests run in serial way [default: parallel] -L, --tests-on-latest-block runs only test on latest block + -M, --max-failures stop after n failures, 0 = unlimited [default: 100] -R, --report-file write summary report to file (.csv or .txt) --cpuprofile write cpu profile to file --memprofile write memory profile to file @@ -225,6 +226,24 @@ Number of success tests: 1188 Number of failed tests: 0 ``` +### Stop after too many failures + +By default the runner stops after 100 failures to keep result artifacts small. Use `-M` to +override the limit or set it to `0` for unlimited: + +```bash +# Stop after 50 failures (stricter than default) +./build/bin/rpc_int -c -f -M 50 + +# Run all tests regardless of failure count +./build/bin/rpc_int -c -f -M 0 +``` + +When the limit is reached the runner prints: +``` +ABORTED: too many failures (100), test sequence stopped early +``` + ### Run CI tests with Erigon Assuming you have `erigon` installed beside `rpc-tests`: diff --git a/internal/config/config.go b/internal/config/config.go index 83379113..63f1f962 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -113,6 +113,9 @@ type Config struct { // Archive handling SanitizeArchiveExt bool + // Failure cap + MaxFailures int // stop after this many failures (0 = unlimited) + // Report ReportFile string @@ -142,6 +145,7 @@ func NewConfig() *Config { DiffKind: JsonDiffGo, TransportType: TransportHTTP, ResultsDir: ResultsDir, + MaxFailures: 100, } } diff --git a/internal/filter/filter_test.go b/internal/filter/filter_test.go index b471e464..3e7cd214 100644 --- a/internal/filter/filter_test.go +++ b/internal/filter/filter_test.go @@ -21,10 +21,6 @@ func TestIsSkipped_DefaultList(t *testing.T) { if !f.IsSkipped("engine_exchangeCapabilities", "engine_exchangeCapabilities/test_01.json", 2) { t.Error("engine_ APIs should be skipped by default") } - if !f.IsSkipped("trace_rawTransaction", "trace_rawTransaction/test_01.json", 3) { - t.Error("trace_rawTransaction should be skipped by default") - } - // Normal API should not be skipped if f.IsSkipped("eth_call", "eth_call/test_01.json", 10) { t.Error("eth_call should not be skipped by default") diff --git a/internal/runner/runner.go b/internal/runner/runner.go index 31dc3817..dbc16094 100644 --- a/internal/runner/runner.go +++ b/internal/runner/runner.go @@ -160,6 +160,9 @@ func Run(ctx context.Context, cancelCtx context.CancelFunc, cfg *config.Config) if cfg.ExitOnFail && stats.FailedTests > 0 { return } + if maxFailuresReached(cfg, stats) { + return + } } case <-ctx.Done(): return @@ -271,7 +274,7 @@ func Run(ctx context.Context, cancelCtx context.CancelFunc, cfg *config.Config) // Print summary elapsed := time.Since(startTime) - stats.PrintSummary(elapsed, cfg.LoopNumber, availableTestedAPIs, globalTestNumber) + stats.PrintSummary(startTime, elapsed, cfg.LoopNumber, availableTestedAPIs, globalTestNumber) reportMu.Lock() entries := reportEntries @@ -296,12 +299,20 @@ func Run(ctx context.Context, cancelCtx context.CancelFunc, cfg *config.Config) } } + if maxFailuresReached(cfg, stats) { + fmt.Printf("\nABORTED: too many failures (%d), test sequence stopped early\n", cfg.MaxFailures) + } + if stats.FailedTests > 0 { return 1, nil } return 0, nil } +func maxFailuresReached(cfg *config.Config, stats *Stats) bool { + return cfg.MaxFailures > 0 && stats.FailedTests >= cfg.MaxFailures +} + func printResult(w *bufio.Writer, result *testdata.TestResult, stats *Stats, cfg *config.Config, cancelCtx context.CancelFunc, reportEntries *[]reportEntry, reportMu *sync.Mutex) { file := fmt.Sprintf("%-60s", result.Test.Name) tt := fmt.Sprintf("%-15s", result.Test.TransportType) @@ -352,7 +363,11 @@ func printResult(w *bufio.Writer, result *testdata.TestResult, stats *Stats, cfg }) reportMu.Unlock() } - if cfg.ExitOnFail { + if maxFailuresReached(cfg, stats) { + fmt.Fprintf(w, "\nABORTED: too many failures (%d), test sequence stopped early\n", cfg.MaxFailures) + w.Flush() + cancelCtx() + } else if cfg.ExitOnFail { w.Flush() cancelCtx() } diff --git a/internal/runner/runner_test.go b/internal/runner/runner_test.go index c98a9c03..de54a215 100644 --- a/internal/runner/runner_test.go +++ b/internal/runner/runner_test.go @@ -154,6 +154,97 @@ func TestCheckTestNameForNumber(t *testing.T) { } } +func TestMaxFailures_StopsAfterLimit(t *testing.T) { + const maxFail = 3 + const totalTests = 10 + + cfg := config.NewConfig() + cfg.ExitOnFail = false + cfg.MaxFailures = maxFail + + var buf bytes.Buffer + w := bufio.NewWriter(&buf) + stats := &Stats{} + var entries []reportEntry + var mu sync.Mutex + + ctxCancelled := false + cancelWrapper := func() { + ctxCancelled = true + } + + for i := range totalTests { + r := testdata.TestResult{ + Outcome: testdata.TestOutcome{Success: false, Error: fmt.Errorf("connection refused")}, + Test: &testdata.TestDescriptor{ + Name: "eth_call/test_01.json", + Number: i + 1, + TransportType: "http", + Index: i, + }, + } + printResult(w, &r, stats, cfg, cancelWrapper, &entries, &mu) + if ctxCancelled { + break + } + } + w.Flush() + + if stats.FailedTests != maxFail { + t.Errorf("FailedTests: got %d, want %d", stats.FailedTests, maxFail) + } + if !ctxCancelled { + t.Error("cancelCtx should have been called when MaxFailures reached") + } + output := buf.String() + if !strings.Contains(output, "ABORTED") { + t.Errorf("expected ABORTED message in output, got:\n%s", output) + } + if !strings.Contains(output, fmt.Sprintf("%d", maxFail)) { + t.Errorf("expected failure count %d in abort message, got:\n%s", maxFail, output) + } +} + +func TestMaxFailures_ZeroMeansUnlimited(t *testing.T) { + const totalTests = 10 + + cfg := config.NewConfig() + cfg.ExitOnFail = false + cfg.MaxFailures = 0 // unlimited + + var buf bytes.Buffer + w := bufio.NewWriter(&buf) + stats := &Stats{} + var entries []reportEntry + var mu sync.Mutex + + ctxCancelled := false + cancelWrapper := func() { + ctxCancelled = true + } + + for i := range totalTests { + r := testdata.TestResult{ + Outcome: testdata.TestOutcome{Success: false, Error: fmt.Errorf("connection refused")}, + Test: &testdata.TestDescriptor{ + Name: "eth_call/test_01.json", + Number: i + 1, + TransportType: "http", + Index: i, + }, + } + printResult(w, &r, stats, cfg, cancelWrapper, &entries, &mu) + } + w.Flush() + + if ctxCancelled { + t.Error("cancelCtx should NOT be called when MaxFailures=0 (unlimited)") + } + if stats.FailedTests != totalTests { + t.Errorf("FailedTests: got %d, want %d", stats.FailedTests, totalTests) + } +} + func TestPrintResult_OrderedOutput(t *testing.T) { const numTests = 50 diff --git a/internal/runner/stats.go b/internal/runner/stats.go index 74b7d32e..a475113e 100644 --- a/internal/runner/stats.go +++ b/internal/runner/stats.go @@ -40,8 +40,9 @@ func (s *Stats) AddFailure() { } // PrintSummary prints the v1-compatible summary output. -func (s *Stats) PrintSummary(elapsed time.Duration, iterations, totalAPIs, totalTests int) { +func (s *Stats) PrintSummary(startTime time.Time, elapsed time.Duration, iterations, totalAPIs, totalTests int) { fmt.Println("\n ") + fmt.Printf("Test execution time: %v\n", startTime.Format("2006-01-02 15:04:05")) fmt.Printf("Total HTTP round-trip time: %v\n", s.TotalRoundTripTime) fmt.Printf("Total Marshalling time: %v\n", s.TotalMarshallingTime) fmt.Printf("Total Unmarshalling time: %v\n", s.TotalUnmarshallingTime)