diff --git a/api/admin/admin.go b/api/admin/admin.go index c04dbf1011..4ece72be39 100644 --- a/api/admin/admin.go +++ b/api/admin/admin.go @@ -13,21 +13,49 @@ import ( "github.com/gorilla/handlers" "github.com/gorilla/mux" - "github.com/vechain/thor/v2/api/admin/apilogs" + "github.com/vechain/thor/v2/api/admin/featuregate" "github.com/vechain/thor/v2/api/admin/loglevel" + "github.com/vechain/thor/v2/api/admin/pprof" "github.com/vechain/thor/v2/cmd/thor/node" healthAPI "github.com/vechain/thor/v2/api/admin/health" ) -func NewHTTPHandler(logLevel *slog.LevelVar, health *healthAPI.Health, apiLogsToggle *atomic.Bool, master *node.Master) http.HandlerFunc { +func NewHTTPHandler( + logLevel *slog.LevelVar, + health *healthAPI.Health, + apiLogsGate *featuregate.Gate, + txpoolAPIGate *featuregate.Gate, + pprofGate *featuregate.Gate, + master *node.Master, +) http.HandlerFunc { router := mux.NewRouter() subRouter := router.PathPrefix("/admin").Subrouter() loglevel.New(logLevel).Mount(subRouter, "/loglevel") healthAPI.NewAPI(health, master).Mount(subRouter, "/health") - apilogs.New(apiLogsToggle).Mount(subRouter, "/apilogs") + + reg := featuregate.NewRegistry() + reg.Add(apiLogsGate) + reg.Add(txpoolAPIGate) + reg.Add(pprofGate) + reg.MountAPI(subRouter, "/features") + + // Legacy alias — /admin/apilogs predates the unified /admin/features + // namespace; kept for backward compatibility with existing clients. + reg.MountLegacyAlias(subRouter, "/apilogs", "apilogs") + + // pprof's /debug/pprof/* handlers must live on the router root + // (net/http/pprof.Index hard-codes the prefix). Gated by pprofGate. + pprof.MountHandlers(router, pprofGate) handler := handlers.CompressHandler(router) return handler.ServeHTTP } + +// NewGate builds a featuregate.Gate pre-wired with the admin audit metric. +// Callers don't need to know about the metric layer; this keeps the +// "every admin toggle is audited" invariant inside this package. +func NewGate(name string, enabled *atomic.Bool) *featuregate.Gate { + return featuregate.New(name, enabled, recordToggle) +} diff --git a/api/admin/admin_test.go b/api/admin/admin_test.go new file mode 100644 index 0000000000..89fae17135 --- /dev/null +++ b/api/admin/admin_test.go @@ -0,0 +1,100 @@ +// Copyright (c) 2026 The VeChainThor developers + +// Distributed under the GNU Lesser General Public License v3.0 software license, see the accompanying +// file LICENSE or + +package admin_test + +import ( + "bytes" + "encoding/json" + "io" + "log/slog" + "net/http" + "net/http/httptest" + "sync/atomic" + "testing" + "time" + + "github.com/ethereum/go-ethereum/crypto" + "github.com/gorilla/mux" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/vechain/thor/v2/api" + "github.com/vechain/thor/v2/api/admin" + healthAPI "github.com/vechain/thor/v2/api/admin/health" + apinode "github.com/vechain/thor/v2/api/node" + "github.com/vechain/thor/v2/cmd/thor/node" + "github.com/vechain/thor/v2/comm" + "github.com/vechain/thor/v2/test/testchain" + "github.com/vechain/thor/v2/thor" + "github.com/vechain/thor/v2/txpool" +) + +// TestAdminToggleAffectsNodeAPI is the e2e contract test: flipping +// /admin/features/txpool-api via the admin server must immediately gate +// /node/txpool on the business API server, via the shared atomic.Bool. +func TestAdminToggleAffectsNodeAPI(t *testing.T) { + chain, err := testchain.NewDefault() + require.NoError(t, err) + pool := txpool.New(chain.Repo(), chain.Stater(), txpool.Options{ + Limit: 100, LimitPerAccount: 16, MaxLifetime: time.Minute, + }, &thor.NoFork) + defer pool.Close() + + enableTxPool := &atomic.Bool{} + enableTxPool.Store(true) + txpoolGate := admin.NewGate("txpool-api", enableTxPool) + apiLogsGate := admin.NewGate("apilogs", &atomic.Bool{}) + pprofGate := admin.NewGate("pprof", &atomic.Bool{}) + + // Admin server + privKey, _ := crypto.HexToECDSA("99f0500549792796c14fed62011a51081dc5b5e68fe8bd8a13b86be829c4fd36") + master := &node.Master{PrivateKey: privKey} + adminHandler := admin.NewHTTPHandler( + &slog.LevelVar{}, + healthAPI.New(chain.Repo(), comm.New(chain.Repo(), pool)), + apiLogsGate, txpoolGate, pprofGate, + master, + ) + adminTS := httptest.NewServer(adminHandler) + defer adminTS.Close() + + // Business API server, sharing enableTxPool with the admin gate + nodeRouter := mux.NewRouter() + apinode.New(comm.New(chain.Repo(), pool), pool, enableTxPool).Mount(nodeRouter, "/node") + nodeTS := httptest.NewServer(nodeRouter) + defer nodeTS.Close() + + // Sanity: initially enabled + require.Equal(t, http.StatusOK, getStatus(t, nodeTS.URL+"/node/txpool")) + + // Toggle off via admin + body, _ := json.Marshal(api.ToggleStatus{Enabled: false}) + resp, err := http.Post(adminTS.URL+"/admin/features/txpool-api", "application/json", bytes.NewReader(body)) + require.NoError(t, err) + defer resp.Body.Close() + require.Equal(t, http.StatusOK, resp.StatusCode) + + // Business endpoint now 503 + assert.Equal(t, http.StatusServiceUnavailable, getStatus(t, nodeTS.URL+"/node/txpool")) + + // Toggle back on via admin + body, _ = json.Marshal(api.ToggleStatus{Enabled: true}) + resp, err = http.Post(adminTS.URL+"/admin/features/txpool-api", "application/json", bytes.NewReader(body)) + require.NoError(t, err) + defer resp.Body.Close() + require.Equal(t, http.StatusOK, resp.StatusCode) + + assert.Equal(t, http.StatusOK, getStatus(t, nodeTS.URL+"/node/txpool")) +} + +func getStatus(t *testing.T, url string) int { + t.Helper() + res, err := http.Get(url) //#nosec G107 + require.NoError(t, err) + defer res.Body.Close() + _, _ = io.Copy(io.Discard, res.Body) + return res.StatusCode +} diff --git a/api/admin/apilogs/api_logs.go b/api/admin/apilogs/api_logs.go deleted file mode 100644 index e755825ee3..0000000000 --- a/api/admin/apilogs/api_logs.go +++ /dev/null @@ -1,68 +0,0 @@ -// Copyright (c) 2024 The VeChainThor developers -// -// Distributed under the GNU Lesser General Public License v3.0 software license, see the accompanying -// file LICENSE or - -package apilogs - -import ( - "net/http" - "sync" - "sync/atomic" - - "github.com/gorilla/mux" - - "github.com/vechain/thor/v2/api" - "github.com/vechain/thor/v2/api/restutil" - "github.com/vechain/thor/v2/log" -) - -type APILogs struct { - enabled *atomic.Bool - mu sync.Mutex -} - -func New(enabled *atomic.Bool) *APILogs { - return &APILogs{ - enabled: enabled, - } -} - -func (a *APILogs) Mount(root *mux.Router, pathPrefix string) { - sub := root.PathPrefix(pathPrefix).Subrouter() - sub.Path(""). - Methods(http.MethodGet). - Name("get-api-logs-enabled"). - HandlerFunc(restutil.WrapHandlerFunc(a.areAPILogsEnabled)) - - sub.Path(""). - Methods(http.MethodPost). - Name("post-api-logs-enabled"). - HandlerFunc(restutil.WrapHandlerFunc(a.setAPILogsEnabled)) -} - -func (a *APILogs) areAPILogsEnabled(w http.ResponseWriter, _ *http.Request) error { - a.mu.Lock() - defer a.mu.Unlock() - - return restutil.WriteJSON(w, api.LogStatus{ - Enabled: a.enabled.Load(), - }) -} - -func (a *APILogs) setAPILogsEnabled(w http.ResponseWriter, r *http.Request) error { - a.mu.Lock() - defer a.mu.Unlock() - - var req api.LogStatus - if err := restutil.ParseJSON(r.Body, &req); err != nil { - return restutil.BadRequest(err) - } - a.enabled.Store(req.Enabled) - - log.Info("api logs updated", "pkg", "apilogs", "enabled", req.Enabled) - - return restutil.WriteJSON(w, api.LogStatus{ - Enabled: a.enabled.Load(), - }) -} diff --git a/api/admin/apilogs/api_logs_test.go b/api/admin/apilogs/api_logs_test.go deleted file mode 100644 index 6081cddb5f..0000000000 --- a/api/admin/apilogs/api_logs_test.go +++ /dev/null @@ -1,93 +0,0 @@ -// Copyright (c) 2024 The VeChainThor developers -// -// Distributed under the GNU Lesser General Public License v3.0 software license, see the accompanying -// file LICENSE or - -package apilogs - -import ( - "bytes" - "encoding/json" - "net/http" - "net/http/httptest" - "sync/atomic" - "testing" - - "github.com/gorilla/mux" - "github.com/stretchr/testify/assert" - - "github.com/vechain/thor/v2/api" -) - -type TestCase struct { - name string - method string - expectedHTTP int - startValue bool - expectedEndValue bool - requestBody bool -} - -func marshalBody(tt TestCase, t *testing.T) []byte { - var reqBody []byte - var err error - if tt.method == "POST" { - reqBody, err = json.Marshal(api.LogStatus{Enabled: tt.requestBody}) - if err != nil { - t.Fatalf("could not marshal request body: %v", err) - } - } - return reqBody -} - -func TestLogLevelHandler(t *testing.T) { - tests := []TestCase{ - { - name: "Valid POST input - set logs to enabled", - method: "POST", - expectedHTTP: http.StatusOK, - startValue: false, - requestBody: true, - expectedEndValue: true, - }, - { - name: "Valid POST input - set logs to disabled", - method: "POST", - expectedHTTP: http.StatusOK, - startValue: true, - requestBody: false, - expectedEndValue: false, - }, - { - name: "GET request - get current level INFO", - method: "GET", - expectedHTTP: http.StatusOK, - startValue: true, - expectedEndValue: true, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - logLevel := atomic.Bool{} - logLevel.Store(tt.startValue) - - reqBodyBytes := marshalBody(tt, t) - - req, err := http.NewRequest(tt.method, "/admin/apilogs", bytes.NewBuffer(reqBodyBytes)) - if err != nil { - t.Fatal(err) - } - - rr := httptest.NewRecorder() - router := mux.NewRouter() - New(&logLevel).Mount(router, "/admin/apilogs") - router.ServeHTTP(rr, req) - - assert.Equal(t, tt.expectedHTTP, rr.Code) - responseBody := api.LogStatus{} - assert.NoError(t, json.Unmarshal(rr.Body.Bytes(), &responseBody)) - assert.Equal(t, tt.expectedEndValue, responseBody.Enabled) - }) - } -} diff --git a/api/admin/featuregate/featuregate.go b/api/admin/featuregate/featuregate.go new file mode 100644 index 0000000000..4d2a161705 --- /dev/null +++ b/api/admin/featuregate/featuregate.go @@ -0,0 +1,257 @@ +// Copyright (c) 2026 The VeChainThor developers + +// Distributed under the GNU Lesser General Public License v3.0 software license, see the accompanying +// file LICENSE or + +// Package featuregate provides a generic runtime on/off toggle backed by +// an *atomic.Bool. Each Gate exposes a Middleware that returns 503 when +// disabled, and an optional TTL after which the gate auto-disables. +// A Registry groups gates under a single REST namespace +// (e.g. /admin/features/{name}) and can also expose per-feature legacy URLs +// as aliases for backward compatibility. +package featuregate + +import ( + "net/http" + "sort" + "sync" + "sync/atomic" + "time" + + "github.com/gorilla/mux" + "github.com/pkg/errors" + + "github.com/vechain/thor/v2/api" + "github.com/vechain/thor/v2/api/restutil" + "github.com/vechain/thor/v2/log" +) + +const maxTTLSeconds = 3600 + +// Observer is invoked on every toggle change (manual or TTL-triggered). +// May be nil. +type Observer func(name string, enabled bool) + +// Gate is a runtime on/off switch. +type Gate struct { + name string + enabled *atomic.Bool + observer Observer + mu sync.Mutex + ttlTimer *time.Timer + // gen is bumped on every Set under mu. Each TTL timer callback captures + // its own birth-gen and bails if it has been superseded — this closes + // the race where Timer.Stop misses an already-dispatched callback. + gen uint64 +} + +// New returns a Gate backed by the provided atomic.Bool. The bool is owned +// by the caller (typically created in main and shared between admin and +// business servers) so its lifecycle is independent of the Gate. +// observer may be nil. +func New(name string, enabled *atomic.Bool, observer Observer) *Gate { + return &Gate{name: name, enabled: enabled, observer: observer} +} + +func (g *Gate) record(enabled bool) { + if g.observer != nil { + g.observer(g.name, enabled) + } +} + +func (g *Gate) Name() string { return g.name } +func (g *Gate) Enabled() bool { return g.enabled.Load() } + +// Middleware returns 503 + Retry-After while the gate is disabled. +func (g *Gate) Middleware(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if !g.enabled.Load() { + w.Header().Set("Retry-After", "1") + http.Error(w, g.name+" is disabled", http.StatusServiceUnavailable) + return + } + next.ServeHTTP(w, r) + }) +} + +// Set atomically toggles the gate, replaces any pending TTL timer, and +// records audit metrics. ttlSeconds is clamped to [0, maxTTLSeconds]; a +// non-zero TTL only takes effect when enabled is true. +func (g *Gate) Set(enabled bool, ttlSeconds int) api.ToggleStatus { + g.mu.Lock() + + if ttlSeconds < 0 { + ttlSeconds = 0 + } else if ttlSeconds > maxTTLSeconds { + ttlSeconds = maxTTLSeconds + } + + if g.ttlTimer != nil { + g.ttlTimer.Stop() + g.ttlTimer = nil + } + g.gen++ + myGen := g.gen + g.enabled.Store(enabled) + if enabled && ttlSeconds > 0 { + g.ttlTimer = time.AfterFunc(time.Duration(ttlSeconds)*time.Second, func() { + g.mu.Lock() + if g.gen != myGen { + // Superseded by a later Set that raced with our dispatch. + g.mu.Unlock() + return + } + g.ttlTimer = nil + g.mu.Unlock() + g.enabled.Store(false) + g.record(false) + log.Info(g.name+" auto-disabled after ttl", "pkg", "featuregate") + }) + } + g.mu.Unlock() + + // Run observer and logging outside the lock so a slow Observer can't + // stall other callers. + g.record(enabled) + log.Info(g.name+" toggled", "pkg", "featuregate", "enabled", enabled, "ttlSeconds", ttlSeconds) + return api.ToggleStatus{Enabled: g.enabled.Load(), TTLSeconds: ttlSeconds} +} + +// Status returns the current enabled state without TTL info. +func (g *Gate) Status() api.ToggleStatus { + return api.ToggleStatus{Enabled: g.enabled.Load()} +} + +// Registry catalogs named Gates. +type Registry struct { + mu sync.Mutex + gates map[string]*Gate +} + +func NewRegistry() *Registry { + return &Registry{gates: map[string]*Gate{}} +} + +// Add registers g and returns it. Panics if a gate with the same name is +// already registered — Registry is expected to be wired once at startup +// and silent overwrite would mask a programmer error. +func (r *Registry) Add(g *Gate) *Gate { + r.mu.Lock() + defer r.mu.Unlock() + if _, exists := r.gates[g.name]; exists { + panic("featuregate: duplicate gate name: " + g.name) + } + r.gates[g.name] = g + return g +} + +// Get returns the named gate. +func (r *Registry) Get(name string) (*Gate, bool) { + r.mu.Lock() + defer r.mu.Unlock() + g, ok := r.gates[name] + return g, ok +} + +// MountAPI mounts the unified REST endpoints under pathPrefix: +// +// GET pathPrefix -> list all +// GET pathPrefix/{name} -> single gate status +// POST pathPrefix/{name} -> toggle (body: api.ToggleStatus) +func (r *Registry) MountAPI(root *mux.Router, pathPrefix string) { + sub := root.PathPrefix(pathPrefix).Subrouter() + sub.Path(""). + Methods(http.MethodGet). + Name("list-features"). + HandlerFunc(restutil.WrapHandlerFunc(r.handleList)) + sub.Path("/{name}"). + Methods(http.MethodGet). + Name("get-feature"). + HandlerFunc(restutil.WrapHandlerFunc(r.handleGet)) + sub.Path("/{name}"). + Methods(http.MethodPost). + Name("post-feature"). + HandlerFunc(restutil.WrapHandlerFunc(r.handleSet)) +} + +// MountLegacyAlias mounts GET/POST at pathPrefix that operate on the named +// gate. Adds a Deprecation response header pointing at the unified URL. +func (r *Registry) MountLegacyAlias(root *mux.Router, pathPrefix, name string) { + g, ok := r.Get(name) + if !ok { + panic("featuregate: legacy alias for unknown gate: " + name) + } + sub := root.PathPrefix(pathPrefix).Subrouter() + sub.Use(deprecationHeader("/admin/features/" + name)) + sub.Path(""). + Methods(http.MethodGet). + Name("legacy-get-" + name). + HandlerFunc(restutil.WrapHandlerFunc(func(w http.ResponseWriter, _ *http.Request) error { + return restutil.WriteJSON(w, g.Status()) + })) + sub.Path(""). + Methods(http.MethodPost). + Name("legacy-post-" + name). + HandlerFunc(restutil.WrapHandlerFunc(func(w http.ResponseWriter, req *http.Request) error { + return handleSetGate(w, req, g) + })) +} + +func (r *Registry) handleList(w http.ResponseWriter, _ *http.Request) error { + r.mu.Lock() + out := make([]namedStatus, 0, len(r.gates)) + for name, g := range r.gates { + out = append(out, namedStatus{Name: name, Enabled: g.Enabled()}) + } + r.mu.Unlock() + sort.Slice(out, func(i, j int) bool { return out[i].Name < out[j].Name }) + return restutil.WriteJSON(w, out) +} + +func (r *Registry) handleGet(w http.ResponseWriter, req *http.Request) error { + g, err := r.resolve(req) + if err != nil { + return err + } + return restutil.WriteJSON(w, g.Status()) +} + +func (r *Registry) handleSet(w http.ResponseWriter, req *http.Request) error { + g, err := r.resolve(req) + if err != nil { + return err + } + return handleSetGate(w, req, g) +} + +func (r *Registry) resolve(req *http.Request) (*Gate, error) { + name := mux.Vars(req)["name"] + g, ok := r.Get(name) + if !ok { + return nil, restutil.HTTPError(errors.New("unknown feature: "+name), http.StatusNotFound) + } + return g, nil +} + +func handleSetGate(w http.ResponseWriter, req *http.Request, g *Gate) error { + var body api.ToggleStatus + if err := restutil.ParseJSON(req.Body, &body); err != nil { + return restutil.BadRequest(err) + } + return restutil.WriteJSON(w, g.Set(body.Enabled, body.TTLSeconds)) +} + +func deprecationHeader(replacement string) mux.MiddlewareFunc { + return func(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Deprecation", "true") + w.Header().Set("Link", "<"+replacement+">; rel=\"successor-version\"") + next.ServeHTTP(w, r) + }) + } +} + +type namedStatus struct { + Name string `json:"name"` + Enabled bool `json:"enabled"` +} diff --git a/api/admin/featuregate/featuregate_test.go b/api/admin/featuregate/featuregate_test.go new file mode 100644 index 0000000000..1649688a8a --- /dev/null +++ b/api/admin/featuregate/featuregate_test.go @@ -0,0 +1,155 @@ +// Copyright (c) 2026 The VeChainThor developers + +// Distributed under the GNU Lesser General Public License v3.0 software license, see the accompanying +// file LICENSE or + +package featuregate + +import ( + "bytes" + "encoding/json" + "net/http" + "net/http/httptest" + "sync/atomic" + "testing" + "time" + + "github.com/gorilla/mux" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/vechain/thor/v2/api" +) + +func newRegistryRouter(gates map[string]bool) (*mux.Router, *Registry, map[string]*atomic.Bool) { + reg := NewRegistry() + flags := map[string]*atomic.Bool{} + for name, initial := range gates { + b := &atomic.Bool{} + b.Store(initial) + flags[name] = b + reg.Add(New(name, b, nil)) + } + router := mux.NewRouter() + reg.MountAPI(router, "/admin/features") + return router, reg, flags +} + +func TestGateMiddleware503WhenDisabled(t *testing.T) { + b := &atomic.Bool{} + g := New("x", b, nil) + inner := http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusOK) }) + rr := httptest.NewRecorder() + g.Middleware(inner).ServeHTTP(rr, httptest.NewRequest(http.MethodGet, "/", nil)) + assert.Equal(t, http.StatusServiceUnavailable, rr.Code) + assert.NotEmpty(t, rr.Header().Get("Retry-After")) +} + +func TestGateMiddlewareOKWhenEnabled(t *testing.T) { + b := &atomic.Bool{} + b.Store(true) + g := New("x", b, nil) + inner := http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusOK) }) + rr := httptest.NewRecorder() + g.Middleware(inner).ServeHTTP(rr, httptest.NewRequest(http.MethodGet, "/", nil)) + assert.Equal(t, http.StatusOK, rr.Code) +} + +func TestRegistryList(t *testing.T) { + router, _, _ := newRegistryRouter(map[string]bool{"a": true, "b": false}) + rr := httptest.NewRecorder() + router.ServeHTTP(rr, httptest.NewRequest(http.MethodGet, "/admin/features", nil)) + assert.Equal(t, http.StatusOK, rr.Code) + + var out []namedStatus + require.NoError(t, json.Unmarshal(rr.Body.Bytes(), &out)) + got := map[string]bool{} + for _, s := range out { + got[s.Name] = s.Enabled + } + assert.Equal(t, map[string]bool{"a": true, "b": false}, got) +} + +func TestRegistryGetOne(t *testing.T) { + router, _, _ := newRegistryRouter(map[string]bool{"a": true}) + rr := httptest.NewRecorder() + router.ServeHTTP(rr, httptest.NewRequest(http.MethodGet, "/admin/features/a", nil)) + assert.Equal(t, http.StatusOK, rr.Code) + + var resp api.ToggleStatus + require.NoError(t, json.Unmarshal(rr.Body.Bytes(), &resp)) + assert.True(t, resp.Enabled) +} + +func TestRegistryGetUnknown(t *testing.T) { + router, _, _ := newRegistryRouter(map[string]bool{"a": true}) + rr := httptest.NewRecorder() + router.ServeHTTP(rr, httptest.NewRequest(http.MethodGet, "/admin/features/missing", nil)) + assert.Equal(t, http.StatusNotFound, rr.Code) +} + +func TestRegistryPostFlip(t *testing.T) { + router, _, flags := newRegistryRouter(map[string]bool{"a": false}) + body, _ := json.Marshal(api.ToggleStatus{Enabled: true}) + rr := httptest.NewRecorder() + router.ServeHTTP(rr, httptest.NewRequest(http.MethodPost, "/admin/features/a", bytes.NewReader(body))) + assert.Equal(t, http.StatusOK, rr.Code) + assert.True(t, flags["a"].Load()) +} + +func TestTTLAutoDisable(t *testing.T) { + b := &atomic.Bool{} + g := New("x", b, nil) + g.Set(true, 1) + assert.True(t, b.Load()) + require.Eventually(t, func() bool { return !b.Load() }, 3*time.Second, 50*time.Millisecond) +} + +func TestTTLClamp(t *testing.T) { + b := &atomic.Bool{} + g := New("x", b, nil) + st := g.Set(true, 99999) + assert.Equal(t, maxTTLSeconds, st.TTLSeconds) + g.Set(false, 0) // stop timer +} + +func TestSecondSetCancelsTimer(t *testing.T) { + b := &atomic.Bool{} + g := New("x", b, nil) + g.Set(true, 1) + g.Set(true, 0) // no TTL, should cancel previous timer + time.Sleep(1500 * time.Millisecond) + assert.True(t, b.Load(), "expected gate still enabled after old timer fired") +} + +// TestSetWhileTimerFiring covers the race where a TTL timer has already +// dispatched its callback by the time a second Set tries to Stop it. Without +// the generation-counter guard the late callback would flip enabled back to +// false even though the latest user intent is "stay enabled". +func TestSetWhileTimerFiring(t *testing.T) { + b := &atomic.Bool{} + g := New("x", b, nil) + g.Set(true, 1) + // Sleep just under the TTL so the timer is about to dispatch (or has + // already dispatched and is racing with our next Set call). + time.Sleep(990 * time.Millisecond) + g.Set(true, 0) + time.Sleep(200 * time.Millisecond) + assert.True(t, b.Load(), "second Set without TTL must win over a racing old timer") +} + +func TestLegacyAlias(t *testing.T) { + router := mux.NewRouter() + reg := NewRegistry() + b := &atomic.Bool{} + reg.Add(New("a", b, nil)) + reg.MountLegacyAlias(router, "/admin/a", "a") + + body, _ := json.Marshal(api.ToggleStatus{Enabled: true}) + rr := httptest.NewRecorder() + router.ServeHTTP(rr, httptest.NewRequest(http.MethodPost, "/admin/a", bytes.NewReader(body))) + assert.Equal(t, http.StatusOK, rr.Code) + assert.True(t, b.Load()) + assert.Equal(t, "true", rr.Header().Get("Deprecation")) + assert.Contains(t, rr.Header().Get("Link"), "/admin/features/a") +} diff --git a/api/admin/metrics.go b/api/admin/metrics.go new file mode 100644 index 0000000000..b15c4e060a --- /dev/null +++ b/api/admin/metrics.go @@ -0,0 +1,20 @@ +// Copyright (c) 2026 The VeChainThor developers + +// Distributed under the GNU Lesser General Public License v3.0 software license, see the accompanying +// file LICENSE or + +package admin + +import "github.com/vechain/thor/v2/metrics" + +// toggleCount audits every toggle change made through a Gate built by NewGate. +// Labels: feature (gate name), to ("enabled" or "disabled"). +var toggleCount = metrics.LazyLoadCounterVec("admin_toggle_count", []string{"feature", "to"}) + +func recordToggle(feature string, enabled bool) { + to := "disabled" + if enabled { + to = "enabled" + } + toggleCount().AddWithLabel(1, map[string]string{"feature": feature, "to": to}) +} diff --git a/api/admin/metrics_test.go b/api/admin/metrics_test.go new file mode 100644 index 0000000000..e26c371b6e --- /dev/null +++ b/api/admin/metrics_test.go @@ -0,0 +1,55 @@ +// Copyright (c) 2026 The VeChainThor developers + +// Distributed under the GNU Lesser General Public License v3.0 software license, see the accompanying +// file LICENSE or + +package admin + +import ( + "sync/atomic" + "testing" + + "github.com/prometheus/client_golang/prometheus" + "github.com/stretchr/testify/require" + + "github.com/vechain/thor/v2/metrics" +) + +// counterValue reads thor_metrics_admin_toggle_count{feature=feature,to=to} +// from the prometheus default registry. Returns 0 if the series doesn't exist. +func counterValue(t *testing.T, feature, to string) float64 { + t.Helper() + mfs, err := prometheus.DefaultGatherer.Gather() + require.NoError(t, err) + for _, mf := range mfs { + if mf.GetName() != "thor_metrics_admin_toggle_count" { + continue + } + for _, m := range mf.GetMetric() { + labels := map[string]string{} + for _, l := range m.GetLabel() { + labels[l.GetName()] = l.GetValue() + } + if labels["feature"] == feature && labels["to"] == to { + return m.GetCounter().GetValue() + } + } + } + return 0 +} + +func TestNewGateAuditsToggles(t *testing.T) { + metrics.InitializePrometheusMetrics() + + gate := NewGate("featuregate-audit-test", &atomic.Bool{}) + + enabledBefore := counterValue(t, "featuregate-audit-test", "enabled") + disabledBefore := counterValue(t, "featuregate-audit-test", "disabled") + + gate.Set(true, 0) + gate.Set(false, 0) + gate.Set(true, 0) + + require.Equal(t, enabledBefore+2, counterValue(t, "featuregate-audit-test", "enabled")) + require.Equal(t, disabledBefore+1, counterValue(t, "featuregate-audit-test", "disabled")) +} diff --git a/api/admin/pprof/pprof.go b/api/admin/pprof/pprof.go new file mode 100644 index 0000000000..1caa74c4c0 --- /dev/null +++ b/api/admin/pprof/pprof.go @@ -0,0 +1,30 @@ +// Copyright (c) 2026 The VeChainThor developers + +// Distributed under the GNU Lesser General Public License v3.0 software license, see the accompanying +// file LICENSE or + +// Package pprof wires net/http/pprof handlers under /debug/pprof/* on a +// gorilla/mux router root, gated by a featuregate.Gate so they only serve +// while pprof is enabled at runtime. +package pprof + +import ( + "net/http/pprof" + + "github.com/gorilla/mux" + + "github.com/vechain/thor/v2/api/admin/featuregate" +) + +// MountHandlers registers the pprof endpoints under /debug/pprof/* on root. +// They are protected by gate.Middleware and return 503 while disabled. +// The prefix is fixed because net/http/pprof.Index hard-codes it. +func MountHandlers(root *mux.Router, gate *featuregate.Gate) { + sub := root.PathPrefix("/debug/pprof").Subrouter() + sub.Use(gate.Middleware) + sub.HandleFunc("/cmdline", pprof.Cmdline) + sub.HandleFunc("/profile", pprof.Profile) + sub.HandleFunc("/symbol", pprof.Symbol) + sub.HandleFunc("/trace", pprof.Trace) + sub.PathPrefix("/").HandlerFunc(pprof.Index) +} diff --git a/api/admin/pprof/pprof_test.go b/api/admin/pprof/pprof_test.go new file mode 100644 index 0000000000..555ac362bc --- /dev/null +++ b/api/admin/pprof/pprof_test.go @@ -0,0 +1,39 @@ +// Copyright (c) 2026 The VeChainThor developers + +// Distributed under the GNU Lesser General Public License v3.0 software license, see the accompanying +// file LICENSE or + +package pprof + +import ( + "net/http" + "net/http/httptest" + "sync/atomic" + "testing" + + "github.com/gorilla/mux" + "github.com/stretchr/testify/assert" + + "github.com/vechain/thor/v2/api/admin/featuregate" +) + +func TestMountHandlersGatedByDefault(t *testing.T) { + router := mux.NewRouter() + flag := &atomic.Bool{} + MountHandlers(router, featuregate.New("pprof", flag, nil)) + + rr := httptest.NewRecorder() + router.ServeHTTP(rr, httptest.NewRequest(http.MethodGet, "/debug/pprof/heap", nil)) + assert.Equal(t, http.StatusServiceUnavailable, rr.Code) +} + +func TestMountHandlersOKWhenEnabled(t *testing.T) { + router := mux.NewRouter() + flag := &atomic.Bool{} + flag.Store(true) + MountHandlers(router, featuregate.New("pprof", flag, nil)) + + rr := httptest.NewRecorder() + router.ServeHTTP(rr, httptest.NewRequest(http.MethodGet, "/debug/pprof/heap", nil)) + assert.Equal(t, http.StatusOK, rr.Code) +} diff --git a/api/admin_types.go b/api/admin_types.go index 74f2a892a4..ee706759ac 100644 --- a/api/admin_types.go +++ b/api/admin_types.go @@ -13,6 +13,14 @@ type LogStatus struct { Enabled bool `json:"enabled"` } +// ToggleStatus carries an enabled flag plus an optional TTL after which +// the flag will automatically be set back to false. A zero TTL means no +// auto-disable. Used by admin toggles like /admin/pprof and /admin/txpool-api. +type ToggleStatus struct { + Enabled bool `json:"enabled"` + TTLSeconds int `json:"ttlSeconds,omitempty"` +} + type HealthStatus struct { Healthy bool `json:"healthy"` BestBlockTime *time.Time `json:"bestBlockTime"` diff --git a/api/node/node.go b/api/node/node.go index 23286c94bd..f4fb1a5171 100644 --- a/api/node/node.go +++ b/api/node/node.go @@ -7,6 +7,7 @@ package node import ( "net/http" + "sync/atomic" "github.com/gorilla/mux" "github.com/pkg/errors" @@ -22,10 +23,10 @@ import ( type Node struct { pool txpool.Pool nw api.Network - enableTxpool bool + enableTxpool *atomic.Bool } -func New(nw api.Network, pool txpool.Pool, enableTxpool bool) *Node { +func New(nw api.Network, pool txpool.Pool, enableTxpool *atomic.Bool) *Node { return &Node{ pool, nw, @@ -95,16 +96,27 @@ func (n *Node) Mount(root *mux.Router, pathPrefix string) { Name("GET /node/network/peers"). HandlerFunc(restutil.WrapHandlerFunc(n.handleNetwork)) - if n.enableTxpool { - sub.Path("/txpool"). - Methods(http.MethodGet). - Name("GET /node/txpool"). - HandlerFunc(restutil.WrapHandlerFunc(n.handleGetTransactions)) - sub.Path("/txpool/status"). - Methods(http.MethodGet). - Name("GET /node/txpool/status"). - HandlerFunc(restutil.WrapHandlerFunc(n.handleGetTxpoolStatus)) - } + txpoolSub := sub.PathPrefix("/txpool").Subrouter() + txpoolSub.Use(n.txpoolGate) + txpoolSub.Path(""). + Methods(http.MethodGet). + Name("GET /node/txpool"). + HandlerFunc(restutil.WrapHandlerFunc(n.handleGetTransactions)) + txpoolSub.Path("/status"). + Methods(http.MethodGet). + Name("GET /node/txpool/status"). + HandlerFunc(restutil.WrapHandlerFunc(n.handleGetTxpoolStatus)) +} + +func (n *Node) txpoolGate(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if !n.enableTxpool.Load() { + w.Header().Set("Retry-After", "1") + http.Error(w, "txpool API is disabled", http.StatusServiceUnavailable) + return + } + next.ServeHTTP(w, r) + }) } func filterTransactions(origin thor.Address, allTransactions tx.Transactions) (tx.Transactions, error) { diff --git a/api/node/node_test.go b/api/node/node_test.go index 7e9435e7e4..2b0cc31921 100644 --- a/api/node/node_test.go +++ b/api/node/node_test.go @@ -6,7 +6,9 @@ package node import ( "encoding/json" + "net/http" "net/http/httptest" + "sync/atomic" "testing" "time" @@ -26,9 +28,10 @@ import ( ) var ( - ts *httptest.Server - tclient *thorclient.Client - pool *txpool.TxPool + ts *httptest.Server + tclient *thorclient.Client + pool *txpool.TxPool + txpoolEnabled *atomic.Bool ) func TestNode(t *testing.T) { @@ -44,6 +47,7 @@ func TestNode(t *testing.T) { t.Run("getTransactionsWithOrigin", testGetTransactionsWithOrigin) t.Run("getTransactionsWithBadExpanded", testGetTransactionsWithBadExpanded) t.Run("getTransactionsWithBadOrigin", testGetTransactionsWithBadOrigin) + t.Run("txpoolDisabledReturns503", testTxpoolDisabled) } func initCommServer(t *testing.T) { @@ -80,8 +84,11 @@ func initCommServer(t *testing.T) { }, &thor.NoFork), ) + txpoolEnabled = &atomic.Bool{} + txpoolEnabled.Store(true) + router := mux.NewRouter() - New(communicator, pool, true).Mount(router, "/node") + New(communicator, pool, txpoolEnabled).Mount(router, "/node") ts = httptest.NewServer(router) } @@ -128,3 +135,14 @@ func testGetTransactionsWithBadExpanded(t *testing.T) { func testGetTransactionsWithBadOrigin(t *testing.T) { httpGetAndCheckResponseStatus(t, "/node/txpool?origin=0xinvalid", 400) } + +func testTxpoolDisabled(t *testing.T) { + txpoolEnabled.Store(false) + defer txpoolEnabled.Store(true) + + for _, path := range []string{"/node/txpool", "/node/txpool/status"} { + _, statusCode, err := tclient.RawHTTPClient().RawHTTPGet(path) + require.NoError(t, err) + assert.Equal(t, http.StatusServiceUnavailable, statusCode, "path=%s", path) + } +} diff --git a/cmd/thor/httpserver/admin_server.go b/cmd/thor/httpserver/admin_server.go index c6ae6c0531..266c03f7c2 100644 --- a/cmd/thor/httpserver/admin_server.go +++ b/cmd/thor/httpserver/admin_server.go @@ -10,12 +10,12 @@ import ( "net" "net/http" "sync" - "sync/atomic" "time" "github.com/pkg/errors" "github.com/vechain/thor/v2/api/admin" + "github.com/vechain/thor/v2/api/admin/featuregate" "github.com/vechain/thor/v2/api/admin/health" "github.com/vechain/thor/v2/chain" "github.com/vechain/thor/v2/cmd/thor/node" @@ -27,7 +27,9 @@ func StartAdminServer( logLevel *slog.LevelVar, repo *chain.Repository, p2p *comm.Communicator, - apiLogs *atomic.Bool, + apiLogsGate *featuregate.Gate, + txpoolAPIGate *featuregate.Gate, + pprofGate *featuregate.Gate, master *node.Master, ) (string, func(), error) { listener, err := net.Listen("tcp", addr) @@ -35,7 +37,7 @@ func StartAdminServer( return "", nil, errors.Wrapf(err, "listen admin API addr [%v]", addr) } - adminHandler := admin.NewHTTPHandler(logLevel, health.New(repo, p2p), apiLogs, master) + adminHandler := admin.NewHTTPHandler(logLevel, health.New(repo, p2p), apiLogsGate, txpoolAPIGate, pprofGate, master) srv := &http.Server{Handler: adminHandler, ReadHeaderTimeout: time.Second, ReadTimeout: 5 * time.Second} var goes sync.WaitGroup diff --git a/cmd/thor/httpserver/api_server.go b/cmd/thor/httpserver/api_server.go index 387b9f046f..b4d449cdde 100644 --- a/cmd/thor/httpserver/api_server.go +++ b/cmd/thor/httpserver/api_server.go @@ -8,7 +8,6 @@ package httpserver import ( "net" "net/http" - "net/http/pprof" "strings" "sync" "sync/atomic" @@ -52,7 +51,6 @@ type APIConfig struct { BacktraceLimit uint32 CallGasLimit uint64 BatchDataMaxSize uint64 - PprofOn bool SkipLogs bool AllowCustomTracer bool EnableReqLogger *atomic.Bool @@ -61,7 +59,7 @@ type APIConfig struct { AllowedTracers []string SoloMode bool EnableDeprecated bool - EnableTxPool bool + EnableTxPool *atomic.Bool APIBacktraceLimit int PriorityIncreasePercentage int Timeout int @@ -131,14 +129,6 @@ func StartAPIServer( subs := subscriptions.New(repo, origins, config.BacktraceLimit, txPool, config.EnableDeprecated) subs.Mount(router, "/subscriptions") - if config.PprofOn { - router.HandleFunc("/debug/pprof/cmdline", pprof.Cmdline) - router.HandleFunc("/debug/pprof/profile", pprof.Profile) - router.HandleFunc("/debug/pprof/symbol", pprof.Symbol) - router.HandleFunc("/debug/pprof/trace", pprof.Trace) - router.PathPrefix("/debug/pprof/").HandlerFunc(pprof.Index) - } - // middlewares // body limit and timeout router.Use(middleware.HandleRequestBodyLimit(defaultRequestBodyLimit)) diff --git a/cmd/thor/main.go b/cmd/thor/main.go index 7da6d1eb1e..2d74ce9d4a 100644 --- a/cmd/thor/main.go +++ b/cmd/thor/main.go @@ -23,6 +23,7 @@ import ( "github.com/pkg/errors" "github.com/urfave/cli/v3" + "github.com/vechain/thor/v2/api/admin" "github.com/vechain/thor/v2/api/doc" "github.com/vechain/thor/v2/bft" "github.com/vechain/thor/v2/chain" @@ -292,13 +293,25 @@ func defaultAction(_ context.Context, ctx *cli.Command) error { adminURL := "" logAPIRequests := &atomic.Bool{} logAPIRequests.Store(ctx.Bool(enableAPILogsFlag.Name)) + enableTxPool := &atomic.Bool{} + enableTxPool.Store(ctx.Bool(apiTxpoolFlag.Name)) + pprofEnabled := &atomic.Bool{} + pprofEnabled.Store(ctx.Bool(pprofFlag.Name)) + apiLogsGate := admin.NewGate("apilogs", logAPIRequests) + txpoolGate := admin.NewGate("txpool-api", enableTxPool) + pprofGate := admin.NewGate("pprof", pprofEnabled) + if pprofEnabled.Load() && !ctx.Bool(enableAdminFlag.Name) { + log.Warn("--pprof has no effect without --enable-admin; pprof endpoints are only served by the admin server") + } if ctx.Bool(enableAdminFlag.Name) { url, closeFunc, err := httpserver.StartAdminServer( ctx.String(adminAddrFlag.Name), logLevel, repo, p2pCommunicator.Communicator(), - logAPIRequests, + apiLogsGate, + txpoolGate, + pprofGate, master, ) if err != nil { @@ -322,7 +335,7 @@ func defaultAction(_ context.Context, ctx *cli.Command) error { bftEngine, p2pCommunicator.Communicator(), forkConfig, - makeAPIConfig(ctx, logAPIRequests, false), + makeAPIConfig(ctx, logAPIRequests, enableTxPool, false), ) if err != nil { return err @@ -449,13 +462,25 @@ func soloAction(_ context.Context, ctx *cli.Command) error { adminURL := "" logAPIRequests := &atomic.Bool{} logAPIRequests.Store(ctx.Bool(enableAPILogsFlag.Name)) + enableTxPool := &atomic.Bool{} + enableTxPool.Store(ctx.Bool(apiTxpoolFlag.Name)) + pprofEnabled := &atomic.Bool{} + pprofEnabled.Store(ctx.Bool(pprofFlag.Name)) + apiLogsGate := admin.NewGate("apilogs", logAPIRequests) + txpoolGate := admin.NewGate("txpool-api", enableTxPool) + pprofGate := admin.NewGate("pprof", pprofEnabled) + if pprofEnabled.Load() && !ctx.Bool(enableAdminFlag.Name) { + log.Warn("--pprof has no effect without --enable-admin; pprof endpoints are only served by the admin server") + } if ctx.Bool(enableAdminFlag.Name) { url, closeFunc, err := httpserver.StartAdminServer( ctx.String(adminAddrFlag.Name), logLevel, repo, nil, - logAPIRequests, + apiLogsGate, + txpoolGate, + pprofGate, nil, ) if err != nil { @@ -518,7 +543,7 @@ func soloAction(_ context.Context, ctx *cli.Command) error { bftEngine, &solo.Communicator{}, forkConfig, - makeAPIConfig(ctx, logAPIRequests, true), + makeAPIConfig(ctx, logAPIRequests, enableTxPool, true), ) if err != nil { return err diff --git a/cmd/thor/utils.go b/cmd/thor/utils.go index be5ed36acd..2aa77eea34 100644 --- a/cmd/thor/utils.go +++ b/cmd/thor/utils.go @@ -257,13 +257,12 @@ func parseGenesisFile(uri string) (*genesis.Genesis, *thor.ForkConfig, error) { return customGen, &forkConfig, nil } -func makeAPIConfig(ctx *cli.Command, logAPIRequests *atomic.Bool, soloMode bool) httpserver.APIConfig { +func makeAPIConfig(ctx *cli.Command, logAPIRequests *atomic.Bool, enableTxPool *atomic.Bool, soloMode bool) httpserver.APIConfig { return httpserver.APIConfig{ AllowedOrigins: ctx.String(apiCorsFlag.Name), BacktraceLimit: uint32(ctx.Uint64(apiBacktraceLimitFlag.Name)), CallGasLimit: ctx.Uint64(apiCallGasLimitFlag.Name), BatchDataMaxSize: ctx.Uint64(apiBatchDataMaxSizeFlag.Name), - PprofOn: ctx.Bool(pprofFlag.Name), SkipLogs: ctx.Bool(skipLogsFlag.Name), APIBacktraceLimit: int(ctx.Uint64(apiBacktraceLimitFlag.Name)), PriorityIncreasePercentage: int(ctx.Uint64(apiPriorityFeesPercentageFlag.Name)), @@ -274,7 +273,7 @@ func makeAPIConfig(ctx *cli.Command, logAPIRequests *atomic.Bool, soloMode bool) AllowedTracers: parseTracerList(strings.TrimSpace(ctx.String(allowedTracersFlag.Name))), EnableDeprecated: ctx.Bool(apiEnableDeprecatedFlag.Name), SoloMode: soloMode, - EnableTxPool: ctx.Bool(apiTxpoolFlag.Name), + EnableTxPool: enableTxPool, Timeout: int(ctx.Uint64(apiTimeoutFlag.Name)), SlowQueriesThreshold: int(ctx.Uint64(apiSlowQueriesThresholdFlag.Name)), Log5XXErrors: ctx.Bool(apiLog5xxErrorsFlag.Name), diff --git a/test/testnode/node.go b/test/testnode/node.go index 3f29fb7860..b75c282ffa 100644 --- a/test/testnode/node.go +++ b/test/testnode/node.go @@ -8,6 +8,7 @@ package testnode import ( "errors" "net/http/httptest" + "sync/atomic" "github.com/gorilla/mux" @@ -87,7 +88,9 @@ func (n *node) Start() error { []string{"all"}, true, ).Mount(router, "/debug") - node2.New(&solo.Communicator{}, n.txPool, true).Mount(router, "/node") + enableTxpool := &atomic.Bool{} + enableTxpool.Store(true) + node2.New(&solo.Communicator{}, n.txPool, enableTxpool).Mount(router, "/node") fees.New(repo, engine, forkConfig, stater, fees.Config{ APIBacktraceLimit: 1000, PriorityIncreasePercentage: 5, diff --git a/txpool/metrics.go b/txpool/metrics.go index f532c0d7dc..b69fcc298b 100644 --- a/txpool/metrics.go +++ b/txpool/metrics.go @@ -13,5 +13,6 @@ var ( metricTxPoolGauge = metrics.LazyLoadGaugeVec("txpool_current_tx_count", []string{"source", "type"}) metricBadTxGauge = metrics.LazyLoadGaugeVec("bad_tx_count", []string{"source"}) metricTxPoolExecutablesGauge = metrics.LazyLoadGauge("txpool_executable_tx_count") + metricTxPoolAllGauge = metrics.LazyLoadGauge("txpool_all_tx_count") metricAccountQuotaExceeded = metrics.LazyLoadCounterVec("account_quota_exceeded", []string{"type"}) ) diff --git a/txpool/tx_pool.go b/txpool/tx_pool.go index e93f190134..d5ba213544 100644 --- a/txpool/tx_pool.go +++ b/txpool/tx_pool.go @@ -150,7 +150,11 @@ func (p *TxPool) housekeeping() { ctx = append(ctx, "err", err) } else { p.executables.Store(executables) + // Post-wash snapshot: report alongside the executables gauge + // from the same wash, so both reflect the consistent state + // that wash observed (not affected by concurrent Adds). metricTxPoolExecutablesGauge().Set(int64(len(executables))) + metricTxPoolAllGauge().Set(int64(poolLen - removedLegacy - removedDynamicFee)) } if removedLegacy > 0 { @@ -320,18 +324,6 @@ func (p *TxPool) checkTxPriority(txObj *TxObject, executable bool) bool { return txObj.priorityGasPrice.Cmp(thresholdTxObj.priorityGasPrice) > 0 } -// validateNonExecutableLimit validates that adding a non-executable transaction won't exceed pool limits. -func (p *TxPool) validateNonExecutableLimit(executable bool) error { - // Check non-executable pool limit (20% of total) - if !executable { - if p.all.Len()-len(p.Executables()) >= p.options.Limit*2/10 { - return txRejectedError{"non executable pool is full"} - } - } - - return nil -} - // addWhenSynced handles transaction addition when the chain is synced. func (p *TxPool) addWhenSynced( newTx *tx.Transaction, @@ -367,8 +359,11 @@ func (p *TxPool) addWhenSynced( return txRejectedError{"tx is not executable"} } - if err := p.validateNonExecutableLimit(executable); err != nil { - return err + // Check non-executable pool limit (20% of total) + if !executable { + if p.all.Len()-len(p.Executables()) >= p.options.Limit*2/10 { + return txRejectedError{"non executable pool is full"} + } } txObj.executable = executable