diff --git a/gno.land/pkg/integration/testdata/addpkg_import_testdep_gas.txtar b/gno.land/pkg/integration/testdata/addpkg_import_testdep_gas.txtar new file mode 100644 index 00000000000..4ed35103d16 --- /dev/null +++ b/gno.land/pkg/integration/testdata/addpkg_import_testdep_gas.txtar @@ -0,0 +1,93 @@ +# Demonstrates the mempackage prod/test storage split's import-time gas win: +# a dependency's _test.gno bytes do NOT inflate the cost of importing it. +# +# depa and depb have byte-identical PRODUCTION code; depb additionally carries a +# large lib_test.gno. usea imports depa, useb imports depb, and usea/useb are +# byte-identical apart from the a<->b in the package name and import path. When +# addpkg type-checks an importer it decodes the imported dependency's stored +# blob; the split stores depb's test file in a pkg:#allbutprod sibling that +# is never decoded here, so importing depb decodes the same bytes as importing +# depa => usea and useb cost the SAME gas. +# +# On the pre-split single-blob layout, depb's mempackage included its _test.gno, +# so type-checking useb decoded those extra bytes and useb cost MORE than usea. +# +# Measured: on this branch usea == useb == 3113401 (equal => the win). On master +# (pre-split) importing depb costs ~+40620 gas more than depa for its _test.gno +# bytes (useb > usea). The split removes that import penalty. +# +# The two GAS USED assertions below MUST stay equal to each other; that equality +# is the regression guard for the split (the absolute value is an ordinary pin). + +gnoland start + +gnokey maketx addpkg -pkgdir $WORK/depa -pkgpath gno.land/p/demo/depa -gas-fee 1000000ugnot -gas-wanted 5000000 -chainid=tendermint_test test1 +gnokey maketx addpkg -pkgdir $WORK/depb -pkgpath gno.land/p/demo/depb -gas-fee 1000000ugnot -gas-wanted 5000000 -chainid=tendermint_test test1 + +gnokey maketx addpkg -pkgdir $WORK/usea -pkgpath gno.land/p/demo/usea -gas-fee 1000000ugnot -gas-wanted 5000000 -chainid=tendermint_test test1 +stdout 'GAS USED:\s+3113401' +gnokey maketx addpkg -pkgdir $WORK/useb -pkgpath gno.land/p/demo/useb -gas-fee 1000000ugnot -gas-wanted 5000000 -chainid=tendermint_test test1 +stdout 'GAS USED:\s+3113401' + +-- depa/gnomod.toml -- +module = "gno.land/p/demo/depa" +gno = "0.9" +-- depa/lib.gno -- +package depa + +func Hi() string { return "hi" } +-- depb/gnomod.toml -- +module = "gno.land/p/demo/depb" +gno = "0.9" +-- depb/lib.gno -- +package depb + +func Hi() string { return "hi" } +-- depb/lib_test.gno -- +package depb + +// This _test.gno exists only to add bytes to depb's test files. Under the +// prod/test storage split these bytes live in the pkg:#allbutprod +// sibling and are NOT decoded when another package imports depb, so they do +// not affect importer gas. Padding below makes the size difference vs the +// prod blob substantial and the master-vs-branch delta clearly measurable. +// +// pad: 0123456789 0123456789 0123456789 0123456789 0123456789 0123456789 0123 +// pad: 0123456789 0123456789 0123456789 0123456789 0123456789 0123456789 0123 +// pad: 0123456789 0123456789 0123456789 0123456789 0123456789 0123456789 0123 +// pad: 0123456789 0123456789 0123456789 0123456789 0123456789 0123456789 0123 +// pad: 0123456789 0123456789 0123456789 0123456789 0123456789 0123456789 0123 +// pad: 0123456789 0123456789 0123456789 0123456789 0123456789 0123456789 0123 +// pad: 0123456789 0123456789 0123456789 0123456789 0123456789 0123456789 0123 +// pad: 0123456789 0123456789 0123456789 0123456789 0123456789 0123456789 0123 +// pad: 0123456789 0123456789 0123456789 0123456789 0123456789 0123456789 0123 +// pad: 0123456789 0123456789 0123456789 0123456789 0123456789 0123456789 0123 +// pad: 0123456789 0123456789 0123456789 0123456789 0123456789 0123456789 0123 +// pad: 0123456789 0123456789 0123456789 0123456789 0123456789 0123456789 0123 +// pad: 0123456789 0123456789 0123456789 0123456789 0123456789 0123456789 0123 +// pad: 0123456789 0123456789 0123456789 0123456789 0123456789 0123456789 0123 +// pad: 0123456789 0123456789 0123456789 0123456789 0123456789 0123456789 0123 +// pad: 0123456789 0123456789 0123456789 0123456789 0123456789 0123456789 0123 +// pad: 0123456789 0123456789 0123456789 0123456789 0123456789 0123456789 0123 +// pad: 0123456789 0123456789 0123456789 0123456789 0123456789 0123456789 0123 +// pad: 0123456789 0123456789 0123456789 0123456789 0123456789 0123456789 0123 +// pad: 0123456789 0123456789 0123456789 0123456789 0123456789 0123456789 0123 +func testPad() string { return Hi() } +-- usea/gnomod.toml -- +module = "gno.land/p/demo/usea" +gno = "0.9" +-- usea/lib.gno -- +package usea + +import "gno.land/p/demo/depa" + +func Use() string { return depa.Hi() } +-- useb/gnomod.toml -- +module = "gno.land/p/demo/useb" +gno = "0.9" +-- useb/lib.gno -- +package useb + +import "gno.land/p/demo/depb" + +func Use() string { return depb.Hi() } diff --git a/gno.land/pkg/integration/testdata/gnokey_gasfee.txtar b/gno.land/pkg/integration/testdata/gnokey_gasfee.txtar index 92efa2d6a22..553d093f80e 100644 --- a/gno.land/pkg/integration/testdata/gnokey_gasfee.txtar +++ b/gno.land/pkg/integration/testdata/gnokey_gasfee.txtar @@ -13,8 +13,8 @@ stdout '"coins": "10000000000000ugnot"' # gas-wanted/gas-fee on the simulate call are generous; simulate reports # the actual gas the real tx would consume. gnokey maketx addpkg -pkgdir $WORK/hello -pkgpath gno.land/r/hello -gas-wanted 3_500_000 -gas-fee 350001ugnot -chainid tendermint_test -simulate only test1 -stdout 'GAS USED:\s+2756592' -stdout 'INFO: estimated gas usage: 2756592 \(suggested, with 5% margin: 2894422\), gas fee: 2894ugnot, current gas price: 1ugnot/1000gas' +stdout 'GAS USED:\s+2756609' +stdout 'INFO: estimated gas usage: 2756609 \(suggested, with 5% margin: 2894440\), gas fee: 2894ugnot, current gas price: 1ugnot/1000gas' ## No fee was charged, and the sequence number did not change. gnokey query auth/accounts/$test1_user_addr diff --git a/gno.land/pkg/integration/testdata/restart_gas.txtar b/gno.land/pkg/integration/testdata/restart_gas.txtar index a7b81ea8fd8..4bb1c681a36 100644 --- a/gno.land/pkg/integration/testdata/restart_gas.txtar +++ b/gno.land/pkg/integration/testdata/restart_gas.txtar @@ -10,15 +10,15 @@ gnoland start gnokey maketx addpkg -pkgdir $WORK/bar -pkgpath gno.land/r/$test1_user_addr/bar -gas-fee 1000000ugnot -gas-wanted 100000000 -chainid=tendermint_test test1 -stdout 'GAS USED: +2780212' +stdout 'GAS USED: +2780229' gnokey maketx addpkg -pkgdir $WORK/foo -pkgpath gno.land/r/$test1_user_addr/foo -gas-fee 1000000ugnot -gas-wanted 100000000 -chainid=tendermint_test test1 -stdout 'GAS USED: +2782531' +stdout 'GAS USED: +2782548' gnoland restart gnokey maketx addpkg -pkgdir $WORK/baz -pkgpath gno.land/r/$test1_user_addr/baz -gas-fee 1000000ugnot -gas-wanted 100000000 -chainid=tendermint_test test1 -stdout 'GAS USED: +2782558' +stdout 'GAS USED: +2782575' -- bar/gnomod.toml -- module = "bar" diff --git a/gno.land/pkg/sdk/vm/apphash_crossrealm38_test.go b/gno.land/pkg/sdk/vm/apphash_crossrealm38_test.go index 8e7b1e01d2f..6497ede0b7b 100644 --- a/gno.land/pkg/sdk/vm/apphash_crossrealm38_test.go +++ b/gno.land/pkg/sdk/vm/apphash_crossrealm38_test.go @@ -68,7 +68,12 @@ import ( // their *_test.gno source bytes), which shifts the iavlStore Merkle root. This // is the only consensus-relevant change in that PR; verified by bisection that // no other change in the PR moves this hash. The shift is therefore expected. -const expectedCrossrealm38Hash = "28f55f0ad9842bc3c4d8984f1a63a709203a1e99fae7e816786825e26629f618" +// +// Hash bumped 2026-06-14: mempackage prod/test storage split — MP*All packages now +// store production files under pkg: (typed MP*Prod) and test/filetest files +// under a pkg:#allbutprod sibling, changing stored package bytes and the +// iavlStore Merkle root. Behavior is unchanged; only the storage encoding shifted. +const expectedCrossrealm38Hash = "adef42a3fcc41839fb59faf81e0190fd1f51fe7afba82f717d4def275c84c977" func TestAppHashCrossrealm38(t *testing.T) { env := setupTestEnv() diff --git a/gno.land/pkg/sdk/vm/keeper.go b/gno.land/pkg/sdk/vm/keeper.go index 67669751e12..d0bd17ec710 100644 --- a/gno.land/pkg/sdk/vm/keeper.go +++ b/gno.land/pkg/sdk/vm/keeper.go @@ -615,6 +615,14 @@ func (vm *VMKeeper) AddPackage(ctx sdk.Context, msg MsgAddPackage) (err error) { if err := gno.ValidateMemPackageAny(msg.Package); err != nil { return ErrInvalidPkgPath(err.Error()) } + // Reject packages with no production .gno files (e.g. only _test.gno + // files). The storage split writes no prod blob for them (store.go + // splitProdAllButProd), so a restarted node would rebuild no PackageNode + // while a non-restarted node still holds the deploy-time node in RAM — + // making call gas depend on restart history. + if gno.MPFProd.FilterMemPackage(memPkg).IsEmpty() { + return ErrInvalidPackage("package has no production .gno files") + } if !strings.HasPrefix(pkgPath, chainDomain+"/") { return ErrInvalidPkgPath("invalid domain: " + pkgPath) @@ -624,6 +632,15 @@ func (vm *VMKeeper) AddPackage(ctx sdk.Context, msg MsgAddPackage) (err error) { if pv != nil && !pv.Private { return ErrPkgAlreadyExists("package already exists: " + pkgPath) } + if pv != nil { + // A private package is being redeployed (non-private re-adds were + // rejected above). Clear its prior mempackage blobs first: AddMemPackage + // stores an MP*All package as a prod blob plus a #allbutprod sibling, and + // its conditional writes don't fully replace across both keys, so a stale + // sibling (or stale prod blob, if redeployed prod-less) could otherwise + // survive the re-add and be served by qfile/GetMemPackage. + gnostore.DeleteMemPackage(pkgPath) + } if !gno.IsRealmPath(pkgPath) && !gno.IsPPackagePath(pkgPath) { return ErrInvalidPkgPath("package path must be valid realm or p package path") @@ -1394,7 +1411,8 @@ func (vm *VMKeeper) QueryFile(ctx sdk.Context, filepath string) (res string, err } return memFile.Body, nil } else { - memPkg := store.GetMemPackage(dirpath) + // GetMemPackageAll so the file listing includes test/filetest files. + memPkg := store.GetMemPackageAll(dirpath) if memPkg == nil { return "", errors.Wrapf(&InvalidPackageError{}, "package %q is not available", dirpath) } @@ -1412,7 +1430,9 @@ func (vm *VMKeeper) QueryDoc(ctx sdk.Context, pkgPath string) (*doc.JSONDocument ctx = ctx.WithGasMeter(store.NewGasMeter(maxGasQuery)) store := vm.newGnoTransactionStore(ctx) // throwaway (never committed) - memPkg := store.GetMemPackage(pkgPath) + // GetMemPackageAll for parity with QueryFile, so doc generation sees test + // files (e.g. for any future test-derived examples). + memPkg := store.GetMemPackageAll(pkgPath) if memPkg == nil { err := ErrInvalidPkgPath(fmt.Sprintf( "package not found: %s", pkgPath)) diff --git a/gno.land/pkg/sdk/vm/keeper_test.go b/gno.land/pkg/sdk/vm/keeper_test.go index 29f208029bf..32731da32bd 100644 --- a/gno.land/pkg/sdk/vm/keeper_test.go +++ b/gno.land/pkg/sdk/vm/keeper_test.go @@ -22,6 +22,7 @@ import ( bft "github.com/gnolang/gno/tm2/pkg/bft/types" "github.com/gnolang/gno/tm2/pkg/crypto" "github.com/gnolang/gno/tm2/pkg/db/memdb" + tmerrors "github.com/gnolang/gno/tm2/pkg/errors" "github.com/gnolang/gno/tm2/pkg/log" "github.com/gnolang/gno/tm2/pkg/sdk" tu "github.com/gnolang/gno/tm2/pkg/sdk/testutils" @@ -126,6 +127,96 @@ func Echo(cur realm) string { assert.Nil(t, memFile) } +// A package with no production .gno files (only _test.gno / _filetest.gno) +// must be rejected: the storage split writes no prod blob for it, so a +// restarted node would rebuild no PackageNode while a non-restarted node +// still holds the deploy-time node in RAM (divergent call gas). +func TestVMKeeperAddPackage_NoProdFiles(t *testing.T) { + env := setupTestEnv() + ctx := env.vmk.MakeGnoTransactionStore(env.ctx) + + addr := crypto.AddressFromPreimage([]byte("addr1")) + acc := env.acck.NewAccountWithAddress(ctx, addr) + env.acck.SetAccount(ctx, acc) + env.bankk.SetCoins(ctx, addr, initialBalance) + + const pkgPath = "gno.land/r/testonly" + files := []*std.MemFile{ + {Name: "gnomod.toml", Body: gnolang.GenGnoModLatest(pkgPath)}, + { + Name: "testonly_test.gno", + Body: `package testonly +import "testing" +func TestNothing(t *testing.T) {}`, + }, + } + + err := env.vmk.AddPackage(ctx, NewMsgAddPackage(addr, pkgPath, files)) + + require.Error(t, err) + assert.Equal(t, InvalidPackageError{}, tmerrors.Cause(err)) + assert.Nil(t, env.vmk.getGnoTransactionStore(ctx).GetPackage(pkgPath, false)) + assert.Nil(t, env.vmk.getGnoTransactionStore(ctx).GetMemPackageAll(pkgPath)) +} + +// Structural companion to addpkg_import_testdep_gas.txtar: importing a +// dependency that carries a _test.gno file must cost exactly the same gas as +// importing one without, because the storage split keeps a dep's test bytes +// in the #allbutprod sibling, which import-time type-checking never decodes. +// The txtar pins two equal literals (which a bulk gas re-pin could silently +// de-equalize); this asserts the equality itself. +func TestVMKeeperAddPackage_ImportTestDepGasEqual(t *testing.T) { + env := setupTestEnv() + + addr := crypto.AddressFromPreimage([]byte("addr1")) + { + ctx := env.vmk.MakeGnoTransactionStore(env.ctx) + acc := env.acck.NewAccountWithAddress(ctx, addr) + env.acck.SetAccount(ctx, acc) + env.bankk.SetCoins(ctx, addr, initialBalance) + env.vmk.CommitGnoTransactionStore(ctx) + } + + // deploy runs msg in its own tx store with a fresh gas meter and + // returns the gas the deploy consumed. + deploy := func(pkgPath, libName, libBody string, extra ...*std.MemFile) int64 { + t.Helper() + files := append([]*std.MemFile{ + {Name: "gnomod.toml", Body: gnolang.GenGnoModLatest(pkgPath)}, + {Name: libName, Body: libBody}, + }, extra...) + gm := types.NewInfiniteGasMeter() + ctx := env.vmk.MakeGnoTransactionStore(env.ctx.WithGasMeter(gm)) + require.NoError(t, env.vmk.AddPackage(ctx, NewMsgAddPackage(addr, pkgPath, files))) + env.vmk.CommitGnoTransactionStore(ctx) + return gm.GasConsumed() + } + + // depa and depb have byte-identical production code (a<->b only); depb + // additionally carries a _test.gno. + deploy("gno.land/p/demo/depa", "lib.gno", + "package depa\n\nfunc Hi() string { return \"hi\" }\n") + deploy("gno.land/p/demo/depb", "lib.gno", + "package depb\n\nfunc Hi() string { return \"hi\" }\n", + &std.MemFile{Name: "lib_test.gno", Body: `package depb + +// Test-only bytes: under the prod/test storage split these live in the +// pkg:#allbutprod sibling and must not affect importer gas. +// pad: 0123456789 0123456789 0123456789 0123456789 0123456789 0123456789 +// pad: 0123456789 0123456789 0123456789 0123456789 0123456789 0123456789 +func testPad() string { return Hi() } +`}) + + // usea and useb are byte-identical apart from a<->b. + useaGas := deploy("gno.land/p/demo/usea", "lib.gno", + "package usea\n\nimport \"gno.land/p/demo/depa\"\n\nfunc Use() string { return depa.Hi() }\n") + usebGas := deploy("gno.land/p/demo/useb", "lib.gno", + "package useb\n\nimport \"gno.land/p/demo/depb\"\n\nfunc Use() string { return depb.Hi() }\n") + + assert.Equal(t, useaGas, usebGas, + "importing a dep with test files must cost the same as importing one without") +} + func TestVMKeeperAddPackage_DraftPackage(t *testing.T) { env := setupTestEnv() ctx := env.vmk.MakeGnoTransactionStore(env.ctx) diff --git a/gno.land/pkg/sdk/vm/params.go b/gno.land/pkg/sdk/vm/params.go index 09d66c29a7f..1dcccfe5985 100644 --- a/gno.land/pkg/sdk/vm/params.go +++ b/gno.land/pkg/sdk/vm/params.go @@ -25,9 +25,9 @@ const ( // Depth floors calibrated for B+32 at 100M items with 10K cache, batched 1000 muts. minGetReadDepth100Default = int64(300) // 3.0 GET read ops minSetReadDepth100Default = int64(200) // 2.0 SET read ops - minWriteDepth100Default = int64(440) // 4.4 write ops (batched) + minWriteDepth100Default = 440 // 4.4 write ops (batched) // Iterator step flat cost; mirrors store.DefaultGasConfig().IterNextCostFlat. - iterNextCostFlatDefault = int64(1_000) + iterNextCostFlatDefault = 1_000 ) var ASCIIDomain = regexp.MustCompile(`^(?:[A-Za-z0-9](?:[A-Za-z0-9-]{0,61}[A-Za-z0-9])?\.)+[A-Za-z]{2,}$`) @@ -152,7 +152,7 @@ func (p Params) Validate() error { // realistic per-step cost while far below the block gas limit // (~3B) so that a single adversarial proposal can't make one // iterator step drain an entire block's gas budget. - const maxIterNextCostFlat = int64(100_000) + const maxIterNextCostFlat = 100_000 if p.IterNextCostFlat <= 0 { return fmt.Errorf("IterNextCostFlat must be positive, got %d", p.IterNextCostFlat) } diff --git a/gnovm/pkg/gnolang/debugger.go b/gnovm/pkg/gnolang/debugger.go index a44b79b347a..4dc475093c4 100644 --- a/gnovm/pkg/gnolang/debugger.go +++ b/gnovm/pkg/gnolang/debugger.go @@ -614,12 +614,17 @@ func debugLineInfo(m *Machine) { func isMemPackage(st Store, pkgPath string) bool { ds, ok := st.(*defaultStore) - return ok && ds.iavlStore.Has(ds.gctx, []byte(backendPackagePathKey(pkgPath))) + return ok && (ds.iavlStore.Has(ds.gctx, []byte(backendPackagePathKey(pkgPath))) || + ds.iavlStore.Has(ds.gctx, []byte(backendPackageAllButProdKey(pkgPath)))) } func fileContent(st Store, pkgPath, name string) (string, error) { if isMemPackage(st, pkgPath) { - return st.GetMemFile(pkgPath, name).Body, nil + // The package is in the store, but the requested file may not be in it; + // guard the nil before dereferencing, then fall through to disk. + if mf := st.GetMemFile(pkgPath, name); mf != nil { + return mf.Body, nil + } } buf, err := os.ReadFile(name) return string(buf), err diff --git a/gnovm/pkg/gnolang/machine.go b/gnovm/pkg/gnolang/machine.go index 1fae70eeedc..d10194e2e29 100644 --- a/gnovm/pkg/gnolang/machine.go +++ b/gnovm/pkg/gnolang/machine.go @@ -327,6 +327,17 @@ func assertBorrowedRealm(pkgPath string, r *Realm) { func (m *Machine) PreprocessAllFilesAndSaveBlockNodes() { ch := m.Store.IterMemPackage() for mpkg := range ch { + if mpkg == nil { + // An indexed package with no production files (e.g. an + // xxx_test-only package) has no prod blob, so GetMemPackage + // returns nil. There are no production block nodes to build; + // its test files live under the #allbutprod sibling. On-chain + // this is unreachable — the vm keeper rejects prod-less packages + // at AddPackage (block-node state would otherwise depend on + // restart history) — so this skip is defensive, for non-chain + // stores. + continue + } mpkg = MPFProd.FilterMemPackage(mpkg) fset := m.ParseMemPackage(mpkg) pn := NewPackageNode(Name(mpkg.Name), mpkg.Path, fset) diff --git a/gnovm/pkg/gnolang/store.go b/gnovm/pkg/gnolang/store.go index 2be66d9f79d..dacf77a66a7 100644 --- a/gnovm/pkg/gnolang/store.go +++ b/gnovm/pkg/gnolang/store.go @@ -80,9 +80,15 @@ type Store interface { // loads BlockNodes and Types onto the store for persistence // version 1. AddMemPackage(mpkg *std.MemPackage, mptype MemPackageType) + DeleteMemPackage(path string) GetMemPackage(path string) *std.MemPackage + GetMemPackageAll(path string) *std.MemPackage GetMemFile(path string, name string) *std.MemFile FindPathsByPrefix(prefix string) iter.Seq[string] + // Yields each indexed package's PROD mempackage (test/filetest files + // live under the #allbutprod sibling and are not included), in index + // order. A package with no production .gno files has no prod blob and + // is yielded as nil. IterMemPackage() <-chan *std.MemPackage ClearObjectCache() // run before processing a message GarbageCollectObjectCache(gcCycle int64) @@ -970,6 +976,14 @@ func (ds *defaultStore) incGetPackageIndexCounter() uint64 { // MPAnyProd and MPAnyTest. // MPFiletests are not allowed, as they are currently only read from disk (e.g. // test/files). However, MP*All may include filetests files. +// +// MP*All packages are stored as two keys (a prod blob plus a #allbutprod +// sibling) and the writes are conditional, so this is NOT a full replace across +// both keys. Re-adding an MP*All package at an existing path (e.g. a private +// redeploy) MUST call DeleteMemPackage(path) first, or a stale sibling/prod blob +// can survive. The keeper's AddPackage does this; new MP*All re-add callers must +// too. (Already-filtered non-All types are stored under a single key and +// overwrite cleanly.) func (ds *defaultStore) AddMemPackage(mpkg *std.MemPackage, mptype MemPackageType) { var size int if bm.Enabled { @@ -990,19 +1004,95 @@ func (ds *defaultStore) AddMemPackage(mpkg *std.MemPackage, mptype MemPackageTyp } ctr := ds.incGetPackageIndexCounter() idxkey := []byte(backendPackageIndexKey(ctr)) + ds.baseStore.Set(ds.gctx, idxkey, []byte(mpkg.Path)) + pathkey := []byte(backendPackagePathKey(mpkg.Path)) + // MP*All packages are split for storage: production files under pathkey + // (the import/preprocess hot path, so its read/decode gas is charged on + // prod bytes only), and the remaining test/filetest files under a sibling + // "#allbutprod" key read only by query paths (see GetMemPackageAll). Other + // (already-filtered) types are stored whole under pathkey as before. + if mpkgtype.IsAll() { + prod, allButProd := splitProdAllButProd(mpkg) + // prod is nil for a package with no production .gno files (e.g. an + // xxx_test-only package): it has no prod blob, and readers treat a + // missing prod blob as nil. splitProdAllButProd folds that package's + // non-.gno files into the #allbutprod sibling so GetMemPackageAll can + // reconstruct the full package losslessly. + if prod != nil { + size += ds.setMemPackageBlob(pathkey, prod) + } + if len(allButProd.Files) > 0 { + size += ds.setMemPackageBlob([]byte(backendPackageAllButProdKey(mpkg.Path)), allButProd) + } + } else { + size += ds.setMemPackageBlob(pathkey, mpkg) + } +} + +// DeleteMemPackage removes both the production blob (pkg:) and the +// #allbutprod sibling for path. It is a no-op for keys that do not exist. Used +// before a private-package redeploy: AddMemPackage stores an MP*All package as +// two keys, and its conditional writes are not a full replace across both, so a +// stale sibling (or, for a now-prod-less package, a stale prod blob) could +// otherwise survive a re-add and be served by GetMemPackage/GetMemPackageAll. +func (ds *defaultStore) DeleteMemPackage(path string) { + ds.iavlStore.Delete(ds.gctx, []byte(backendPackagePathKey(path))) + ds.iavlStore.Delete(ds.gctx, []byte(backendPackageAllButProdKey(path))) +} + +// setMemPackageBlob amino-marshals mpkg, charges encode gas, writes it under key +// in the iavl store, and returns the encoded byte length. +func (ds *defaultStore) setMemPackageBlob(key []byte, mpkg *std.MemPackage) int { bz := amino.MustMarshal(mpkg) gas := overflow.Mulp(ds.gasConfig.GasAminoEncode, store.Gas(len(bz))) ds.consumeGas(gas, GasAminoEncodeDesc) - ds.baseStore.Set(ds.gctx, idxkey, []byte(mpkg.Path)) - pathkey := []byte(backendPackagePathKey(mpkg.Path)) if trace.StoreGasEnabled { - trace.Store("ENCODE_MEMPKG", gas, pathkey, len(bz), "none") + trace.Store("ENCODE_MEMPKG", gas, key, len(bz), "none") } - ds.iavlStore.Set(ds.gctx, pathkey, bz) + ds.iavlStore.Set(ds.gctx, key, bz) if trace.StoreGasEnabled { - trace.Store("IAVL_SET_MEMPKG", 0, pathkey, len(bz), "none") + trace.Store("IAVL_SET_MEMPKG", 0, key, len(bz), "none") + } + return len(bz) +} + +// splitProdAllButProd partitions an MP*All mempackage into its production blob +// and the complement ("all but prod") sibling so that prod ∪ allButProd == +// mpkg.Files exactly, with no overlap and no drops. +// +// The production blob holds the importable subset: non-.gno files plus non-test +// .gno files, typed MP*Prod. However, a package with no production .gno files +// (e.g. an xxx_test-only package) has no valid prod blob — an empty mempackage +// fails validation, and readers treat a missing prod blob as nil — so in that +// case prod is returned nil and ALL of its files (including non-.gno files such +// as gnomod.toml, LICENSE, README, *.md, *.toml) are folded into the complement +// instead, otherwise they would be silently dropped from storage. +// +// The complement keeps the package's Name/Path/Info and the original MP*All type +// as an inert sentinel: it is written and read only by the store and never +// enters the MemPackageType dispatch paths. +func splitProdAllButProd(mpkg *std.MemPackage) (prod, allButProd *std.MemPackage) { + prod = MPFProd.FilterMemPackage(mpkg) + allButProd = &std.MemPackage{ + Name: mpkg.Name, + Path: mpkg.Path, + Info: mpkg.Info, + Type: mpkg.Type, + } + // If there are no production .gno files, the prod blob will not be written + // (it would fail validation). Fold every non-prod-.gno file into the + // complement so reconstruction stays lossless. + prodSkipped := prod.IsEmpty() + if prodSkipped { + prod = nil + } + for _, mfile := range mpkg.Files { + if IsTestFile(mfile.Name) || (prodSkipped && !strings.HasSuffix(mfile.Name, ".gno")) { + allButProd.Files = append(allButProd.Files, mfile.Copy()) + } } - size = len(bz) + allButProd.Sort() + return prod, allButProd } // GetMemPackage retrieves the MemPackage at the given path. @@ -1047,11 +1137,57 @@ func (ds *defaultStore) getMemPackage(path string, isRetry bool) *std.MemPackage return mpkg } +// getMemPackageAllButProd reads and decodes the "#allbutprod" sibling blob (the +// test/filetest files) for path, or nil if there is none. Charges decode gas. +func (ds *defaultStore) getMemPackageAllButProd(path string) *std.MemPackage { + bz := ds.iavlStore.Get(ds.gctx, []byte(backendPackageAllButProdKey(path))) + if bz == nil { + return nil + } + gas := overflow.Mulp(ds.gasConfig.GasAminoDecode, store.Gas(len(bz))) + ds.consumeGas(gas, GasAminoDecodeDesc) + var mpkg *std.MemPackage + amino.MustUnmarshal(bz, &mpkg) + return mpkg +} + +// GetMemPackageAll retrieves the complete MemPackage at path, including test and +// filetest files, by merging the production blob with its "#allbutprod" sibling. +// It returns nil if the package does not exist. The import/run hot path uses the +// prod-only GetMemPackage; GetMemPackageAll is for query/tooling paths that must +// see test files (e.g. vm/qfile). +func (ds *defaultStore) GetMemPackageAll(path string) *std.MemPackage { + prod := ds.GetMemPackage(path) + allButProd := ds.getMemPackageAllButProd(path) + if prod == nil && allButProd == nil { + return nil + } + base := prod + if base == nil { + base = allButProd + } + merged := &std.MemPackage{ + Name: base.Name, + Path: base.Path, + Info: base.Info, + Type: MPAnyAll.Decide(path), // MPUserAll or MPStdlibAll. + } + if prod != nil { + merged.Files = append(merged.Files, prod.Files...) + } + if allButProd != nil { + merged.Files = append(merged.Files, allButProd.Files...) + } + merged.Sort() + return merged +} + // GetMemFile retrieves the MemFile with the given name, contained in the // MemPackage at the given path. It returns nil if the file or the package -// do not exist. +// do not exist. It consults the full package (prod + test files) so that +// test/filetest files remain retrievable. func (ds *defaultStore) GetMemFile(path string, name string) *std.MemFile { - mpkg := ds.GetMemPackage(path) + mpkg := ds.GetMemPackageAll(path) if mpkg == nil { return nil } @@ -1075,8 +1211,31 @@ func (ds *defaultStore) FindPathsByPrefix(prefix string) iter.Seq[string] { iter := ds.iavlStore.Iterator(ds.gctx, startKey, endKey) defer iter.Close() + var last string + var hasLast bool for ; iter.Valid(); iter.Next() { - path := decodeBackendPackagePathKey(string(iter.Key())) + key := string(iter.Key()) + // A package's prod key (pkg:) and its #allbutprod sibling map + // to the same path. Strip the sibling suffix and de-dup so each + // package is yielded exactly once — including an empty-prod package + // (only a sibling key). Prod and sibling keys for a path are + // adjacent in iavl order, so de-dup against the previous suffices. + key = strings.TrimSuffix(key, "#allbutprod") + // A prefix containing '#' (impossible in a valid package path, + // but reachable from raw query input, e.g. vm/qpaths) can range + // over a sibling key whose trimmed form no longer carries the + // requested prefix; don't yield such a path. Compared in key + // space (against startKey): stdlib paths decode without their + // "_/" key marker, so path-space comparison would wrongly drop + // legitimate stdlib matches. + if len(prefix) > 0 && !strings.HasPrefix(key, string(startKey)) { + continue + } + path := decodeBackendPackagePathKey(key) + if hasLast && path == last { + continue + } + last, hasLast = path, true if !yield(path) { return } @@ -1303,6 +1462,14 @@ func backendPackageStdlibPath(path string) string { return "pkg:_/" + path } func backendPackageGlobalPath(path string) string { return "pkg:" + path } +// backendPackageAllButProdKey returns the sibling key holding a package's +// test/filetest files (everything in an MP*All package but its production +// subset). It suffixes the package path key with "#allbutprod"; "#" cannot +// appear in a valid package path, so it never collides with a real package key. +func backendPackageAllButProdKey(path string) string { + return backendPackagePathKey(path) + "#allbutprod" +} + func decodeBackendPackagePathKey(key string) string { path := strings.TrimPrefix(key, "pkg:") return strings.TrimPrefix(path, "_/") diff --git a/gnovm/pkg/gnolang/store_test.go b/gnovm/pkg/gnolang/store_test.go index b5b18a0b666..95efadb5012 100644 --- a/gnovm/pkg/gnolang/store_test.go +++ b/gnovm/pkg/gnolang/store_test.go @@ -112,6 +112,54 @@ func TestCopyFromCachedStore(t *testing.T) { assert.Equal(t, cachedStore.cacheNodes, destStore.cacheNodes, "cacheNodes should match") } +func TestDeleteMemPackageClearsStaleBlobsOnReAdd(t *testing.T) { + newStore := func() *defaultStore { + d1, d2 := memdb.NewMemDB(), memdb.NewMemDB() + d1s := dbadapter.StoreConstructor(d1, storetypes.StoreOptions{}) + d2s := dbadapter.StoreConstructor(d2, storetypes.StoreOptions{}) + return NewStore(nil, d1s, d2s) + } + const pkgPath = "gno.land/r/demo/foo" + withTests := func() *std.MemPackage { + return &std.MemPackage{ + Type: MPUserAll, Name: "foo", Path: pkgPath, + Files: []*std.MemFile{ + {Name: "foo.gno", Body: "package foo\n"}, + {Name: "foo_test.gno", Body: "package foo\n"}, + }, + } + } + prodOnly := func() *std.MemPackage { + return &std.MemPackage{ + Type: MPUserAll, Name: "foo", Path: pkgPath, + Files: []*std.MemFile{ + {Name: "foo.gno", Body: "package foo\n"}, + }, + } + } + + // DeleteMemPackage removes both the prod blob and the #allbutprod sibling. + st := newStore() + st.AddMemPackage(withTests(), MPUserAll) + require.NotNil(t, st.GetMemFile(pkgPath, "foo_test.gno")) + st.DeleteMemPackage(pkgPath) + assert.Nil(t, st.GetMemPackage(pkgPath)) + assert.Nil(t, st.GetMemPackageAll(pkgPath)) + assert.Nil(t, st.GetMemFile(pkgPath, "foo_test.gno")) + + // Re-add idempotency (mirrors the keeper clearing a private package before + // redeploy): dropping the test file must not leave a stale #allbutprod sibling. + st = newStore() + st.AddMemPackage(withTests(), MPUserAll) + st.DeleteMemPackage(pkgPath) + st.AddMemPackage(prodOnly(), MPUserAll) + all := st.GetMemPackageAll(pkgPath) + require.NotNil(t, all) + require.Len(t, all.Files, 1) + assert.Equal(t, "foo.gno", all.Files[0].Name) + assert.Nil(t, st.GetMemFile(pkgPath, "foo_test.gno"), "stale test file must not survive re-add") +} + func TestFindByPrefix(t *testing.T) { stdlibs := []string{"abricot", "balloon", "call", "dingdong", "gnocchi"} pkgs := []string{ @@ -198,3 +246,48 @@ func TestFindByPrefix(t *testing.T) { }) } } + +func TestFindByPrefixDeDupesSplitPackages(t *testing.T) { + d1, d2 := memdb.NewMemDB(), memdb.NewMemDB() + d1s := dbadapter.StoreConstructor(d1, storetypes.StoreOptions{}) + d2s := dbadapter.StoreConstructor(d2, storetypes.StoreOptions{}) + store := NewStore(nil, d1s, d2s) + + add := func(name string, files ...*std.MemFile) { + store.AddMemPackage(&std.MemPackage{ + Type: MPUserAll, + Name: name, + Path: "gno.land/r/demo/" + name, + Files: files, + }, MPUserAll) + } + // alpha: prod + test -> prod blob + #allbutprod sibling (must list once, not twice). + add("alpha", &std.MemFile{Name: "alpha.gno", Body: "package alpha\n"}, + &std.MemFile{Name: "alpha_test.gno", Body: "package alpha\n"}) + // beta: test-only -> #allbutprod sibling only (empty prod; must still list once). + add("beta", &std.MemFile{Name: "beta_test.gno", Body: "package beta\n"}) + // gamma: prod only -> prod blob only. + add("gamma", &std.MemFile{Name: "gamma.gno", Body: "package gamma\n"}) + + var got []string + store.FindPathsByPrefix("gno.land")(func(p string) bool { + got = append(got, p) + return true + }) + require.Equal(t, []string{ + "gno.land/r/demo/alpha", + "gno.land/r/demo/beta", + "gno.land/r/demo/gamma", + }, got) + + // A '#'-containing prefix (impossible in a valid package path, reachable + // from raw query input) ranges over alpha's #allbutprod sibling key; the + // trimmed path "gno.land/r/demo/alpha" does not carry the prefix and must + // not be yielded. + for _, prefix := range []string{"gno.land/r/demo/alpha#", "gno.land/r/demo/alpha#allbutprod"} { + store.FindPathsByPrefix(prefix)(func(p string) bool { + t.Fatalf("prefix %q must yield nothing, got %q", prefix, p) + return true + }) + } +}