diff --git a/core/commands/dag/dag.go b/core/commands/dag/dag.go index a256213ecd0..eab70aa3332 100644 --- a/core/commands/dag/dag.go +++ b/core/commands/dag/dag.go @@ -23,6 +23,7 @@ const ( statsOptionName = "stats" fastProvideRootOptionName = "fast-provide-root" fastProvideWaitOptionName = "fast-provide-wait" + localOnlyOptionName = "local-only" ) // DagCmd provides a subset of commands for interacting with ipld dag objects @@ -192,6 +193,9 @@ Note: currently present in the blockstore does not represent a complete DAG, pinning of that individual root will fail. + Use --local-only and --pin-roots=false for partial CARs (e.g. from + 'dag export --local-only'). + FAST PROVIDE OPTIMIZATION: Root CIDs from CAR headers are immediately provided to the DHT in addition @@ -213,6 +217,7 @@ Specification of CAR formats: https://ipld.io/specs/transport/car/ }, Options: []cmds.Option{ cmds.BoolOption(pinRootsOptionName, "Pin optional roots listed in the .car headers after importing.").WithDefault(true), + cmds.BoolOption(localOnlyOptionName, "Import partial CAR without pinning roots (e.g. from dag export --local-only)."), cmds.BoolOption(silentOptionName, "No output."), cmds.BoolOption(statsOptionName, "Output stats."), cmds.BoolOption(fastProvideRootOptionName, "Immediately provide root CIDs to DHT in addition to regular queue, for faster discovery. Default: Import.FastProvideRoot"), @@ -285,6 +290,7 @@ CAR file follows the CARv1 format: https://ipld.io/specs/transport/car/carv1/ }, Options: []cmds.Option{ cmds.BoolOption(progressOptionName, "p", "Display progress on CLI. Defaults to true when STDERR is a TTY."), + cmds.BoolOption(localOnlyOptionName, "If set, only blocks present locally are exported; missing blocks are skipped (partial CAR). Use with --offline for a local-only DAG walk."), }, Run: dagExport, PostRun: cmds.PostRunMap{ diff --git a/core/commands/dag/export.go b/core/commands/dag/export.go index 48223f86083..4967b47ab41 100644 --- a/core/commands/dag/export.go +++ b/core/commands/dag/export.go @@ -1,6 +1,7 @@ package dagcmd import ( + "bytes" "context" "errors" "fmt" @@ -16,7 +17,10 @@ import ( "github.com/ipfs/kubo/core/commands/cmdutils" iface "github.com/ipfs/kubo/core/coreiface" gocar "github.com/ipld/go-car/v2" + "github.com/ipld/go-ipld-prime/datamodel" + "github.com/ipld/go-ipld-prime/linking" cidlink "github.com/ipld/go-ipld-prime/linking/cid" + "github.com/ipld/go-ipld-prime/traversal" selectorparse "github.com/ipld/go-ipld-prime/traversal/selector/parse" ) @@ -27,6 +31,7 @@ func dagExport(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment return err } + localOnly, _ := req.Options[localOnlyOptionName].(bool) api, err := cmdenv.GetApi(env, req) if err != nil { return err @@ -51,7 +56,25 @@ func dagExport(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment }() lsys := cidlink.DefaultLinkSystem() - lsys.SetReadStorage(&dagStore{dag: api.Dag(), ctx: req.Context}) + ds := &dagStore{dag: api.Dag(), ctx: req.Context} + if localOnly { + lsys.StorageReadOpener = func(lctx linking.LinkContext, lnk datamodel.Link) (io.Reader, error) { + cl, ok := lnk.(cidlink.Link) + if !ok { + return nil, fmt.Errorf("unsupported link type: %T", lnk) + } + block, err := ds.dag.Get(lctx.Ctx, cl.Cid) + if err != nil { + if ipld.IsNotFound(err) { + return nil, traversal.SkipMe{} + } + return nil, fmt.Errorf("local block read failed: %w", err) + } + return bytes.NewReader(block.RawData()), nil + } + } else { + lsys.SetReadStorage(ds) + } // Uncomment the following to support CARv2 output. /* @@ -194,7 +217,7 @@ func cidFromBinString(key string) (cid.Cid, error) { return cid.Undef, fmt.Errorf("dagStore: key was not a cid: %w", err) } if l != len(key) { - return cid.Undef, fmt.Errorf("dagSore: key was not a cid: had %d bytes leftover", len(key)-l) + return cid.Undef, fmt.Errorf("dagStore: key was not a cid: had %d bytes leftover", len(key)-l) } return k, nil } diff --git a/core/commands/dag/import.go b/core/commands/dag/import.go index 032b9e52a6c..fde3e9157f8 100644 --- a/core/commands/dag/import.go +++ b/core/commands/dag/import.go @@ -49,7 +49,11 @@ func dagImport(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment } doPinRoots, _ := req.Options[pinRootsOptionName].(bool) + localOnly, _ := req.Options[localOnlyOptionName].(bool) + if doPinRoots && localOnly { + return fmt.Errorf("cannot pass both --%s and --%s", pinRootsOptionName, localOnlyOptionName) + } fastProvideRoot, fastProvideRootSet := req.Options[fastProvideRootOptionName].(bool) fastProvideWait, fastProvideWaitSet := req.Options[fastProvideWaitOptionName].(bool) diff --git a/docs/examples/kubo-as-a-library/go.mod b/docs/examples/kubo-as-a-library/go.mod index 154def8bb50..9f6fefd5f84 100644 --- a/docs/examples/kubo-as-a-library/go.mod +++ b/docs/examples/kubo-as-a-library/go.mod @@ -101,7 +101,7 @@ require ( github.com/ipfs/go-peertaskqueue v0.8.3 // indirect github.com/ipfs/go-test v0.2.3 // indirect github.com/ipfs/go-unixfsnode v1.10.3 // indirect - github.com/ipld/go-car/v2 v2.16.0 // indirect + github.com/ipld/go-car/v2 v2.16.1-0.20260306172652-7d2f4aceb070 // indirect github.com/ipld/go-codec-dagpb v1.7.0 // indirect github.com/ipld/go-ipld-prime v0.22.0 // indirect github.com/ipshipyard/p2p-forge v0.7.0 // indirect diff --git a/docs/examples/kubo-as-a-library/go.sum b/docs/examples/kubo-as-a-library/go.sum index fe4f15b4ee7..6c7acb70cb8 100644 --- a/docs/examples/kubo-as-a-library/go.sum +++ b/docs/examples/kubo-as-a-library/go.sum @@ -422,8 +422,8 @@ github.com/ipfs/go-test v0.2.3 h1:Z/jXNAReQFtCYyn7bsv/ZqUwS6E7iIcSpJ2CuzCvnrc= github.com/ipfs/go-test v0.2.3/go.mod h1:QW8vSKkwYvWFwIZQLGQXdkt9Ud76eQXRQ9Ao2H+cA1o= github.com/ipfs/go-unixfsnode v1.10.3 h1:c8sJjuGNkxXAQH75P+f5ngPda/9T+DrboVA0TcDGvGI= github.com/ipfs/go-unixfsnode v1.10.3/go.mod h1:2Jlc7DoEwr12W+7l8Hr6C7XF4NHST3gIkqSArLhGSxU= -github.com/ipld/go-car/v2 v2.16.0 h1:LWe0vmN/QcQmUU4tr34W5Nv5mNraW+G6jfN2s+ndBco= -github.com/ipld/go-car/v2 v2.16.0/go.mod h1:RqFGWN9ifcXVmCrTAVnfnxiWZk1+jIx67SYhenlmL34= +github.com/ipld/go-car/v2 v2.16.1-0.20260306172652-7d2f4aceb070 h1:aUJhNVqQZV3PhxQX/BPAkurYnsxBmMLx2a7YPsgd5NU= +github.com/ipld/go-car/v2 v2.16.1-0.20260306172652-7d2f4aceb070/go.mod h1:eN3661vGFqfxzIfPLrCQIdKcY9iIOCgZON1CedVtvwk= github.com/ipld/go-codec-dagpb v1.7.0 h1:hpuvQjCSVSLnTnHXn+QAMR0mLmb1gA6wl10LExo2Ts0= github.com/ipld/go-codec-dagpb v1.7.0/go.mod h1:rD3Zg+zub9ZnxcLwfol/OTQRVjaLzXypgy4UqHQvilM= github.com/ipld/go-ipld-prime v0.11.0/go.mod h1:+WIAkokurHmZ/KwzDOMUuoeJgaRQktHtEaLglS3ZeV8= diff --git a/go.mod b/go.mod index ce8499eaec1..711730d48dd 100644 --- a/go.mod +++ b/go.mod @@ -43,7 +43,7 @@ require ( github.com/ipfs/go-metrics-prometheus v0.1.0 github.com/ipfs/go-test v0.2.3 github.com/ipfs/go-unixfsnode v1.10.3 - github.com/ipld/go-car/v2 v2.16.0 + github.com/ipld/go-car/v2 v2.16.1-0.20260306172652-7d2f4aceb070 github.com/ipld/go-codec-dagpb v1.7.0 github.com/ipld/go-ipld-prime v0.22.0 github.com/ipshipyard/p2p-forge v0.7.0 diff --git a/go.sum b/go.sum index 85fe6f8839a..221d5e48083 100644 --- a/go.sum +++ b/go.sum @@ -464,8 +464,8 @@ github.com/ipfs/go-test v0.2.3 h1:Z/jXNAReQFtCYyn7bsv/ZqUwS6E7iIcSpJ2CuzCvnrc= github.com/ipfs/go-test v0.2.3/go.mod h1:QW8vSKkwYvWFwIZQLGQXdkt9Ud76eQXRQ9Ao2H+cA1o= github.com/ipfs/go-unixfsnode v1.10.3 h1:c8sJjuGNkxXAQH75P+f5ngPda/9T+DrboVA0TcDGvGI= github.com/ipfs/go-unixfsnode v1.10.3/go.mod h1:2Jlc7DoEwr12W+7l8Hr6C7XF4NHST3gIkqSArLhGSxU= -github.com/ipld/go-car/v2 v2.16.0 h1:LWe0vmN/QcQmUU4tr34W5Nv5mNraW+G6jfN2s+ndBco= -github.com/ipld/go-car/v2 v2.16.0/go.mod h1:RqFGWN9ifcXVmCrTAVnfnxiWZk1+jIx67SYhenlmL34= +github.com/ipld/go-car/v2 v2.16.1-0.20260306172652-7d2f4aceb070 h1:aUJhNVqQZV3PhxQX/BPAkurYnsxBmMLx2a7YPsgd5NU= +github.com/ipld/go-car/v2 v2.16.1-0.20260306172652-7d2f4aceb070/go.mod h1:eN3661vGFqfxzIfPLrCQIdKcY9iIOCgZON1CedVtvwk= github.com/ipld/go-codec-dagpb v1.7.0 h1:hpuvQjCSVSLnTnHXn+QAMR0mLmb1gA6wl10LExo2Ts0= github.com/ipld/go-codec-dagpb v1.7.0/go.mod h1:rD3Zg+zub9ZnxcLwfol/OTQRVjaLzXypgy4UqHQvilM= github.com/ipld/go-ipld-prime v0.11.0/go.mod h1:+WIAkokurHmZ/KwzDOMUuoeJgaRQktHtEaLglS3ZeV8= diff --git a/test/cli/dag_test.go b/test/cli/dag_test.go index 38457318a0c..a834767ba20 100644 --- a/test/cli/dag_test.go +++ b/test/cli/dag_test.go @@ -1,9 +1,14 @@ package cli import ( + "context" "encoding/json" + "fmt" "io" "os" + "os/exec" + "path/filepath" + "strings" "testing" "time" @@ -305,3 +310,153 @@ func TestDagImportFastProvide(t *testing.T) { require.Contains(t, daemonLog, "fast-provide-root: skipped") }) } + +// dagRefs returns root plus recursive ref CIDs from "ipfs refs -r --unique root". +func dagRefs(node *harness.Node, root string) []string { + refsRes := node.IPFS("refs", "-r", "--unique", root) + refs := []string{root} + for _, line := range testutils.SplitLines(strings.TrimSpace(refsRes.Stdout.String())) { + if line != "" { + refs = append(refs, line) + } + } + return refs +} + +func parseImportedBlockCount(stdout string) int { + var n int + for _, line := range testutils.SplitLines(stdout) { + if _, err := fmt.Sscanf(line, "Imported %d blocks", &n); err == nil { + return n + } + } + return 0 +} + +func TestDagExportLocalOnly(t *testing.T) { + t.Parallel() + node := harness.NewT(t).NewNode().Init().StartDaemon() + defer node.StopDaemon() + + root := node.IPFSAddDeterministic("300KiB", "dag-export-local-only", "--raw-leaves") + refs := dagRefs(node, root) + require.GreaterOrEqual(t, len(refs), 2, "need at least root and one child block") + + fullCarPath := filepath.Join(node.Dir, "full.car") + require.NoError(t, node.IPFSDagExport(root, fullCarPath)) + require.Equal(t, 0, node.RunIPFS("pin", "rm", root).ExitCode()) + require.Equal(t, 0, node.RunIPFS("block", "rm", refs[1]).ExitCode()) + + // Export --offline should fail; discard output (no file needed). + res := node.Runner.Run(harness.RunRequest{ + Path: node.IPFSBin, + Args: []string{"dag", "export", "--offline", root}, + CmdOpts: []harness.CmdOpt{harness.RunWithStdout(io.Discard)}, + }) + require.NotEqual(t, 0, res.ExitCode(), "export --offline without --local-only should fail when a block is missing") + require.Contains(t, res.Stderr.String(), "block was not found locally") + + partialCarPath := filepath.Join(node.Dir, "partial.car") + require.NoError(t, node.IPFSDagExport(root, partialCarPath, "--local-only", "--offline")) + + nodeFull := harness.NewT(t).NewNode().Init().StartDaemon() + defer nodeFull.StopDaemon() + fullCAR, err := os.Open(fullCarPath) + require.NoError(t, err) + defer fullCAR.Close() + fullRes := nodeFull.Runner.Run(harness.RunRequest{ + Path: nodeFull.IPFSBin, + Args: []string{"dag", "import", "--pin-roots=false", "--stats"}, + CmdOpts: []harness.CmdOpt{harness.RunWithStdin(fullCAR)}, + }) + require.Equal(t, 0, fullRes.ExitCode()) + fullCount := parseImportedBlockCount(fullRes.Stdout.String()) + require.Greater(t, fullCount, 0, "expected 'Imported N blocks' in output: %s", fullRes.Stdout.String()) + + nodePartial := harness.NewT(t).NewNode().Init().StartDaemon() + defer nodePartial.StopDaemon() + partialCAR, err := os.Open(partialCarPath) + require.NoError(t, err) + defer partialCAR.Close() + partialRes := nodePartial.Runner.Run(harness.RunRequest{ + Path: nodePartial.IPFSBin, + Args: []string{"dag", "import", "--pin-roots=false", "--stats"}, + CmdOpts: []harness.CmdOpt{harness.RunWithStdin(partialCAR)}, + }) + require.Equal(t, 0, partialRes.ExitCode()) + partialCount := parseImportedBlockCount(partialRes.Stdout.String()) + require.Greater(t, partialCount, 0, "expected 'Imported N blocks' in output: %s", partialRes.Stdout.String()) + + require.Less(t, partialCount, fullCount, "partial CAR should have fewer blocks than full DAG") +} + +func TestDagExportLocalOnlyRequiresOffline(t *testing.T) { + t.Parallel() + node := harness.NewT(t).NewNode().Init().StartDaemon() + defer node.StopDaemon() + + root := node.IPFSAddDeterministic("300KiB", "dag-local-only-requires-offline", "--raw-leaves") + refs := dagRefs(node, root) + + require.GreaterOrEqual(t, len(refs), 2) + require.Equal(t, 0, node.RunIPFS("pin", "rm", root).ExitCode()) + require.Equal(t, 0, node.RunIPFS("block", "rm", refs[1]).ExitCode()) + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + cmd := exec.CommandContext(ctx, node.IPFSBin, "dag", "export", "--local-only", root) + cmd.Env = append(os.Environ(), "IPFS_PATH="+node.Dir) + cmd.Stdout = io.Discard + + err := cmd.Run() + + require.Error(t, err) // command should fail +} + +func TestDagImportPartialCAR(t *testing.T) { + t.Parallel() + node := harness.NewT(t).NewNode().Init().StartDaemon() + defer node.StopDaemon() + + root := node.IPFSAddDeterministic("300KiB", "dag-import-partial", "--raw-leaves") + refs := dagRefs(node, root) + require.GreaterOrEqual(t, len(refs), 2) + + require.Equal(t, 0, node.RunIPFS("pin", "rm", root).ExitCode()) + require.Equal(t, 0, node.RunIPFS("block", "rm", refs[1]).ExitCode()) + + partialCarPath := filepath.Join(node.Dir, "partial.car") + require.NoError(t, node.IPFSDagExport(root, partialCarPath, "--local-only", "--offline")) + + imp := harness.NewT(t).NewNode().Init().StartDaemon() + defer imp.StopDaemon() + partialCAR, err := os.Open(partialCarPath) + require.NoError(t, err) + defer partialCAR.Close() + require.NoError(t, imp.IPFSDagImport(partialCAR, root)) +} +func TestDagImportLocalOnlyPinRootsConflict(t *testing.T) { + t.Parallel() + node := harness.NewT(t).NewNode().Init().StartDaemon() + defer node.StopDaemon() + + r, err := os.Open(fixtureFile) + require.NoError(t, err) + defer r.Close() + + res := node.Runner.Run(harness.RunRequest{ + Path: node.IPFSBin, + Args: []string{"dag", "import", "--local-only", "--pin-roots"}, + CmdOpts: []harness.CmdOpt{harness.RunWithStdin(r)}, + }) + + require.Equal(t, 1, res.ExitCode()) + require.Error(t, res.Err) + + errOutput := res.Stderr.String() + + require.Contains(t, errOutput, "cannot pass both") + require.Contains(t, errOutput, "pin-roots") + require.Contains(t, errOutput, "local-only") +} diff --git a/test/cli/harness/ipfs.go b/test/cli/harness/ipfs.go index d7470b4e764..637b316867b 100644 --- a/test/cli/harness/ipfs.go +++ b/test/cli/harness/ipfs.go @@ -162,17 +162,19 @@ func (n *Node) IPFSDagImport(content io.Reader, cid string, args ...string) erro } // IPFSDagExport exports a DAG rooted at cid to a CAR file at carPath. -func (n *Node) IPFSDagExport(cid string, carPath string) error { - log.Debugf("node %d dag export of %s to %q", n.ID, cid, carPath) +func (n *Node) IPFSDagExport(cid string, carPath string, args ...string) error { + log.Debugf("node %d dag export of %s to %q with args: %v", n.ID, cid, carPath, args) car, err := os.Create(carPath) if err != nil { return err } defer car.Close() + fullArgs := append([]string{"dag", "export"}, args...) + fullArgs = append(fullArgs, cid) res := n.Runner.MustRun(RunRequest{ Path: n.IPFSBin, - Args: []string{"dag", "export", cid}, + Args: fullArgs, CmdOpts: []CmdOpt{RunWithStdout(car)}, }) return res.Err diff --git a/test/dependencies/go.mod b/test/dependencies/go.mod index 8bfa80cc98f..9a9fa579f26 100644 --- a/test/dependencies/go.mod +++ b/test/dependencies/go.mod @@ -149,7 +149,7 @@ require ( github.com/ipfs/go-metrics-interface v0.3.0 // indirect github.com/ipfs/go-unixfsnode v1.10.3 // indirect github.com/ipfs/kubo v0.31.0 // indirect - github.com/ipld/go-car/v2 v2.16.0 // indirect + github.com/ipld/go-car/v2 v2.16.1-0.20260306172652-7d2f4aceb070 // indirect github.com/ipld/go-codec-dagpb v1.7.0 // indirect github.com/ipld/go-ipld-prime v0.22.0 // indirect github.com/ipshipyard/p2p-forge v0.7.0 // indirect diff --git a/test/dependencies/go.sum b/test/dependencies/go.sum index b0fb90e97ab..5d950c9d9bd 100644 --- a/test/dependencies/go.sum +++ b/test/dependencies/go.sum @@ -500,8 +500,8 @@ github.com/ipfs/iptb v1.4.1 h1:faXd3TKGPswbHyZecqqg6UfbES7RDjTKQb+6VFPKDUo= github.com/ipfs/iptb v1.4.1/go.mod h1:nTsBMtVYFEu0FjC5DgrErnABm3OG9ruXkFXGJoTV5OA= github.com/ipfs/iptb-plugins v0.5.1 h1:11PNTNEt2+SFxjUcO5qpyCTXqDj6T8Tx9pU/G4ytCIQ= github.com/ipfs/iptb-plugins v0.5.1/go.mod h1:mscJAjRnu4g16QK6oUBn9RGpcp8ueJmLfmPxIG/At78= -github.com/ipld/go-car/v2 v2.16.0 h1:LWe0vmN/QcQmUU4tr34W5Nv5mNraW+G6jfN2s+ndBco= -github.com/ipld/go-car/v2 v2.16.0/go.mod h1:RqFGWN9ifcXVmCrTAVnfnxiWZk1+jIx67SYhenlmL34= +github.com/ipld/go-car/v2 v2.16.1-0.20260306172652-7d2f4aceb070 h1:aUJhNVqQZV3PhxQX/BPAkurYnsxBmMLx2a7YPsgd5NU= +github.com/ipld/go-car/v2 v2.16.1-0.20260306172652-7d2f4aceb070/go.mod h1:eN3661vGFqfxzIfPLrCQIdKcY9iIOCgZON1CedVtvwk= github.com/ipld/go-codec-dagpb v1.7.0 h1:hpuvQjCSVSLnTnHXn+QAMR0mLmb1gA6wl10LExo2Ts0= github.com/ipld/go-codec-dagpb v1.7.0/go.mod h1:rD3Zg+zub9ZnxcLwfol/OTQRVjaLzXypgy4UqHQvilM= github.com/ipld/go-ipld-prime v0.22.0 h1:YJhDhjEOvOYaqshd3b4atIWUoRg/rKrgmwCyUHwlbuY=