From caf0f954ae063be6002cddf4e0f3b7d4ac1ef3c7 Mon Sep 17 00:00:00 2001 From: Will Scott Date: Thu, 26 May 2022 16:59:04 +0200 Subject: [PATCH 1/6] complete reads of lazy nodes in walks --- v2/selective.go | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/v2/selective.go b/v2/selective.go index 22351e71..68629e74 100644 --- a/v2/selective.go +++ b/v2/selective.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "io" + "io/ioutil" "math" "os" @@ -12,6 +13,7 @@ import ( "github.com/ipld/go-car/v2/internal/carv1" "github.com/ipld/go-car/v2/internal/loader" ipld "github.com/ipld/go-ipld-prime" + "github.com/ipld/go-ipld-prime/datamodel" "github.com/ipld/go-ipld-prime/linking" cidlink "github.com/ipld/go-ipld-prime/linking/cid" "github.com/ipld/go-ipld-prime/node/basicnode" @@ -259,7 +261,17 @@ func traverse(ctx context.Context, ls *ipld.LinkSystem, root cid.Cid, s ipld.Nod if err != nil { return fmt.Errorf("root blk load failed: %s", err) } - err = progress.WalkMatching(rootNode, sel, func(_ traversal.Progress, _ ipld.Node) error { + err = progress.WalkMatching(rootNode, sel, func(_ traversal.Progress, node ipld.Node) error { + if lbn, ok := node.(datamodel.LargeBytesNode); ok { + s, err := lbn.AsLargeBytes() + if err != nil { + return err + } + _, err = io.Copy(ioutil.Discard, s) + if err != nil { + return err + } + } return nil }) if err != nil { From 3cebd852f924dfedab23dd4ae29cdbb4f5b12011 Mon Sep 17 00:00:00 2001 From: Will Scott Date: Fri, 27 May 2022 22:17:51 +0200 Subject: [PATCH 2/6] * add option to set prototype chooser in traversals * writer won't double-count replicated loads (maybe needs to respect option for this) --- v2/internal/loader/writing_loader.go | 38 ++++++++++++++++++---------- v2/options.go | 10 ++++++++ v2/selective.go | 23 ++++++++++++----- 3 files changed, 50 insertions(+), 21 deletions(-) diff --git a/v2/internal/loader/writing_loader.go b/v2/internal/loader/writing_loader.go index 13236f1c..b4d0e6ef 100644 --- a/v2/internal/loader/writing_loader.go +++ b/v2/internal/loader/writing_loader.go @@ -16,7 +16,7 @@ type writerOutput struct { w io.Writer size uint64 code multicodec.Code - rcrds []index.Record + rcrds map[cid.Cid]index.Record } func (w *writerOutput) Size() uint64 { @@ -28,7 +28,11 @@ func (w *writerOutput) Index() (index.Index, error) { if err != nil { return nil, err } - if err := idx.Load(w.rcrds); err != nil { + rcrds := make([]index.Record, 0, len(w.rcrds)) + for _, r := range w.rcrds { + rcrds = append(rcrds, r) + } + if err := idx.Load(rcrds); err != nil { return nil, err } @@ -63,17 +67,13 @@ func (w *writingReader) Read(p []byte) (int, error) { if _, err := cpy.WriteTo(w.wo.w); err != nil { return 0, err } - - // maybe write the index. - if w.wo.code != index.CarIndexNone { - _, c, err := cid.CidFromBytes([]byte(w.cid)) - if err != nil { - return 0, err - } - w.wo.rcrds = append(w.wo.rcrds, index.Record{ - Cid: c, - Offset: w.wo.size, - }) + _, c, err := cid.CidFromBytes([]byte(w.cid)) + if err != nil { + return 0, err + } + w.wo.rcrds[c] = index.Record{ + Cid: c, + Offset: w.wo.size, } w.wo.size += uint64(w.len) + uint64(len(size)+len(w.cid)) @@ -94,11 +94,21 @@ func TeeingLinkSystem(ls ipld.LinkSystem, w io.Writer, initialOffset uint64, ind w: w, size: initialOffset, code: indexCodec, - rcrds: make([]index.Record, 0), + rcrds: make(map[cid.Cid]index.Record), } tls := ls tls.StorageReadOpener = func(lc linking.LinkContext, l ipld.Link) (io.Reader, error) { + _, c, err := cid.CidFromBytes([]byte(l.Binary())) + if err != nil { + return nil, err + } + + // if we've already read this cid in this session, don't re-write it. + if _, ok := wo.rcrds[c]; ok { + return ls.StorageReadOpener(lc, l) + } + r, err := ls.StorageReadOpener(lc, l) if err != nil { return nil, err diff --git a/v2/options.go b/v2/options.go index 228b359b..d2923b21 100644 --- a/v2/options.go +++ b/v2/options.go @@ -4,6 +4,7 @@ import ( "math" "github.com/ipld/go-car/v2/index" + "github.com/ipld/go-ipld-prime/traversal" "github.com/multiformats/go-multicodec" ) @@ -40,6 +41,7 @@ type Options struct { BlockstoreUseWholeCIDs bool MaxTraversalLinks uint64 WriteAsCarV1 bool + TraversalPrototypeChooser traversal.LinkTargetNodePrototypeChooser } // ApplyOptions applies given opts and returns the resulting Options. @@ -118,3 +120,11 @@ func MaxIndexCidSize(s uint64) Option { o.MaxIndexCidSize = s } } + +// WithTraversalPrototypeChooser specifies the prototype chooser that should be used +// when performing traversals in writes from a linksystem. +func WithTraversalPrototypeChooser(t traversal.LinkTargetNodePrototypeChooser) Option { + return func(o *Options) { + o.TraversalPrototypeChooser = t + } +} diff --git a/v2/selective.go b/v2/selective.go index 68629e74..39bb5f91 100644 --- a/v2/selective.go +++ b/v2/selective.go @@ -238,14 +238,19 @@ func traverse(ctx context.Context, ls *ipld.LinkSystem, root cid.Cid, s ipld.Nod return err } + chooser := func(_ ipld.Link, _ linking.LinkContext) (ipld.NodePrototype, error) { + return basicnode.Prototype.Any, nil + } + if opts.TraversalPrototypeChooser != nil { + chooser = opts.TraversalPrototypeChooser + } + progress := traversal.Progress{ Cfg: &traversal.Config{ - Ctx: ctx, - LinkSystem: *ls, - LinkTargetNodePrototypeChooser: func(_ ipld.Link, _ linking.LinkContext) (ipld.NodePrototype, error) { - return basicnode.Prototype.Any, nil - }, - LinkVisitOnlyOnce: !opts.BlockstoreAllowDuplicatePuts, + Ctx: ctx, + LinkSystem: *ls, + LinkTargetNodePrototypeChooser: chooser, + LinkVisitOnlyOnce: !opts.BlockstoreAllowDuplicatePuts, }, } if opts.MaxTraversalLinks < math.MaxInt64 { @@ -257,7 +262,11 @@ func traverse(ctx context.Context, ls *ipld.LinkSystem, root cid.Cid, s ipld.Nod lnk := cidlink.Link{Cid: root} ls.TrustedStorage = true - rootNode, err := ls.Load(ipld.LinkContext{}, lnk, basicnode.Prototype.Any) + rp, err := chooser(lnk, ipld.LinkContext{}) + if err != nil { + return err + } + rootNode, err := ls.Load(ipld.LinkContext{}, lnk, rp) if err != nil { return fmt.Errorf("root blk load failed: %s", err) } From bbd97806bc7505da79921a890786aebaed293d24 Mon Sep 17 00:00:00 2001 From: Will Scott Date: Sun, 29 May 2022 06:25:14 +0200 Subject: [PATCH 3/6] add partial write test --- v2/go.mod | 8 +++++++- v2/go.sum | 29 ++++++++++++++++++++++++++++- v2/selective_test.go | 36 ++++++++++++++++++++++++++++++++++++ 3 files changed, 71 insertions(+), 2 deletions(-) diff --git a/v2/go.mod b/v2/go.mod index 5afda64b..ff3ef47b 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -9,10 +9,11 @@ require ( github.com/ipfs/go-ipld-cbor v0.0.5 github.com/ipfs/go-ipld-format v0.3.0 github.com/ipfs/go-merkledag v0.6.0 + github.com/ipfs/go-unixfsnode v1.4.0 github.com/ipld/go-codec-dagpb v1.3.1 github.com/ipld/go-ipld-prime v0.16.0 github.com/ipld/go-ipld-prime/storage/bsadapter v0.0.0-20211210234204-ce2a1c70cd73 - github.com/multiformats/go-multicodec v0.3.1-0.20210902112759-1539a079fd61 + github.com/multiformats/go-multicodec v0.3.1-0.20211210143421-a526f306ed2c github.com/multiformats/go-multihash v0.1.0 github.com/multiformats/go-varint v0.0.6 github.com/petar/GoLLRB v0.0.0-20210522233825-ae3b015fd3e9 @@ -22,13 +23,16 @@ require ( ) require ( + github.com/Stebalien/go-bitfield v0.0.1 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/google/uuid v1.2.0 // indirect github.com/hashicorp/golang-lru v0.5.4 // indirect github.com/ipfs/bbloom v0.0.4 // indirect + github.com/ipfs/go-bitfield v1.0.0 // indirect github.com/ipfs/go-blockservice v0.3.0 // indirect github.com/ipfs/go-datastore v0.5.0 // indirect + github.com/ipfs/go-ipfs-chunker v0.0.1 // indirect github.com/ipfs/go-ipfs-ds-help v1.1.0 // indirect github.com/ipfs/go-ipfs-exchange-interface v0.1.0 // indirect github.com/ipfs/go-ipfs-exchange-offline v0.2.0 // indirect @@ -40,6 +44,7 @@ require ( github.com/ipfs/go-verifcid v0.0.1 // indirect github.com/jbenet/goprocess v0.1.4 // indirect github.com/klauspost/cpuid/v2 v2.0.9 // indirect + github.com/libp2p/go-buffer-pool v0.0.2 // indirect github.com/mattn/go-isatty v0.0.13 // indirect github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1 // indirect github.com/minio/sha256-simd v1.0.0 // indirect @@ -52,6 +57,7 @@ require ( github.com/polydawn/refmt v0.0.0-20201211092308-30ac6d18308e // indirect github.com/spaolacci/murmur3 v1.1.0 // indirect github.com/whyrusleeping/cbor-gen v0.0.0-20200123233031-1cdf64d27158 // indirect + github.com/whyrusleeping/chunker v0.0.0-20181014151217-fe64bd25879f // indirect go.uber.org/atomic v1.7.0 // indirect go.uber.org/multierr v1.7.0 // indirect go.uber.org/zap v1.16.0 // indirect diff --git a/v2/go.sum b/v2/go.sum index 1930c938..4273ced4 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -18,6 +18,8 @@ github.com/Kubuxu/go-os-helper v0.0.1/go.mod h1:N8B+I7vPCT80IcP58r50u4+gEEcsZETF github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo= github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI= +github.com/Stebalien/go-bitfield v0.0.1 h1:X3kbSSPUaJK60wV2hjOPZwmpljr6VGCqdq4cBLhbQBo= +github.com/Stebalien/go-bitfield v0.0.1/go.mod h1:GNjFpasyUVkHMsfEOk8EFLJ9syQ6SI+XWrX9Wf2XH0s= github.com/VividCortex/gohistogram v1.0.0/go.mod h1:Pf5mBqqDxYaXu3hDrrU+w6nw50o/4+TcAqDqk/vUH7g= github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII= github.com/afex/hystrix-go v0.0.0-20180502004556-fa1af6a1f4f5/go.mod h1:SkGFH1ia65gfNATL8TAiHDNxPzPdmEL5uirI2Uyuz6c= @@ -26,6 +28,7 @@ github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuy github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= +github.com/alecthomas/units v0.0.0-20210927113745-59d0afb8317a/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE= github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c= github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= github.com/apache/thrift v0.13.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= @@ -120,6 +123,7 @@ github.com/francoispqt/gojay v1.2.13/go.mod h1:ehT5mTG4ua4581f1++1WLG0vPdaA9HaiD github.com/franela/goblin v0.0.0-20200105215937-c9ffbefa60db/go.mod h1:7dvUGVsVBjqR7JHJk0brhHOZYGmfBYOrK0ZhYMEtBr4= github.com/franela/goreq v0.0.0-20171204163338-bcd34c9993f8/go.mod h1:ZhphrRTfi2rbfLwlschooIH4+wKKDR4Pdxhh+TRoA20= github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k= +github.com/frankban/quicktest v1.14.0/go.mod h1:NeW+ay9A/U67EYXNFA1nPE8e/tnQv/09mUdL/ijj8og= github.com/frankban/quicktest v1.14.2 h1:SPb1KFFmM+ybpEjPUhCCkZOM5xlovT5UbrMvWnXyBns= github.com/frankban/quicktest v1.14.2/go.mod h1:mgiwOwqx65TmIk1wJ6Q7wvnVMocbUorkibMOrVTHZps= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= @@ -181,6 +185,7 @@ github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.7 h1:81/ik6ipDQS2aGcBfIN5dHDB36BwrStyeAQquSYCV4o= github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE= github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ= @@ -246,11 +251,15 @@ github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANyt github.com/influxdata/influxdb1-client v0.0.0-20191209144304-8bf82d3c094d/go.mod h1:qj24IKcXYK6Iy9ceXlo3Tc+vtHo9lIhSX5JddghvEPo= github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs= github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0= +github.com/ipfs/go-bitfield v1.0.0 h1:y/XHm2GEmD9wKngheWNNCNL0pzrWXZwCdQGv1ikXknQ= +github.com/ipfs/go-bitfield v1.0.0/go.mod h1:N/UiujQy+K+ceU1EF5EkVd1TNqevLrCQMIcAEPrdtus= +github.com/ipfs/go-bitswap v0.5.1/go.mod h1:P+ckC87ri1xFLvk74NlXdP0Kj9RmWAh4+H78sC6Qopo= github.com/ipfs/go-bitswap v0.6.0 h1:f2rc6GZtoSFhEIzQmddgGiel9xntj02Dg0ZNf2hSC+w= github.com/ipfs/go-bitswap v0.6.0/go.mod h1:Hj3ZXdOC5wBJvENtdqsixmzzRukqd8EHLxZLZc3mzRA= github.com/ipfs/go-block-format v0.0.2/go.mod h1:AWR46JfpcObNfg3ok2JHDUfdiHRgWhJgCQF+KIgOPJY= github.com/ipfs/go-block-format v0.0.3 h1:r8t66QstRp/pd/or4dpnbVfXT5Gt7lOqRvC+/dDTpMc= github.com/ipfs/go-block-format v0.0.3/go.mod h1:4LmD4ZUw0mhO+JSKdpWwrzATiEfM7WWgQ8H5l6P8MVk= +github.com/ipfs/go-blockservice v0.2.1/go.mod h1:k6SiwmgyYgs4M/qt+ww6amPeUH9EISLRBnvUurKJhi8= github.com/ipfs/go-blockservice v0.3.0 h1:cDgcZ+0P0Ih3sl8+qjFr2sVaMdysg/YZpLj5WJ8kiiw= github.com/ipfs/go-blockservice v0.3.0/go.mod h1:P5ppi8IHDC7O+pA0AlGTF09jruB2h+oP3wVVaZl8sfk= github.com/ipfs/go-cid v0.0.1/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM= @@ -279,11 +288,14 @@ github.com/ipfs/go-ds-badger v0.2.3/go.mod h1:pEYw0rgg3FIrywKKnL+Snr+w/LjJZVMTBR github.com/ipfs/go-ds-leveldb v0.0.1/go.mod h1:feO8V3kubwsEF22n0YRQCffeb79OOYIykR4L04tMOYc= github.com/ipfs/go-ds-leveldb v0.4.1/go.mod h1:jpbku/YqBSsBc1qgME8BkWS4AxzF2cEu1Ii2r79Hh9s= github.com/ipfs/go-ds-leveldb v0.4.2/go.mod h1:jpbku/YqBSsBc1qgME8BkWS4AxzF2cEu1Ii2r79Hh9s= +github.com/ipfs/go-ipfs-blockstore v0.2.1/go.mod h1:jGesd8EtCM3/zPgx+qr0/feTXGUeRai6adgwC+Q+JvE= github.com/ipfs/go-ipfs-blockstore v1.1.2/go.mod h1:w51tNR9y5+QXB0wkNcHt4O2aSZjTdqaEWaQdSxEyUOY= github.com/ipfs/go-ipfs-blockstore v1.2.0 h1:n3WTeJ4LdICWs/0VSfjHrlqpPpl6MZ+ySd3j8qz0ykw= github.com/ipfs/go-ipfs-blockstore v1.2.0/go.mod h1:eh8eTFLiINYNSNawfZOC7HOxNTxpB1PFuA5E1m/7exE= github.com/ipfs/go-ipfs-blocksutil v0.0.1 h1:Eh/H4pc1hsvhzsQoMEP3Bke/aW5P5rVM1IWFJMcGIPQ= github.com/ipfs/go-ipfs-blocksutil v0.0.1/go.mod h1:Yq4M86uIOmxmGPUHv/uI7uKqZNtLb449gwKqXjIsnRk= +github.com/ipfs/go-ipfs-chunker v0.0.1 h1:cHUUxKFQ99pozdahi+uSC/3Y6HeRpi9oTeUHbE27SEw= +github.com/ipfs/go-ipfs-chunker v0.0.1/go.mod h1:tWewYK0we3+rMbOh7pPFGDyypCtvGcBFymgY4rSDLAw= github.com/ipfs/go-ipfs-delay v0.0.0-20181109222059-70721b86a9a8/go.mod h1:8SP1YXK1M1kXuc4KJZINY3TQQ03J2rwBG9QfXmbRPrw= github.com/ipfs/go-ipfs-delay v0.0.1 h1:r/UXYyRcddO6thwOnhiznIAiSvxMECGgtv35Xs1IeRQ= github.com/ipfs/go-ipfs-delay v0.0.1/go.mod h1:8SP1YXK1M1kXuc4KJZINY3TQQ03J2rwBG9QfXmbRPrw= @@ -292,8 +304,11 @@ github.com/ipfs/go-ipfs-ds-help v1.1.0 h1:yLE2w9RAsl31LtfMt91tRZcrx+e61O5mDxFRR9 github.com/ipfs/go-ipfs-ds-help v1.1.0/go.mod h1:YR5+6EaebOhfcqVCyqemItCLthrpVNot+rsOU/5IatU= github.com/ipfs/go-ipfs-exchange-interface v0.1.0 h1:TiMekCrOGQuWYtZO3mf4YJXDIdNgnKWZ9IE3fGlnWfo= github.com/ipfs/go-ipfs-exchange-interface v0.1.0/go.mod h1:ych7WPlyHqFvCi/uQI48zLZuAWVP5iTQPXEfVaw5WEI= +github.com/ipfs/go-ipfs-exchange-offline v0.1.1/go.mod h1:vTiBRIbzSwDD0OWm+i3xeT0mO7jG2cbJYatp3HPk5XY= github.com/ipfs/go-ipfs-exchange-offline v0.2.0 h1:2PF4o4A7W656rC0RxuhUace997FTcDTcIQ6NoEtyjAI= github.com/ipfs/go-ipfs-exchange-offline v0.2.0/go.mod h1:HjwBeW0dvZvfOMwDP0TSKXIHf2s+ksdP4E3MLDRtLKY= +github.com/ipfs/go-ipfs-files v0.0.3/go.mod h1:INEFm0LL2LWXBhNJ2PMIIb2w45hpXgPjNoE7yA8Y1d4= +github.com/ipfs/go-ipfs-posinfo v0.0.1/go.mod h1:SwyeVP+jCwiDu0C313l/8jg6ZxM0qqtlt2a0vILTc1A= github.com/ipfs/go-ipfs-pq v0.0.2 h1:e1vOOW6MuOwG2lqxcLA+wEn93i/9laCY8sXAw76jFOY= github.com/ipfs/go-ipfs-pq v0.0.2/go.mod h1:LWIqQpqfRG3fNc5XsnIhz/wQ2XXGyugQwls7BgUmUfY= github.com/ipfs/go-ipfs-routing v0.2.1 h1:E+whHWhJkdN9YeoHZNj5itzc+OR292AJ2uE9FFiW0BY= @@ -322,19 +337,26 @@ github.com/ipfs/go-log/v2 v2.1.1/go.mod h1:2v2nsGfZsvvAJz13SyFzf9ObaqwHiHxsPLEHn github.com/ipfs/go-log/v2 v2.1.3/go.mod h1:/8d0SH3Su5Ooc31QlL1WysJhvyOTDCjcCZ9Axpmri6g= github.com/ipfs/go-log/v2 v2.3.0 h1:31Re/cPqFHpsRHgyVwjWADPoF0otB1WrjTy8ZFYwEZU= github.com/ipfs/go-log/v2 v2.3.0/go.mod h1:QqGoj30OTpnKaG/LKTGTxoP2mmQtjVMEnK72gynbe/g= +github.com/ipfs/go-merkledag v0.5.1/go.mod h1:cLMZXx8J08idkp5+id62iVftUQV+HlYJ3PIhDfZsjA4= github.com/ipfs/go-merkledag v0.6.0 h1:oV5WT2321tS4YQVOPgIrWHvJ0lJobRTerU+i9nmUCuA= github.com/ipfs/go-merkledag v0.6.0/go.mod h1:9HSEwRd5sV+lbykiYP+2NC/3o6MZbKNaa4hfNcH5iH0= github.com/ipfs/go-metrics-interface v0.0.1 h1:j+cpbjYvu4R8zbleSs36gvB7jR+wsL2fGD6n0jO4kdg= github.com/ipfs/go-metrics-interface v0.0.1/go.mod h1:6s6euYU4zowdslK0GKHmqaIZ3j/b/tL7HTWtJ4VPgWY= github.com/ipfs/go-peertaskqueue v0.7.0 h1:VyO6G4sbzX80K58N60cCaHsSsypbUNs1GjO5seGNsQ0= github.com/ipfs/go-peertaskqueue v0.7.0/go.mod h1:M/akTIE/z1jGNXMU7kFB4TeSEFvj68ow0Rrb04donIU= +github.com/ipfs/go-unixfs v0.3.1 h1:LrfED0OGfG98ZEegO4/xiprx2O+yS+krCMQSp7zLVv8= +github.com/ipfs/go-unixfs v0.3.1/go.mod h1:h4qfQYzghiIc8ZNFKiLMFWOTzrWIAtzYQ59W/pCFf1o= +github.com/ipfs/go-unixfsnode v1.4.0 h1:9BUxHBXrbNi8mWHc6j+5C580WJqtVw9uoeEKn4tMhwA= +github.com/ipfs/go-unixfsnode v1.4.0/go.mod h1:qc7YFFZ8tABc58p62HnIYbUMwj9chhUuFWmxSokfePo= github.com/ipfs/go-verifcid v0.0.1 h1:m2HI7zIuR5TFyQ1b79Da5N9dnnCP1vcu2QqawmWlK2E= github.com/ipfs/go-verifcid v0.0.1/go.mod h1:5Hrva5KBeIog4A+UpqlaIU+DEstipcJYQQZc0g37pY0= +github.com/ipld/go-car/v2 v2.1.1/go.mod h1:+2Yvf0Z3wzkv7NeI69i8tuZ+ft7jyjPYIWZzeVNeFcI= github.com/ipld/go-codec-dagpb v1.3.0/go.mod h1:ga4JTU3abYApDC3pZ00BC2RSvC3qfBb9MSJkMLSwnhA= github.com/ipld/go-codec-dagpb v1.3.1 h1:yVNlWRQexCa54ln3MSIiUN++ItH7pdhBFhh0hSgZu1w= github.com/ipld/go-codec-dagpb v1.3.1/go.mod h1:ErNNglIi5KMur/MfFE/svtgQthzVvf+43MrzLbpcIZY= github.com/ipld/go-ipld-prime v0.9.1-0.20210324083106-dc342a9917db/go.mod h1:KvBLMr4PX1gWptgkzRjVZCrLmSGcZCb/jioOQwCqZN8= github.com/ipld/go-ipld-prime v0.11.0/go.mod h1:+WIAkokurHmZ/KwzDOMUuoeJgaRQktHtEaLglS3ZeV8= +github.com/ipld/go-ipld-prime v0.14.0/go.mod h1:9ASQLwUFLptCov6lIYc70GRB4V7UTyLD0IJtrDJe6ZM= github.com/ipld/go-ipld-prime v0.16.0 h1:RS5hhjB/mcpeEPJvfyj0qbOj/QL+/j05heZ0qa97dVo= github.com/ipld/go-ipld-prime v0.16.0/go.mod h1:axSCuOCBPqrH+gvXr2w9uAOulJqBPhHPT2PjoiiU1qA= github.com/ipld/go-ipld-prime/storage/bsadapter v0.0.0-20211210234204-ce2a1c70cd73 h1:TsyATB2ZRRQGTwafJdgEUQkmjOExRV0DNokcihZxbnQ= @@ -659,8 +681,9 @@ github.com/multiformats/go-multibase v0.0.1/go.mod h1:bja2MqRZ3ggyXtZSEDKpl0uO/g github.com/multiformats/go-multibase v0.0.3 h1:l/B6bJDQjvQ5G52jw4QGSYeOTZoAwIO77RblWplfIqk= github.com/multiformats/go-multibase v0.0.3/go.mod h1:5+1R4eQrT3PkYZ24C3W2Ue2tPwIdYQD509ZjSb5y9Oc= github.com/multiformats/go-multicodec v0.3.0/go.mod h1:qGGaQmioCDh+TeFOnxrbU0DaIPw8yFgAZgFG0V7p1qQ= -github.com/multiformats/go-multicodec v0.3.1-0.20210902112759-1539a079fd61 h1:ZrUuMKNgJ52qHPoQ+bx0h0uBfcWmN7Px+4uKSZeesiI= github.com/multiformats/go-multicodec v0.3.1-0.20210902112759-1539a079fd61/go.mod h1:1Hj/eHRaVWSXiSNNfcEPcwZleTmdNP81xlxDLnWU9GQ= +github.com/multiformats/go-multicodec v0.3.1-0.20211210143421-a526f306ed2c h1:VyANTtZ0wsx0IAZnCZhfMmAmfUyzJq/5JQi2hHOtKS0= +github.com/multiformats/go-multicodec v0.3.1-0.20211210143421-a526f306ed2c/go.mod h1:1Hj/eHRaVWSXiSNNfcEPcwZleTmdNP81xlxDLnWU9GQ= github.com/multiformats/go-multihash v0.0.1/go.mod h1:w/5tugSrLEbWqlcgJabL3oHFKTwfvkofsjW2Qa1ct4U= github.com/multiformats/go-multihash v0.0.5/go.mod h1:lt/HCbqlQwlPBz7lv0sQCdtfcMtlJvakRUn/0Ual8po= github.com/multiformats/go-multihash v0.0.8/go.mod h1:YSLudS+Pi8NHE7o6tb3D8vrpKa63epEDmG8nTduyAew= @@ -860,6 +883,8 @@ github.com/whyrusleeping/cbor v0.0.0-20171005072247-63513f603b11 h1:5HZfQkwe0mIf github.com/whyrusleeping/cbor v0.0.0-20171005072247-63513f603b11/go.mod h1:Wlo/SzPmxVp6vXpGt/zaXhHH0fn4IxgqZc82aKg6bpQ= github.com/whyrusleeping/cbor-gen v0.0.0-20200123233031-1cdf64d27158 h1:WXhVOwj2USAXB5oMDwRl3piOux2XMV9TANaYxXHdkoE= github.com/whyrusleeping/cbor-gen v0.0.0-20200123233031-1cdf64d27158/go.mod h1:Xj/M2wWU+QdTdRbu/L/1dIZY8/Wb2K9pAhtroQuxJJI= +github.com/whyrusleeping/chunker v0.0.0-20181014151217-fe64bd25879f h1:jQa4QT2UP9WYv2nzyawpKMOCl+Z/jW7djv2/J50lj9E= +github.com/whyrusleeping/chunker v0.0.0-20181014151217-fe64bd25879f/go.mod h1:p9UJB6dDgdPgMJZs7UjUOdulKyRr9fqkS+6JKAInPy8= github.com/whyrusleeping/go-keyspace v0.0.0-20160322163242-5b898ac5add1/go.mod h1:8UvriyWtv5Q5EOgjHaSseUEdkQfvwFv1I/In/O2M9gc= github.com/whyrusleeping/go-logging v0.0.0-20170515211332-0457bb6b88fc/go.mod h1:bopw91TMyo8J3tvftk8xmU2kPmlrt4nScJQZU2hE5EM= github.com/whyrusleeping/go-logging v0.0.1/go.mod h1:lDPYj54zutzG1XYfHAhcc7oNXEburHQBn+Iqd4yS4vE= @@ -1009,6 +1034,7 @@ golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -1024,6 +1050,7 @@ golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20190219092855-153ac476189d/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190228124157-a34e9553db1e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190302025703-b6889370fb10/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190316082340-a2f829d7f35f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190405154228-4b34438f7a67/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= diff --git a/v2/selective_test.go b/v2/selective_test.go index f2bba6f8..7d89bcae 100644 --- a/v2/selective_test.go +++ b/v2/selective_test.go @@ -7,10 +7,18 @@ import ( "path" "testing" + "github.com/ipfs/go-cid" + "github.com/ipfs/go-unixfsnode" + "github.com/ipfs/go-unixfsnode/data/builder" "github.com/ipld/go-car/v2" "github.com/ipld/go-car/v2/blockstore" + dagpb "github.com/ipld/go-codec-dagpb" + "github.com/ipld/go-ipld-prime/datamodel" + "github.com/ipld/go-ipld-prime/linking" cidlink "github.com/ipld/go-ipld-prime/linking/cid" + basicnode "github.com/ipld/go-ipld-prime/node/basic" "github.com/ipld/go-ipld-prime/storage/bsadapter" + sb "github.com/ipld/go-ipld-prime/traversal/selector/builder" selectorparse "github.com/ipld/go-ipld-prime/traversal/selector/parse" "github.com/stretchr/testify/require" @@ -81,3 +89,31 @@ func TestV1Traversal(t *testing.T) { fa, _ := os.Stat("testdata/sample-v1.car") require.Equal(t, fa.Size(), int64(n)) } + +func TestPartialTraversal(t *testing.T) { + store := cidlink.Memory{Bag: make(map[string][]byte)} + ls := cidlink.DefaultLinkSystem() + ls.StorageReadOpener = store.OpenRead + ls.StorageWriteOpener = store.OpenWrite + unixfsnode.AddUnixFSReificationToLinkSystem(&ls) + + // write a unixfs file. + initBuf := bytes.Buffer{} + _, _ = initBuf.Write(make([]byte, 1000000)) + rt, _, err := builder.BuildUnixFSFile(&initBuf, "", &ls) + require.NoError(t, err) + + // read a subset of the file. + _, rts, err := cid.CidFromBytes([]byte(rt.Binary())) + ssb := sb.NewSelectorSpecBuilder(basicnode.Prototype.Any) + sel := ssb.ExploreInterpretAs("unixfs", ssb.MatcherSubset(0, 256*1000)) + buf := bytes.Buffer{} + chooser := dagpb.AddSupportToChooser(func(l datamodel.Link, lc linking.LinkContext) (datamodel.NodePrototype, error) { + return basicnode.Prototype.Any, nil + }) + _, err = car.TraverseV1(context.Background(), &ls, rts, sel.Node(), &buf, car.WithTraversalPrototypeChooser(chooser)) + require.NoError(t, err) + + fb := len(buf.Bytes()) + require.Less(t, fb, 1000000) +} From c37e919f33a941e73b6d6e0f063d9391d88ca6cd Mon Sep 17 00:00:00 2001 From: Will Scott Date: Sun, 29 May 2022 06:29:13 +0200 Subject: [PATCH 4/6] fix static checks --- v2/selective_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/v2/selective_test.go b/v2/selective_test.go index 7d89bcae..cd65e6fb 100644 --- a/v2/selective_test.go +++ b/v2/selective_test.go @@ -22,7 +22,6 @@ import ( selectorparse "github.com/ipld/go-ipld-prime/traversal/selector/parse" "github.com/stretchr/testify/require" - _ "github.com/ipld/go-codec-dagpb" _ "github.com/ipld/go-ipld-prime/codec/dagcbor" _ "github.com/ipld/go-ipld-prime/codec/raw" ) @@ -105,6 +104,7 @@ func TestPartialTraversal(t *testing.T) { // read a subset of the file. _, rts, err := cid.CidFromBytes([]byte(rt.Binary())) + require.NoError(t, err) ssb := sb.NewSelectorSpecBuilder(basicnode.Prototype.Any) sel := ssb.ExploreInterpretAs("unixfs", ssb.MatcherSubset(0, 256*1000)) buf := bytes.Buffer{} From 88b2295cd45dbb38a9be9695276ca2c26fc2dcb9 Mon Sep 17 00:00:00 2001 From: Will Scott Date: Mon, 30 May 2022 10:17:59 +0200 Subject: [PATCH 5/6] stronger test --- v2/selective_test.go | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/v2/selective_test.go b/v2/selective_test.go index cd65e6fb..924351d4 100644 --- a/v2/selective_test.go +++ b/v2/selective_test.go @@ -3,10 +3,12 @@ package car_test import ( "bytes" "context" + "io" "os" "path" "testing" + blocks "github.com/ipfs/go-block-format" "github.com/ipfs/go-cid" "github.com/ipfs/go-unixfsnode" "github.com/ipfs/go-unixfsnode/data/builder" @@ -116,4 +118,20 @@ func TestPartialTraversal(t *testing.T) { fb := len(buf.Bytes()) require.Less(t, fb, 1000000) + + loaded, err := car.NewBlockReader(&buf) + require.NoError(t, err) + fnd := make(map[cid.Cid]struct{}) + var b blocks.Block + for err == nil { + b, err = loaded.Next() + if err == io.EOF { + break + } + if _, ok := fnd[b.Cid()]; ok { + require.Fail(t, "duplicate block present", b.Cid()) + } + fnd[b.Cid()] = struct{}{} + } + require.Equal(t, 2, len(fnd)) } From 14256034d0067385c00356478d6ec3325d631ce2 Mon Sep 17 00:00:00 2001 From: Will Scott Date: Tue, 31 May 2022 08:51:35 +0200 Subject: [PATCH 6/6] bump go-multicodec --- v2/go.mod | 2 +- v2/go.sum | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/v2/go.mod b/v2/go.mod index ff3ef47b..b2686a7c 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -13,7 +13,7 @@ require ( github.com/ipld/go-codec-dagpb v1.3.1 github.com/ipld/go-ipld-prime v0.16.0 github.com/ipld/go-ipld-prime/storage/bsadapter v0.0.0-20211210234204-ce2a1c70cd73 - github.com/multiformats/go-multicodec v0.3.1-0.20211210143421-a526f306ed2c + github.com/multiformats/go-multicodec v0.5.0 github.com/multiformats/go-multihash v0.1.0 github.com/multiformats/go-varint v0.0.6 github.com/petar/GoLLRB v0.0.0-20210522233825-ae3b015fd3e9 diff --git a/v2/go.sum b/v2/go.sum index 4273ced4..1b7eb9f9 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -682,8 +682,9 @@ github.com/multiformats/go-multibase v0.0.3 h1:l/B6bJDQjvQ5G52jw4QGSYeOTZoAwIO77 github.com/multiformats/go-multibase v0.0.3/go.mod h1:5+1R4eQrT3PkYZ24C3W2Ue2tPwIdYQD509ZjSb5y9Oc= github.com/multiformats/go-multicodec v0.3.0/go.mod h1:qGGaQmioCDh+TeFOnxrbU0DaIPw8yFgAZgFG0V7p1qQ= github.com/multiformats/go-multicodec v0.3.1-0.20210902112759-1539a079fd61/go.mod h1:1Hj/eHRaVWSXiSNNfcEPcwZleTmdNP81xlxDLnWU9GQ= -github.com/multiformats/go-multicodec v0.3.1-0.20211210143421-a526f306ed2c h1:VyANTtZ0wsx0IAZnCZhfMmAmfUyzJq/5JQi2hHOtKS0= github.com/multiformats/go-multicodec v0.3.1-0.20211210143421-a526f306ed2c/go.mod h1:1Hj/eHRaVWSXiSNNfcEPcwZleTmdNP81xlxDLnWU9GQ= +github.com/multiformats/go-multicodec v0.5.0 h1:EgU6cBe/D7WRwQb1KmnBvU7lrcFGMggZVTPtOW9dDHs= +github.com/multiformats/go-multicodec v0.5.0/go.mod h1:DiY2HFaEp5EhEXb/iYzVAunmyX/aSFMxq2KMKfWEues= github.com/multiformats/go-multihash v0.0.1/go.mod h1:w/5tugSrLEbWqlcgJabL3oHFKTwfvkofsjW2Qa1ct4U= github.com/multiformats/go-multihash v0.0.5/go.mod h1:lt/HCbqlQwlPBz7lv0sQCdtfcMtlJvakRUn/0Ual8po= github.com/multiformats/go-multihash v0.0.8/go.mod h1:YSLudS+Pi8NHE7o6tb3D8vrpKa63epEDmG8nTduyAew=