From bc4f854ade2c5713bb9bba2039d3ee8fc03c6441 Mon Sep 17 00:00:00 2001 From: Jorropo Date: Wed, 6 Jul 2022 15:05:14 +0200 Subject: [PATCH 1/3] chore: update go-car (cherry picked from commit cbf5fdae5517b06491deba2d3cc31b7ff41451fe) --- core/commands/dag/export.go | 6 +++--- core/corehttp/gateway_handler_car.go | 6 +++--- go.mod | 6 +++--- go.sum | 12 ++++++------ 4 files changed, 15 insertions(+), 15 deletions(-) diff --git a/core/commands/dag/export.go b/core/commands/dag/export.go index 530bf4d5788..8de39c5326f 100644 --- a/core/commands/dag/export.go +++ b/core/commands/dag/export.go @@ -134,12 +134,12 @@ func finishCLIExport(res cmds.Response, re cmds.ResponseEmitter) error { } } +// FIXME(@Jorropo): https://github.com/ipld/go-car/issues/315 type dagStore struct { dag iface.APIDagService ctx context.Context } -func (ds dagStore) Get(c cid.Cid) (blocks.Block, error) { - obj, err := ds.dag.Get(ds.ctx, c) - return obj, err +func (ds dagStore) Get(_ context.Context, c cid.Cid) (blocks.Block, error) { + return ds.dag.Get(ds.ctx, c) } diff --git a/core/corehttp/gateway_handler_car.go b/core/corehttp/gateway_handler_car.go index 1958088706e..5294d659840 100644 --- a/core/corehttp/gateway_handler_car.go +++ b/core/corehttp/gateway_handler_car.go @@ -81,12 +81,12 @@ func (i *gatewayHandler) serveCAR(ctx context.Context, w http.ResponseWriter, r i.carStreamGetMetric.WithLabelValues(contentPath.Namespace()).Observe(time.Since(begin).Seconds()) } +// FIXME(@Jorropo): https://github.com/ipld/go-car/issues/315 type dagStore struct { dag coreiface.APIDagService ctx context.Context } -func (ds dagStore) Get(c cid.Cid) (blocks.Block, error) { - obj, err := ds.dag.Get(ds.ctx, c) - return obj, err +func (ds dagStore) Get(_ context.Context, c cid.Cid) (blocks.Block, error) { + return ds.dag.Get(ds.ctx, c) } diff --git a/go.mod b/go.mod index bcdd9069ec7..fd41e737e8d 100644 --- a/go.mod +++ b/go.mod @@ -60,8 +60,8 @@ require ( github.com/ipfs/go-verifcid v0.0.1 github.com/ipfs/interface-go-ipfs-core v0.7.0 github.com/ipfs/tar-utils v0.0.2 - github.com/ipld/go-car v0.3.2 - github.com/ipld/go-car/v2 v2.1.1 + github.com/ipld/go-car v0.4.0 + github.com/ipld/go-car/v2 v2.4.0 github.com/ipld/go-codec-dagpb v1.4.0 github.com/ipld/go-ipld-prime v0.16.0 github.com/jbenet/go-random v0.0.0-20190219211222-123a90aedc0c @@ -96,7 +96,7 @@ require ( github.com/multiformats/go-multiaddr v0.5.0 github.com/multiformats/go-multiaddr-dns v0.3.1 github.com/multiformats/go-multibase v0.0.3 - github.com/multiformats/go-multicodec v0.4.1 + github.com/multiformats/go-multicodec v0.5.0 github.com/multiformats/go-multihash v0.1.0 github.com/opentracing/opentracing-go v1.2.0 github.com/pkg/errors v0.9.1 diff --git a/go.sum b/go.sum index 09757be3eb4..9235d48947d 100644 --- a/go.sum +++ b/go.sum @@ -646,11 +646,11 @@ github.com/ipfs/interface-go-ipfs-core v0.7.0 h1:7tb+2upz8oCcjIyjo1atdMk+P+u7wPm github.com/ipfs/interface-go-ipfs-core v0.7.0/go.mod h1:lF27E/nnSPbylPqKVXGZghal2hzifs3MmjyiEjnc9FY= github.com/ipfs/tar-utils v0.0.2 h1:UNgHB4x/PPzbMkmJi+7EqC9LNMPDztOVSnx1HAqSNg4= github.com/ipfs/tar-utils v0.0.2/go.mod h1:4qlnRWgTVljIMhSG2SqRYn66NT+3wrv/kZt9V+eqxDM= -github.com/ipld/go-car v0.3.2 h1:V9wt/80FNfbMRWSD98W5br6fyjUAyVgI2lDOTZX16Lg= -github.com/ipld/go-car v0.3.2/go.mod h1:WEjynkVt04dr0GwJhry0KlaTeSDEiEYyMPOxDBQ17KE= -github.com/ipld/go-car/v2 v2.1.1 h1:saaKz4nC0AdfCGHLYKeXLGn8ivoPC54fyS55uyOLKwA= +github.com/ipld/go-car v0.4.0 h1:U6W7F1aKF/OJMHovnOVdst2cpQE5GhmHibQkAixgNcQ= +github.com/ipld/go-car v0.4.0/go.mod h1:Uslcn4O9cBKK9wqHm/cLTFacg6RAPv6LZx2mxd2Ypl4= github.com/ipld/go-car/v2 v2.1.1/go.mod h1:+2Yvf0Z3wzkv7NeI69i8tuZ+ft7jyjPYIWZzeVNeFcI= -github.com/ipld/go-codec-dagpb v1.2.0/go.mod h1:6nBN7X7h8EOsEejZGqC7tej5drsdBAXbMHyBT+Fne5s= +github.com/ipld/go-car/v2 v2.4.0 h1:8jI6/iKlyLqRZzLz31jFWTqKvslaVzFsin305sOuqNQ= +github.com/ipld/go-car/v2 v2.4.0/go.mod h1:zjpRf0Jew9gHqSvjsKVyoq9OY9SWoEKdYCQUKVaaPT0= github.com/ipld/go-codec-dagpb v1.3.0/go.mod h1:ga4JTU3abYApDC3pZ00BC2RSvC3qfBb9MSJkMLSwnhA= github.com/ipld/go-codec-dagpb v1.3.1/go.mod h1:ErNNglIi5KMur/MfFE/svtgQthzVvf+43MrzLbpcIZY= github.com/ipld/go-codec-dagpb v1.4.0 h1:VqADPIFng8G4vz5EQytmmcx/2gEgOHfBuw/kIuCgDAY= @@ -658,7 +658,6 @@ github.com/ipld/go-codec-dagpb v1.4.0/go.mod h1:ErNNglIi5KMur/MfFE/svtgQthzVvf+4 github.com/ipld/go-ipld-prime v0.9.0/go.mod h1:KvBLMr4PX1gWptgkzRjVZCrLmSGcZCb/jioOQwCqZN8= github.com/ipld/go-ipld-prime v0.9.1-0.20210324083106-dc342a9917db/go.mod h1:KvBLMr4PX1gWptgkzRjVZCrLmSGcZCb/jioOQwCqZN8= github.com/ipld/go-ipld-prime v0.11.0/go.mod h1:+WIAkokurHmZ/KwzDOMUuoeJgaRQktHtEaLglS3ZeV8= -github.com/ipld/go-ipld-prime v0.12.3/go.mod h1:PaeLYq8k6dJLmDUSLrzkEpoGV4PEfe/1OtFN/eALOc8= github.com/ipld/go-ipld-prime v0.14.0/go.mod h1:9ASQLwUFLptCov6lIYc70GRB4V7UTyLD0IJtrDJe6ZM= github.com/ipld/go-ipld-prime v0.14.1/go.mod h1:QcE4Y9n/ZZr8Ijg5bGPT0GqYWgZ1704nH0RDcQtgTP0= github.com/ipld/go-ipld-prime v0.16.0 h1:RS5hhjB/mcpeEPJvfyj0qbOj/QL+/j05heZ0qa97dVo= @@ -1261,8 +1260,9 @@ github.com/multiformats/go-multicodec v0.2.0/go.mod h1:/y4YVwkfMyry5kFbMTbLJKErh github.com/multiformats/go-multicodec v0.3.0/go.mod h1:qGGaQmioCDh+TeFOnxrbU0DaIPw8yFgAZgFG0V7p1qQ= github.com/multiformats/go-multicodec v0.3.1-0.20210902112759-1539a079fd61/go.mod h1:1Hj/eHRaVWSXiSNNfcEPcwZleTmdNP81xlxDLnWU9GQ= github.com/multiformats/go-multicodec v0.3.1-0.20211210143421-a526f306ed2c/go.mod h1:1Hj/eHRaVWSXiSNNfcEPcwZleTmdNP81xlxDLnWU9GQ= -github.com/multiformats/go-multicodec v0.4.1 h1:BSJbf+zpghcZMZrwTYBGwy0CPcVZGWiC72Cp8bBd4R4= github.com/multiformats/go-multicodec v0.4.1/go.mod h1:1Hj/eHRaVWSXiSNNfcEPcwZleTmdNP81xlxDLnWU9GQ= +github.com/multiformats/go-multicodec v0.5.0 h1:EgU6cBe/D7WRwQb1KmnBvU7lrcFGMggZVTPtOW9dDHs= +github.com/multiformats/go-multicodec v0.5.0/go.mod h1:DiY2HFaEp5EhEXb/iYzVAunmyX/aSFMxq2KMKfWEues= github.com/multiformats/go-multihash v0.0.1/go.mod h1:w/5tugSrLEbWqlcgJabL3oHFKTwfvkofsjW2Qa1ct4U= github.com/multiformats/go-multihash v0.0.5/go.mod h1:lt/HCbqlQwlPBz7lv0sQCdtfcMtlJvakRUn/0Ual8po= github.com/multiformats/go-multihash v0.0.8/go.mod h1:YSLudS+Pi8NHE7o6tb3D8vrpKa63epEDmG8nTduyAew= From af27e43632da2db676cadc3b6069dd6a64f7d4a5 Mon Sep 17 00:00:00 2001 From: Jorropo Date: Wed, 6 Jul 2022 15:25:55 +0200 Subject: [PATCH 2/3] chore: Add release notes for 0.13.1 --- CHANGELOG.md | 117 ++++++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 102 insertions(+), 15 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fd796ff66ee..75f357aaf29 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,92 @@ # go-ipfs changelog +## v0.13.1 2022-07-06 + +This release includes security fixes for various DOS vectors when importing untrusted user input with `ipfs dag import` +and the [`v0/dag/import`](https://docs.ipfs.io/reference/http/api/#api-v0-dag-import) endpoint. + +View the linked [security advisory](https://github.com/ipfs/go-ipfs/security/advisories/GHSA-f2gr-7299-487h) for more information. + +### Changelog + +- github.com/ipfs/go-ipfs: + - chore: update car +- github.com/ipld/go-car (v0.3.2 -> v0.4.0) & (v2.1.1 -> v2.4.0): + - Bump version in prep for releasing go-car `v0` + - Revert changes to `insertionindex` + - Revert changes to `index.Index` while keeping most of security fixes + - Return error when section length is invalid `varint` + - Drop repeated package name from `CarStats` + - Benchmark `Reader.Inspect` with and without hash validation + - Use consistent CID mismatch error in `Inspect` and `BlockReader.Next` + - Use streaming APIs to verify the hash of blocks in CAR `Inspect` + - test: add fuzzing for reader#Inspect + - feat: add block hash validation to Inspect() + - feat: add Reader#Inspect() function to check basic validity of a CAR and return stats + - Remove support for `ForEach` enumeration from car-index-sorted + - Use a fix code as the multihash code for `CarIndexSorted` + - Fix testutil assertion logic and update index generation tests + - fix: tighter constraint of singleWidthIndex width, add index recommentation docs + - fix: explicitly disable serialization of insertionindex + - feat: MaxAllowed{Header,Section}Size option + - feat: MaxAllowedSectionSize default to 32M + - fix: use CidFromReader() which has overread and OOM protection + - fix: staticcheck catches + - fix: revert to internalio.NewOffsetReadSeeker in Reader#IndexReader + - fix index comparisons + - feat: Refactor indexes to put storage considerations on consumers + - test: v2 add fuzzing of the index + - fix: v2 don't divide by zero in width indexes + - fix: v2 don't allocate indexes too big + - test: v2 add fuzzing to Reader + - fix: v2 don't accept overflowing offsets while reading v2 headers + - test: v2 add fuzzing to BlockReader + - fix: v2 don't OOM if the header size is too big + - test: add fuzzing of NewCarReader + - fix: do bound check while checking for CIDv0 + - fix: don't OOM if the header size is too big + - Add API to regenerate index from CARv1 or CARv2 + - PrototypeChooser support (#305) ([ipld/go-car#305](https://github.com/ipld/go-car/pull/305)) + - bump to newer blockstore err not found (#301) ([ipld/go-car#301](https://github.com/ipld/go-car/pull/301)) + - Car command supports for `largebytes` nodes (#296) ([ipld/go-car#296](https://github.com/ipld/go-car/pull/296)) + - fix(test): rootless fixture should have no roots, not null roots + - Allow extracton of a raw unixfs file (#284) ([ipld/go-car#284](https://github.com/ipld/go-car/pull/284)) + - cmd/car: use a better install command in the README + - feat: --version selector for `car create` & update deps + - feat: add option to create blockstore that writes a plain CARv1 (#288) ([ipld/go-car#288](https://github.com/ipld/go-car/pull/288)) + - add `car detach-index list` to list detached index contents (#287) ([ipld/go-car#287](https://github.com/ipld/go-car/pull/287)) + - add `car root` command (#283) ([ipld/go-car#283](https://github.com/ipld/go-car/pull/283)) + - make specification of root cid in get-dag command optional (#281) ([ipld/go-car#281](https://github.com/ipld/go-car/pull/281)) + - Update `version.json` after manual tag push + - Update v2 to context datastores (#275) ([ipld/go-car#275](https://github.com/ipld/go-car/pull/275)) + - update context datastore ([ipld/go-car#273](https://github.com/ipld/go-car/pull/273)) + - Traversal-based car creation (#269) ([ipld/go-car#269](https://github.com/ipld/go-car/pull/269)) + - Seek to start before index generation in `ReadOnly` blockstore + - support extraction of unixfs content stored in car files (#263) ([ipld/go-car#263](https://github.com/ipld/go-car/pull/263)) + - Add a barebones readme to the car CLI (#262) ([ipld/go-car#262](https://github.com/ipld/go-car/pull/262)) + - sync: update CI config files (#261) ([ipld/go-car#261](https://github.com/ipld/go-car/pull/261)) + - fix!: use -version=n instead of -v1 for index command + - feat: fix get-dag and add version=1 option + - creation of car from file / directory (#246) ([ipld/go-car#246](https://github.com/ipld/go-car/pull/246)) + - forEach iterates over index in stable order (#258) ([ipld/go-car#258](https://github.com/ipld/go-car/pull/258)) +- github.com/multiformats/go-multicodec (v0.4.1 -> v0.5.0): + - Bump version to 0.5.0 + - Bump version to 0.4.2 + - deps: update stringer version in go generate command + - docs(readme): improved usage examples (#66) ([multiformats/go-multicodec#66](https://github.com/multiformats/go-multicodec/pull/66)) + +### ❤ Contributors + +| Contributor | Commits | Lines ± | Files Changed | +|-------------|---------|---------|---------------| +| Masih H. Derkani | 27 | +1494/-1446 | 100 | +| Rod Vagg | 31 | +2021/-606 | 105 | +| Will | 19 | +1898/-151 | 69 | +| Jorropo | 27 | +1638/-248 | 76 | +| Aayush Rajasekaran | 1 | +130/-100 | 10 | +| whyrusleeping | 1 | +24/-22 | 4 | +| Marcin Rataj | 1 | +27/-1 | 1 | + ## v0.13.0 2022-05-04 We're happy to announce go-ipfs 0.13.0, packed full of changes and improvements! @@ -140,7 +227,7 @@ There are two ways for requesting CID specific response format: ```console $ curl -H 'Accept: application/vnd.ipld.raw' "http://127.0.0.1:8080/ipfs/QmZULkCELmmk5XNfCgTnCyFgAVxBRBXyDHGGMVoLFLiXEN" --output block.bin -$ cat block.bin | ipfs block put +$ cat block.bin | ipfs block put $ ipfs cat QmZULkCELmmk5XNfCgTnCyFgAVxBRBXyDHGGMVoLFLiXEN hello ``` @@ -2123,7 +2210,7 @@ As usual, this release includes important fixes, some of which may be critical f - `ipfs dag put` - defaults changed to reduce ambiguity and surprises: input is now assumed to be [`dag-json`](https://ipld.io/specs/codecs/dag-json/spec/), and data is serialized to [`dag-cbor`](https://ipld.io/specs/codecs/dag-cbor/spec/) at rest. - `--format` and `--input-enc` were removed and replaced with `--store-codec` and `--input-codec` - - codec names now match the ones defined in the [multicodec table](https://github.com/multiformats/multicodec/blob/master/table.csv) + - codec names now match the ones defined in the [multicodec table](https://github.com/multiformats/multicodec/blob/master/table.csv) - dag-pb (e.g. unixfs) field names changed - impacts userland code that works with `dag-pb` objects stored via `dag put` Keep reading to learn more details. @@ -3507,7 +3594,7 @@ Examples include: ``` ipfs pin remote service add myservice https://myservice.tld:1234/api/path myaccess key -ipfs pin remote add /ipfs/bafymydata --service=myservice --name=myfile +ipfs pin remote add /ipfs/bafymydata --service=myservice --name=myfile ipfs pin remote ls --service=myservice --name=myfile ipfs pin remote ls --service=myservice --cid=bafymydata ipfs pin remote rm --serivce=myservice --name=myfile @@ -3530,7 +3617,7 @@ Previously DNSLink names would have trouble loading over subdomain gateways with `/ipns/en.wikipedia-on-ipfs.org` → `ipns://en.wikipedia-on-ipfs.org` → -`https://dweb.link/ipns/en.wikipedia-on-ipfs.org` +`https://dweb.link/ipns/en.wikipedia-on-ipfs.org` `https://en-wikipedia--on--ipfs-org.ipns.dweb.link` :point_left: _a single DNS label, no TLS error_ #### QUIC update @@ -5258,7 +5345,7 @@ default. This: * Ensures that every dapp gets its own browser origin. * Makes it easier to write websites that "just work" with IPFS because absolute paths will now work (though you should still use relative links because they're better). - + Paths addressing the gateway by IP address (`http://127.0.0.1:5001/ipfs/CID`) will not be altered as IP addresses can't have subdomains. Note: cURL doesn't follow redirects by default. To avoid breaking cURL and other clients that don't support redirects, go-ipfs will return the requested file along with the redirect. Browsers will follow the redirect and abort the download while cURL will ignore the redirect and finish the download. @@ -5287,7 +5374,7 @@ This feature will be enabled by default in 0.6.0. ##### IPNS with base32 PIDs -IPNS names can now be expressed as special multibase CIDs. E.g., +IPNS names can now be expressed as special multibase CIDs. E.g., > /ipns/bafzbeibxfjp4gaxc4cdn57257cyvc7jfa4rlp4e5min6geg44m57g6nx7e @@ -5385,7 +5472,7 @@ IPFS now allows rapid reading and writing of blocks in [`.car` format](https://g ~$ ipfs dag export QmQPeNsJPyVWPFDVHb77w8G42Fvo15z4bG2X8D2GhfbSXc \ | xz > welcome_to_ipfs.car.xz - 0s 6.73 KiB / ? [-------=-------------------------------------] 5.16 MiB/s 0s + 0s 6.73 KiB / ? [-------=-------------------------------------] 5.16 MiB/s 0s ``` Then on another `ipfs` instance, not even connected to the network: @@ -5413,7 +5500,7 @@ To enable structured logging, set the `IPFS_LOGGING_FMT` environment variable to Note: while we've switched to using Zap as the logging backend, most of go-ipfs still logs strings. -##### Systemd Support +##### Systemd Support For Linux users, this release includes support for two systemd features: socket activation and startup/shutdown notifications. This makes it possible to: @@ -5470,7 +5557,7 @@ type Plugin interface { } ``` -to +to ```go type Environment struct { @@ -6480,7 +6567,7 @@ This release includes fixes for the following regressions: networking module, causing it to announce the wrong addresses and sometimes complain about not being able to determine the IP address ([ipfs/go-ipfs#6415](https://github.com/ipfs/go-ipfs/pull/6415)). - + It also includes fixes for: 1. Pins not being persisted after `ipfs block add --pin` @@ -6765,13 +6852,13 @@ rce is stdin. -p, --progress bool - Stream progress data. -t, --trickle bool - Use trickle-dag format for da g generation. - -n, --only-hash bool - Only chunk and hash - do not + -n, --only-hash bool - Only chunk and hash - do not write to disk. -w, --wrap-with-directory bool - Wrap files with a directory o bject. -s, --chunker string - Chunking algorithm, size-[byt es] or rabin-[min]-[avg]-[max]. Default: size-262144. - --pin bool - Pin this object when adding. + --pin bool - Pin this object when adding. Default: true. --raw-leaves bool - Use raw blocks for leaf nodes . (experimental). @@ -7269,7 +7356,7 @@ performance issues that we've now fixed in this release. Specifically: * We've fixed quite a number of performance issues that only show up when rapidly forming new connections. This will actually help _all_ nodes but will especially help relays. - + If you've enabled relay _hop_ (`Swarm.EnableRelayHop`) in go-ipfs 0.4.19 and it hasn't burned down your machine yet, this release should improve things significantly. However, relays are still under heavy load so running an open @@ -8336,7 +8423,7 @@ bafybeicg2rebjoofv4kbyovkw7af3rpiitvnl6i7ckcywaq6xjcxnc2mby The refactored `ipfs p2p` command allows forwarding TCP streams through two IPFS nodes from one host to another. It's `ssh -L` but for IPFS. You can find -documentation +documentation [here](https://github.com/ipfs/go-ipfs/blob/master/docs/experimental-features.md#ipfs-p2p). It's still experimental but we don't expect too many breaking changes at this point (it will very likely be stabilized in the next release). Quick summary of @@ -9974,7 +10061,7 @@ few other improvements to other parts of the codebase. Notably: - 81% coverage on blockstore ([ipfs/go-ipfs#3074](https://github.com/ipfs/go-ipfs/pull/3074)) - 80% coverage of unixfs/mod ([ipfs/go-ipfs#3096](https://github.com/ipfs/go-ipfs/pull/3096)) - 82% coverage on blocks ([ipfs/go-ipfs#3086](https://github.com/ipfs/go-ipfs/pull/3086)) - - 87% coverage on unixfs ([ipfs/go-ipfs#3492](https://github.com/ipfs/go-ipfs/pull/3492)) + - 87% coverage on unixfs ([ipfs/go-ipfs#3492](https://github.com/ipfs/go-ipfs/pull/3492)) - Improve coverage on routing/offline ([ipfs/go-ipfs#3516](https://github.com/ipfs/go-ipfs/pull/3516)) - Add test for flags package ([ipfs/go-ipfs#3449](https://github.com/ipfs/go-ipfs/pull/3449)) - improve test coverage on merkledag package ([ipfs/go-ipfs#3113](https://github.com/ipfs/go-ipfs/pull/3113)) From 1b60d07d6c137af401e629bf201d2cbfd8e1c3ab Mon Sep 17 00:00:00 2001 From: Jorropo Date: Wed, 6 Jul 2022 15:28:44 +0200 Subject: [PATCH 3/3] chore: bump version number to v0.13.1 --- version.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version.go b/version.go index 944af1c3898..c85ed8665ca 100644 --- a/version.go +++ b/version.go @@ -11,7 +11,7 @@ import ( var CurrentCommit string // CurrentVersionNumber is the current application's version literal -const CurrentVersionNumber = "0.13.0" +const CurrentVersionNumber = "0.13.1" const ApiVersion = "/go-ipfs/" + CurrentVersionNumber + "/"