Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
feat(gateway): TAR response format (#9029)
Implementation of IPIP-288 (ipfs/specs#288) Co-authored-by: Marcin Rataj <lidel@lidel.org>
- Loading branch information
Showing
10 changed files
with
222 additions
and
12 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,92 @@ | ||
| package corehttp | ||
|
|
||
| import ( | ||
| "context" | ||
| "html" | ||
| "net/http" | ||
| "time" | ||
|
|
||
| files "github.com/ipfs/go-ipfs-files" | ||
| ipath "github.com/ipfs/interface-go-ipfs-core/path" | ||
| "github.com/ipfs/kubo/tracing" | ||
| "go.opentelemetry.io/otel/attribute" | ||
| "go.opentelemetry.io/otel/trace" | ||
| "go.uber.org/zap" | ||
| ) | ||
|
|
||
| var unixEpochTime = time.Unix(0, 0) | ||
|
|
||
| func (i *gatewayHandler) serveTAR(ctx context.Context, w http.ResponseWriter, r *http.Request, resolvedPath ipath.Resolved, contentPath ipath.Path, begin time.Time, logger *zap.SugaredLogger) { | ||
| ctx, span := tracing.Span(ctx, "Gateway", "ServeTAR", trace.WithAttributes(attribute.String("path", resolvedPath.String()))) | ||
| defer span.End() | ||
|
|
||
| ctx, cancel := context.WithCancel(ctx) | ||
| defer cancel() | ||
|
|
||
| // Get Unixfs file | ||
| file, err := i.api.Unixfs().Get(ctx, resolvedPath) | ||
| if err != nil { | ||
| webError(w, "ipfs cat "+html.EscapeString(contentPath.String()), err, http.StatusBadRequest) | ||
| return | ||
| } | ||
| defer file.Close() | ||
|
|
||
| rootCid := resolvedPath.Cid() | ||
|
|
||
| // Set Cache-Control and read optional Last-Modified time | ||
| modtime := addCacheControlHeaders(w, r, contentPath, rootCid) | ||
|
|
||
| // Weak Etag W/ because we can't guarantee byte-for-byte identical | ||
| // responses, but still want to benefit from HTTP Caching. Two TAR | ||
| // responses for the same CID will be logically equivalent, | ||
| // but when TAR is streamed, then in theory, files and directories | ||
| // may arrive in different order (depends on TAR lib and filesystem/inodes). | ||
| etag := `W/` + getEtag(r, rootCid) | ||
| w.Header().Set("Etag", etag) | ||
|
|
||
| // Finish early if Etag match | ||
| if r.Header.Get("If-None-Match") == etag { | ||
| w.WriteHeader(http.StatusNotModified) | ||
| return | ||
| } | ||
|
|
||
| // Set Content-Disposition | ||
| var name string | ||
| if urlFilename := r.URL.Query().Get("filename"); urlFilename != "" { | ||
| name = urlFilename | ||
| } else { | ||
| name = rootCid.String() + ".tar" | ||
| } | ||
| setContentDispositionHeader(w, name, "attachment") | ||
|
|
||
| // Construct the TAR writer | ||
| tarw, err := files.NewTarWriter(w) | ||
| if err != nil { | ||
| webError(w, "could not build tar writer", err, http.StatusInternalServerError) | ||
| return | ||
| } | ||
| defer tarw.Close() | ||
|
|
||
| // Sets correct Last-Modified header. This code is borrowed from the standard | ||
| // library (net/http/server.go) as we cannot use serveFile without throwing the entire | ||
| // TAR into the memory first. | ||
| if !(modtime.IsZero() || modtime.Equal(unixEpochTime)) { | ||
| w.Header().Set("Last-Modified", modtime.UTC().Format(http.TimeFormat)) | ||
| } | ||
|
|
||
| w.Header().Set("Content-Type", "application/x-tar") | ||
| w.Header().Set("X-Content-Type-Options", "nosniff") // no funny business in the browsers :^) | ||
|
|
||
| // The TAR has a top-level directory (or file) named by the CID. | ||
| if err := tarw.WriteFile(file, rootCid.String()); err != nil { | ||
| w.Header().Set("X-Stream-Error", err.Error()) | ||
| // Trailer headers do not work in web browsers | ||
| // (see https://github.com/mdn/browser-compat-data/issues/14703) | ||
| // and we have limited options around error handling in browser contexts. | ||
| // To improve UX/DX, we finish response stream with error message, allowing client to | ||
| // (1) detect error by having corrupted TAR | ||
| // (2) be able to reason what went wrong by instecting the tail of TAR stream | ||
| _, _ = w.Write([]byte(err.Error())) | ||
| return | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Binary file not shown.
Binary file not shown.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,91 @@ | ||
| #!/usr/bin/env bash | ||
|
|
||
| test_description="Test HTTP Gateway TAR (application/x-tar) Support" | ||
|
|
||
| . lib/test-lib.sh | ||
|
|
||
| test_init_ipfs | ||
| test_launch_ipfs_daemon_without_network | ||
|
|
||
| OUTSIDE_ROOT_CID="bafybeicaj7kvxpcv4neaqzwhrqqmdstu4dhrwfpknrgebq6nzcecfucvyu" | ||
| INSIDE_ROOT_CID="bafybeibfevfxlvxp5vxobr5oapczpf7resxnleb7tkqmdorc4gl5cdva3y" | ||
|
|
||
| test_expect_success "Add the test directory" ' | ||
| mkdir -p rootDir/ipfs && | ||
| mkdir -p rootDir/ipns && | ||
| mkdir -p rootDir/api && | ||
| mkdir -p rootDir/ą/ę && | ||
| echo "I am a txt file on path with utf8" > rootDir/ą/ę/file-źł.txt && | ||
| echo "I am a txt file in confusing /api dir" > rootDir/api/file.txt && | ||
| echo "I am a txt file in confusing /ipfs dir" > rootDir/ipfs/file.txt && | ||
| echo "I am a txt file in confusing /ipns dir" > rootDir/ipns/file.txt && | ||
| DIR_CID=$(ipfs add -Qr --cid-version 1 rootDir) && | ||
| FILE_CID=$(ipfs files stat --enc=json /ipfs/$DIR_CID/ą/ę/file-źł.txt | jq -r .Hash) && | ||
| FILE_SIZE=$(ipfs files stat --enc=json /ipfs/$DIR_CID/ą/ę/file-źł.txt | jq -r .Size) | ||
| echo "$FILE_CID / $FILE_SIZE" | ||
| ' | ||
|
|
||
| test_expect_success "GET TAR with format=tar and extract" ' | ||
| curl "http://127.0.0.1:$GWAY_PORT/ipfs/$FILE_CID?format=tar" | tar -x | ||
| ' | ||
|
|
||
| test_expect_success "GET TAR with 'Accept: application/x-tar' and extract" ' | ||
| curl -H "Accept: application/x-tar" "http://127.0.0.1:$GWAY_PORT/ipfs/$FILE_CID" | tar -x | ||
| ' | ||
|
|
||
| test_expect_success "GET TAR with format=tar has expected Content-Type" ' | ||
| curl -sD - "http://127.0.0.1:$GWAY_PORT/ipfs/$FILE_CID?format=tar" > curl_output_filename 2>&1 && | ||
| test_should_contain "Content-Disposition: attachment;" curl_output_filename && | ||
| test_should_contain "Etag: W/\"$FILE_CID.x-tar" curl_output_filename && | ||
| test_should_contain "Content-Type: application/x-tar" curl_output_filename | ||
| ' | ||
|
|
||
| test_expect_success "GET TAR with 'Accept: application/x-tar' has expected Content-Type" ' | ||
| curl -sD - -H "Accept: application/x-tar" "http://127.0.0.1:$GWAY_PORT/ipfs/$FILE_CID" > curl_output_filename 2>&1 && | ||
| test_should_contain "Content-Disposition: attachment;" curl_output_filename && | ||
| test_should_contain "Etag: W/\"$FILE_CID.x-tar" curl_output_filename && | ||
| test_should_contain "Content-Type: application/x-tar" curl_output_filename | ||
| ' | ||
|
|
||
| test_expect_success "GET TAR has expected root file" ' | ||
| rm -rf outputDir && mkdir outputDir && | ||
| curl "http://127.0.0.1:$GWAY_PORT/ipfs/$FILE_CID?format=tar" | tar -x -C outputDir && | ||
| test -f "outputDir/$FILE_CID" && | ||
| echo "I am a txt file on path with utf8" > expected && | ||
| test_cmp expected outputDir/$FILE_CID | ||
| ' | ||
|
|
||
| test_expect_success "GET TAR has expected root directory" ' | ||
| rm -rf outputDir && mkdir outputDir && | ||
| curl "http://127.0.0.1:$GWAY_PORT/ipfs/$DIR_CID?format=tar" | tar -x -C outputDir && | ||
| test -d "outputDir/$DIR_CID" && | ||
| echo "I am a txt file on path with utf8" > expected && | ||
| test_cmp expected outputDir/$DIR_CID/ą/ę/file-źł.txt | ||
| ' | ||
|
|
||
| test_expect_success "GET TAR with explicit ?filename= succeeds with modified Content-Disposition header" " | ||
| curl -fo actual -D actual_headers 'http://127.0.0.1:$GWAY_PORT/ipfs/$DIR_CID?filename=testтест.tar&format=tar' && | ||
| grep -F 'Content-Disposition: attachment; filename=\"test____.tar\"; filename*=UTF-8'\'\''test%D1%82%D0%B5%D1%81%D1%82.tar' actual_headers | ||
| " | ||
|
|
||
| test_expect_success "Add CARs with relative paths to test with" ' | ||
| ipfs dag import ../t0122-gateway-tar-data/outside-root.car > import_output && | ||
| test_should_contain $OUTSIDE_ROOT_CID import_output && | ||
| ipfs dag import ../t0122-gateway-tar-data/inside-root.car > import_output && | ||
| test_should_contain $INSIDE_ROOT_CID import_output | ||
| ' | ||
|
|
||
| test_expect_success "GET TAR with relative paths outside root fails" ' | ||
| curl -o - "http://127.0.0.1:$GWAY_PORT/ipfs/$OUTSIDE_ROOT_CID?format=tar" > curl_output_filename && | ||
| test_should_contain "relative UnixFS paths outside the root are now allowed" curl_output_filename | ||
| ' | ||
|
|
||
| test_expect_success "GET TAR with relative paths inside root works" ' | ||
| rm -rf outputDir && mkdir outputDir && | ||
| curl "http://127.0.0.1:$GWAY_PORT/ipfs/$INSIDE_ROOT_CID?format=tar" | tar -x -C outputDir && | ||
| test -f outputDir/$INSIDE_ROOT_CID/foobar/file | ||
| ' | ||
|
|
||
| test_kill_ipfs_daemon | ||
|
|
||
| test_done |