From de53536e68d7c7ca81cb248ef67181428cd3752b Mon Sep 17 00:00:00 2001
From: Henrique Dias <hacdias@gmail.com>
Date: Thu, 9 Jun 2022 15:35:53 +0200
Subject: [PATCH 01/30] feat: gateway support for tar

---
 core/corehttp/gateway_handler.go     | 12 ++++-
 core/corehttp/gateway_handler_tar.go | 77 ++++++++++++++++++++++++++++
 test/sharness/t0110-gateway.sh       | 10 ++++
 3 files changed, 98 insertions(+), 1 deletion(-)
 create mode 100644 core/corehttp/gateway_handler_tar.go

diff --git a/core/corehttp/gateway_handler.go b/core/corehttp/gateway_handler.go
index a96799f5875..04a55e391d4 100644
--- a/core/corehttp/gateway_handler.go
+++ b/core/corehttp/gateway_handler.go
@@ -430,6 +430,10 @@ func (i *gatewayHandler) getOrHeadHandler(w http.ResponseWriter, r *http.Request
 		carVersion := formatParams["version"]
 		i.serveCAR(r.Context(), w, r, resolvedPath, contentPath, carVersion, begin)
 		return
+	case "application/x-tar":
+		logger.Debugw("serving tar file", "path", contentPath)
+		i.serveTAR(r.Context(), w, r, resolvedPath, contentPath, begin, logger)
+		return
 	default: // catch-all for unsuported application/vnd.*
 		err := fmt.Errorf("unsupported format %q", responseFormat)
 		webError(w, "failed respond with requested content type", err, http.StatusBadRequest)
@@ -845,6 +849,10 @@ func getEtag(r *http.Request, cid cid.Cid) string {
 		f := responseFormat[strings.LastIndex(responseFormat, ".")+1:]
 		// Etag: "cid.foo" (gives us nice compression together with Content-Disposition in block (raw) and car responses)
 		suffix = `.` + f + suffix
+		// Since different TAR implementations may produce different byte-for-byte responses, we define a weak Etag.
+		if responseFormat == "application/x-tar" {
+			prefix = "W/" + prefix
+		}
 	}
 	// TODO: include selector suffix when https://github.com/ipfs/kubo/issues/8769 lands
 	return prefix + cid.String() + suffix
@@ -859,11 +867,13 @@ func customResponseFormat(r *http.Request) (mediaType string, params map[string]
 			return "application/vnd.ipld.raw", nil, nil
 		case "car":
 			return "application/vnd.ipld.car", nil, nil
+		case "tar":
+			return "application/x-tar", nil, nil
 		}
 	}
 	// Browsers and other user agents will send Accept header with generic types like:
 	// Accept:text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8
-	// We only care about explciit, vendor-specific content-types.
+	// We only care about explicit, vendor-specific content-types.
 	for _, accept := range r.Header.Values("Accept") {
 		// respond to the very first ipld content type
 		if strings.HasPrefix(accept, "application/vnd.ipld") {
diff --git a/core/corehttp/gateway_handler_tar.go b/core/corehttp/gateway_handler_tar.go
new file mode 100644
index 00000000000..bbf60cefeda
--- /dev/null
+++ b/core/corehttp/gateway_handler_tar.go
@@ -0,0 +1,77 @@
+package corehttp
+
+import (
+	"context"
+	"html"
+	"net/http"
+	"time"
+
+	files "github.com/ipfs/go-ipfs-files"
+	"github.com/ipfs/go-ipfs/tracing"
+	ipath "github.com/ipfs/interface-go-ipfs-core/path"
+	"go.opentelemetry.io/otel/attribute"
+	"go.opentelemetry.io/otel/trace"
+	"go.uber.org/zap"
+)
+
+var unixEpochTime = time.Unix(0, 0)
+
+func (i *gatewayHandler) serveTAR(ctx context.Context, w http.ResponseWriter, r *http.Request, resolvedPath ipath.Resolved, contentPath ipath.Path, begin time.Time, logger *zap.SugaredLogger) {
+	ctx, span := tracing.Span(ctx, "Gateway", "ServeTAR", trace.WithAttributes(attribute.String("path", resolvedPath.String())))
+	defer span.End()
+
+	ctx, cancel := context.WithCancel(ctx)
+	defer cancel()
+
+	// Set Cache-Control and read optional Last-Modified time
+	modtime := addCacheControlHeaders(w, r, contentPath, resolvedPath.Cid())
+
+	// Finish early if Etag match
+	if r.Header.Get("If-None-Match") == getEtag(r, resolvedPath.Cid()) {
+		w.WriteHeader(http.StatusNotModified)
+		return
+	}
+
+	// Set Content-Disposition
+	var name string
+	if urlFilename := r.URL.Query().Get("filename"); urlFilename != "" {
+		name = urlFilename
+	} else {
+		name = resolvedPath.Cid().String() + ".tar"
+	}
+	setContentDispositionHeader(w, name, "attachment")
+
+	// Get Unixfs file
+	file, err := i.api.Unixfs().Get(ctx, resolvedPath)
+	if err != nil {
+		webError(w, "ipfs cat "+html.EscapeString(contentPath.String()), err, http.StatusNotFound)
+		return
+	}
+	defer file.Close()
+
+	// Construct the TAR writer
+	tarw, err := files.NewTarWriter(w)
+	if err != nil {
+		webError(w, "could not build tar writer", err, http.StatusInternalServerError)
+		return
+	}
+	defer tarw.Close()
+
+	// Sets correct Last-Modified header. This code is borrowed from the standard
+	// library (net/http/server.go) as we cannot use serveFile without throwing the entire
+	// TAR into the memory first.
+	if !(modtime.IsZero() || modtime.Equal(unixEpochTime)) {
+		w.Header().Set("Last-Modified", modtime.UTC().Format(http.TimeFormat))
+	}
+
+	w.Header().Set("Content-Type", "application/x-tar")
+
+	if err := tarw.WriteFile(file, name); err != nil {
+		// We return error as a trailer, however it is not something browsers can access
+		// (https://github.com/mdn/browser-compat-data/issues/14703)
+		// Due to this, we suggest client always verify that
+		// the received CAR stream response is matching requested DAG selector
+		w.Header().Set("X-Stream-Error", err.Error())
+		return
+	}
+}
diff --git a/test/sharness/t0110-gateway.sh b/test/sharness/t0110-gateway.sh
index 87aa61c70a2..0de68b35c5a 100755
--- a/test/sharness/t0110-gateway.sh
+++ b/test/sharness/t0110-gateway.sh
@@ -301,6 +301,16 @@ test_expect_success "Verify gateway file" '
   test_cmp gateway_daemon_actual gateway_file_actual
 '
 
+test_expect_success "GET TAR file from gateway and extract" '
+  curl "http://127.0.0.1:$port/ipfs/$FOO2_HASH?format=tar" | tar -x
+'
+
+test_expect_success "GET TAR file has expected Content-Type" '
+  curl -svX GET "http://127.0.0.1:$port/ipfs/$FOO2_HASH?format=tar" > curl_output_filename 2>&1 &&
+  cat curl_output_filename &&
+  grep "< Content-Type: application/x-tar" curl_output_filename
+'
+
 test_kill_ipfs_daemon
 
 GWPORT=32563

From f79610ae435a82b5e6c000a7675029494520214f Mon Sep 17 00:00:00 2001
From: Henrique Dias <hacdias@gmail.com>
Date: Mon, 15 Aug 2022 16:46:59 +0200
Subject: [PATCH 02/30] fix: kubo namings

---
 core/corehttp/gateway_handler_tar.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/core/corehttp/gateway_handler_tar.go b/core/corehttp/gateway_handler_tar.go
index bbf60cefeda..acfa5effc26 100644
--- a/core/corehttp/gateway_handler_tar.go
+++ b/core/corehttp/gateway_handler_tar.go
@@ -7,8 +7,8 @@ import (
 	"time"
 
 	files "github.com/ipfs/go-ipfs-files"
-	"github.com/ipfs/go-ipfs/tracing"
 	ipath "github.com/ipfs/interface-go-ipfs-core/path"
+	"github.com/ipfs/kubo/tracing"
 	"go.opentelemetry.io/otel/attribute"
 	"go.opentelemetry.io/otel/trace"
 	"go.uber.org/zap"

From e8180ea580684c00d1d776e67d46d277ee31ae65 Mon Sep 17 00:00:00 2001
From: Henrique Dias <hacdias@gmail.com>
Date: Mon, 3 Oct 2022 13:50:10 +0200
Subject: [PATCH 03/30] fix: accept 'Accept: application/x-tar' header

---
 core/corehttp/gateway_handler.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/core/corehttp/gateway_handler.go b/core/corehttp/gateway_handler.go
index 04a55e391d4..8f75a76df33 100644
--- a/core/corehttp/gateway_handler.go
+++ b/core/corehttp/gateway_handler.go
@@ -876,7 +876,7 @@ func customResponseFormat(r *http.Request) (mediaType string, params map[string]
 	// We only care about explicit, vendor-specific content-types.
 	for _, accept := range r.Header.Values("Accept") {
 		// respond to the very first ipld content type
-		if strings.HasPrefix(accept, "application/vnd.ipld") {
+		if strings.HasPrefix(accept, "application/vnd.ipld") || strings.HasPrefix(accept, "application/x-tar") {
 			mediatype, params, err := mime.ParseMediaType(accept)
 			if err != nil {
 				return "", nil, err

From c5b773f4ed99bb797ac3b934f38a733c29fa883e Mon Sep 17 00:00:00 2001
From: Henrique Dias <hacdias@gmail.com>
Date: Mon, 3 Oct 2022 13:52:56 +0200
Subject: [PATCH 04/30] test: add tar file tests with accept header

---
 test/sharness/t0110-gateway.sh | 14 ++++++++++++--
 1 file changed, 12 insertions(+), 2 deletions(-)

diff --git a/test/sharness/t0110-gateway.sh b/test/sharness/t0110-gateway.sh
index 0de68b35c5a..b8b5db4c2c3 100755
--- a/test/sharness/t0110-gateway.sh
+++ b/test/sharness/t0110-gateway.sh
@@ -301,16 +301,26 @@ test_expect_success "Verify gateway file" '
   test_cmp gateway_daemon_actual gateway_file_actual
 '
 
-test_expect_success "GET TAR file from gateway and extract" '
+test_expect_success "GET TAR with format=tar and extract" '
   curl "http://127.0.0.1:$port/ipfs/$FOO2_HASH?format=tar" | tar -x
 '
 
-test_expect_success "GET TAR file has expected Content-Type" '
+test_expect_success "GET TAR with 'Accept: application/x-tar' and extract" '
+  curl -H "Accept: application/x-tar" "http://127.0.0.1:$port/ipfs/$FOO2_HASH" | tar -x
+'
+
+test_expect_success "GET TAR with format=tar has expected Content-Type" '
   curl -svX GET "http://127.0.0.1:$port/ipfs/$FOO2_HASH?format=tar" > curl_output_filename 2>&1 &&
   cat curl_output_filename &&
   grep "< Content-Type: application/x-tar" curl_output_filename
 '
 
+test_expect_success "GET TAR with 'Accept: application/x-tar' has expected Content-Type" '
+  curl -svX GET -H "Accept: application/x-tar" "http://127.0.0.1:$port/ipfs/$FOO2_HASH" > curl_output_filename 2>&1 &&
+  cat curl_output_filename &&
+  grep "< Content-Type: application/x-tar" curl_output_filename
+'
+
 test_kill_ipfs_daemon
 
 GWPORT=32563

From 4d232c639721d5674a255bc41aba6cd36c62c8ef Mon Sep 17 00:00:00 2001
From: Henrique Dias <hacdias@gmail.com>
Date: Tue, 4 Oct 2022 12:14:39 +0200
Subject: [PATCH 05/30] wip: basepath tar writer

---
 core/corehttp/gateway_handler_tar.go        |   3 +-
 core/corehttp/gateway_handler_tar_writer.go | 114 ++++++++++++++++++++
 2 files changed, 115 insertions(+), 2 deletions(-)
 create mode 100644 core/corehttp/gateway_handler_tar_writer.go

diff --git a/core/corehttp/gateway_handler_tar.go b/core/corehttp/gateway_handler_tar.go
index acfa5effc26..90e5135a5d1 100644
--- a/core/corehttp/gateway_handler_tar.go
+++ b/core/corehttp/gateway_handler_tar.go
@@ -6,7 +6,6 @@ import (
 	"net/http"
 	"time"
 
-	files "github.com/ipfs/go-ipfs-files"
 	ipath "github.com/ipfs/interface-go-ipfs-core/path"
 	"github.com/ipfs/kubo/tracing"
 	"go.opentelemetry.io/otel/attribute"
@@ -50,7 +49,7 @@ func (i *gatewayHandler) serveTAR(ctx context.Context, w http.ResponseWriter, r
 	defer file.Close()
 
 	// Construct the TAR writer
-	tarw, err := files.NewTarWriter(w)
+	tarw, err := NewTarWriter(w)
 	if err != nil {
 		webError(w, "could not build tar writer", err, http.StatusInternalServerError)
 		return
diff --git a/core/corehttp/gateway_handler_tar_writer.go b/core/corehttp/gateway_handler_tar_writer.go
new file mode 100644
index 00000000000..bce9ecc0093
--- /dev/null
+++ b/core/corehttp/gateway_handler_tar_writer.go
@@ -0,0 +1,114 @@
+package corehttp
+
+import (
+	"archive/tar"
+	"fmt"
+	"io"
+	"path"
+	"strings"
+	"time"
+
+	files "github.com/ipfs/go-ipfs-files"
+)
+
+type TarWriter struct {
+	TarW *tar.Writer
+	root string
+}
+
+// NewTarWriter wraps given io.Writer into a new tar writer
+func NewTarWriter(w io.Writer) (*TarWriter, error) {
+	return &TarWriter{
+		TarW: tar.NewWriter(w),
+	}, nil
+}
+
+func (w *TarWriter) writeDir(f files.Directory, fpath string) error {
+	if err := writeDirHeader(w.TarW, fpath); err != nil {
+		return err
+	}
+
+	it := f.Entries()
+	for it.Next() {
+		if err := w.WriteFile(it.Node(), path.Join(fpath, it.Name())); err != nil {
+			return err
+		}
+	}
+	return it.Err()
+}
+
+func (w *TarWriter) writeFile(f files.File, fpath string) error {
+	size, err := f.Size()
+	if err != nil {
+		return err
+	}
+
+	if err := writeFileHeader(w.TarW, fpath, uint64(size)); err != nil {
+		return err
+	}
+
+	if _, err := io.Copy(w.TarW, f); err != nil {
+		return err
+	}
+	w.TarW.Flush()
+	return nil
+}
+
+// WriteNode adds a node to the archive.
+func (w *TarWriter) WriteFile(nd files.Node, fpath string) error {
+	if w.root == "" {
+		w.root = fpath
+	}
+
+	if !strings.HasPrefix(fpath, w.root) {
+		fpath = strings.Replace(fpath, ".", "", -1)
+		fpath = strings.Replace(fpath, "..", "", -1)
+		fpath = path.Join(w.root, fpath)
+	}
+
+	switch nd := nd.(type) {
+	case *files.Symlink:
+		return writeSymlinkHeader(w.TarW, nd.Target, fpath)
+	case files.File:
+		return w.writeFile(nd, fpath)
+	case files.Directory:
+		return w.writeDir(nd, fpath)
+	default:
+		return fmt.Errorf("file type %T is not supported", nd)
+	}
+}
+
+// Close closes the tar writer.
+func (w *TarWriter) Close() error {
+	return w.TarW.Close()
+}
+
+func writeDirHeader(w *tar.Writer, fpath string) error {
+	return w.WriteHeader(&tar.Header{
+		Name:     fpath,
+		Typeflag: tar.TypeDir,
+		Mode:     0777,
+		ModTime:  time.Now().Truncate(time.Second),
+		// TODO: set mode, dates, etc. when added to unixFS
+	})
+}
+
+func writeFileHeader(w *tar.Writer, fpath string, size uint64) error {
+	return w.WriteHeader(&tar.Header{
+		Name:     fpath,
+		Size:     int64(size),
+		Typeflag: tar.TypeReg,
+		Mode:     0644,
+		ModTime:  time.Now().Truncate(time.Second),
+		// TODO: set mode, dates, etc. when added to unixFS
+	})
+}
+
+func writeSymlinkHeader(w *tar.Writer, target, fpath string) error {
+	return w.WriteHeader(&tar.Header{
+		Name:     fpath,
+		Linkname: target,
+		Mode:     0777,
+		Typeflag: tar.TypeSymlink,
+	})
+}

From fd195f3902de7f31d82593d7dad2c1b3d4b2e474 Mon Sep 17 00:00:00 2001
From: Henrique Dias <hacdias@gmail.com>
Date: Tue, 4 Oct 2022 16:25:15 +0200
Subject: [PATCH 06/30] =?UTF-8?q?test:=20separate=20tar=20testfile=C2=A7?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 test/sharness/t0110-gateway.sh               |  20 ----------
 test/sharness/t0122-gateway-tar-data/foo.car | Bin 0 -> 203 bytes
 test/sharness/t0122-gateway-tar.sh           |  37 +++++++++++++++++++
 3 files changed, 37 insertions(+), 20 deletions(-)
 create mode 100644 test/sharness/t0122-gateway-tar-data/foo.car
 create mode 100755 test/sharness/t0122-gateway-tar.sh

diff --git a/test/sharness/t0110-gateway.sh b/test/sharness/t0110-gateway.sh
index b8b5db4c2c3..87aa61c70a2 100755
--- a/test/sharness/t0110-gateway.sh
+++ b/test/sharness/t0110-gateway.sh
@@ -301,26 +301,6 @@ test_expect_success "Verify gateway file" '
   test_cmp gateway_daemon_actual gateway_file_actual
 '
 
-test_expect_success "GET TAR with format=tar and extract" '
-  curl "http://127.0.0.1:$port/ipfs/$FOO2_HASH?format=tar" | tar -x
-'
-
-test_expect_success "GET TAR with 'Accept: application/x-tar' and extract" '
-  curl -H "Accept: application/x-tar" "http://127.0.0.1:$port/ipfs/$FOO2_HASH" | tar -x
-'
-
-test_expect_success "GET TAR with format=tar has expected Content-Type" '
-  curl -svX GET "http://127.0.0.1:$port/ipfs/$FOO2_HASH?format=tar" > curl_output_filename 2>&1 &&
-  cat curl_output_filename &&
-  grep "< Content-Type: application/x-tar" curl_output_filename
-'
-
-test_expect_success "GET TAR with 'Accept: application/x-tar' has expected Content-Type" '
-  curl -svX GET -H "Accept: application/x-tar" "http://127.0.0.1:$port/ipfs/$FOO2_HASH" > curl_output_filename 2>&1 &&
-  cat curl_output_filename &&
-  grep "< Content-Type: application/x-tar" curl_output_filename
-'
-
 test_kill_ipfs_daemon
 
 GWPORT=32563
diff --git a/test/sharness/t0122-gateway-tar-data/foo.car b/test/sharness/t0122-gateway-tar-data/foo.car
new file mode 100644
index 0000000000000000000000000000000000000000..f1e26614d35ce9bda912488db029f73c1087591f
GIT binary patch
literal 203
zcmcColv<RZUsBw7Ln}g+fw4eHA+o79@653|{h<%0_U*8krC8|gWHT%F#FLZT5_3=P
zD?XcEmReMtnV-iPMvQ799WEsyg-=eG&nD}yt`l*QS@O<lwU9~s29e8JkNhWX<ElPu
spgU2BB`q^2RYHu5iG$ICuyQUD4kjT!kJOx;d>w`I{Gyx`MJ@?G0B`tH0{{R3

literal 0
HcmV?d00001

diff --git a/test/sharness/t0122-gateway-tar.sh b/test/sharness/t0122-gateway-tar.sh
new file mode 100755
index 00000000000..3313300b48e
--- /dev/null
+++ b/test/sharness/t0122-gateway-tar.sh
@@ -0,0 +1,37 @@
+#!/usr/bin/env bash
+
+test_description="Test HTTP Gateway TAR (application/x-tar) Support"
+
+. lib/test-lib.sh
+
+test_init_ipfs
+test_launch_ipfs_daemon_without_network
+
+DIR_HASH="bafybeiczqj6w5tggtshvlyevr24drgrboffuepe2lxeojsnwmfw4tpttzu"
+
+test_expect_success "Add directory to test with" '
+  ipfs dag import ../t0122-gateway-tar-data/foo.car
+'
+test_expect_success "GET TAR with format=tar and extract" '
+  curl "http://127.0.0.1:$GWAY_PORT/ipfs/$DIR_HASH?format=tar" | tar -x
+'
+
+test_expect_success "GET TAR with 'Accept: application/x-tar' and extract" '
+  curl -H "Accept: application/x-tar" "http://127.0.0.1:$GWAY_PORT/ipfs/$DIR_HASH" | tar -x
+'
+
+test_expect_success "GET TAR with format=tar has expected Content-Type" '
+  curl -svX GET "http://127.0.0.1:$GWAY_PORT/ipfs/$DIR_HASH?format=tar" > curl_output_filename 2>&1 &&
+  cat curl_output_filename &&
+  grep "< Content-Type: application/x-tar" curl_output_filename
+'
+
+test_expect_success "GET TAR with 'Accept: application/x-tar' has expected Content-Type" '
+  curl -svX GET -H "Accept: application/x-tar" "http://127.0.0.1:$GWAY_PORT/ipfs/$DIR_HASH" > curl_output_filename 2>&1 &&
+  cat curl_output_filename &&
+  grep "< Content-Type: application/x-tar" curl_output_filename
+'
+
+test_kill_ipfs_daemon
+
+test_done

From 72af6b26862bc19c4fef8bfe8b6e9ba8c7ffa41c Mon Sep 17 00:00:00 2001
From: Henrique Dias <hacdias@gmail.com>
Date: Tue, 4 Oct 2022 16:45:07 +0200
Subject: [PATCH 07/30] wip: base dir tar writer

---
 core/corehttp/gateway_handler_tar.go          |   2 +-
 core/corehttp/gateway_handler_tar_writer.go   |  32 ++++++++----------
 .../t0122-gateway-tar-data/path-overwrite.car | Bin 0 -> 551 bytes
 3 files changed, 16 insertions(+), 18 deletions(-)
 create mode 100644 test/sharness/t0122-gateway-tar-data/path-overwrite.car

diff --git a/core/corehttp/gateway_handler_tar.go b/core/corehttp/gateway_handler_tar.go
index 90e5135a5d1..9351d50e1b4 100644
--- a/core/corehttp/gateway_handler_tar.go
+++ b/core/corehttp/gateway_handler_tar.go
@@ -49,7 +49,7 @@ func (i *gatewayHandler) serveTAR(ctx context.Context, w http.ResponseWriter, r
 	defer file.Close()
 
 	// Construct the TAR writer
-	tarw, err := NewTarWriter(w)
+	tarw, err := NewBaseDirTarWriter(w, name)
 	if err != nil {
 		webError(w, "could not build tar writer", err, http.StatusInternalServerError)
 		return
diff --git a/core/corehttp/gateway_handler_tar_writer.go b/core/corehttp/gateway_handler_tar_writer.go
index bce9ecc0093..b5d7dce3ff3 100644
--- a/core/corehttp/gateway_handler_tar_writer.go
+++ b/core/corehttp/gateway_handler_tar_writer.go
@@ -11,19 +11,21 @@ import (
 	files "github.com/ipfs/go-ipfs-files"
 )
 
-type TarWriter struct {
-	TarW *tar.Writer
-	root string
+// Adapted from: https://github.com/ipfs/go-ipfs-files/blob/master/tarwriter.go
+
+type BaseDirTarWriter struct {
+	TarW    *tar.Writer
+	baseDir string
 }
 
-// NewTarWriter wraps given io.Writer into a new tar writer
-func NewTarWriter(w io.Writer) (*TarWriter, error) {
-	return &TarWriter{
-		TarW: tar.NewWriter(w),
+func NewBaseDirTarWriter(w io.Writer, baseDir string) (*BaseDirTarWriter, error) {
+	return &BaseDirTarWriter{
+		TarW:    tar.NewWriter(w),
+		baseDir: baseDir,
 	}, nil
 }
 
-func (w *TarWriter) writeDir(f files.Directory, fpath string) error {
+func (w *BaseDirTarWriter) writeDir(f files.Directory, fpath string) error {
 	if err := writeDirHeader(w.TarW, fpath); err != nil {
 		return err
 	}
@@ -37,7 +39,7 @@ func (w *TarWriter) writeDir(f files.Directory, fpath string) error {
 	return it.Err()
 }
 
-func (w *TarWriter) writeFile(f files.File, fpath string) error {
+func (w *BaseDirTarWriter) writeFile(f files.File, fpath string) error {
 	size, err := f.Size()
 	if err != nil {
 		return err
@@ -55,15 +57,11 @@ func (w *TarWriter) writeFile(f files.File, fpath string) error {
 }
 
 // WriteNode adds a node to the archive.
-func (w *TarWriter) WriteFile(nd files.Node, fpath string) error {
-	if w.root == "" {
-		w.root = fpath
-	}
-
-	if !strings.HasPrefix(fpath, w.root) {
+func (w *BaseDirTarWriter) WriteFile(nd files.Node, fpath string) error {
+	if !strings.HasPrefix(fpath, w.baseDir) {
 		fpath = strings.Replace(fpath, ".", "", -1)
 		fpath = strings.Replace(fpath, "..", "", -1)
-		fpath = path.Join(w.root, fpath)
+		fpath = path.Join(w.baseDir, fpath)
 	}
 
 	switch nd := nd.(type) {
@@ -79,7 +77,7 @@ func (w *TarWriter) WriteFile(nd files.Node, fpath string) error {
 }
 
 // Close closes the tar writer.
-func (w *TarWriter) Close() error {
+func (w *BaseDirTarWriter) Close() error {
 	return w.TarW.Close()
 }
 
diff --git a/test/sharness/t0122-gateway-tar-data/path-overwrite.car b/test/sharness/t0122-gateway-tar-data/path-overwrite.car
new file mode 100644
index 0000000000000000000000000000000000000000..db074c77b2e48fd58b29d435d0443af9390c28ca
GIT binary patch
literal 551
zcmcColv<RZUsBw7Ln}g+fw4eHVZuAD2hM`J1>H_7em*EEZ$HA{;Ol$)jL3cN+GFQB
zZRe+#r4|)u=I1d^VI)SmkTsVINL^%8ZQhw<bNWLcOzqoYF-x(~+sS5D?1?8Qw<YGD
z+*f>7NLWu#ABb`jb25`N^Gl0$Q!<N$bRnAUzcI@^%P#LdADPFvcB}Ta#TrvwbavY8
zDt|mrHC=DvR3T=dVhJuTCJx3hqAb(lQW8@5<aGILvi|Bi5f_;y@0?Z(nY3>Zxvcfb
zf6_Lt>azyA6NOmPGILTT#6ZSc5LV75!oeiO=aHI|ldq#to?n!cqR1t|Hys>G1U+C6
za**bklw+Cddo-jw>utX8cX*j+zT*Cp<KHU;-j+;izRUYZNE{MID8U4FBbN}^M+6Ok
V_y`<OI)v@x;sE-HDTzyh2>^fL-NXO@

literal 0
HcmV?d00001


From 97d2023420bf1f091a8a8136fe840fb2606dd1c5 Mon Sep 17 00:00:00 2001
From: Henrique Dias <hacdias@gmail.com>
Date: Wed, 5 Oct 2022 16:37:22 +0200
Subject: [PATCH 08/30] wip: remove duplicate code, cleanup docs and text

---
 core/corehttp/gateway_handler_tar.go        |   9 +-
 core/corehttp/gateway_handler_tar_writer.go | 112 --------------------
 go.mod                                      |   2 +-
 go.sum                                      |   2 +
 4 files changed, 8 insertions(+), 117 deletions(-)
 delete mode 100644 core/corehttp/gateway_handler_tar_writer.go

diff --git a/core/corehttp/gateway_handler_tar.go b/core/corehttp/gateway_handler_tar.go
index 9351d50e1b4..b15241097d4 100644
--- a/core/corehttp/gateway_handler_tar.go
+++ b/core/corehttp/gateway_handler_tar.go
@@ -6,6 +6,7 @@ import (
 	"net/http"
 	"time"
 
+	files "github.com/ipfs/go-ipfs-files"
 	ipath "github.com/ipfs/interface-go-ipfs-core/path"
 	"github.com/ipfs/kubo/tracing"
 	"go.opentelemetry.io/otel/attribute"
@@ -49,7 +50,7 @@ func (i *gatewayHandler) serveTAR(ctx context.Context, w http.ResponseWriter, r
 	defer file.Close()
 
 	// Construct the TAR writer
-	tarw, err := NewBaseDirTarWriter(w, name)
+	tarw, err := files.NewTarWriter(w)
 	if err != nil {
 		webError(w, "could not build tar writer", err, http.StatusInternalServerError)
 		return
@@ -63,13 +64,13 @@ func (i *gatewayHandler) serveTAR(ctx context.Context, w http.ResponseWriter, r
 		w.Header().Set("Last-Modified", modtime.UTC().Format(http.TimeFormat))
 	}
 
+	// Configure trailing error header in case it happens.
+	w.Header().Set("Trailer", "X-Stream-Error")
 	w.Header().Set("Content-Type", "application/x-tar")
 
 	if err := tarw.WriteFile(file, name); err != nil {
 		// We return error as a trailer, however it is not something browsers can access
-		// (https://github.com/mdn/browser-compat-data/issues/14703)
-		// Due to this, we suggest client always verify that
-		// the received CAR stream response is matching requested DAG selector
+		// (https://github.com/mdn/browser-compat-data/issues/14703).
 		w.Header().Set("X-Stream-Error", err.Error())
 		return
 	}
diff --git a/core/corehttp/gateway_handler_tar_writer.go b/core/corehttp/gateway_handler_tar_writer.go
deleted file mode 100644
index b5d7dce3ff3..00000000000
--- a/core/corehttp/gateway_handler_tar_writer.go
+++ /dev/null
@@ -1,112 +0,0 @@
-package corehttp
-
-import (
-	"archive/tar"
-	"fmt"
-	"io"
-	"path"
-	"strings"
-	"time"
-
-	files "github.com/ipfs/go-ipfs-files"
-)
-
-// Adapted from: https://github.com/ipfs/go-ipfs-files/blob/master/tarwriter.go
-
-type BaseDirTarWriter struct {
-	TarW    *tar.Writer
-	baseDir string
-}
-
-func NewBaseDirTarWriter(w io.Writer, baseDir string) (*BaseDirTarWriter, error) {
-	return &BaseDirTarWriter{
-		TarW:    tar.NewWriter(w),
-		baseDir: baseDir,
-	}, nil
-}
-
-func (w *BaseDirTarWriter) writeDir(f files.Directory, fpath string) error {
-	if err := writeDirHeader(w.TarW, fpath); err != nil {
-		return err
-	}
-
-	it := f.Entries()
-	for it.Next() {
-		if err := w.WriteFile(it.Node(), path.Join(fpath, it.Name())); err != nil {
-			return err
-		}
-	}
-	return it.Err()
-}
-
-func (w *BaseDirTarWriter) writeFile(f files.File, fpath string) error {
-	size, err := f.Size()
-	if err != nil {
-		return err
-	}
-
-	if err := writeFileHeader(w.TarW, fpath, uint64(size)); err != nil {
-		return err
-	}
-
-	if _, err := io.Copy(w.TarW, f); err != nil {
-		return err
-	}
-	w.TarW.Flush()
-	return nil
-}
-
-// WriteNode adds a node to the archive.
-func (w *BaseDirTarWriter) WriteFile(nd files.Node, fpath string) error {
-	if !strings.HasPrefix(fpath, w.baseDir) {
-		fpath = strings.Replace(fpath, ".", "", -1)
-		fpath = strings.Replace(fpath, "..", "", -1)
-		fpath = path.Join(w.baseDir, fpath)
-	}
-
-	switch nd := nd.(type) {
-	case *files.Symlink:
-		return writeSymlinkHeader(w.TarW, nd.Target, fpath)
-	case files.File:
-		return w.writeFile(nd, fpath)
-	case files.Directory:
-		return w.writeDir(nd, fpath)
-	default:
-		return fmt.Errorf("file type %T is not supported", nd)
-	}
-}
-
-// Close closes the tar writer.
-func (w *BaseDirTarWriter) Close() error {
-	return w.TarW.Close()
-}
-
-func writeDirHeader(w *tar.Writer, fpath string) error {
-	return w.WriteHeader(&tar.Header{
-		Name:     fpath,
-		Typeflag: tar.TypeDir,
-		Mode:     0777,
-		ModTime:  time.Now().Truncate(time.Second),
-		// TODO: set mode, dates, etc. when added to unixFS
-	})
-}
-
-func writeFileHeader(w *tar.Writer, fpath string, size uint64) error {
-	return w.WriteHeader(&tar.Header{
-		Name:     fpath,
-		Size:     int64(size),
-		Typeflag: tar.TypeReg,
-		Mode:     0644,
-		ModTime:  time.Now().Truncate(time.Second),
-		// TODO: set mode, dates, etc. when added to unixFS
-	})
-}
-
-func writeSymlinkHeader(w *tar.Writer, target, fpath string) error {
-	return w.WriteHeader(&tar.Header{
-		Name:     fpath,
-		Linkname: target,
-		Mode:     0777,
-		Typeflag: tar.TypeSymlink,
-	})
-}
diff --git a/go.mod b/go.mod
index 1ccd10ae0e1..a8368ff3bfa 100644
--- a/go.mod
+++ b/go.mod
@@ -37,7 +37,7 @@ require (
 	github.com/ipfs/go-ipfs-cmds v0.8.1
 	github.com/ipfs/go-ipfs-exchange-interface v0.2.0
 	github.com/ipfs/go-ipfs-exchange-offline v0.3.0
-	github.com/ipfs/go-ipfs-files v0.1.1
+	github.com/ipfs/go-ipfs-files v0.1.2-0.20221005143422-2acf76c5d599
 	github.com/ipfs/go-ipfs-keystore v0.0.2
 	github.com/ipfs/go-ipfs-pinner v0.2.1
 	github.com/ipfs/go-ipfs-posinfo v0.0.1
diff --git a/go.sum b/go.sum
index f1917d039d0..33584cca44e 100644
--- a/go.sum
+++ b/go.sum
@@ -554,6 +554,8 @@ github.com/ipfs/go-ipfs-files v0.0.3/go.mod h1:INEFm0LL2LWXBhNJ2PMIIb2w45hpXgPjN
 github.com/ipfs/go-ipfs-files v0.0.8/go.mod h1:wiN/jSG8FKyk7N0WyctKSvq3ljIa2NNTiZB55kpTdOs=
 github.com/ipfs/go-ipfs-files v0.1.1 h1:/MbEowmpLo9PJTEQk16m9rKzUHjeP4KRU9nWJyJO324=
 github.com/ipfs/go-ipfs-files v0.1.1/go.mod h1:8xkIrMWH+Y5P7HvJ4Yc5XWwIW2e52dyXUiC0tZyjDbM=
+github.com/ipfs/go-ipfs-files v0.1.2-0.20221005143422-2acf76c5d599 h1:Q8zjRB6RBXw+B4/7aHSOMueHZOTo1jaN37daSr75ca8=
+github.com/ipfs/go-ipfs-files v0.1.2-0.20221005143422-2acf76c5d599/go.mod h1:vT7uaQfIsprKktzbTPLnIsd+NGw9ZbYwSq0g3N74u0M=
 github.com/ipfs/go-ipfs-keystore v0.0.2 h1:Fa9xg9IFD1VbiZtrNLzsD0GuELVHUFXCWF64kCPfEXU=
 github.com/ipfs/go-ipfs-keystore v0.0.2/go.mod h1:H49tRmibOEs7gLMgbOsjC4dqh1u5e0R/SWuc2ScfgSo=
 github.com/ipfs/go-ipfs-pinner v0.2.1 h1:kw9hiqh2p8TatILYZ3WAfQQABby7SQARdrdA+5Z5QfY=

From 6165b62542f4c9ca138d47cf92bff809e0ca887b Mon Sep 17 00:00:00 2001
From: Henrique Dias <hacdias@gmail.com>
Date: Wed, 5 Oct 2022 16:40:54 +0200
Subject: [PATCH 09/30] wip: go mod tidy

---
 go.sum | 2 --
 1 file changed, 2 deletions(-)

diff --git a/go.sum b/go.sum
index 33584cca44e..b6f96151202 100644
--- a/go.sum
+++ b/go.sum
@@ -552,8 +552,6 @@ github.com/ipfs/go-ipfs-exchange-offline v0.3.0 h1:c/Dg8GDPzixGd0MC8Jh6mjOwU57uY
 github.com/ipfs/go-ipfs-exchange-offline v0.3.0/go.mod h1:MOdJ9DChbb5u37M1IcbrRB02e++Z7521fMxqCNRrz9s=
 github.com/ipfs/go-ipfs-files v0.0.3/go.mod h1:INEFm0LL2LWXBhNJ2PMIIb2w45hpXgPjNoE7yA8Y1d4=
 github.com/ipfs/go-ipfs-files v0.0.8/go.mod h1:wiN/jSG8FKyk7N0WyctKSvq3ljIa2NNTiZB55kpTdOs=
-github.com/ipfs/go-ipfs-files v0.1.1 h1:/MbEowmpLo9PJTEQk16m9rKzUHjeP4KRU9nWJyJO324=
-github.com/ipfs/go-ipfs-files v0.1.1/go.mod h1:8xkIrMWH+Y5P7HvJ4Yc5XWwIW2e52dyXUiC0tZyjDbM=
 github.com/ipfs/go-ipfs-files v0.1.2-0.20221005143422-2acf76c5d599 h1:Q8zjRB6RBXw+B4/7aHSOMueHZOTo1jaN37daSr75ca8=
 github.com/ipfs/go-ipfs-files v0.1.2-0.20221005143422-2acf76c5d599/go.mod h1:vT7uaQfIsprKktzbTPLnIsd+NGw9ZbYwSq0g3N74u0M=
 github.com/ipfs/go-ipfs-keystore v0.0.2 h1:Fa9xg9IFD1VbiZtrNLzsD0GuELVHUFXCWF64kCPfEXU=

From 3a228e4eaa56992d70459cea9ef3f68020d034c2 Mon Sep 17 00:00:00 2001
From: Henrique Dias <hacdias@gmail.com>
Date: Thu, 6 Oct 2022 11:49:23 +0200
Subject: [PATCH 10/30] chore: go mod tidy

---
 docs/examples/kubo-as-a-library/go.mod | 2 +-
 docs/examples/kubo-as-a-library/go.sum | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/docs/examples/kubo-as-a-library/go.mod b/docs/examples/kubo-as-a-library/go.mod
index 950800dd0fe..020d9699f2f 100644
--- a/docs/examples/kubo-as-a-library/go.mod
+++ b/docs/examples/kubo-as-a-library/go.mod
@@ -7,7 +7,7 @@ go 1.17
 replace github.com/ipfs/kubo => ./../../..
 
 require (
-	github.com/ipfs/go-ipfs-files v0.1.1
+	github.com/ipfs/go-ipfs-files v0.1.2-0.20221005143422-2acf76c5d599
 	github.com/ipfs/interface-go-ipfs-core v0.7.0
 	github.com/ipfs/kubo v0.14.0-rc1
 	github.com/libp2p/go-libp2p v0.23.2
diff --git a/docs/examples/kubo-as-a-library/go.sum b/docs/examples/kubo-as-a-library/go.sum
index 76400b4106c..43308afaab0 100644
--- a/docs/examples/kubo-as-a-library/go.sum
+++ b/docs/examples/kubo-as-a-library/go.sum
@@ -559,8 +559,8 @@ github.com/ipfs/go-ipfs-exchange-offline v0.3.0 h1:c/Dg8GDPzixGd0MC8Jh6mjOwU57uY
 github.com/ipfs/go-ipfs-exchange-offline v0.3.0/go.mod h1:MOdJ9DChbb5u37M1IcbrRB02e++Z7521fMxqCNRrz9s=
 github.com/ipfs/go-ipfs-files v0.0.3/go.mod h1:INEFm0LL2LWXBhNJ2PMIIb2w45hpXgPjNoE7yA8Y1d4=
 github.com/ipfs/go-ipfs-files v0.0.8/go.mod h1:wiN/jSG8FKyk7N0WyctKSvq3ljIa2NNTiZB55kpTdOs=
-github.com/ipfs/go-ipfs-files v0.1.1 h1:/MbEowmpLo9PJTEQk16m9rKzUHjeP4KRU9nWJyJO324=
-github.com/ipfs/go-ipfs-files v0.1.1/go.mod h1:8xkIrMWH+Y5P7HvJ4Yc5XWwIW2e52dyXUiC0tZyjDbM=
+github.com/ipfs/go-ipfs-files v0.1.2-0.20221005143422-2acf76c5d599 h1:Q8zjRB6RBXw+B4/7aHSOMueHZOTo1jaN37daSr75ca8=
+github.com/ipfs/go-ipfs-files v0.1.2-0.20221005143422-2acf76c5d599/go.mod h1:vT7uaQfIsprKktzbTPLnIsd+NGw9ZbYwSq0g3N74u0M=
 github.com/ipfs/go-ipfs-keystore v0.0.2 h1:Fa9xg9IFD1VbiZtrNLzsD0GuELVHUFXCWF64kCPfEXU=
 github.com/ipfs/go-ipfs-keystore v0.0.2/go.mod h1:H49tRmibOEs7gLMgbOsjC4dqh1u5e0R/SWuc2ScfgSo=
 github.com/ipfs/go-ipfs-pinner v0.2.1 h1:kw9hiqh2p8TatILYZ3WAfQQABby7SQARdrdA+5Z5QfY=

From 31ee4e5c495316c372296c80da8d3bc209bb4d93 Mon Sep 17 00:00:00 2001
From: Henrique Dias <hacdias@gmail.com>
Date: Thu, 6 Oct 2022 12:07:45 +0200
Subject: [PATCH 11/30] wip: add inside and outside root cars for testing

---
 .../t0122-gateway-tar-data/inside-root.car        | Bin 0 -> 383 bytes
 .../t0122-gateway-tar-data/outside-root.car       | Bin 0 -> 249 bytes
 .../t0122-gateway-tar-data/path-overwrite.car     | Bin 551 -> 0 bytes
 3 files changed, 0 insertions(+), 0 deletions(-)
 create mode 100644 test/sharness/t0122-gateway-tar-data/inside-root.car
 create mode 100644 test/sharness/t0122-gateway-tar-data/outside-root.car
 delete mode 100644 test/sharness/t0122-gateway-tar-data/path-overwrite.car

diff --git a/test/sharness/t0122-gateway-tar-data/inside-root.car b/test/sharness/t0122-gateway-tar-data/inside-root.car
new file mode 100644
index 0000000000000000000000000000000000000000..c37b594f8d6d132f3d3ef955b7b2270302f3b574
GIT binary patch
literal 383
zcmcColv<RZUsBw7Ln}g+fw4eHK~>ee^xFHkc@K`SV?H{4`bU*@S0~udT5xce(!=Q&
z`4-$uFH0>d&dkqaj37p}kRF!`NF7V9N~fsTRp-u?;^j5_^n32ysn~q^+RC22)y!qz
zgARrYv8CnbCnXkfF>x?P6Q#irV(`1uv2n9Mx30H1R<)FoX-6hgLt#_gqO@SXiDDIH
zM?cOJ;!MdbN=+`wFRFx_O;8WW>`)<vn~XkgDlJ}1cO4cfx6RI&btV0mpR82C!W}E;
oMos;$|5b=xPftHBGbdGo4`j9xF?u{wb8_-^6w339a#9qz07cTFga7~l

literal 0
HcmV?d00001

diff --git a/test/sharness/t0122-gateway-tar-data/outside-root.car b/test/sharness/t0122-gateway-tar-data/outside-root.car
new file mode 100644
index 0000000000000000000000000000000000000000..7587dcb7efae5c7a5fac5d94539c131eacc83537
GIT binary patch
literal 249
zcmcColv<RZUsBw7Ln}g+fw4eH!NLD(^q$bi9vo@MdlU|yDtT~TTJ%+pkHg`!Cpwfa
zgdR;VOD!tS%+F)&WF$tpkO7wpW2lhAO-3I#l@_n1yAF$#+h%9Xx|06OPgW{m;f|Ga
zqo#h>|0=|$r>CEmpD)2Dq)Vg@=A^_T2|g|+4n`wlZ170U$;sDID9<m-Nm1ki0CQ(#
A%K!iX

literal 0
HcmV?d00001

diff --git a/test/sharness/t0122-gateway-tar-data/path-overwrite.car b/test/sharness/t0122-gateway-tar-data/path-overwrite.car
deleted file mode 100644
index db074c77b2e48fd58b29d435d0443af9390c28ca..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 551
zcmcColv<RZUsBw7Ln}g+fw4eHVZuAD2hM`J1>H_7em*EEZ$HA{;Ol$)jL3cN+GFQB
zZRe+#r4|)u=I1d^VI)SmkTsVINL^%8ZQhw<bNWLcOzqoYF-x(~+sS5D?1?8Qw<YGD
z+*f>7NLWu#ABb`jb25`N^Gl0$Q!<N$bRnAUzcI@^%P#LdADPFvcB}Ta#TrvwbavY8
zDt|mrHC=DvR3T=dVhJuTCJx3hqAb(lQW8@5<aGILvi|Bi5f_;y@0?Z(nY3>Zxvcfb
zf6_Lt>azyA6NOmPGILTT#6ZSc5LV75!oeiO=aHI|ldq#to?n!cqR1t|Hys>G1U+C6
za**bklw+Cddo-jw>utX8cX*j+zT*Cp<KHU;-j+;izRUYZNE{MID8U4FBbN}^M+6Ok
V_y`<OI)v@x;sE-HDTzyh2>^fL-NXO@


From 766fa01a335b4de096d56b448ef62ba0736396c1 Mon Sep 17 00:00:00 2001
From: Henrique Dias <hacdias@gmail.com>
Date: Thu, 6 Oct 2022 12:07:53 +0200
Subject: [PATCH 12/30] test: cleanup

---
 test/sharness/t0122-gateway-tar.sh | 17 +++++++++++------
 1 file changed, 11 insertions(+), 6 deletions(-)

diff --git a/test/sharness/t0122-gateway-tar.sh b/test/sharness/t0122-gateway-tar.sh
index 3313300b48e..dc59ff42c83 100755
--- a/test/sharness/t0122-gateway-tar.sh
+++ b/test/sharness/t0122-gateway-tar.sh
@@ -7,31 +7,36 @@ test_description="Test HTTP Gateway TAR (application/x-tar) Support"
 test_init_ipfs
 test_launch_ipfs_daemon_without_network
 
-DIR_HASH="bafybeiczqj6w5tggtshvlyevr24drgrboffuepe2lxeojsnwmfw4tpttzu"
+FOO_HASH="bafybeiczqj6w5tggtshvlyevr24drgrboffuepe2lxeojsnwmfw4tpttzu"
 
 test_expect_success "Add directory to test with" '
-  ipfs dag import ../t0122-gateway-tar-data/foo.car
+  ipfs dag import ../t0122-gateway-tar-data/foo.car > import_output &&
+  grep -q $FOO_HASH import_output
 '
+
 test_expect_success "GET TAR with format=tar and extract" '
-  curl "http://127.0.0.1:$GWAY_PORT/ipfs/$DIR_HASH?format=tar" | tar -x
+  curl "http://127.0.0.1:$GWAY_PORT/ipfs/$FOO_HASH?format=tar" | tar -x
 '
 
 test_expect_success "GET TAR with 'Accept: application/x-tar' and extract" '
-  curl -H "Accept: application/x-tar" "http://127.0.0.1:$GWAY_PORT/ipfs/$DIR_HASH" | tar -x
+  curl -H "Accept: application/x-tar" "http://127.0.0.1:$GWAY_PORT/ipfs/$FOO_HASH" | tar -x
 '
 
 test_expect_success "GET TAR with format=tar has expected Content-Type" '
-  curl -svX GET "http://127.0.0.1:$GWAY_PORT/ipfs/$DIR_HASH?format=tar" > curl_output_filename 2>&1 &&
+  curl -svX GET "http://127.0.0.1:$GWAY_PORT/ipfs/$FOO_HASH?format=tar" > curl_output_filename 2>&1 &&
   cat curl_output_filename &&
   grep "< Content-Type: application/x-tar" curl_output_filename
 '
 
 test_expect_success "GET TAR with 'Accept: application/x-tar' has expected Content-Type" '
-  curl -svX GET -H "Accept: application/x-tar" "http://127.0.0.1:$GWAY_PORT/ipfs/$DIR_HASH" > curl_output_filename 2>&1 &&
+  curl -svX GET -H "Accept: application/x-tar" "http://127.0.0.1:$GWAY_PORT/ipfs/$FOO_HASH" > curl_output_filename 2>&1 &&
   cat curl_output_filename &&
   grep "< Content-Type: application/x-tar" curl_output_filename
 '
 
+# outside-root.car --> bafybeicaj7kvxpcv4neaqzwhrqqmdstu4dhrwfpknrgebq6nzcecfucvyu
+# inside-root.car --> bafybeibfevfxlvxp5vxobr5oapczpf7resxnleb7tkqmdorc4gl5cdva3y
+
 test_kill_ipfs_daemon
 
 test_done

From 56798ed4a8cb53366e55ec5fa03a9745805d7844 Mon Sep 17 00:00:00 2001
From: Henrique Dias <hacdias@gmail.com>
Date: Thu, 6 Oct 2022 12:20:09 +0200
Subject: [PATCH 13/30] tests: test with relative paths inside and outside
 directory

---
 test/sharness/t0122-gateway-tar.sh | 21 +++++++++++++++++++--
 1 file changed, 19 insertions(+), 2 deletions(-)

diff --git a/test/sharness/t0122-gateway-tar.sh b/test/sharness/t0122-gateway-tar.sh
index dc59ff42c83..a56e8c3b5a2 100755
--- a/test/sharness/t0122-gateway-tar.sh
+++ b/test/sharness/t0122-gateway-tar.sh
@@ -8,6 +8,8 @@ test_init_ipfs
 test_launch_ipfs_daemon_without_network
 
 FOO_HASH="bafybeiczqj6w5tggtshvlyevr24drgrboffuepe2lxeojsnwmfw4tpttzu"
+OUTSIDE_ROOT_HASH="bafybeicaj7kvxpcv4neaqzwhrqqmdstu4dhrwfpknrgebq6nzcecfucvyu"
+INSIDE_ROOT_HASH="bafybeibfevfxlvxp5vxobr5oapczpf7resxnleb7tkqmdorc4gl5cdva3y"
 
 test_expect_success "Add directory to test with" '
   ipfs dag import ../t0122-gateway-tar-data/foo.car > import_output &&
@@ -34,8 +36,23 @@ test_expect_success "GET TAR with 'Accept: application/x-tar' has expected Conte
   grep "< Content-Type: application/x-tar" curl_output_filename
 '
 
-# outside-root.car --> bafybeicaj7kvxpcv4neaqzwhrqqmdstu4dhrwfpknrgebq6nzcecfucvyu
-# inside-root.car --> bafybeibfevfxlvxp5vxobr5oapczpf7resxnleb7tkqmdorc4gl5cdva3y
+test_expect_success "Add directories with relative paths to test with" '
+  ipfs dag import ../t0122-gateway-tar-data/outside-root.car > import_output &&
+  grep -q $OUTSIDE_ROOT_HASH import_output &&
+  ipfs dag import ../t0122-gateway-tar-data/inside-root.car > import_output &&
+  grep -q $INSIDE_ROOT_HASH import_output
+'
+
+test_expect_success "GET TAR with relative paths outside root fails" '
+  curl -i "http://127.0.0.1:$GWAY_PORT/ipfs/$OUTSIDE_ROOT_HASH?format=tar" > curl_output_filename &&
+  grep -q "Trailer: X-Stream-Error" curl_output_filename &&
+  grep -q "X-Stream-Error:" curl_output_filename
+'
+
+test_expect_success "GET TAR with relative paths inside root works" '
+  curl -i "http://127.0.0.1:$GWAY_PORT/ipfs/$INSIDE_ROOT_HASH?format=tar" > curl_output_filename &&
+  ! grep -q "X-Stream-Error:" curl_output_filename
+'
 
 test_kill_ipfs_daemon
 

From 0d6fad123b4745312985b73a68a27f36e461885c Mon Sep 17 00:00:00 2001
From: Henrique Dias <hacdias@gmail.com>
Date: Thu, 6 Oct 2022 13:28:58 +0200
Subject: [PATCH 14/30] test: add missing line

---
 test/sharness/t0122-gateway-tar.sh | 1 +
 1 file changed, 1 insertion(+)

diff --git a/test/sharness/t0122-gateway-tar.sh b/test/sharness/t0122-gateway-tar.sh
index a56e8c3b5a2..1051fe02b92 100755
--- a/test/sharness/t0122-gateway-tar.sh
+++ b/test/sharness/t0122-gateway-tar.sh
@@ -51,6 +51,7 @@ test_expect_success "GET TAR with relative paths outside root fails" '
 
 test_expect_success "GET TAR with relative paths inside root works" '
   curl -i "http://127.0.0.1:$GWAY_PORT/ipfs/$INSIDE_ROOT_HASH?format=tar" > curl_output_filename &&
+  grep -q "Trailer: X-Stream-Error" curl_output_filename &&
   ! grep -q "X-Stream-Error:" curl_output_filename
 '
 

From 51bf57281f6fa10e98361e456fe51bafdfca8d92 Mon Sep 17 00:00:00 2001
From: Henrique Dias <hacdias@gmail.com>
Date: Fri, 7 Oct 2022 08:52:39 +0200
Subject: [PATCH 15/30] tar: force close connection upon error

---
 core/corehttp/gateway_handler_tar.go | 25 +++++++++++++++++++------
 test/sharness/t0122-gateway-tar.sh   |  8 ++------
 2 files changed, 21 insertions(+), 12 deletions(-)

diff --git a/core/corehttp/gateway_handler_tar.go b/core/corehttp/gateway_handler_tar.go
index b15241097d4..4b1fb5715be 100644
--- a/core/corehttp/gateway_handler_tar.go
+++ b/core/corehttp/gateway_handler_tar.go
@@ -64,14 +64,27 @@ func (i *gatewayHandler) serveTAR(ctx context.Context, w http.ResponseWriter, r
 		w.Header().Set("Last-Modified", modtime.UTC().Format(http.TimeFormat))
 	}
 
-	// Configure trailing error header in case it happens.
-	w.Header().Set("Trailer", "X-Stream-Error")
 	w.Header().Set("Content-Type", "application/x-tar")
 
-	if err := tarw.WriteFile(file, name); err != nil {
-		// We return error as a trailer, however it is not something browsers can access
-		// (https://github.com/mdn/browser-compat-data/issues/14703).
-		w.Header().Set("X-Stream-Error", err.Error())
+	if tarErr := tarw.WriteFile(file, name); tarErr != nil {
+		// There are no good ways of showing an error during a stream. Therefore, we try
+		// to hijack the connection to forcefully close it, causing a network error.
+		hj, ok := w.(http.Hijacker)
+		if !ok {
+			// If we could not Hijack the connection, we write the original error. This will hopefully
+			// corrupt the generated TAR file, such that the client will receive an error unpacking.
+			webError(w, "could not build tar archive", tarErr, http.StatusInternalServerError)
+			return
+		}
+
+		conn, _, err := hj.Hijack()
+		if err != nil {
+			// Deliberately pass the original tar error here instead of the hijacking error.
+			webError(w, "could not build tar archive", tarErr, http.StatusInternalServerError)
+			return
+		}
+
+		conn.Close()
 		return
 	}
 }
diff --git a/test/sharness/t0122-gateway-tar.sh b/test/sharness/t0122-gateway-tar.sh
index 1051fe02b92..0dfd7e71a13 100755
--- a/test/sharness/t0122-gateway-tar.sh
+++ b/test/sharness/t0122-gateway-tar.sh
@@ -44,15 +44,11 @@ test_expect_success "Add directories with relative paths to test with" '
 '
 
 test_expect_success "GET TAR with relative paths outside root fails" '
-  curl -i "http://127.0.0.1:$GWAY_PORT/ipfs/$OUTSIDE_ROOT_HASH?format=tar" > curl_output_filename &&
-  grep -q "Trailer: X-Stream-Error" curl_output_filename &&
-  grep -q "X-Stream-Error:" curl_output_filename
+  ! curl "http://127.0.0.1:$GWAY_PORT/ipfs/$OUTSIDE_ROOT_HASH?format=tar"
 '
 
 test_expect_success "GET TAR with relative paths inside root works" '
-  curl -i "http://127.0.0.1:$GWAY_PORT/ipfs/$INSIDE_ROOT_HASH?format=tar" > curl_output_filename &&
-  grep -q "Trailer: X-Stream-Error" curl_output_filename &&
-  ! grep -q "X-Stream-Error:" curl_output_filename
+  curl "http://127.0.0.1:$GWAY_PORT/ipfs/$INSIDE_ROOT_HASH?format=tar" | tar -x
 '
 
 test_kill_ipfs_daemon

From 8eee4934fd12b28602c65bbbf0eb26f22ec08c2d Mon Sep 17 00:00:00 2001
From: Henrique Dias <hacdias@gmail.com>
Date: Mon, 10 Oct 2022 11:12:29 +0200
Subject: [PATCH 16/30] style: wrap line

---
 core/corehttp/gateway_handler.go | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/core/corehttp/gateway_handler.go b/core/corehttp/gateway_handler.go
index 8f75a76df33..521df920455 100644
--- a/core/corehttp/gateway_handler.go
+++ b/core/corehttp/gateway_handler.go
@@ -876,7 +876,8 @@ func customResponseFormat(r *http.Request) (mediaType string, params map[string]
 	// We only care about explicit, vendor-specific content-types.
 	for _, accept := range r.Header.Values("Accept") {
 		// respond to the very first ipld content type
-		if strings.HasPrefix(accept, "application/vnd.ipld") || strings.HasPrefix(accept, "application/x-tar") {
+		if strings.HasPrefix(accept, "application/vnd.ipld") ||
+			strings.HasPrefix(accept, "application/x-tar") {
 			mediatype, params, err := mime.ParseMediaType(accept)
 			if err != nil {
 				return "", nil, err

From e6b099a8233f6f9de24fa6c8f2e4f6dc84495470 Mon Sep 17 00:00:00 2001
From: Henrique Dias <hacdias@gmail.com>
Date: Mon, 10 Oct 2022 11:56:46 +0200
Subject: [PATCH 17/30] test: use test_should_contain

---
 test/sharness/t0122-gateway-tar.sh | 12 +++++-------
 1 file changed, 5 insertions(+), 7 deletions(-)

diff --git a/test/sharness/t0122-gateway-tar.sh b/test/sharness/t0122-gateway-tar.sh
index 0dfd7e71a13..5c9ce957cc0 100755
--- a/test/sharness/t0122-gateway-tar.sh
+++ b/test/sharness/t0122-gateway-tar.sh
@@ -13,7 +13,7 @@ INSIDE_ROOT_HASH="bafybeibfevfxlvxp5vxobr5oapczpf7resxnleb7tkqmdorc4gl5cdva3y"
 
 test_expect_success "Add directory to test with" '
   ipfs dag import ../t0122-gateway-tar-data/foo.car > import_output &&
-  grep -q $FOO_HASH import_output
+  test_should_contain $FOO_HASH import_output
 '
 
 test_expect_success "GET TAR with format=tar and extract" '
@@ -26,21 +26,19 @@ test_expect_success "GET TAR with 'Accept: application/x-tar' and extract" '
 
 test_expect_success "GET TAR with format=tar has expected Content-Type" '
   curl -svX GET "http://127.0.0.1:$GWAY_PORT/ipfs/$FOO_HASH?format=tar" > curl_output_filename 2>&1 &&
-  cat curl_output_filename &&
-  grep "< Content-Type: application/x-tar" curl_output_filename
+  test_should_contain "< Content-Type: application/x-tar" curl_output_filename
 '
 
 test_expect_success "GET TAR with 'Accept: application/x-tar' has expected Content-Type" '
   curl -svX GET -H "Accept: application/x-tar" "http://127.0.0.1:$GWAY_PORT/ipfs/$FOO_HASH" > curl_output_filename 2>&1 &&
-  cat curl_output_filename &&
-  grep "< Content-Type: application/x-tar" curl_output_filename
+  test_should_contain "< Content-Type: application/x-tar" curl_output_filename
 '
 
 test_expect_success "Add directories with relative paths to test with" '
   ipfs dag import ../t0122-gateway-tar-data/outside-root.car > import_output &&
-  grep -q $OUTSIDE_ROOT_HASH import_output &&
+  test_should_contain $OUTSIDE_ROOT_HASH import_output &&
   ipfs dag import ../t0122-gateway-tar-data/inside-root.car > import_output &&
-  grep -q $INSIDE_ROOT_HASH import_output
+  test_should_contain $INSIDE_ROOT_HASH import_output
 '
 
 test_expect_success "GET TAR with relative paths outside root fails" '

From f792fdbf342600219bc40cd84f7b68df262d0420 Mon Sep 17 00:00:00 2001
From: Henrique Dias <hacdias@gmail.com>
Date: Mon, 10 Oct 2022 12:33:29 +0200
Subject: [PATCH 18/30] test: simplify test and s/HASH/CID/g

---
 test/sharness/t0122-gateway-tar.sh | 37 ++++++++++++++----------------
 1 file changed, 17 insertions(+), 20 deletions(-)

diff --git a/test/sharness/t0122-gateway-tar.sh b/test/sharness/t0122-gateway-tar.sh
index 5c9ce957cc0..4ca3b56ea65 100755
--- a/test/sharness/t0122-gateway-tar.sh
+++ b/test/sharness/t0122-gateway-tar.sh
@@ -7,46 +7,43 @@ test_description="Test HTTP Gateway TAR (application/x-tar) Support"
 test_init_ipfs
 test_launch_ipfs_daemon_without_network
 
-FOO_HASH="bafybeiczqj6w5tggtshvlyevr24drgrboffuepe2lxeojsnwmfw4tpttzu"
-OUTSIDE_ROOT_HASH="bafybeicaj7kvxpcv4neaqzwhrqqmdstu4dhrwfpknrgebq6nzcecfucvyu"
-INSIDE_ROOT_HASH="bafybeibfevfxlvxp5vxobr5oapczpf7resxnleb7tkqmdorc4gl5cdva3y"
+FOO_CID="bafybeiczqj6w5tggtshvlyevr24drgrboffuepe2lxeojsnwmfw4tpttzu"
+OUTSIDE_ROOT_CID="bafybeicaj7kvxpcv4neaqzwhrqqmdstu4dhrwfpknrgebq6nzcecfucvyu"
+INSIDE_ROOT_CID="bafybeibfevfxlvxp5vxobr5oapczpf7resxnleb7tkqmdorc4gl5cdva3y"
 
-test_expect_success "Add directory to test with" '
+test_expect_success "Add files and directories to test with" '
   ipfs dag import ../t0122-gateway-tar-data/foo.car > import_output &&
-  test_should_contain $FOO_HASH import_output
+  test_should_contain $FOO_CID import_output &&
+  ipfs dag import ../t0122-gateway-tar-data/outside-root.car > import_output &&
+  test_should_contain $OUTSIDE_ROOT_CID import_output &&
+  ipfs dag import ../t0122-gateway-tar-data/inside-root.car > import_output &&
+  test_should_contain $INSIDE_ROOT_CID import_output
 '
 
 test_expect_success "GET TAR with format=tar and extract" '
-  curl "http://127.0.0.1:$GWAY_PORT/ipfs/$FOO_HASH?format=tar" | tar -x
+  curl "http://127.0.0.1:$GWAY_PORT/ipfs/$FOO_CID?format=tar" | tar -x
 '
 
 test_expect_success "GET TAR with 'Accept: application/x-tar' and extract" '
-  curl -H "Accept: application/x-tar" "http://127.0.0.1:$GWAY_PORT/ipfs/$FOO_HASH" | tar -x
+  curl -H "Accept: application/x-tar" "http://127.0.0.1:$GWAY_PORT/ipfs/$FOO_CID" | tar -x
 '
 
 test_expect_success "GET TAR with format=tar has expected Content-Type" '
-  curl -svX GET "http://127.0.0.1:$GWAY_PORT/ipfs/$FOO_HASH?format=tar" > curl_output_filename 2>&1 &&
-  test_should_contain "< Content-Type: application/x-tar" curl_output_filename
+  curl -sD - "http://127.0.0.1:$GWAY_PORT/ipfs/$FOO_CID?format=tar" > curl_output_filename 2>&1 &&
+  test_should_contain "Content-Type: application/x-tar" curl_output_filename
 '
 
 test_expect_success "GET TAR with 'Accept: application/x-tar' has expected Content-Type" '
-  curl -svX GET -H "Accept: application/x-tar" "http://127.0.0.1:$GWAY_PORT/ipfs/$FOO_HASH" > curl_output_filename 2>&1 &&
-  test_should_contain "< Content-Type: application/x-tar" curl_output_filename
-'
-
-test_expect_success "Add directories with relative paths to test with" '
-  ipfs dag import ../t0122-gateway-tar-data/outside-root.car > import_output &&
-  test_should_contain $OUTSIDE_ROOT_HASH import_output &&
-  ipfs dag import ../t0122-gateway-tar-data/inside-root.car > import_output &&
-  test_should_contain $INSIDE_ROOT_HASH import_output
+  curl -sD - -H "Accept: application/x-tar" "http://127.0.0.1:$GWAY_PORT/ipfs/$FOO_CID" > curl_output_filename 2>&1 &&
+  test_should_contain "Content-Type: application/x-tar" curl_output_filename
 '
 
 test_expect_success "GET TAR with relative paths outside root fails" '
-  ! curl "http://127.0.0.1:$GWAY_PORT/ipfs/$OUTSIDE_ROOT_HASH?format=tar"
+  ! curl "http://127.0.0.1:$GWAY_PORT/ipfs/$OUTSIDE_ROOT_CID?format=tar"
 '
 
 test_expect_success "GET TAR with relative paths inside root works" '
-  curl "http://127.0.0.1:$GWAY_PORT/ipfs/$INSIDE_ROOT_HASH?format=tar" | tar -x
+  curl "http://127.0.0.1:$GWAY_PORT/ipfs/$INSIDE_ROOT_CID?format=tar" | tar -x
 '
 
 test_kill_ipfs_daemon

From 2059d36cd2903e2a53e86c719cc45a96f7305115 Mon Sep 17 00:00:00 2001
From: Henrique Dias <hacdias@gmail.com>
Date: Mon, 10 Oct 2022 13:47:04 +0200
Subject: [PATCH 19/30] fix: add CID as top-level name for consistency

---
 core/corehttp/gateway_handler_tar.go | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/core/corehttp/gateway_handler_tar.go b/core/corehttp/gateway_handler_tar.go
index 4b1fb5715be..87936e8200d 100644
--- a/core/corehttp/gateway_handler_tar.go
+++ b/core/corehttp/gateway_handler_tar.go
@@ -66,7 +66,8 @@ func (i *gatewayHandler) serveTAR(ctx context.Context, w http.ResponseWriter, r
 
 	w.Header().Set("Content-Type", "application/x-tar")
 
-	if tarErr := tarw.WriteFile(file, name); tarErr != nil {
+	// The TAR has a top-level directory (or file) named by the CID.
+	if tarErr := tarw.WriteFile(file, resolvedPath.Cid().String()); tarErr != nil {
 		// There are no good ways of showing an error during a stream. Therefore, we try
 		// to hijack the connection to forcefully close it, causing a network error.
 		hj, ok := w.(http.Hijacker)

From bc410af93e82c586f8431f99860cf9c201728bf1 Mon Sep 17 00:00:00 2001
From: Henrique Dias <hacdias@gmail.com>
Date: Mon, 10 Oct 2022 13:59:51 +0200
Subject: [PATCH 20/30] test: add utf-8 testing and root directory/file name
 check

---
 test/sharness/t0122-gateway-tar-data/foo.car | Bin 203 -> 0 bytes
 test/sharness/t0122-gateway-tar.sh           |  57 +++++++++++++++----
 2 files changed, 45 insertions(+), 12 deletions(-)
 delete mode 100644 test/sharness/t0122-gateway-tar-data/foo.car

diff --git a/test/sharness/t0122-gateway-tar-data/foo.car b/test/sharness/t0122-gateway-tar-data/foo.car
deleted file mode 100644
index f1e26614d35ce9bda912488db029f73c1087591f..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 203
zcmcColv<RZUsBw7Ln}g+fw4eHA+o79@653|{h<%0_U*8krC8|gWHT%F#FLZT5_3=P
zD?XcEmReMtnV-iPMvQ799WEsyg-=eG&nD}yt`l*QS@O<lwU9~s29e8JkNhWX<ElPu
spgU2BB`q^2RYHu5iG$ICuyQUD4kjT!kJOx;d>w`I{Gyx`MJ@?G0B`tH0{{R3

diff --git a/test/sharness/t0122-gateway-tar.sh b/test/sharness/t0122-gateway-tar.sh
index 4ca3b56ea65..5f9d74c889e 100755
--- a/test/sharness/t0122-gateway-tar.sh
+++ b/test/sharness/t0122-gateway-tar.sh
@@ -7,37 +7,70 @@ test_description="Test HTTP Gateway TAR (application/x-tar) Support"
 test_init_ipfs
 test_launch_ipfs_daemon_without_network
 
-FOO_CID="bafybeiczqj6w5tggtshvlyevr24drgrboffuepe2lxeojsnwmfw4tpttzu"
 OUTSIDE_ROOT_CID="bafybeicaj7kvxpcv4neaqzwhrqqmdstu4dhrwfpknrgebq6nzcecfucvyu"
 INSIDE_ROOT_CID="bafybeibfevfxlvxp5vxobr5oapczpf7resxnleb7tkqmdorc4gl5cdva3y"
 
-test_expect_success "Add files and directories to test with" '
-  ipfs dag import ../t0122-gateway-tar-data/foo.car > import_output &&
-  test_should_contain $FOO_CID import_output &&
-  ipfs dag import ../t0122-gateway-tar-data/outside-root.car > import_output &&
-  test_should_contain $OUTSIDE_ROOT_CID import_output &&
-  ipfs dag import ../t0122-gateway-tar-data/inside-root.car > import_output &&
-  test_should_contain $INSIDE_ROOT_CID import_output
+test_expect_success "Add the test directory" '
+  mkdir -p rootDir/ipfs &&
+  mkdir -p rootDir/ipns &&
+  mkdir -p rootDir/api &&
+  mkdir -p rootDir/ą/ę &&
+  echo "I am a txt file on path with utf8" > rootDir/ą/ę/file-źł.txt &&
+  echo "I am a txt file in confusing /api dir" > rootDir/api/file.txt &&
+  echo "I am a txt file in confusing /ipfs dir" > rootDir/ipfs/file.txt &&
+  echo "I am a txt file in confusing /ipns dir" > rootDir/ipns/file.txt &&
+  DIR_CID=$(ipfs add -Qr --cid-version 1 rootDir) &&
+  FILE_CID=$(ipfs files stat --enc=json /ipfs/$DIR_CID/ą/ę/file-źł.txt | jq -r .Hash) &&
+  FILE_SIZE=$(ipfs files stat --enc=json /ipfs/$DIR_CID/ą/ę/file-źł.txt | jq -r .Size)
+  echo "$FILE_CID / $FILE_SIZE"
 '
 
 test_expect_success "GET TAR with format=tar and extract" '
-  curl "http://127.0.0.1:$GWAY_PORT/ipfs/$FOO_CID?format=tar" | tar -x
+  curl "http://127.0.0.1:$GWAY_PORT/ipfs/$FILE_CID?format=tar" | tar -x
 '
 
 test_expect_success "GET TAR with 'Accept: application/x-tar' and extract" '
-  curl -H "Accept: application/x-tar" "http://127.0.0.1:$GWAY_PORT/ipfs/$FOO_CID" | tar -x
+  curl -H "Accept: application/x-tar" "http://127.0.0.1:$GWAY_PORT/ipfs/$FILE_CID" | tar -x
 '
 
 test_expect_success "GET TAR with format=tar has expected Content-Type" '
-  curl -sD - "http://127.0.0.1:$GWAY_PORT/ipfs/$FOO_CID?format=tar" > curl_output_filename 2>&1 &&
+  curl -sD - "http://127.0.0.1:$GWAY_PORT/ipfs/$FILE_CID?format=tar" > curl_output_filename 2>&1 &&
   test_should_contain "Content-Type: application/x-tar" curl_output_filename
 '
 
 test_expect_success "GET TAR with 'Accept: application/x-tar' has expected Content-Type" '
-  curl -sD - -H "Accept: application/x-tar" "http://127.0.0.1:$GWAY_PORT/ipfs/$FOO_CID" > curl_output_filename 2>&1 &&
+  curl -sD - -H "Accept: application/x-tar" "http://127.0.0.1:$GWAY_PORT/ipfs/$FILE_CID" > curl_output_filename 2>&1 &&
   test_should_contain "Content-Type: application/x-tar" curl_output_filename
 '
 
+test_expect_success "GET TAR has expected root file" '
+  rm -rf outputDir && mkdir outputDir &&
+  curl "http://127.0.0.1:$GWAY_PORT/ipfs/$FILE_CID?format=tar" | tar -x -C outputDir &&
+  test -f "outputDir/$FILE_CID" &&
+  echo "I am a txt file on path with utf8" > expected &&
+  test_cmp expected outputDir/$FILE_CID
+'
+
+test_expect_success "GET TAR has expected root directory" '
+  rm -rf outputDir && mkdir outputDir &&
+  curl "http://127.0.0.1:$GWAY_PORT/ipfs/$DIR_CID?format=tar" | tar -x -C outputDir &&
+  test -d "outputDir/$DIR_CID" &&
+  echo "I am a txt file on path with utf8" > expected &&
+  test_cmp expected outputDir/$DIR_CID/ą/ę/file-źł.txt
+'
+
+test_expect_success "GET TAR with explicit ?filename= succeeds with proper header" "
+  curl -fo actual -D actual_headers 'http://127.0.0.1:$GWAY_PORT/ipfs/$DIR_CID?filename=testтест.tar&format=tar' &&
+  grep -F 'Content-Disposition: attachment; filename=\"test____.tar\"; filename*=UTF-8'\'\''test%D1%82%D0%B5%D1%81%D1%82.tar' actual_headers
+"
+
+test_expect_success "Add CARs with relative paths to test with" '
+  ipfs dag import ../t0122-gateway-tar-data/outside-root.car > import_output &&
+  test_should_contain $OUTSIDE_ROOT_CID import_output &&
+  ipfs dag import ../t0122-gateway-tar-data/inside-root.car > import_output &&
+  test_should_contain $INSIDE_ROOT_CID import_output
+'
+
 test_expect_success "GET TAR with relative paths outside root fails" '
   ! curl "http://127.0.0.1:$GWAY_PORT/ipfs/$OUTSIDE_ROOT_CID?format=tar"
 '

From bd4fd1f76348b7e4e44d22cd3c644f6f961565c7 Mon Sep 17 00:00:00 2001
From: Henrique Dias <hacdias@gmail.com>
Date: Mon, 10 Oct 2022 14:09:17 +0200
Subject: [PATCH 21/30] fix: return 400 if not unixfs

---
 core/corehttp/gateway_handler_tar.go | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/core/corehttp/gateway_handler_tar.go b/core/corehttp/gateway_handler_tar.go
index 87936e8200d..7b7bb75f1ff 100644
--- a/core/corehttp/gateway_handler_tar.go
+++ b/core/corehttp/gateway_handler_tar.go
@@ -23,6 +23,14 @@ func (i *gatewayHandler) serveTAR(ctx context.Context, w http.ResponseWriter, r
 	ctx, cancel := context.WithCancel(ctx)
 	defer cancel()
 
+	// Get Unixfs file
+	file, err := i.api.Unixfs().Get(ctx, resolvedPath)
+	if err != nil {
+		webError(w, "ipfs cat "+html.EscapeString(contentPath.String()), err, http.StatusBadRequest)
+		return
+	}
+	defer file.Close()
+
 	// Set Cache-Control and read optional Last-Modified time
 	modtime := addCacheControlHeaders(w, r, contentPath, resolvedPath.Cid())
 
@@ -41,14 +49,6 @@ func (i *gatewayHandler) serveTAR(ctx context.Context, w http.ResponseWriter, r
 	}
 	setContentDispositionHeader(w, name, "attachment")
 
-	// Get Unixfs file
-	file, err := i.api.Unixfs().Get(ctx, resolvedPath)
-	if err != nil {
-		webError(w, "ipfs cat "+html.EscapeString(contentPath.String()), err, http.StatusNotFound)
-		return
-	}
-	defer file.Close()
-
 	// Construct the TAR writer
 	tarw, err := files.NewTarWriter(w)
 	if err != nil {

From 568789787bd080f9fe521e56ba91fb94f4aff113 Mon Sep 17 00:00:00 2001
From: Henrique Dias <hacdias@gmail.com>
Date: Mon, 10 Oct 2022 14:20:41 +0200
Subject: [PATCH 22/30] test: check if file exists and is on correct directory

---
 test/sharness/t0122-gateway-tar.sh | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/test/sharness/t0122-gateway-tar.sh b/test/sharness/t0122-gateway-tar.sh
index 5f9d74c889e..6bfa9cd7a09 100755
--- a/test/sharness/t0122-gateway-tar.sh
+++ b/test/sharness/t0122-gateway-tar.sh
@@ -76,7 +76,9 @@ test_expect_success "GET TAR with relative paths outside root fails" '
 '
 
 test_expect_success "GET TAR with relative paths inside root works" '
-  curl "http://127.0.0.1:$GWAY_PORT/ipfs/$INSIDE_ROOT_CID?format=tar" | tar -x
+  rm -rf outputDir && mkdir outputDir &&
+  curl "http://127.0.0.1:$GWAY_PORT/ipfs/$INSIDE_ROOT_CID?format=tar" | tar -x -C outputDir &&
+  test -f outputDir/$INSIDE_ROOT_CID/foobar/file
 '
 
 test_kill_ipfs_daemon

From 7a352a98125376bf25dc464cf3ff2c0072162208 Mon Sep 17 00:00:00 2001
From: Henrique Dias <hacdias@gmail.com>
Date: Thu, 13 Oct 2022 17:47:57 +0200
Subject: [PATCH 23/30] refactor: use abort handler from #9333

---
 core/corehttp/gateway_handler.go     | 23 +++++++++++++++++++++++
 core/corehttp/gateway_handler_tar.go | 22 ++--------------------
 2 files changed, 25 insertions(+), 20 deletions(-)

diff --git a/core/corehttp/gateway_handler.go b/core/corehttp/gateway_handler.go
index 521df920455..715987865eb 100644
--- a/core/corehttp/gateway_handler.go
+++ b/core/corehttp/gateway_handler.go
@@ -1085,3 +1085,26 @@ func (i *gatewayHandler) setCommonHeaders(w http.ResponseWriter, r *http.Request
 
 	return nil
 }
+
+// abortConn forcefully closes an HTTP connection, leading to a network error on the
+// client side. This is a way of showing the client that there was an error while streaming
+// the contents since there are no good ways of showing an error during a stream.
+func abortConn(w http.ResponseWriter) {
+	hj, ok := w.(http.Hijacker)
+	if !ok {
+		// If we could not Hijack the connection (such as in HTTP/2.x) connections, we
+		// panic using http.ErrAbortHandler, which aborts the response.
+		// https://pkg.go.dev/net/http#ErrAbortHandler
+		panic(http.ErrAbortHandler)
+	}
+
+	conn, _, err := hj.Hijack()
+	if err != nil {
+		panic(http.ErrAbortHandler)
+	}
+
+	err = conn.Close()
+	if err != nil {
+		panic(http.ErrAbortHandler)
+	}
+}
diff --git a/core/corehttp/gateway_handler_tar.go b/core/corehttp/gateway_handler_tar.go
index 7b7bb75f1ff..bbb72510ad6 100644
--- a/core/corehttp/gateway_handler_tar.go
+++ b/core/corehttp/gateway_handler_tar.go
@@ -67,25 +67,7 @@ func (i *gatewayHandler) serveTAR(ctx context.Context, w http.ResponseWriter, r
 	w.Header().Set("Content-Type", "application/x-tar")
 
 	// The TAR has a top-level directory (or file) named by the CID.
-	if tarErr := tarw.WriteFile(file, resolvedPath.Cid().String()); tarErr != nil {
-		// There are no good ways of showing an error during a stream. Therefore, we try
-		// to hijack the connection to forcefully close it, causing a network error.
-		hj, ok := w.(http.Hijacker)
-		if !ok {
-			// If we could not Hijack the connection, we write the original error. This will hopefully
-			// corrupt the generated TAR file, such that the client will receive an error unpacking.
-			webError(w, "could not build tar archive", tarErr, http.StatusInternalServerError)
-			return
-		}
-
-		conn, _, err := hj.Hijack()
-		if err != nil {
-			// Deliberately pass the original tar error here instead of the hijacking error.
-			webError(w, "could not build tar archive", tarErr, http.StatusInternalServerError)
-			return
-		}
-
-		conn.Close()
-		return
+	if err := tarw.WriteFile(file, resolvedPath.Cid().String()); err != nil {
+		abortConn(w)
 	}
 }

From be7cf70d7c8aa4b9f11211dd48cfb211019decfa Mon Sep 17 00:00:00 2001
From: Henrique Dias <hacdias@gmail.com>
Date: Tue, 8 Nov 2022 11:25:23 +0100
Subject: [PATCH 24/30] refactor: print error instead of aborting

---
 core/corehttp/gateway_handler.go     | 23 -----------------------
 core/corehttp/gateway_handler_tar.go |  4 +++-
 test/sharness/t0122-gateway-tar.sh   |  3 ++-
 3 files changed, 5 insertions(+), 25 deletions(-)

diff --git a/core/corehttp/gateway_handler.go b/core/corehttp/gateway_handler.go
index 715987865eb..521df920455 100644
--- a/core/corehttp/gateway_handler.go
+++ b/core/corehttp/gateway_handler.go
@@ -1085,26 +1085,3 @@ func (i *gatewayHandler) setCommonHeaders(w http.ResponseWriter, r *http.Request
 
 	return nil
 }
-
-// abortConn forcefully closes an HTTP connection, leading to a network error on the
-// client side. This is a way of showing the client that there was an error while streaming
-// the contents since there are no good ways of showing an error during a stream.
-func abortConn(w http.ResponseWriter) {
-	hj, ok := w.(http.Hijacker)
-	if !ok {
-		// If we could not Hijack the connection (such as in HTTP/2.x) connections, we
-		// panic using http.ErrAbortHandler, which aborts the response.
-		// https://pkg.go.dev/net/http#ErrAbortHandler
-		panic(http.ErrAbortHandler)
-	}
-
-	conn, _, err := hj.Hijack()
-	if err != nil {
-		panic(http.ErrAbortHandler)
-	}
-
-	err = conn.Close()
-	if err != nil {
-		panic(http.ErrAbortHandler)
-	}
-}
diff --git a/core/corehttp/gateway_handler_tar.go b/core/corehttp/gateway_handler_tar.go
index bbb72510ad6..d222bf80d2a 100644
--- a/core/corehttp/gateway_handler_tar.go
+++ b/core/corehttp/gateway_handler_tar.go
@@ -68,6 +68,8 @@ func (i *gatewayHandler) serveTAR(ctx context.Context, w http.ResponseWriter, r
 
 	// The TAR has a top-level directory (or file) named by the CID.
 	if err := tarw.WriteFile(file, resolvedPath.Cid().String()); err != nil {
-		abortConn(w)
+		w.Header().Set("X-Stream-Error", err.Error())
+		w.Write([]byte(err.Error()))
+		return
 	}
 }
diff --git a/test/sharness/t0122-gateway-tar.sh b/test/sharness/t0122-gateway-tar.sh
index 6bfa9cd7a09..75fd9560ab5 100755
--- a/test/sharness/t0122-gateway-tar.sh
+++ b/test/sharness/t0122-gateway-tar.sh
@@ -72,7 +72,8 @@ test_expect_success "Add CARs with relative paths to test with" '
 '
 
 test_expect_success "GET TAR with relative paths outside root fails" '
-  ! curl "http://127.0.0.1:$GWAY_PORT/ipfs/$OUTSIDE_ROOT_CID?format=tar"
+  curl -o - "http://127.0.0.1:$GWAY_PORT/ipfs/$OUTSIDE_ROOT_CID?format=tar" > curl_output_filename &&
+  test_should_contain "relative UnixFS paths outside the root are now allowed" curl_output_filename
 '
 
 test_expect_success "GET TAR with relative paths inside root works" '

From 0a54117c6a99751a1fa09e9057838d799c15569d Mon Sep 17 00:00:00 2001
From: Henrique Dias <hacdias@gmail.com>
Date: Tue, 8 Nov 2022 11:31:24 +0100
Subject: [PATCH 25/30] ignore w.Write error as not much can be done now

---
 core/corehttp/gateway_handler_tar.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/core/corehttp/gateway_handler_tar.go b/core/corehttp/gateway_handler_tar.go
index d222bf80d2a..cedc57a6b08 100644
--- a/core/corehttp/gateway_handler_tar.go
+++ b/core/corehttp/gateway_handler_tar.go
@@ -69,7 +69,7 @@ func (i *gatewayHandler) serveTAR(ctx context.Context, w http.ResponseWriter, r
 	// The TAR has a top-level directory (or file) named by the CID.
 	if err := tarw.WriteFile(file, resolvedPath.Cid().String()); err != nil {
 		w.Header().Set("X-Stream-Error", err.Error())
-		w.Write([]byte(err.Error()))
+		_, _ = w.Write([]byte(err.Error()))
 		return
 	}
 }

From 8870480f23c7ddc02e23ed61c4ef49696fdc58f3 Mon Sep 17 00:00:00 2001
From: Henrique Dias <hacdias@gmail.com>
Date: Tue, 8 Nov 2022 11:58:20 +0100
Subject: [PATCH 26/30] update to go-ipfs-files@619bbe4

---
 go.mod | 2 +-
 go.sum | 2 ++
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/go.mod b/go.mod
index a8368ff3bfa..a36c8b503ef 100644
--- a/go.mod
+++ b/go.mod
@@ -37,7 +37,7 @@ require (
 	github.com/ipfs/go-ipfs-cmds v0.8.1
 	github.com/ipfs/go-ipfs-exchange-interface v0.2.0
 	github.com/ipfs/go-ipfs-exchange-offline v0.3.0
-	github.com/ipfs/go-ipfs-files v0.1.2-0.20221005143422-2acf76c5d599
+	github.com/ipfs/go-ipfs-files v0.1.2-0.20221107134911-619bbe4ac790
 	github.com/ipfs/go-ipfs-keystore v0.0.2
 	github.com/ipfs/go-ipfs-pinner v0.2.1
 	github.com/ipfs/go-ipfs-posinfo v0.0.1
diff --git a/go.sum b/go.sum
index b6f96151202..dd6517c7464 100644
--- a/go.sum
+++ b/go.sum
@@ -554,6 +554,8 @@ github.com/ipfs/go-ipfs-files v0.0.3/go.mod h1:INEFm0LL2LWXBhNJ2PMIIb2w45hpXgPjN
 github.com/ipfs/go-ipfs-files v0.0.8/go.mod h1:wiN/jSG8FKyk7N0WyctKSvq3ljIa2NNTiZB55kpTdOs=
 github.com/ipfs/go-ipfs-files v0.1.2-0.20221005143422-2acf76c5d599 h1:Q8zjRB6RBXw+B4/7aHSOMueHZOTo1jaN37daSr75ca8=
 github.com/ipfs/go-ipfs-files v0.1.2-0.20221005143422-2acf76c5d599/go.mod h1:vT7uaQfIsprKktzbTPLnIsd+NGw9ZbYwSq0g3N74u0M=
+github.com/ipfs/go-ipfs-files v0.1.2-0.20221107134911-619bbe4ac790 h1:oOOTxRZm77AyCttD493JoQEvhQGlhQRIVSNMl1AjX9s=
+github.com/ipfs/go-ipfs-files v0.1.2-0.20221107134911-619bbe4ac790/go.mod h1:vT7uaQfIsprKktzbTPLnIsd+NGw9ZbYwSq0g3N74u0M=
 github.com/ipfs/go-ipfs-keystore v0.0.2 h1:Fa9xg9IFD1VbiZtrNLzsD0GuELVHUFXCWF64kCPfEXU=
 github.com/ipfs/go-ipfs-keystore v0.0.2/go.mod h1:H49tRmibOEs7gLMgbOsjC4dqh1u5e0R/SWuc2ScfgSo=
 github.com/ipfs/go-ipfs-pinner v0.2.1 h1:kw9hiqh2p8TatILYZ3WAfQQABby7SQARdrdA+5Z5QfY=

From 53861364e349fc155c5c63b703a91c8228f74112 Mon Sep 17 00:00:00 2001
From: Henrique Dias <hacdias@gmail.com>
Date: Tue, 8 Nov 2022 12:07:19 +0100
Subject: [PATCH 27/30] go mod tidy

---
 docs/examples/kubo-as-a-library/go.mod | 2 +-
 docs/examples/kubo-as-a-library/go.sum | 4 ++--
 go.sum                                 | 2 --
 3 files changed, 3 insertions(+), 5 deletions(-)

diff --git a/docs/examples/kubo-as-a-library/go.mod b/docs/examples/kubo-as-a-library/go.mod
index 020d9699f2f..8fec53c965f 100644
--- a/docs/examples/kubo-as-a-library/go.mod
+++ b/docs/examples/kubo-as-a-library/go.mod
@@ -7,7 +7,7 @@ go 1.17
 replace github.com/ipfs/kubo => ./../../..
 
 require (
-	github.com/ipfs/go-ipfs-files v0.1.2-0.20221005143422-2acf76c5d599
+	github.com/ipfs/go-ipfs-files v0.1.2-0.20221107134911-619bbe4ac790
 	github.com/ipfs/interface-go-ipfs-core v0.7.0
 	github.com/ipfs/kubo v0.14.0-rc1
 	github.com/libp2p/go-libp2p v0.23.2
diff --git a/docs/examples/kubo-as-a-library/go.sum b/docs/examples/kubo-as-a-library/go.sum
index 43308afaab0..ae1a6ef3a7c 100644
--- a/docs/examples/kubo-as-a-library/go.sum
+++ b/docs/examples/kubo-as-a-library/go.sum
@@ -559,8 +559,8 @@ github.com/ipfs/go-ipfs-exchange-offline v0.3.0 h1:c/Dg8GDPzixGd0MC8Jh6mjOwU57uY
 github.com/ipfs/go-ipfs-exchange-offline v0.3.0/go.mod h1:MOdJ9DChbb5u37M1IcbrRB02e++Z7521fMxqCNRrz9s=
 github.com/ipfs/go-ipfs-files v0.0.3/go.mod h1:INEFm0LL2LWXBhNJ2PMIIb2w45hpXgPjNoE7yA8Y1d4=
 github.com/ipfs/go-ipfs-files v0.0.8/go.mod h1:wiN/jSG8FKyk7N0WyctKSvq3ljIa2NNTiZB55kpTdOs=
-github.com/ipfs/go-ipfs-files v0.1.2-0.20221005143422-2acf76c5d599 h1:Q8zjRB6RBXw+B4/7aHSOMueHZOTo1jaN37daSr75ca8=
-github.com/ipfs/go-ipfs-files v0.1.2-0.20221005143422-2acf76c5d599/go.mod h1:vT7uaQfIsprKktzbTPLnIsd+NGw9ZbYwSq0g3N74u0M=
+github.com/ipfs/go-ipfs-files v0.1.2-0.20221107134911-619bbe4ac790 h1:oOOTxRZm77AyCttD493JoQEvhQGlhQRIVSNMl1AjX9s=
+github.com/ipfs/go-ipfs-files v0.1.2-0.20221107134911-619bbe4ac790/go.mod h1:vT7uaQfIsprKktzbTPLnIsd+NGw9ZbYwSq0g3N74u0M=
 github.com/ipfs/go-ipfs-keystore v0.0.2 h1:Fa9xg9IFD1VbiZtrNLzsD0GuELVHUFXCWF64kCPfEXU=
 github.com/ipfs/go-ipfs-keystore v0.0.2/go.mod h1:H49tRmibOEs7gLMgbOsjC4dqh1u5e0R/SWuc2ScfgSo=
 github.com/ipfs/go-ipfs-pinner v0.2.1 h1:kw9hiqh2p8TatILYZ3WAfQQABby7SQARdrdA+5Z5QfY=
diff --git a/go.sum b/go.sum
index dd6517c7464..e7e4278469f 100644
--- a/go.sum
+++ b/go.sum
@@ -552,8 +552,6 @@ github.com/ipfs/go-ipfs-exchange-offline v0.3.0 h1:c/Dg8GDPzixGd0MC8Jh6mjOwU57uY
 github.com/ipfs/go-ipfs-exchange-offline v0.3.0/go.mod h1:MOdJ9DChbb5u37M1IcbrRB02e++Z7521fMxqCNRrz9s=
 github.com/ipfs/go-ipfs-files v0.0.3/go.mod h1:INEFm0LL2LWXBhNJ2PMIIb2w45hpXgPjNoE7yA8Y1d4=
 github.com/ipfs/go-ipfs-files v0.0.8/go.mod h1:wiN/jSG8FKyk7N0WyctKSvq3ljIa2NNTiZB55kpTdOs=
-github.com/ipfs/go-ipfs-files v0.1.2-0.20221005143422-2acf76c5d599 h1:Q8zjRB6RBXw+B4/7aHSOMueHZOTo1jaN37daSr75ca8=
-github.com/ipfs/go-ipfs-files v0.1.2-0.20221005143422-2acf76c5d599/go.mod h1:vT7uaQfIsprKktzbTPLnIsd+NGw9ZbYwSq0g3N74u0M=
 github.com/ipfs/go-ipfs-files v0.1.2-0.20221107134911-619bbe4ac790 h1:oOOTxRZm77AyCttD493JoQEvhQGlhQRIVSNMl1AjX9s=
 github.com/ipfs/go-ipfs-files v0.1.2-0.20221107134911-619bbe4ac790/go.mod h1:vT7uaQfIsprKktzbTPLnIsd+NGw9ZbYwSq0g3N74u0M=
 github.com/ipfs/go-ipfs-keystore v0.0.2 h1:Fa9xg9IFD1VbiZtrNLzsD0GuELVHUFXCWF64kCPfEXU=

From 7411e703062d0e92126b24f8b8b5ab74ff2fa5ec Mon Sep 17 00:00:00 2001
From: Marcin Rataj <lidel@lidel.org>
Date: Wed, 9 Nov 2022 17:41:51 +0100
Subject: [PATCH 28/30] chore: go-ipfs-files v0.2.0

https://github.com/ipfs/go-ipfs-files/releases/tag/v0.2.0
---
 docs/examples/kubo-as-a-library/go.mod | 2 +-
 docs/examples/kubo-as-a-library/go.sum | 4 ++--
 go.mod                                 | 2 +-
 go.sum                                 | 4 ++--
 4 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/docs/examples/kubo-as-a-library/go.mod b/docs/examples/kubo-as-a-library/go.mod
index 8fec53c965f..de0527efd8d 100644
--- a/docs/examples/kubo-as-a-library/go.mod
+++ b/docs/examples/kubo-as-a-library/go.mod
@@ -7,7 +7,7 @@ go 1.17
 replace github.com/ipfs/kubo => ./../../..
 
 require (
-	github.com/ipfs/go-ipfs-files v0.1.2-0.20221107134911-619bbe4ac790
+	github.com/ipfs/go-ipfs-files v0.2.0
 	github.com/ipfs/interface-go-ipfs-core v0.7.0
 	github.com/ipfs/kubo v0.14.0-rc1
 	github.com/libp2p/go-libp2p v0.23.2
diff --git a/docs/examples/kubo-as-a-library/go.sum b/docs/examples/kubo-as-a-library/go.sum
index ae1a6ef3a7c..5345acee3b8 100644
--- a/docs/examples/kubo-as-a-library/go.sum
+++ b/docs/examples/kubo-as-a-library/go.sum
@@ -559,8 +559,8 @@ github.com/ipfs/go-ipfs-exchange-offline v0.3.0 h1:c/Dg8GDPzixGd0MC8Jh6mjOwU57uY
 github.com/ipfs/go-ipfs-exchange-offline v0.3.0/go.mod h1:MOdJ9DChbb5u37M1IcbrRB02e++Z7521fMxqCNRrz9s=
 github.com/ipfs/go-ipfs-files v0.0.3/go.mod h1:INEFm0LL2LWXBhNJ2PMIIb2w45hpXgPjNoE7yA8Y1d4=
 github.com/ipfs/go-ipfs-files v0.0.8/go.mod h1:wiN/jSG8FKyk7N0WyctKSvq3ljIa2NNTiZB55kpTdOs=
-github.com/ipfs/go-ipfs-files v0.1.2-0.20221107134911-619bbe4ac790 h1:oOOTxRZm77AyCttD493JoQEvhQGlhQRIVSNMl1AjX9s=
-github.com/ipfs/go-ipfs-files v0.1.2-0.20221107134911-619bbe4ac790/go.mod h1:vT7uaQfIsprKktzbTPLnIsd+NGw9ZbYwSq0g3N74u0M=
+github.com/ipfs/go-ipfs-files v0.2.0 h1:z6MCYHQSZpDWpUSK59Kf0ajP1fi4gLCf6fIulVsp8A8=
+github.com/ipfs/go-ipfs-files v0.2.0/go.mod h1:vT7uaQfIsprKktzbTPLnIsd+NGw9ZbYwSq0g3N74u0M=
 github.com/ipfs/go-ipfs-keystore v0.0.2 h1:Fa9xg9IFD1VbiZtrNLzsD0GuELVHUFXCWF64kCPfEXU=
 github.com/ipfs/go-ipfs-keystore v0.0.2/go.mod h1:H49tRmibOEs7gLMgbOsjC4dqh1u5e0R/SWuc2ScfgSo=
 github.com/ipfs/go-ipfs-pinner v0.2.1 h1:kw9hiqh2p8TatILYZ3WAfQQABby7SQARdrdA+5Z5QfY=
diff --git a/go.mod b/go.mod
index a36c8b503ef..43688f8bfe0 100644
--- a/go.mod
+++ b/go.mod
@@ -37,7 +37,7 @@ require (
 	github.com/ipfs/go-ipfs-cmds v0.8.1
 	github.com/ipfs/go-ipfs-exchange-interface v0.2.0
 	github.com/ipfs/go-ipfs-exchange-offline v0.3.0
-	github.com/ipfs/go-ipfs-files v0.1.2-0.20221107134911-619bbe4ac790
+	github.com/ipfs/go-ipfs-files v0.2.0
 	github.com/ipfs/go-ipfs-keystore v0.0.2
 	github.com/ipfs/go-ipfs-pinner v0.2.1
 	github.com/ipfs/go-ipfs-posinfo v0.0.1
diff --git a/go.sum b/go.sum
index e7e4278469f..348263ffefe 100644
--- a/go.sum
+++ b/go.sum
@@ -552,8 +552,8 @@ github.com/ipfs/go-ipfs-exchange-offline v0.3.0 h1:c/Dg8GDPzixGd0MC8Jh6mjOwU57uY
 github.com/ipfs/go-ipfs-exchange-offline v0.3.0/go.mod h1:MOdJ9DChbb5u37M1IcbrRB02e++Z7521fMxqCNRrz9s=
 github.com/ipfs/go-ipfs-files v0.0.3/go.mod h1:INEFm0LL2LWXBhNJ2PMIIb2w45hpXgPjNoE7yA8Y1d4=
 github.com/ipfs/go-ipfs-files v0.0.8/go.mod h1:wiN/jSG8FKyk7N0WyctKSvq3ljIa2NNTiZB55kpTdOs=
-github.com/ipfs/go-ipfs-files v0.1.2-0.20221107134911-619bbe4ac790 h1:oOOTxRZm77AyCttD493JoQEvhQGlhQRIVSNMl1AjX9s=
-github.com/ipfs/go-ipfs-files v0.1.2-0.20221107134911-619bbe4ac790/go.mod h1:vT7uaQfIsprKktzbTPLnIsd+NGw9ZbYwSq0g3N74u0M=
+github.com/ipfs/go-ipfs-files v0.2.0 h1:z6MCYHQSZpDWpUSK59Kf0ajP1fi4gLCf6fIulVsp8A8=
+github.com/ipfs/go-ipfs-files v0.2.0/go.mod h1:vT7uaQfIsprKktzbTPLnIsd+NGw9ZbYwSq0g3N74u0M=
 github.com/ipfs/go-ipfs-keystore v0.0.2 h1:Fa9xg9IFD1VbiZtrNLzsD0GuELVHUFXCWF64kCPfEXU=
 github.com/ipfs/go-ipfs-keystore v0.0.2/go.mod h1:H49tRmibOEs7gLMgbOsjC4dqh1u5e0R/SWuc2ScfgSo=
 github.com/ipfs/go-ipfs-pinner v0.2.1 h1:kw9hiqh2p8TatILYZ3WAfQQABby7SQARdrdA+5Z5QfY=

From 4ca3940368d57d9d1341f6e125f5e5c1cfc59e4f Mon Sep 17 00:00:00 2001
From: Marcin Rataj <lidel@lidel.org>
Date: Wed, 9 Nov 2022 18:23:03 +0100
Subject: [PATCH 29/30] refactor: cleanup Etag handling with TAR

- shortened etag
- added tests
- moved weak etag to _tar.go file + added bunch of comments why
---
 core/corehttp/gateway_handler.go     | 11 ++++-------
 core/corehttp/gateway_handler_tar.go | 25 +++++++++++++++++++++----
 test/sharness/t0122-gateway-tar.sh   |  6 +++++-
 3 files changed, 30 insertions(+), 12 deletions(-)

diff --git a/core/corehttp/gateway_handler.go b/core/corehttp/gateway_handler.go
index 521df920455..7f0f11885e6 100644
--- a/core/corehttp/gateway_handler.go
+++ b/core/corehttp/gateway_handler.go
@@ -846,13 +846,10 @@ func getEtag(r *http.Request, cid cid.Cid) string {
 	responseFormat, _, err := customResponseFormat(r)
 	if err == nil && responseFormat != "" {
 		// application/vnd.ipld.foo → foo
-		f := responseFormat[strings.LastIndex(responseFormat, ".")+1:]
-		// Etag: "cid.foo" (gives us nice compression together with Content-Disposition in block (raw) and car responses)
-		suffix = `.` + f + suffix
-		// Since different TAR implementations may produce different byte-for-byte responses, we define a weak Etag.
-		if responseFormat == "application/x-tar" {
-			prefix = "W/" + prefix
-		}
+		// application/x-bar → x-bar
+		shortFormat := responseFormat[strings.LastIndexAny(responseFormat, "/.")+1:]
+		// Etag: "cid.shortFmt" (gives us nice compression together with Content-Disposition in block (raw) and car responses)
+		suffix = `.` + shortFormat + suffix
 	}
 	// TODO: include selector suffix when https://github.com/ipfs/kubo/issues/8769 lands
 	return prefix + cid.String() + suffix
diff --git a/core/corehttp/gateway_handler_tar.go b/core/corehttp/gateway_handler_tar.go
index cedc57a6b08..532d8875760 100644
--- a/core/corehttp/gateway_handler_tar.go
+++ b/core/corehttp/gateway_handler_tar.go
@@ -31,11 +31,21 @@ func (i *gatewayHandler) serveTAR(ctx context.Context, w http.ResponseWriter, r
 	}
 	defer file.Close()
 
+	rootCid := resolvedPath.Cid()
+
 	// Set Cache-Control and read optional Last-Modified time
-	modtime := addCacheControlHeaders(w, r, contentPath, resolvedPath.Cid())
+	modtime := addCacheControlHeaders(w, r, contentPath, rootCid)
+
+	// Weak Etag W/ because we can't guarantee byte-for-byte identical
+	// responses, but still want to benefit from HTTP Caching. Two TAR
+	// responses for the same CID will be logically equivalent,
+	// but when TAR is streamed, then in theory, files and directories
+	// may arrive in different order (depends on TAR lib and filesystem/inodes).
+	etag := `W/` + getEtag(r, rootCid)
+	w.Header().Set("Etag", etag)
 
 	// Finish early if Etag match
-	if r.Header.Get("If-None-Match") == getEtag(r, resolvedPath.Cid()) {
+	if r.Header.Get("If-None-Match") == etag {
 		w.WriteHeader(http.StatusNotModified)
 		return
 	}
@@ -45,7 +55,7 @@ func (i *gatewayHandler) serveTAR(ctx context.Context, w http.ResponseWriter, r
 	if urlFilename := r.URL.Query().Get("filename"); urlFilename != "" {
 		name = urlFilename
 	} else {
-		name = resolvedPath.Cid().String() + ".tar"
+		name = rootCid.String() + ".tar"
 	}
 	setContentDispositionHeader(w, name, "attachment")
 
@@ -65,10 +75,17 @@ func (i *gatewayHandler) serveTAR(ctx context.Context, w http.ResponseWriter, r
 	}
 
 	w.Header().Set("Content-Type", "application/x-tar")
+	w.Header().Set("X-Content-Type-Options", "nosniff") // no funny business in the browsers :^)
 
 	// The TAR has a top-level directory (or file) named by the CID.
-	if err := tarw.WriteFile(file, resolvedPath.Cid().String()); err != nil {
+	if err := tarw.WriteFile(file, rootCid.String()); err != nil {
 		w.Header().Set("X-Stream-Error", err.Error())
+		// Trailer headers do not work in web browsers
+		// (see https://github.com/mdn/browser-compat-data/issues/14703)
+		// and we have limited options around error handling in browser contexts.
+		// To improve UX/DX, we finish response stream with error message, allowing client to
+		// (1) detect error by having corrupted TAR
+		// (2) be able to reason what went wrong by instecting the tail of TAR stream
 		_, _ = w.Write([]byte(err.Error()))
 		return
 	}
diff --git a/test/sharness/t0122-gateway-tar.sh b/test/sharness/t0122-gateway-tar.sh
index 75fd9560ab5..34dc1ba12c8 100755
--- a/test/sharness/t0122-gateway-tar.sh
+++ b/test/sharness/t0122-gateway-tar.sh
@@ -35,11 +35,15 @@ test_expect_success "GET TAR with 'Accept: application/x-tar' and extract" '
 
 test_expect_success "GET TAR with format=tar has expected Content-Type" '
   curl -sD - "http://127.0.0.1:$GWAY_PORT/ipfs/$FILE_CID?format=tar" > curl_output_filename 2>&1 &&
+  test_should_contain "Content-Disposition: attachment;" curl_output_filename &&
+  test_should_contain "Etag: W/\"$FILE_CID.x-tar" curl_output_filename &&
   test_should_contain "Content-Type: application/x-tar" curl_output_filename
 '
 
 test_expect_success "GET TAR with 'Accept: application/x-tar' has expected Content-Type" '
   curl -sD - -H "Accept: application/x-tar" "http://127.0.0.1:$GWAY_PORT/ipfs/$FILE_CID" > curl_output_filename 2>&1 &&
+  test_should_contain "Content-Disposition: attachment;" curl_output_filename &&
+  test_should_contain "Etag: W/\"$FILE_CID.x-tar" curl_output_filename &&
   test_should_contain "Content-Type: application/x-tar" curl_output_filename
 '
 
@@ -59,7 +63,7 @@ test_expect_success "GET TAR has expected root directory" '
   test_cmp expected outputDir/$DIR_CID/ą/ę/file-źł.txt
 '
 
-test_expect_success "GET TAR with explicit ?filename= succeeds with proper header" "
+test_expect_success "GET TAR with explicit ?filename= succeeds with modified Content-Disposition header" "
   curl -fo actual -D actual_headers 'http://127.0.0.1:$GWAY_PORT/ipfs/$DIR_CID?filename=testтест.tar&format=tar' &&
   grep -F 'Content-Disposition: attachment; filename=\"test____.tar\"; filename*=UTF-8'\'\''test%D1%82%D0%B5%D1%81%D1%82.tar' actual_headers
 "

From ea6bb9ac1b71e8e0fba79498a746f26a1e971a7e Mon Sep 17 00:00:00 2001
From: Marcin Rataj <lidel@lidel.org>
Date: Wed, 9 Nov 2022 19:11:37 +0100
Subject: [PATCH 30/30] docs: add TAR to changelog

---
 docs/changelogs/v0.17.md | 21 ++++++++++++++++++++-
 1 file changed, 20 insertions(+), 1 deletion(-)

diff --git a/docs/changelogs/v0.17.md b/docs/changelogs/v0.17.md
index 7fec0b24312..efe0099395b 100644
--- a/docs/changelogs/v0.17.md
+++ b/docs/changelogs/v0.17.md
@@ -9,15 +9,34 @@ Below is an outline of all that is in this release, so you get a sense of all th
 - [Kubo changelog v0.17](#kubo-changelog-v017)
   - [v0.17.0](#v0170)
     - [Overview](#overview)
+    - [TOC](#toc)
     - [🔦 Highlights](#-highlights)
+      - [TAR Response Format on Gateways](#tar-response-format-on-gateways)
     - [Changelog](#changelog)
     - [Contributors](#contributors)
 
-
 ### 🔦 Highlights
 
 <!-- TODO -->
 
+#### TAR Response Format on Gateways
+
+Implemented [IPIP-288](https://github.com/ipfs/specs/pull/288) which adds
+support for requesting deserialized UnixFS directory as a TAR stream.
+
+HTTP clients can request TAR response by passing the `?format=tar` URL
+parameter, or setting `Accept: application/x-tar` HTTP header:
+
+```console
+$ export DIR_CID=bafybeigccimv3zqm5g4jt363faybagywkvqbrismoquogimy7kvz2sj7sq
+$ curl -H "Accept: application/x-tar" "http://127.0.0.1:8080/ipfs/$DIR_CID" > dir.tar
+$ curl "http://127.0.0.1:8080/ipfs/$DIR_CID?format=tar" | tar xv
+bafybeigccimv3zqm5g4jt363faybagywkvqbrismoquogimy7kvz2sj7sq
+bafybeigccimv3zqm5g4jt363faybagywkvqbrismoquogimy7kvz2sj7sq/1 - Barrel - Part 1 - alt.txt
+bafybeigccimv3zqm5g4jt363faybagywkvqbrismoquogimy7kvz2sj7sq/1 - Barrel - Part 1 - transcript.txt
+bafybeigccimv3zqm5g4jt363faybagywkvqbrismoquogimy7kvz2sj7sq/1 - Barrel - Part 1.png
+```
+
 ### Changelog
 
 <!-- TODO -->