blob: Add support for key/value Metadata

blob/blob.go

@@ -23,6 +23,7 @@ import (
 	"io/ioutil"
 	"mime"
 	"net/http"
+	"strings"
 	"time"
 
 	"github.com/google/go-cloud/blob/driver"
@@ -165,6 +166,16 @@ func (b *Bucket) Attributes(ctx context.Context, key string) (Attributes, error)
 	if err != nil {
 		return Attributes{}, err
 	}
+	if len(a.Metadata) > 0 {
+		// Providers are inconsistent, but at least some treat keys
+		// as case-insensitive. To make the behavior consistent, we
+		// force-lowercase them when writing and reading.
+		md := make(map[string]string, len(a.Metadata))
+		for k, v := range a.Metadata {
+			md[strings.ToLower(k)] = v
+		}
+		a.Metadata = md
+	}
 	return Attributes(a), nil
 }
 
@@ -238,6 +249,19 @@ func (b *Bucket) NewWriter(ctx context.Context, key string, opt *WriterOptions)
 			w, err = b.b.NewTypedWriter(ctx, key, ct, dopt)
 			return &Writer{w: w}, err
 		}
+		if len(opt.Metadata) > 0 {
+			// Providers are inconsistent, but at least some treat keys
+			// as case-insensitive. To make the behavior consistent, we
+			// force-lowercase them when writing and reading.
+			md := make(map[string]string, len(opt.Metadata))
+			for k, v := range opt.Metadata {
+				if k == "" {
+					return nil, errors.New("WriterOptions.Metadata keys may not be empty strings")
+				}
+				md[strings.ToLower(k)] = v
+			}
+			dopt.Metadata = md
+		}
 	}
 	return &Writer{
 		ctx:    ctx,
@@ -273,6 +297,10 @@ type WriterOptions struct {
 	// then it will be inferred from the content using the algorithm described at
 	// http://mimesniff.spec.whatwg.org/
 	ContentType string
+
+	// Metadata are key/value strings to be associated with the blob, or nil.
+	// Keys may not be empty, and are lowercased before being written.
+	Metadata map[string]string
 }
 
 type blobError struct {

blob/driver/driver.go

@@ -58,6 +58,8 @@ type WriterOptions struct {
 	// write in a single request, if supported. Larger objects will be split into
 	// multiple requests.
 	BufferSize int
+	// Metadata holds key/value strings to be associated with the blob.
+	Metadata map[string]string
 }
 
 // ReaderAttributes contains a subset of attributes about a blob that are
@@ -76,6 +78,9 @@ type ReaderAttributes struct {
 type Attributes struct {
 	// ContentType is the MIME type of the blob object. It must not be empty.
 	ContentType string
+	// Metadata holds key/value pairs associated with the blob.
+	// Keys are case-insensitive and will always be returned in lowercase.
+	Metadata map[string]string
 	// ModTime is the time the blob object was last modified.
 	ModTime time.Time
 	// Size is the size of the object in bytes.

blob/drivertest/drivertest.go

@@ -59,6 +59,9 @@ func RunConformanceTests(t *testing.T, newHarness HarnessMaker, pathToTestdata s
 	t.Run("TestCanceledWrite", func(t *testing.T) {
 		testCanceledWrite(t, newHarness, pathToTestdata)
 	})
+	t.Run("TestMetadata", func(t *testing.T) {
+		testMetadata(t, newHarness)
+	})
 	t.Run("TestDelete", func(t *testing.T) {
 		testDelete(t, newHarness)
 	})
@@ -465,6 +468,89 @@ func testCanceledWrite(t *testing.T, newHarness HarnessMaker, pathToTestdata str
 	}
 }
 
+// testMetadata tests writing and reading the key/value metadata for a blob.
+func testMetadata(t *testing.T, newHarness HarnessMaker) {
+	const key = "blob-for-metadata"
+	hello := []byte("hello")
+
+	tests := []struct {
+		name     string
+		metadata map[string]string
+		content  []byte
+		want     map[string]string
+		wantErr  bool
+	}{
+		{
+			name:     "empty",
+			content:  hello,
+			metadata: map[string]string{},
+			want:     nil,
+		},
+		{
+			name:     "empty key fails",
+			content:  hello,
+			metadata: map[string]string{"": "empty key value"},
+			wantErr:  true,
+		},
+		{
+			name:    "valid metadata",
+			content: hello,
+			metadata: map[string]string{
+				"key-a": "value-a",
+				"kEy-B": "value-b",
+				"key-c": "vAlUe-c",
+			},
+			want: map[string]string{
+				"key-a": "value-a",
+				"key-b": "value-b",
+				"key-c": "vAlUe-c",
+			},
+		},
+		{
+			name:     "valid metadata with empty body",
+			content:  nil,
+			metadata: map[string]string{"foo": "bar"},
+			want:     map[string]string{"foo": "bar"},
+		},
+	}
+
+	ctx := context.Background()
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			h, err := newHarness(ctx, t)
+			if err != nil {
+				t.Fatal(err)
+			}
+			defer h.Close()
+
+			b, err := h.MakeBucket(ctx)
+			if err != nil {
+				t.Fatal(err)
+			}
+			opts := &blob.WriterOptions{
+				Metadata: tc.metadata,
+			}
+			err = b.WriteAll(ctx, key, hello, opts)
+			if (err != nil) != tc.wantErr {
+				t.Errorf("got error %v want error %v", err, tc.wantErr)
+			}
+			if err != nil {
+				return
+			}
+			defer func() {
+				_ = b.Delete(ctx, key)
+			}()
+			a, err := b.Attributes(ctx, key)
+			if err != nil {
+				t.Fatal(err)
+			}
+			if diff := cmp.Diff(a.Metadata, tc.want); diff != "" {
+				t.Errorf("got\n%v\nwant\n%v\ndiff\n%s", a.Metadata, tc.want, diff)
+			}
+		})
+	}
+}
+
 // testDelete tests the functionality of Delete.
 func testDelete(t *testing.T, newHarness HarnessMaker) {
 	const key = "blob-for-deleting"

blob/fileblob/attrs.go

@@ -25,7 +25,8 @@ const attrsExt = ".attrs"
 // filesystem extended attributes, see
 // https://www.freedesktop.org/wiki/CommonExtendedAttributes.
 type xattrs struct {
-	ContentType string `json:"user.content_type"`
+	ContentType string            `json:"user.content_type"`
+	Metadata    map[string]string `json:"user.metadata"`
 }
 
 // setAttrs creates a "path.attrs" file along with blob to store the attributes,
@@ -49,7 +50,7 @@ func getAttrs(path string) (xattrs, error) {
 	f, err := os.Open(path + attrsExt)
 	if err != nil {
 		if os.IsNotExist(err) {
-			// Handle gracefully for non-existing .attr files.
+			// Handle gracefully for non-existent .attr files.
 			return xattrs{
 				ContentType: "application/octet-stream",
 			}, nil

blob/fileblob/fileblob.go

@@ -107,6 +107,7 @@ func (b *bucket) Attributes(ctx context.Context, key string) (driver.Attributes,
 	}
 	return driver.Attributes{
 		ContentType: xa.ContentType,
+		Metadata:    xa.Metadata,
 		ModTime:     info.ModTime(),
 		Size:        info.Size(),
 	}, nil
@@ -183,8 +184,13 @@ func (b *bucket) NewTypedWriter(ctx context.Context, key string, contentType str
 	if err != nil {
 		return nil, fmt.Errorf("open file blob %s: %v", key, err)
 	}
+	var metadata map[string]string
+	if opt != nil && len(opt.Metadata) > 0 {
+		metadata = opt.Metadata
+	}
 	attrs := xattrs{
 		ContentType: contentType,
+		Metadata:    metadata,
 	}
 	return &writer{
 		ctx:   ctx,

blob/gcsblob/gcsblob.go

@@ -85,6 +85,7 @@ func (b *bucket) Attributes(ctx context.Context, key string) (driver.Attributes,
 	}
 	return driver.Attributes{
 		ContentType: attrs.ContentType,
+		Metadata:    attrs.Metadata,
 		ModTime:     attrs.Updated,
 		Size:        attrs.Size,
 	}, nil
@@ -120,6 +121,7 @@ func (b *bucket) NewTypedWriter(ctx context.Context, key string, contentType str
 	w.ContentType = contentType
 	if opts != nil {
 		w.ChunkSize = bufferSize(opts.BufferSize)
+		w.Metadata = opts.Metadata
 	}
 	return w, nil
 }