Add support for extra signed headers in Presign() API

Fixes #1415

.golangci.yml

@@ -8,8 +8,8 @@ linters:
     - typecheck
     - goimports
     - misspell
+    - revive
     - govet
-    - golint
     - ineffassign
     - gosimple
     - deadcode

Makefile

@@ -9,7 +9,7 @@ checks: lint vet test examples functional-test
 
 lint:
 	@mkdir -p ${GOPATH}/bin
-	@which golangci-lint 1>/dev/null || (echo "Installing golangci-lint" && curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(GOPATH)/bin v1.27.0)
+	@echo "Installing golangci-lint" && curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(GOPATH)/bin v1.40.1
 	@echo "Running $@ check"
 	@GO111MODULE=on ${GOPATH}/bin/golangci-lint cache clean
 	@GO111MODULE=on ${GOPATH}/bin/golangci-lint run --timeout=5m --config ./.golangci.yml

api-bucket-replication.go

@@ -208,11 +208,10 @@ func (c *Client) ResetBucketReplication(ctx context.Context, bucketName string,
 	return rID, nil
 }
 
-// ResetBucketReplication kicks off replication of previously replicated objects if ExistingObjectReplication
-// is enabled in the replication config
-func (c *Client) ResetBucketReplicationOnTarget(ctx context.Context, bucketName string, olderThan time.Duration, tgtArn string) (rinfo replication.ResyncTargetsInfo, err error) {
-	rID := mustGetUUID()
-	return c.resetBucketReplicationOnTarget(ctx, bucketName, olderThan, tgtArn, rID)
+// ResetBucketReplicationOnTarget kicks off replication of previously replicated objects if
+// ExistingObjectReplication is enabled in the replication config
+func (c *Client) ResetBucketReplicationOnTarget(ctx context.Context, bucketName string, olderThan time.Duration, tgtArn string) (replication.ResyncTargetsInfo, error) {
+	return c.resetBucketReplicationOnTarget(ctx, bucketName, olderThan, tgtArn, mustGetUUID())
 }
 
 // ResetBucketReplication kicks off replication of previously replicated objects if ExistingObjectReplication

api-presigned.go

@@ -30,7 +30,7 @@ import (
 
 // presignURL - Returns a presigned URL for an input 'method'.
 // Expires maximum is 7days - ie. 604800 and minimum is 1.
-func (c *Client) presignURL(ctx context.Context, method string, bucketName string, objectName string, expires time.Duration, reqParams url.Values) (u *url.URL, err error) {
+func (c *Client) presignURL(ctx context.Context, method string, bucketName string, objectName string, expires time.Duration, reqParams url.Values, extraHeaders http.Header) (u *url.URL, err error) {
 	// Input validation.
 	if method == "" {
 		return nil, errInvalidArgument("method cannot be empty.")
@@ -45,11 +45,12 @@ func (c *Client) presignURL(ctx context.Context, method string, bucketName strin
 	// Convert expires into seconds.
 	expireSeconds := int64(expires / time.Second)
 	reqMetadata := requestMetadata{
-		presignURL:  true,
-		bucketName:  bucketName,
-		objectName:  objectName,
-		expires:     expireSeconds,
-		queryValues: reqParams,
+		presignURL:         true,
+		bucketName:         bucketName,
+		objectName:         objectName,
+		expires:            expireSeconds,
+		queryValues:        reqParams,
+		extraPresignHeader: extraHeaders,
 	}
 
 	// Instantiate a new request.
@@ -69,7 +70,7 @@ func (c *Client) PresignedGetObject(ctx context.Context, bucketName string, obje
 	if err = s3utils.CheckValidObjectName(objectName); err != nil {
 		return nil, err
 	}
-	return c.presignURL(ctx, http.MethodGet, bucketName, objectName, expires, reqParams)
+	return c.presignURL(ctx, http.MethodGet, bucketName, objectName, expires, reqParams, nil)
 }
 
 // PresignedHeadObject - Returns a presigned URL to access
@@ -80,7 +81,7 @@ func (c *Client) PresignedHeadObject(ctx context.Context, bucketName string, obj
 	if err = s3utils.CheckValidObjectName(objectName); err != nil {
 		return nil, err
 	}
-	return c.presignURL(ctx, http.MethodHead, bucketName, objectName, expires, reqParams)
+	return c.presignURL(ctx, http.MethodHead, bucketName, objectName, expires, reqParams, nil)
 }
 
 // PresignedPutObject - Returns a presigned URL to upload an object
@@ -90,14 +91,25 @@ func (c *Client) PresignedPutObject(ctx context.Context, bucketName string, obje
 	if err = s3utils.CheckValidObjectName(objectName); err != nil {
 		return nil, err
 	}
-	return c.presignURL(ctx, http.MethodPut, bucketName, objectName, expires, nil)
+	return c.presignURL(ctx, http.MethodPut, bucketName, objectName, expires, nil, nil)
 }
 
-// Presign - returns a presigned URL for any http method of your choice
-// along with custom request params. URL can have a maximum expiry of
-// upto 7days or a minimum of 1sec.
+// PresignHeader - similar to Presign() but allows including HTTP headers that
+// will be used to build the signature. The request using the resulting URL will
+// need to have the exact same headers to be added for signature validation to
+// pass.
+//
+// FIXME: The extra header parameter should be included in Presign() in the next
+// major version bump, and this function should then be deprecated.
+func (c *Client) PresignHeader(ctx context.Context, method string, bucketName string, objectName string, expires time.Duration, reqParams url.Values, extraHeaders http.Header) (u *url.URL, err error) {
+	return c.presignURL(ctx, method, bucketName, objectName, expires, reqParams, extraHeaders)
+}
+
+// Presign - returns a presigned URL for any http method of your choice along
+// with custom request params and extra signed headers. URL can have a maximum
+// expiry of upto 7days or a minimum of 1sec.
 func (c *Client) Presign(ctx context.Context, method string, bucketName string, objectName string, expires time.Duration, reqParams url.Values) (u *url.URL, err error) {
-	return c.presignURL(ctx, method, bucketName, objectName, expires, reqParams)
+	return c.presignURL(ctx, method, bucketName, objectName, expires, reqParams, nil)
 }
 
 // PresignedPostPolicy - Returns POST urlString, form data to upload an object.

api-remove.go

@@ -29,7 +29,7 @@ import (
 	"github.com/minio/minio-go/v7/pkg/s3utils"
 )
 
-// Deprecated: BucketOptions will be renamed to RemoveBucketOptions in future versions.
+// BucketOptions - deprecated: will be renamed to RemoveBucketOptions in future versions.
 type BucketOptions = RemoveBucketOptions
 
 // RemoveBucketOptions special headers to purge buckets, only

api-restore.go

@@ -110,7 +110,7 @@ func (r *RestoreRequest) SetDays(v int) {
 	r.Days = &v
 }
 
-// SetDays sets the GlacierJobParameters of the restore request
+// SetGlacierJobParameters sets the GlacierJobParameters of the restore request
 func (r *RestoreRequest) SetGlacierJobParameters(v GlacierJobParameters) {
 	r.GlacierJobParameters = &v
 }

api.go

@@ -463,11 +463,12 @@ type requestMetadata struct {
 	presignURL bool
 
 	// User supplied.
-	bucketName   string
-	objectName   string
-	queryValues  url.Values
-	customHeader http.Header
-	expires      int64
+	bucketName         string
+	objectName         string
+	queryValues        url.Values
+	customHeader       http.Header
+	extraPresignHeader http.Header
+	expires            int64
 
 	// Generated by our internal code.
 	bucketLocation   string
@@ -813,6 +814,14 @@ func (c *Client) newRequest(ctx context.Context, method string, metadata request
 		if signerType.IsAnonymous() {
 			return nil, errInvalidArgument("Presigned URLs cannot be generated with anonymous credentials.")
 		}
+		if metadata.extraPresignHeader != nil {
+			if signerType.IsV2() {
+				return nil, errInvalidArgument("Extra signed headers for Presign with Signature V2 is not supported.")
+			}
+			for k, v := range metadata.extraPresignHeader {
+				req.Header.Set(k, v[0])
+			}
+		}
 		if signerType.IsV2() {
 			// Presign URL with signature v2.
 			req = signer.PreSignV2(*req, accessKeyID, secretAccessKey, metadata.expires, isVirtualHost)

functional_tests.go

@@ -6093,6 +6093,63 @@ func testFunctional() {
 		return
 	}
 
+	function = "PresignHeader(method, bucketName, objectName, expires, reqParams, extraHeaders)"
+	functionAll += ", " + function
+	presignExtraHeaders := map[string][]string{
+		"mysecret": {"abcxxx"},
+	}
+	args = map[string]interface{}{
+		"method":       "PUT",
+		"bucketName":   bucketName,
+		"objectName":   objectName + "-presign-custom",
+		"expires":      3600 * time.Second,
+		"extraHeaders": presignExtraHeaders,
+	}
+	presignedURL, err := c.PresignHeader(context.Background(), "PUT", bucketName, objectName+"-presign-custom", 3600*time.Second, nil, presignExtraHeaders)
+	if err != nil {
+		logError(testName, function, args, startTime, "", "Presigned failed", err)
+		return
+	}
+
+	// Generate data more than 32K
+	buf = bytes.Repeat([]byte("1"), rand.Intn(1<<10)+32*1024)
+
+	req, err = http.NewRequest(http.MethodPut, presignedURL.String(), bytes.NewReader(buf))
+	if err != nil {
+		logError(testName, function, args, startTime, "", "HTTP request to Presigned URL failed", err)
+		return
+	}
+
+	req.Header.Add("mysecret", "abcxxx")
+	resp, err = httpClient.Do(req)
+	if err != nil {
+		logError(testName, function, args, startTime, "", "HTTP request to Presigned URL failed", err)
+		return
+	}
+
+	// Download the uploaded object to verify
+	args = map[string]interface{}{
+		"bucketName": bucketName,
+		"objectName": objectName + "-presign-custom",
+	}
+	newReader, err = c.GetObject(context.Background(), bucketName, objectName+"-presign-custom", minio.GetObjectOptions{})
+	if err != nil {
+		logError(testName, function, args, startTime, "", "GetObject of uploaded custom-presigned object failed", err)
+		return
+	}
+
+	newReadBytes, err = ioutil.ReadAll(newReader)
+	if err != nil {
+		logError(testName, function, args, startTime, "", "ReadAll failed during get on custom-presigned put object", err)
+		return
+	}
+	newReader.Close()
+
+	if !bytes.Equal(newReadBytes, buf) {
+		logError(testName, function, args, startTime, "", "Bytes mismatch on custom-presigned object upload verification", err)
+		return
+	}
+
 	function = "RemoveObject(bucketName, objectName)"
 	functionAll += ", " + function
 	args = map[string]interface{}{
@@ -6129,6 +6186,14 @@ func testFunctional() {
 		return
 	}
 
+	args["objectName"] = objectName + "-presign-custom"
+	err = c.RemoveObject(context.Background(), bucketName, objectName+"-presign-custom", minio.RemoveObjectOptions{})
+
+	if err != nil {
+		logError(testName, function, args, startTime, "", "RemoveObject failed", err)
+		return
+	}
+
 	function = "RemoveBucket(bucketName)"
 	functionAll += ", " + function
 	args = map[string]interface{}{
@@ -10871,27 +10936,44 @@ func testFunctionalV2() {
 		return
 	}
 
-	function = "GetObject(bucketName, objectName)"
-	functionAll += ", " + function
+	// Download the uploaded object to verify
 	args = map[string]interface{}{
 		"bucketName": bucketName,
 		"objectName": objectName + "-presigned",
 	}
 	newReader, err = c.GetObject(context.Background(), bucketName, objectName+"-presigned", minio.GetObjectOptions{})
 	if err != nil {
-		logError(testName, function, args, startTime, "", "GetObject failed", err)
+		logError(testName, function, args, startTime, "", "GetObject of uploaded presigned object failed", err)
 		return
 	}
 
 	newReadBytes, err = ioutil.ReadAll(newReader)
 	if err != nil {
-		logError(testName, function, args, startTime, "", "ReadAll failed", err)
+		logError(testName, function, args, startTime, "", "ReadAll failed during get on presigned put object", err)
 		return
 	}
 	newReader.Close()
 
 	if !bytes.Equal(newReadBytes, buf) {
-		logError(testName, function, args, startTime, "", "Bytes mismatch", err)
+		logError(testName, function, args, startTime, "", "Bytes mismatch on presigned object upload verification", err)
+		return
+	}
+
+	function = "PresignHeader(method, bucketName, objectName, expires, reqParams, extraHeaders)"
+	functionAll += ", " + function
+	presignExtraHeaders := map[string][]string{
+		"mysecret": {"abcxxx"},
+	}
+	args = map[string]interface{}{
+		"method":       "PUT",
+		"bucketName":   bucketName,
+		"objectName":   objectName + "-presign-custom",
+		"expires":      3600 * time.Second,
+		"extraHeaders": presignExtraHeaders,
+	}
+	_, err = c.PresignHeader(context.Background(), "PUT", bucketName, objectName+"-presign-custom", 3600*time.Second, nil, presignExtraHeaders)
+	if err == nil {
+		logError(testName, function, args, startTime, "", "Presigned with extra headers succeeded", err)
 		return
 	}
 

pkg/replication/replication.go

@@ -494,10 +494,7 @@ func (r Rule) validateStatus() error {
 }
 
 func (r Rule) validateFilter() error {
-	if err := r.Filter.Validate(); err != nil {
-		return err
-	}
-	return nil
+	return r.Filter.Validate()
 }
 
 // Prefix - a rule can either have prefix under <filter></filter> or under