Test GetObject/ListObjects with s3zip enabled. (#1570)

functional_tests.go

@@ -20,6 +20,7 @@
 package main
 
 import (
+	"archive/zip"
 	"bytes"
 	"context"
 	"errors"
@@ -32,6 +33,7 @@ import (
 	"net/http"
 	"net/url"
 	"os"
+	"path"
 	"path/filepath"
 	"reflect"
 	"runtime"
@@ -3180,6 +3182,189 @@ func testPutObjectContext() {
 
 }
 
+// Tests get object with s3zip extensions.
+func testGetObjectS3Zip() {
+	// initialize logging params
+	startTime := time.Now()
+	testName := getFuncName()
+	function := "GetObject(bucketName, objectName)"
+	args := map[string]interface{}{"x-minio-extract": true}
+
+	// Seed random based on current time.
+	rand.Seed(time.Now().Unix())
+
+	// Instantiate new minio client object.
+	c, err := minio.New(os.Getenv(serverEndpoint),
+		&minio.Options{
+			Creds:  credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
+			Secure: mustParseBool(os.Getenv(enableHTTPS)),
+		})
+	if err != nil {
+		logError(testName, function, args, startTime, "", "MinIO client object creation failed", err)
+		return
+	}
+
+	// Enable tracing, write to stderr.
+	// c.TraceOn(os.Stderr)
+
+	// Set user agent.
+	c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0")
+
+	// Generate a new random bucket name.
+	bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
+	args["bucketName"] = bucketName
+
+	// Make a new bucket.
+	err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"})
+	if err != nil {
+		logError(testName, function, args, startTime, "", "MakeBucket failed", err)
+		return
+	}
+
+	defer func() {
+		// Delete all objects and buckets
+		if err = cleanupBucket(bucketName, c); err != nil {
+			logError(testName, function, args, startTime, "", "CleanupBucket failed", err)
+			return
+		}
+	}()
+
+	objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") + ".zip"
+	args["objectName"] = objectName
+
+	var zipFile bytes.Buffer
+	zw := zip.NewWriter(&zipFile)
+	rng := rand.New(rand.NewSource(0xc0cac01a))
+	const nFiles = 500
+	for i := 0; i <= nFiles; i++ {
+		if i == nFiles {
+			// Make one large, compressible file.
+			i = 1000000
+		}
+		b := make([]byte, i)
+		if i < nFiles {
+			rng.Read(b)
+		}
+		wc, err := zw.Create(fmt.Sprintf("test/small/file-%d.bin", i))
+		if err != nil {
+			logError(testName, function, args, startTime, "", "zw.Create failed", err)
+			return
+		}
+		wc.Write(b)
+	}
+	err = zw.Close()
+	if err != nil {
+		logError(testName, function, args, startTime, "", "zw.Close failed", err)
+		return
+	}
+	buf := zipFile.Bytes()
+
+	// Save the data
+	_, err = c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{ContentType: "binary/octet-stream"})
+	if err != nil {
+		logError(testName, function, args, startTime, "", "PutObject failed", err)
+		return
+	}
+
+	// Read the data back
+	r, err := c.GetObject(context.Background(), bucketName, objectName, minio.GetObjectOptions{})
+	if err != nil {
+		logError(testName, function, args, startTime, "", "GetObject failed", err)
+		return
+	}
+
+	st, err := r.Stat()
+	if err != nil {
+		logError(testName, function, args, startTime, "", "Stat object failed", err)
+		return
+	}
+
+	if st.Size != int64(len(buf)) {
+		logError(testName, function, args, startTime, "", "Number of bytes does not match, expected "+string(len(buf))+", got "+string(st.Size), err)
+		return
+	}
+	r.Close()
+
+	zr, err := zip.NewReader(bytes.NewReader(buf), int64(len(buf)))
+	if err != nil {
+		logError(testName, function, args, startTime, "", "zip.NewReader failed", err)
+		return
+	}
+	lOpts := minio.ListObjectsOptions{}
+	lOpts.Set("x-minio-extract", "true")
+	lOpts.Prefix = objectName + "/"
+	lOpts.Recursive = true
+	list := c.ListObjects(context.Background(), bucketName, lOpts)
+	var listed = map[string]minio.ObjectInfo{}
+	for item := range list {
+		if item.Err != nil {
+			break
+		}
+		listed[item.Key] = item
+	}
+	if len(listed) == 0 {
+		// Assume we are running against non-minio.
+		args["SKIPPED"] = true
+		ignoredLog(testName, function, args, startTime, "s3zip does not appear to be present").Info()
+		return
+	}
+
+	for _, file := range zr.File {
+		if file.FileInfo().IsDir() {
+			continue
+		}
+		args["zipfile"] = file.Name
+		zfr, err := file.Open()
+		if err != nil {
+			logError(testName, function, args, startTime, "", "file.Open failed", err)
+			return
+		}
+		want, err := ioutil.ReadAll(zfr)
+		if err != nil {
+			logError(testName, function, args, startTime, "", "fzip file read failed", err)
+			return
+		}
+
+		opts := minio.GetObjectOptions{}
+		opts.Set("x-minio-extract", "true")
+		key := path.Join(objectName, file.Name)
+		r, err = c.GetObject(context.Background(), bucketName, key, opts)
+		if err != nil {
+			terr := minio.ToErrorResponse(err)
+			if terr.StatusCode != http.StatusNotFound {
+				logError(testName, function, args, startTime, "", "GetObject failed", err)
+			}
+			return
+		}
+		got, err := ioutil.ReadAll(r)
+		if err != nil {
+			logError(testName, function, args, startTime, "", "ReadAll failed", err)
+			return
+		}
+		r.Close()
+		if !bytes.Equal(want, got) {
+			logError(testName, function, args, startTime, "", "Content mismatch", err)
+			return
+		}
+		oi, ok := listed[key]
+		if !ok {
+			logError(testName, function, args, startTime, "", "Object Missing", fmt.Errorf("%s not present in listing", key))
+			return
+		}
+		if int(oi.Size) != len(got) {
+			logError(testName, function, args, startTime, "", "Object Size Incorrect", fmt.Errorf("listing %d, read %d", oi.Size, len(got)))
+			return
+		}
+		delete(listed, key)
+	}
+	delete(args, "zipfile")
+	if len(listed) > 0 {
+		logError(testName, function, args, startTime, "", "Extra listed objects", fmt.Errorf("left over: %v", listed))
+		return
+	}
+	successLogger(testName, function, args, startTime).Info()
+}
+
 // Tests get object ReaderSeeker interface methods.
 func testGetObjectReadSeekFunctional() {
 	// initialize logging params
@@ -11736,6 +11921,7 @@ func main() {
 		testPutObjectStreaming()
 		testGetObjectSeekEnd()
 		testGetObjectClosedTwice()
+		testGetObjectS3Zip()
 		testRemoveMultipleObjects()
 		testFPutObjectMultipart()
 		testFPutObject()