Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Support XML API (#331) #1164

Open
wants to merge 4 commits into
base: main
Choose a base branch
from
+149 −30
Open

Support XML API (#331) #1164

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
0a1124f
Support XML API (#331)
tustvold May 12, 2023
fb93ee2
Check bucket exists
tustvold May 12, 2023
02e0ce0
Return ETag
tustvold Oct 17, 2023
46f2f72
Support list start-after
tustvold Oct 27, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
89 changes: 86 additions & 3 deletions fakestorage/object.go
View file
Open in desktop
Expand Up @@ -13,6 +13,7 @@
"fmt"
"io"
"net/http"
"net/url"
"sort"
"strconv"
"strings"
Expand Down Expand Up @@ -324,6 +325,7 @@
StartOffset string
EndOffset string
IncludeTrailingDelimiter bool
StartExclusive bool
}

// ListObjects returns a sorted list of objects that match the given criteria,
Expand Down Expand Up @@ -352,6 +354,9 @@
if !strings.HasPrefix(obj.Name, options.Prefix) {
continue
}
if options.StartExclusive && obj.Name == options.StartOffset {
continue
}
objName := strings.Replace(obj.Name, options.Prefix, "", 1)
delimPos := strings.Index(objName, options.Delimiter)
if options.Delimiter != "" && delimPos > -1 {
Expand Down Expand Up @@ -576,9 +581,11 @@
bucketName := unescapeMuxVars(mux.Vars(r))["bucketName"]

opts := ListOptions{
Prefix: r.URL.Query().Get("prefix"),
Delimiter: r.URL.Query().Get("delimiter"),
Versions: r.URL.Query().Get("versions") == "true",
Prefix: r.URL.Query().Get("prefix"),
Delimiter: r.URL.Query().Get("delimiter"),
Versions: r.URL.Query().Get("versions") == "true",
StartOffset: r.URL.Query().Get("start-after"),
StartExclusive: true,
}

objs, prefixes, err := s.ListObjectsWithOptions(bucketName, opts)
Expand Down Expand Up @@ -626,6 +633,81 @@
}
}

func (s *Server) xmlPutObject(r *http.Request) xmlResponse {
// https://cloud.google.com/storage/docs/xml-api/put-object-upload
vars := unescapeMuxVars(mux.Vars(r))
defer r.Body.Close()

if _, err := s.backend.GetBucket(vars["bucketName"]); err != nil {
return xmlResponse{status: http.StatusNotFound}
}

metaData := make(map[string]string)
for key := range r.Header {
lowerKey := strings.ToLower(key)
if metaDataKey := strings.TrimPrefix(lowerKey, "x-goog-meta-"); metaDataKey != lowerKey {
metaData[metaDataKey] = r.Header.Get(key)
}
}

obj := StreamingObject{
ObjectAttrs: ObjectAttrs{
BucketName: vars["bucketName"],
Name: vars["objectName"],
ContentType: r.Header.Get(contentTypeHeader),
ContentEncoding: r.Header.Get(contentEncodingHeader),
Metadata: metaData,
},
}
if source := r.Header.Get("x-goog-copy-source"); source != "" {
escaped, err := url.PathUnescape(source)
if err != nil {
return xmlResponse{status: http.StatusBadRequest}
}

split := strings.SplitN(escaped, "/", 2)
if len(split) != 2 {
return xmlResponse{status: http.StatusBadRequest}
}

sourceObject, err := s.GetObjectStreaming(split[0], split[1])
if err != nil {
return xmlResponse{status: http.StatusNotFound}
}
obj.Content = sourceObject.Content
} else {
obj.Content = notImplementedSeeker{r.Body}
}

obj, err := s.createObject(obj, backend.NoConditions{})

Check failure on line 683 in fakestorage/object.go

View workflow job for this annotation

GitHub Actions / lint 

File is not `gofumpt`-ed (gofumpt)
if err != nil {
return xmlResponse{
status: http.StatusInternalServerError,
errorMessage: err.Error(),
}
}

obj.Close()

header := make(http.Header)
header.Set("ETag", obj.Etag)

return xmlResponse{
status: http.StatusOK,
header: header,
}
}

func (s *Server) xmlDeleteObject(r *http.Request) xmlResponse {
resp := s.deleteObject(r)
return xmlResponse{
status: resp.status,
errorMessage: resp.errorMessage,
header: resp.header,
}
}

func (s *Server) getObject(w http.ResponseWriter, r *http.Request) {
if alt := r.URL.Query().Get("alt"); alt == "media" || r.Method == http.MethodHead {
s.downloadObject(w, r)
Expand Down Expand Up @@ -867,6 +949,7 @@
w.Header().Set("X-Goog-Generation", strconv.FormatInt(obj.Generation, 10))
w.Header().Set("X-Goog-Hash", fmt.Sprintf("crc32c=%s,md5=%s", obj.Crc32c, obj.Md5Hash))
w.Header().Set("Last-Modified", obj.Updated.Format(http.TimeFormat))
w.Header().Set("ETag", obj.Etag)
for name, value := range obj.Metadata {
w.Header().Set("X-Goog-Meta-"+name, value)
}
Expand Down
9 changes: 3 additions & 6 deletions fakestorage/server.go
View file
Open in desktop
Expand Up @@ -276,6 +276,9 @@ func (s *Server) buildMuxer() {
for _, r := range xmlApiRouters {
r.Path("/").Methods(http.MethodGet).HandlerFunc(xmlToHTTPHandler(s.xmlListObjects))
r.Path("").Methods(http.MethodGet).HandlerFunc(xmlToHTTPHandler(s.xmlListObjects))
r.Path("/{objectName:.+}").Methods(http.MethodPut).HandlerFunc(xmlToHTTPHandler(s.xmlPutObject))
r.Path("/{objectName:.+}").Methods(http.MethodGet, http.MethodHead).HandlerFunc(s.downloadObject)
r.Path("/{objectName:.+}").Methods(http.MethodDelete).HandlerFunc(xmlToHTTPHandler(s.xmlDeleteObject))
}

bucketHost := fmt.Sprintf("{bucketName}.%s", s.publicHost)
Expand All @@ -296,12 +299,6 @@ func (s *Server) buildMuxer() {
handler.Host(s.publicHost).Path("/{bucketName}").MatcherFunc(matchFormData).Methods(http.MethodPost, http.MethodPut).HandlerFunc(xmlToHTTPHandler(s.insertFormObject))
handler.Host(bucketHost).MatcherFunc(matchFormData).Methods(http.MethodPost, http.MethodPut).HandlerFunc(xmlToHTTPHandler(s.insertFormObject))

// Signed URLs (upload and download)
handler.MatcherFunc(s.publicHostMatcher).Path("/{bucketName}/{objectName:.+}").Methods(http.MethodPost, http.MethodPut).HandlerFunc(jsonToHTTPHandler(s.insertObject))
Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This can't have ever worked as insertObject expects query parameters encoding the object name, etc...

Copy link
Owner

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

insertObject does not necessarily read the object name from query parameters:

fake-gcs-server/fakestorage/upload.go

Lines 85 to 95 in d7832ec

default:
// Support Signed URL Uploads
if r.URL.Query().Get("X-Goog-Algorithm") != "" {
switch r.Method {
case http.MethodPost:
return s.resumableUpload(bucketName, r)
case http.MethodPut:
return s.signedUpload(bucketName, r)
}
}
return jsonResponse{errorMessage: "invalid uploadType", status: http.StatusBadRequest}

I need to take a deeper look at this, but curious if you're basing on comment on the docs? There are many undocumented things in the GCS API that we run into when someone tries to integrate with some different SDK, and I've ironically done a poor job in documenting those in fake-gcs-server 🙈

Copy link
Author

@tustvold tustvold May 12, 2023 •
edited

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Indeed, insertObject has a peculiar special case that uses X-Goog-Algorithm to detect a signed URL and then falls back to implementing something that looks like the XML API, because that is what signed URLs are. With this PR that can probably be removed 🤔

Copy link
Owner

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yeah, absolutely. insertObject definitely looks like a little monster doing too many things. Thank you very much! I'll go over the PR in details later today.

handler.MatcherFunc(s.publicHostMatcher).Path("/{bucketName}/{objectName:.+}").Methods(http.MethodGet, http.MethodHead).HandlerFunc(s.getObject)
handler.Host(bucketHost).Path("/{objectName:.+}").Methods(http.MethodPost, http.MethodPut).HandlerFunc(jsonToHTTPHandler(s.insertObject))
handler.Host("{bucketName:.+}").Path("/{objectName:.+}").Methods(http.MethodPost, http.MethodPut).HandlerFunc(jsonToHTTPHandler(s.insertObject))

s.handler = handler
}

Expand Down
2 changes: 2 additions & 0 deletions fakestorage/upload.go
View file
Open in desktop
Expand Up @@ -27,6 +27,8 @@ import (

const contentTypeHeader = "Content-Type"

const contentEncodingHeader = "Content-Encoding"

const (
uploadTypeMedia = "media"
uploadTypeMultipart = "multipart"
Expand Down
79 changes: 58 additions & 21 deletions fakestorage/upload_test.go
View file
Open in desktop
Expand Up @@ -10,7 +10,6 @@
"context"
"crypto/tls"
"encoding/binary"
"encoding/json"
"fmt"
"io"
"mime/multipart"
Expand Down Expand Up @@ -570,9 +569,6 @@

func TestServerClientSignedUploadBucketCNAME(t *testing.T) {
url := "https://mybucket.mydomain.com:4443/files/txt/text-02.txt?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=fake-gcs&X-Goog-Expires=3600&X-Goog-SignedHeaders=host&X-Goog-Signature=fake-gc"
expectedName := "files/txt/text-02.txt"
expectedContentType := "text/plain"
expectedHash := "bHupxaFBQh4cA8uYB8l8dA=="
opts := Options{
InitialObjects: []Object{
{ObjectAttrs: ObjectAttrs{BucketName: "mybucket.mydomain.com", Name: "files/txt/text-01.txt"}, Content: []byte("something")},
Expand All @@ -596,23 +592,6 @@
if resp.StatusCode != http.StatusOK {
t.Errorf("wrong status returned\nwant %d\ngot %d", http.StatusOK, resp.StatusCode)
}
data, err := io.ReadAll(resp.Body)
Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This was incorrect, the XML APIs should not return a body and certainly shouldn't return JSON

Copy link
Owner

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🤔 Oh are signed uploads powered by the XML API? Interesting!

if err != nil {
t.Fatal(err)
}
var obj Object
if err := json.Unmarshal(data, &obj); err != nil {
t.Fatal(err)
}
if obj.Name != expectedName {
t.Errorf("wrong filename\nwant %q\ngot %q", expectedName, obj.Name)
}
if obj.ContentType != expectedContentType {
t.Errorf("wrong content type\nwant %q\ngot %q", expectedContentType, obj.ContentType)
}
if obj.Md5Hash != expectedHash {
t.Errorf("wrong md5 hash\nwant %q\ngot %q", expectedHash, obj.Md5Hash)
}
}

func TestServerClientUploadWithPredefinedAclPublicRead(t *testing.T) {
Expand Down Expand Up @@ -700,6 +679,64 @@
}
}

func TestServerXMLPut(t *testing.T) {
server, err := NewServerWithOptions(Options{
PublicHost: "test",
})
if err != nil {
t.Fatal(err)
}
defer server.Stop()
server.CreateBucketWithOpts(CreateBucketOpts{Name: "bucket1"})
server.CreateBucketWithOpts(CreateBucketOpts{Name: "bucket2"})

const data = "some nice content"
req, err := http.NewRequest("PUT", server.URL()+"/bucket1/path", strings.NewReader(data))
req.Host = "test"
if err != nil {
t.Fatal(err)
}
client := http.Client{
Transport: &http.Transport{
TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
},
}
resp, err := client.Do(req)
if err != nil {
t.Fatal(err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
t.Errorf("got %d expected %d", resp.StatusCode, http.StatusOK)
}

req, err = http.NewRequest("PUT", server.URL()+"/bucket2/path", nil)
req.Host = "test"
req.Header.Set("x-goog-copy-source", "bucket1/path")

resp, err = client.Do(req)
if err != nil {
t.Fatal(err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
t.Errorf("got %d expected %d", resp.StatusCode, http.StatusOK)

Check failure on line 723 in fakestorage/upload_test.go

View workflow job for this annotation

GitHub Actions / staticcheck 

this value of err is never used (SA4006)
}

req, err = http.NewRequest("PUT", server.URL()+"/bucket2/path2", nil)
req.Host = "test"
req.Header.Set("x-goog-copy-source", "bucket1/nonexistent")

resp, err = client.Do(req)
if err != nil {
t.Fatal(err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusNotFound {
t.Errorf("got %d expected %d", resp.StatusCode, http.StatusNotFound)

Check failure on line 736 in fakestorage/upload_test.go

View workflow job for this annotation

GitHub Actions / staticcheck 

this value of err is never used (SA4006)
}
}

func TestServerInvalidUploadType(t *testing.T) {
server := NewServer(nil)
defer server.Stop()
Expand Down