Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

feat(internal/gensupport): use gax DetermineContentType #1716

Merged
merged 3 commits into from Oct 18, 2022
Merged

feat(internal/gensupport): use gax DetermineContentType #1716

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
beef5e9
feat(internal/gensupport): use gax DetermineContentType
noahdietz Oct 17, 2022
af6e455
remove deprecated DetermineContentType
noahdietz Oct 18, 2022
1483d54
Merge branch 'main' into gax-determine-ct
noahdietz Oct 18, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
95 changes: 10 additions & 85 deletions internal/gensupport/media.go
View file
Open in desktop
Expand Up @@ -17,92 +17,10 @@ import (
"sync"
"time"

gax "github.com/googleapis/gax-go/v2"
"google.golang.org/api/googleapi"
)

const sniffBuffSize = 512

func newContentSniffer(r io.Reader) *contentSniffer {
return &contentSniffer{r: r}
}

// contentSniffer wraps a Reader, and reports the content type determined by sniffing up to 512 bytes from the Reader.
type contentSniffer struct {
r io.Reader
start []byte // buffer for the sniffed bytes.
err error // set to any error encountered while reading bytes to be sniffed.

ctype string // set on first sniff.
sniffed bool // set to true on first sniff.
}

func (cs *contentSniffer) Read(p []byte) (n int, err error) {
// Ensure that the content type is sniffed before any data is consumed from Reader.
_, _ = cs.ContentType()

if len(cs.start) > 0 {
n := copy(p, cs.start)
cs.start = cs.start[n:]
return n, nil
}

// We may have read some bytes into start while sniffing, even if the read ended in an error.
// We should first return those bytes, then the error.
if cs.err != nil {
return 0, cs.err
}

// Now we have handled all bytes that were buffered while sniffing. Now just delegate to the underlying reader.
return cs.r.Read(p)
}

// ContentType returns the sniffed content type, and whether the content type was successfully sniffed.
func (cs *contentSniffer) ContentType() (string, bool) {
if cs.sniffed {
return cs.ctype, cs.ctype != ""
}
cs.sniffed = true
// If ReadAll hits EOF, it returns err==nil.
cs.start, cs.err = ioutil.ReadAll(io.LimitReader(cs.r, sniffBuffSize))

// Don't try to detect the content type based on possibly incomplete data.
if cs.err != nil {
return "", false
}

cs.ctype = http.DetectContentType(cs.start)
return cs.ctype, true
}

// DetermineContentType determines the content type of the supplied reader.
// If the content type is already known, it can be specified via ctype.
// Otherwise, the content of media will be sniffed to determine the content type.
// If media implements googleapi.ContentTyper (deprecated), this will be used
// instead of sniffing the content.
// After calling DetectContentType the caller must not perform further reads on
// media, but rather read from the Reader that is returned.
func DetermineContentType(media io.Reader, ctype string) (io.Reader, string) {
// Note: callers could avoid calling DetectContentType if ctype != "",
// but doing the check inside this function reduces the amount of
// generated code.
if ctype != "" {
return media, ctype
}

// For backwards compatibility, allow clients to set content
// type by providing a ContentTyper for media.
if typer, ok := media.(googleapi.ContentTyper); ok {
return media, typer.ContentType()
}

sniffer := newContentSniffer(media)
if ctype, ok := sniffer.ContentType(); ok {
return sniffer, ctype
}
// If content type could not be sniffed, reads from sniffer will eventually fail with an error.
return sniffer, ""
}

type typeReader struct {
io.Reader
typ string
Expand Down Expand Up @@ -234,7 +152,10 @@ func NewInfoFromMedia(r io.Reader, options []googleapi.MediaOption) *MediaInfo {
mi := &MediaInfo{}
opts := googleapi.ProcessMediaOptions(options)
if !opts.ForceEmptyContentType {
r, mi.mType = DetermineContentType(r, opts.ContentType)
mi.mType = opts.ContentType
if mi.mType == "" {
r, mi.mType = gax.DetermineContentType(r)
}
}
mi.chunkRetryDeadline = opts.ChunkRetryDeadline
mi.media, mi.buffer, mi.singleChunk = PrepareUpload(r, opts.ChunkSize)
Expand All @@ -245,7 +166,11 @@ func NewInfoFromMedia(r io.Reader, options []googleapi.MediaOption) *MediaInfo {
// call. It returns a MediaInfo using the given reader, size and media type.
func NewInfoFromResumableMedia(r io.ReaderAt, size int64, mediaType string) *MediaInfo {
rdr := ReaderAtToReader(r, size)
rdr, mType := DetermineContentType(rdr, mediaType)
mType := mediaType
if mType == "" {
rdr, mType = gax.DetermineContentType(rdr)
}

return &MediaInfo{
size: size,
mType: mType,
Expand Down
130 changes: 0 additions & 130 deletions internal/gensupport/media_test.go
View file
Open in desktop
Expand Up @@ -11,143 +11,13 @@ import (
"io/ioutil"
mathrand "math/rand"
"net/http"
"reflect"
"strings"
"testing"
"time"

"google.golang.org/api/googleapi"
)

func TestContentSniffing(t *testing.T) {
type testCase struct {
data []byte // the data to read from the Reader
finalErr error // error to return after data has been read

wantContentType string
wantContentTypeResult bool
}

for _, tc := range []testCase{
{
data: []byte{0, 0, 0, 0},
finalErr: nil,
wantContentType: "application/octet-stream",
wantContentTypeResult: true,
},
{
data: []byte(""),
finalErr: nil,
wantContentType: "text/plain; charset=utf-8",
wantContentTypeResult: true,
},
{
data: []byte(""),
finalErr: io.ErrUnexpectedEOF,
wantContentType: "text/plain; charset=utf-8",
wantContentTypeResult: false,
},
{
data: []byte("abc"),
finalErr: nil,
wantContentType: "text/plain; charset=utf-8",
wantContentTypeResult: true,
},
{
data: []byte("abc"),
finalErr: io.ErrUnexpectedEOF,
wantContentType: "text/plain; charset=utf-8",
wantContentTypeResult: false,
},
// The following examples contain more bytes than are buffered for sniffing.
{
data: bytes.Repeat([]byte("a"), 513),
finalErr: nil,
wantContentType: "text/plain; charset=utf-8",
wantContentTypeResult: true,
},
{
data: bytes.Repeat([]byte("a"), 513),
finalErr: io.ErrUnexpectedEOF,
wantContentType: "text/plain; charset=utf-8",
wantContentTypeResult: true, // true because error is after first 512 bytes.
},
} {
er := &errReader{buf: tc.data, err: tc.finalErr}

sct := newContentSniffer(er)

// Even if was an error during the first 512 bytes, we should still be able to read those bytes.
buf, err := ioutil.ReadAll(sct)

if !reflect.DeepEqual(buf, tc.data) {
t.Fatalf("Failed reading buffer: got: %q; want:%q", buf, tc.data)
}

if err != tc.finalErr {
t.Fatalf("Reading buffer error: got: %v; want: %v", err, tc.finalErr)
}

ct, ok := sct.ContentType()
if ok != tc.wantContentTypeResult {
t.Fatalf("Content type result got: %v; want: %v", ok, tc.wantContentTypeResult)
}
if ok && ct != tc.wantContentType {
t.Fatalf("Content type got: %q; want: %q", ct, tc.wantContentType)
}
}
}

type staticContentTyper struct {
io.Reader
}

func (sct staticContentTyper) ContentType() string {
return "static content type"
}

func TestDetermineContentType(t *testing.T) {
data := []byte("abc")
rdr := func() io.Reader {
return bytes.NewBuffer(data)
}

type testCase struct {
r io.Reader
explicitConentType string
wantContentType string
}

for _, tc := range []testCase{
{
r: rdr(),
wantContentType: "text/plain; charset=utf-8",
},
{
r: staticContentTyper{rdr()},
wantContentType: "static content type",
},
{
r: staticContentTyper{rdr()},
explicitConentType: "explicit",
wantContentType: "explicit",
},
} {
r, ctype := DetermineContentType(tc.r, tc.explicitConentType)
got, err := ioutil.ReadAll(r)
if err != nil {
t.Fatalf("Failed reading buffer: %v", err)
}
if !reflect.DeepEqual(got, data) {
t.Fatalf("Failed reading buffer: got: %q; want:%q", got, data)
}

if ctype != tc.wantContentType {
t.Fatalf("Content type got: %q; want: %q", ctype, tc.wantContentType)
}
}
}

func TestNewInfoFromMedia(t *testing.T) {
const textType = "text/plain; charset=utf-8"
for _, test := range []struct {
Expand Down