From 979f1c99509b9320f986278e5f5a34505f9ce3b6 Mon Sep 17 00:00:00 2001 From: Sander van Harmelen Date: Wed, 29 Dec 2021 15:43:38 +0100 Subject: [PATCH] Refactor all the upload related bits --- gitlab.go | 88 ++++++++++++++++++++++++-- project_import_export.go | 17 +++-- project_import_export_test.go | 21 ++----- projects.go | 113 +++++++++++----------------------- projects_test.go | 35 +++++------ types.go | 9 +++ 6 files changed, 162 insertions(+), 121 deletions(-) diff --git a/gitlab.go b/gitlab.go index 1ccf5ddbd..7c6277712 100644 --- a/gitlab.go +++ b/gitlab.go @@ -18,12 +18,14 @@ package gitlab import ( + "bytes" "context" "encoding/json" "fmt" "io" "io/ioutil" "math/rand" + "mime/multipart" "net/http" "net/url" "sort" @@ -520,11 +522,11 @@ func (c *Client) setBaseURL(urlStr string) error { return nil } -// NewRequest creates an API request. An optional relative URL path can be -// provided in path, in which case it is resolved relative to the base URL -// of the Client. -// Paths should always be specified without a preceding slash. If specified, -// the value pointed to by body is JSON encoded and included as request body. +// NewRequest creates a new API request. The method expects a relative URL +// path that will be resolved relative to the base URL of the Client. +// Relative URL paths should always be specified without a preceding slash. +// If specified, the value pointed to by body is JSON encoded and included +// as the request body. func (c *Client) NewRequest(method, path string, opt interface{}, options []RequestOptionFunc) (*retryablehttp.Request, error) { u := *c.baseURL unescaped, err := url.PathUnescape(path) @@ -585,6 +587,82 @@ func (c *Client) NewRequest(method, path string, opt interface{}, options []Requ return req, nil } +// UploadRequest creates an API request for uploading a file. The method +// expects a relative URL path that will be resolved relative to the base +// URL of the Client. Relative URL paths should always be specified without +// a preceding slash. If specified, the value pointed to by body is JSON +// encoded and included as the request body. +func (c *Client) UploadRequest(method, path string, content io.Reader, filename string, uploadType UploadType, opt interface{}, options []RequestOptionFunc) (*retryablehttp.Request, error) { + u := *c.baseURL + unescaped, err := url.PathUnescape(path) + if err != nil { + return nil, err + } + + // Set the encoded path data + u.RawPath = c.baseURL.Path + path + u.Path = c.baseURL.Path + unescaped + + // Create a request specific headers map. + reqHeaders := make(http.Header) + reqHeaders.Set("Accept", "application/json") + + if c.UserAgent != "" { + reqHeaders.Set("User-Agent", c.UserAgent) + } + + b := new(bytes.Buffer) + w := multipart.NewWriter(b) + + fw, err := w.CreateFormFile(string(uploadType), filename) + if err != nil { + return nil, err + } + + if _, err := io.Copy(fw, content); err != nil { + return nil, err + } + + if opt != nil { + fields, err := query.Values(opt) + if err != nil { + return nil, err + } + for name := range fields { + if err = w.WriteField(name, fmt.Sprintf("%v", fields.Get(name))); err != nil { + return nil, err + } + } + } + + if err = w.Close(); err != nil { + return nil, err + } + + reqHeaders.Set("Content-Type", w.FormDataContentType()) + + req, err := retryablehttp.NewRequest(method, u.String(), b) + if err != nil { + return nil, err + } + + for _, fn := range options { + if fn == nil { + continue + } + if err := fn(req); err != nil { + return nil, err + } + } + + // Set the request specific headers. + for k, v := range reqHeaders { + req.Header[k] = v + } + + return req, nil +} + // Response is a GitLab API response. This wraps the standard http.Response // returned from GitLab and provides convenient access to things like // pagination links. diff --git a/project_import_export.go b/project_import_export.go index 6caef3923..67f8a620a 100644 --- a/project_import_export.go +++ b/project_import_export.go @@ -19,6 +19,7 @@ package gitlab import ( "bytes" "fmt" + "io" "net/http" "time" ) @@ -162,18 +163,26 @@ func (s *ProjectImportExportService) ExportDownload(pid interface{}, options ... // https://docs.gitlab.com/ce/api/project_import_export.html#import-a-file type ImportFileOptions struct { Namespace *string `url:"namespace,omitempty" json:"namespace,omitempty"` - File *string `url:"file,omitempty" json:"file,omitempty"` + Name *string `url:"name,omitempty" json:"name,omitempty"` Path *string `url:"path,omitempty" json:"path,omitempty"` Overwrite *bool `url:"overwrite,omitempty" json:"overwrite,omitempty"` OverrideParams *CreateProjectOptions `url:"override_params,omitempty" json:"override_params,omitempty"` } -// ImportFile import a file. +// Import a project from an archive file. // // GitLab API docs: // https://docs.gitlab.com/ce/api/project_import_export.html#import-a-file -func (s *ProjectImportExportService) ImportFile(opt *ImportFileOptions, options ...RequestOptionFunc) (*ImportStatus, *Response, error) { - req, err := s.client.NewRequest(http.MethodPost, "projects/import", opt, options) +func (s *ProjectImportExportService) ImportFromFile(archive io.Reader, opt *ImportFileOptions, options ...RequestOptionFunc) (*ImportStatus, *Response, error) { + req, err := s.client.UploadRequest( + http.MethodPost, + "projects/import", + archive, + "archive.tar.gz", + UploadFile, + opt, + options, + ) if err != nil { return nil, nil, err } diff --git a/project_import_export_test.go b/project_import_export_test.go index c497b4cd8..723eefd90 100644 --- a/project_import_export_test.go +++ b/project_import_export_test.go @@ -1,6 +1,7 @@ package gitlab import ( + "bytes" "fmt" "net/http" "testing" @@ -159,32 +160,18 @@ func TestProjectImportExportService_ImportFile(t *testing.T) { ImportStatus: "scheduled", } - es, resp, err := client.ProjectImportExport.ImportFile(nil, nil) + file := bytes.NewBufferString("dummy") + es, resp, err := client.ProjectImportExport.ImportFromFile(file, nil, nil) require.NoError(t, err) require.NotNil(t, resp) require.Equal(t, want, es) - es, resp, err = client.ProjectImportExport.ImportFile(nil, errorOption) + es, resp, err = client.ProjectImportExport.ImportFromFile(file, nil, errorOption) require.EqualError(t, err, "RequestOptionFunc returns an error") require.Nil(t, resp) require.Nil(t, es) } -func TestProjectImportExportService_ImportFile_NotFound(t *testing.T) { - mux, server, client := setup(t) - defer teardown(server) - - mux.HandleFunc("/api/v4/projects/import", func(w http.ResponseWriter, r *http.Request) { - testMethod(t, r, http.MethodPost) - w.WriteHeader(http.StatusNotFound) - }) - - es, resp, err := client.ProjectImportExport.ImportFile(nil, nil) - require.Error(t, err) - require.Nil(t, es) - require.Equal(t, http.StatusNotFound, resp.StatusCode) -} - func TestProjectImportExportService_ImportStatus(t *testing.T) { mux, server, client := setup(t) defer teardown(server) diff --git a/projects.go b/projects.go index 58a43ec9b..6eca9bca7 100644 --- a/projects.go +++ b/projects.go @@ -17,13 +17,9 @@ package gitlab import ( - "bytes" "fmt" "io" - "mime/multipart" "net/http" - "os" - "path/filepath" "time" ) @@ -1278,7 +1274,7 @@ func (s *ProjectsService) DeleteProjectForkRelation(pid interface{}, options ... return s.client.Do(req, nil) } -// ProjectFile represents an uploaded project file +// ProjectFile represents an uploaded project file. // // GitLab API docs: https://docs.gitlab.com/ce/api/projects.html#upload-a-file type ProjectFile struct { @@ -1287,57 +1283,69 @@ type ProjectFile struct { Markdown string `json:"markdown"` } -// UploadFile upload a file from disk +// UploadFile uploads a file. // // GitLab API docs: https://docs.gitlab.com/ce/api/projects.html#upload-a-file -func (s *ProjectsService) UploadFile(pid interface{}, file string, options ...RequestOptionFunc) (*ProjectFile, *Response, error) { +func (s *ProjectsService) UploadFile(pid interface{}, content io.Reader, filename string, options ...RequestOptionFunc) (*ProjectFile, *Response, error) { project, err := parseID(pid) if err != nil { return nil, nil, err } u := fmt.Sprintf("projects/%s/uploads", PathEscape(project)) - f, err := os.Open(file) + req, err := s.client.UploadRequest( + http.MethodPost, + u, + content, + filename, + UploadFile, + nil, + options, + ) if err != nil { return nil, nil, err } - defer f.Close() - b := &bytes.Buffer{} - w := multipart.NewWriter(b) - - _, filename := filepath.Split(file) - fw, err := w.CreateFormFile("file", filename) + pf := new(ProjectFile) + resp, err := s.client.Do(req, pf) if err != nil { - return nil, nil, err + return nil, resp, err } - _, err = io.Copy(fw, f) - if err != nil { - return nil, nil, err - } - w.Close() + return pf, resp, nil +} - req, err := s.client.NewRequest(http.MethodPost, u, nil, options) +// UploadAvatar uploads an avatar. +// +// GitLab API docs: +// https://docs.gitlab.com/ee/api/projects.html#upload-a-project-avatar +func (s *ProjectsService) UploadAvatar(pid interface{}, avatar io.Reader, options ...RequestOptionFunc) (*Project, *Response, error) { + project, err := parseID(pid) if err != nil { return nil, nil, err } + u := fmt.Sprintf("projects/%s", PathEscape(project)) - // Set the buffer as the request body. - if err = req.SetBody(b); err != nil { + req, err := s.client.UploadRequest( + http.MethodPut, + u, + avatar, + "avatar.png", + UploadAvatar, + nil, + options, + ) + if err != nil { return nil, nil, err } - // Overwrite the default content type. - req.Header.Set("Content-Type", w.FormDataContentType()) - - uf := &ProjectFile{} - resp, err := s.client.Do(req, uf) + p := new(Project) + resp, err := s.client.Do(req, p) if err != nil { return nil, resp, err } - return uf, resp, nil + return p, resp, err } // ListProjectForks gets a list of project forks. @@ -1808,50 +1816,3 @@ func (s *ProjectsService) TransferProject(pid interface{}, opt *TransferProjectO return p, resp, err } - -// UploadAvatar uploads an avatar for the project -// -// GitLab API docs: https://docs.gitlab.com/ee/api/projects.html#upload-a-project-avatar -func (s *ProjectsService) UploadAvatar(pid interface{}, avatar io.Reader, filename string, options ...RequestOptionFunc) (*Project, *Response, error) { - project, err := parseID(pid) - if err != nil { - return nil, nil, err - } - u := fmt.Sprintf("projects/%s", PathEscape(project)) - - b := &bytes.Buffer{} - w := multipart.NewWriter(b) - - _, filename = filepath.Split(filename) - fw, err := w.CreateFormFile("avatar", filename) - if err != nil { - return nil, nil, err - } - - _, err = io.Copy(fw, avatar) - if err != nil { - return nil, nil, err - } - w.Close() - - req, err := s.client.NewRequest(http.MethodPut, u, nil, options) - if err != nil { - return nil, nil, err - } - - // Set the buffer as the request body. - if err = req.SetBody(b); err != nil { - return nil, nil, err - } - - // Overwrite the default content type. - req.Header.Set("Content-Type", w.FormDataContentType()) - - p := new(Project) - resp, err := s.client.Do(req, p) - if err != nil { - return nil, resp, err - } - - return p, resp, err -} diff --git a/projects_test.go b/projects_test.go index 56bd363b7..9d58d03e9 100644 --- a/projects_test.go +++ b/projects_test.go @@ -385,12 +385,9 @@ func TestUploadFile(t *testing.T) { mux, server, client := setup(t) defer teardown(server) - tf, _ := ioutil.TempFile(os.TempDir(), "test") - defer os.Remove(tf.Name()) - mux.HandleFunc("/api/v4/projects/1/uploads", func(w http.ResponseWriter, r *http.Request) { testMethod(t, r, http.MethodPost) - if false == strings.Contains(r.Header.Get("Content-Type"), "multipart/form-data;") { + if !strings.Contains(r.Header.Get("Content-Type"), "multipart/form-data;") { t.Fatalf("Projects.UploadFile request content-type %+v want multipart/form-data;", r.Header.Get("Content-Type")) } if r.ContentLength == -1 { @@ -409,14 +406,15 @@ func TestUploadFile(t *testing.T) { Markdown: "![dk](/uploads/66dbcd21ec5d24ed6ea225176098d52b/dk.png)", } - file, _, err := client.Projects.UploadFile(1, tf.Name()) + file := bytes.NewBufferString("dummy") + projectFile, _, err := client.Projects.UploadFile(1, file, "test.txt") if err != nil { t.Fatalf("Projects.UploadFile returns an error: %v", err) } - if !reflect.DeepEqual(want, file) { - t.Errorf("Projects.UploadFile returned %+v, want %+v", file, want) + if !reflect.DeepEqual(want, projectFile) { + t.Errorf("Projects.UploadFile returned %+v, want %+v", projectFile, want) } } @@ -434,7 +432,7 @@ func TestUploadFile_Retry(t *testing.T) { isFirstRequest = false return } - if false == strings.Contains(r.Header.Get("Content-Type"), "multipart/form-data;") { + if !strings.Contains(r.Header.Get("Content-Type"), "multipart/form-data;") { t.Fatalf("Projects.UploadFile request content-type %+v want multipart/form-data;", r.Header.Get("Content-Type")) } if r.ContentLength == -1 { @@ -453,14 +451,15 @@ func TestUploadFile_Retry(t *testing.T) { Markdown: "![dk](/uploads/66dbcd21ec5d24ed6ea225176098d52b/dk.png)", } - file, _, err := client.Projects.UploadFile(1, tf.Name()) + file := bytes.NewBufferString("dummy") + projectFile, _, err := client.Projects.UploadFile(1, file, "test.txt") if err != nil { t.Fatalf("Projects.UploadFile returns an error: %v", err) } - if !reflect.DeepEqual(want, file) { - t.Errorf("Projects.UploadFile returned %+v, want %+v", file, want) + if !reflect.DeepEqual(want, projectFile) { + t.Errorf("Projects.UploadFile returned %+v, want %+v", projectFile, want) } } @@ -470,7 +469,7 @@ func TestUploadAvatar(t *testing.T) { mux.HandleFunc("/api/v4/projects/1", func(w http.ResponseWriter, r *http.Request) { testMethod(t, r, http.MethodPut) - if false == strings.Contains(r.Header.Get("Content-Type"), "multipart/form-data;") { + if !strings.Contains(r.Header.Get("Content-Type"), "multipart/form-data;") { t.Fatalf("Projects.UploadAvatar request content-type %+v want multipart/form-data;", r.Header.Get("Content-Type")) } if r.ContentLength == -1 { @@ -479,9 +478,8 @@ func TestUploadAvatar(t *testing.T) { fmt.Fprint(w, `{}`) }) - b := bytes.Buffer{} - - _, _, err := client.Projects.UploadAvatar(1, &b, "image.png") + avatar := new(bytes.Buffer) + _, _, err := client.Projects.UploadAvatar(1, avatar) if err != nil { t.Fatalf("Projects.UploadAvatar returns an error: %v", err) @@ -499,7 +497,7 @@ func TestUploadAvatar_Retry(t *testing.T) { isFirstRequest = false return } - if false == strings.Contains(r.Header.Get("Content-Type"), "multipart/form-data;") { + if !strings.Contains(r.Header.Get("Content-Type"), "multipart/form-data;") { t.Fatalf("Projects.UploadAvatar request content-type %+v want multipart/form-data;", r.Header.Get("Content-Type")) } if r.ContentLength == -1 { @@ -508,9 +506,8 @@ func TestUploadAvatar_Retry(t *testing.T) { fmt.Fprint(w, `{}`) }) - b := bytes.Buffer{} - - _, _, err := client.Projects.UploadAvatar(1, &b, "image.png") + avatar := new(bytes.Buffer) + _, _, err := client.Projects.UploadAvatar(1, avatar) if err != nil { t.Fatalf("Projects.UploadAvatar returns an error: %v", err) diff --git a/types.go b/types.go index 679cb9ef1..cff36e991 100644 --- a/types.go +++ b/types.go @@ -556,6 +556,15 @@ const ( TodoTargetMergeRequest TodoTargetType = "MergeRequest" ) +// UploadType represents the available upload types. +type UploadType string + +// The available upload types. +const ( + UploadAvatar UploadType = "avatar" + UploadFile UploadType = "file" +) + // VariableTypeValue represents a variable type within GitLab. // // GitLab API docs: https://docs.gitlab.com/ce/api/