From 4bf9f9abce0b27b40bb68f7b827ad8a38646fcef Mon Sep 17 00:00:00 2001 From: Timo Furrer Date: Mon, 13 Jun 2022 17:02:34 +0200 Subject: [PATCH] Add pagination helper to easily collect all items The https://github.com/xanzy/go-gitlab/pull/1496 Pull Request triggered some thinking for how to do this in a more "generic" way. Thus, I drafted this `gitlab.Collect()` function, which can be used to "easily" collect all items of a specific "list" API (I've implement a *collect* function which eagerly collects all items, but something similar can be implemented for channels to stream). The only downside of the current implementation so far is that Go doesn't really support generics / interfaces with direct field access as of now ... effectively, you'll always need methods. This means that, because the `ListOptions` field is somewhat inconsistently (sometimes as subfield, sometimes it's promoted) implemented over this package every `gitlab.List*Options` struct needs it's own `GetPage() int` and `SetPage(int)` implementation ... One improvement for this could be that we split out the `ListOptions` from the API endpoint specific options and make it an additional argument to the list methods - so that `Collect()` could profit from this and `GetPage() int` and `SetPage(int)` would only need to be implemented once. Another one would be to just use `gitlab.ListOptions` for endpoints which don't have additional parameters instead of doing this `type ListInstanceVariablesOptions ListOptions`. However, I think best would be to separate the generic `ListOptions` to a separate argument in the long term ... --- examples/pagination_helper.go | 67 +++++++++++++++++++++++++++++++++++ instance_variables.go | 8 +++++ pagination_helpers.go | 31 ++++++++++++++++ projects.go | 7 ++++ 4 files changed, 113 insertions(+) create mode 100644 examples/pagination_helper.go create mode 100644 pagination_helpers.go diff --git a/examples/pagination_helper.go b/examples/pagination_helper.go new file mode 100644 index 000000000..60b21ae98 --- /dev/null +++ b/examples/pagination_helper.go @@ -0,0 +1,67 @@ +// +// Copyright 2021, Timo Furrer +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +package main + +import ( + "fmt" + "log" + + "github.com/xanzy/go-gitlab" +) + +func main() { + git, err := gitlab.NewClient("yourtokengoeshere") + if err != nil { + log.Fatal(err) + } + + options := &gitlab.ListInstanceVariablesOptions{ + Page: 1, + } + instanceVariables, err := gitlab.Collect( + git.InstanceVariables.ListVariables, + options, + ) + fmt.Printf("%d\n", len(instanceVariables)) + + projectOptions := &gitlab.ListProjectsOptions{ + ListOptions: gitlab.ListOptions{ + Page: 1, + }, + Archived: gitlab.Bool(false), + } + projects, err := gitlab.Collect( + git.Projects.ListProjects, + projectOptions, + ) + fmt.Printf("%d\n", len(projects)) + + + userProjectOptions := &gitlab.ListProjectsOptions{ + ListOptions: gitlab.ListOptions{ + Page: 1, + }, + Archived: gitlab.Bool(false), + } + userProjects, err := gitlab.Collect( + func(opt *gitlab.ListProjectsOptions, options ...gitlab.RequestOptionFunc) ([]*gitlab.Project, *gitlab.Response, error) { + return git.Projects.ListUserProjects(1, opt, options...) + }, + userProjectOptions, + ) + fmt.Printf("%d\n", len(userProjects)) +} diff --git a/instance_variables.go b/instance_variables.go index f196a9c0f..aa29c9ede 100644 --- a/instance_variables.go +++ b/instance_variables.go @@ -54,6 +54,14 @@ func (v InstanceVariable) String() string { // https://docs.gitlab.com/ee/api/instance_level_ci_variables.html#list-all-instance-variables type ListInstanceVariablesOptions ListOptions +func (o *ListInstanceVariablesOptions) GetPage() int { + return o.Page +} + +func (o *ListInstanceVariablesOptions) SetPage(page int) { + o.Page = page +} + // ListVariables gets a list of all variables for an instance. // // GitLab API docs: diff --git a/pagination_helpers.go b/pagination_helpers.go new file mode 100644 index 000000000..045084a46 --- /dev/null +++ b/pagination_helpers.go @@ -0,0 +1,31 @@ + +package gitlab + +type ListableOptions interface{ + GetPage() int + SetPage(int) +} + +func (o *ListOptions) GetPage() int { + return o.Page +} + +func (o *ListOptions) SetPage(page int) { + o.Page = page +} + +func Collect[T any, U ListableOptions](f func(U, ...RequestOptionFunc) ([]*T, *Response, error), opt U, options ...RequestOptionFunc) ([]*T, error) { + var collection []*T + opt.SetPage(1) + for opt.GetPage() != 0 { + page, resp, err := f(opt, options...) + if err != nil { + return nil, err + } + + collection = append(collection, page...) + opt.SetPage(resp.NextPage) + } + + return collection, nil +} \ No newline at end of file diff --git a/projects.go b/projects.go index d9c83d252..1b8bcfd16 100644 --- a/projects.go +++ b/projects.go @@ -316,6 +316,13 @@ type ListProjectsOptions struct { WithMergeRequestsEnabled *bool `url:"with_merge_requests_enabled,omitempty" json:"with_merge_requests_enabled,omitempty"` WithProgrammingLanguage *string `url:"with_programming_language,omitempty" json:"with_programming_language,omitempty"` } +func (o *ListProjectsOptions) GetPage() int { + return o.ListOptions.Page +} + +func (o *ListProjectsOptions) SetPage(p int) { + o.ListOptions.Page = p +} // ListProjects gets a list of projects accessible by the authenticated user. //