Skip to content

Commit

Permalink
fix: pagination offset (#642)
Browse files Browse the repository at this point in the history 
See ory/hydra#3362
  • Loading branch information
@aeneasr
aeneasr committed Nov 28, 2022
1 parent 37a7ded commit 6e01212
Show file tree
Hide file tree
Showing 18 changed files with 68 additions and 61 deletions.
28 changes: 15 additions & 13 deletions pagination/header.go
View file
Expand Up @@ -20,8 +20,10 @@ func header(u *url.URL, rel string, limit, offset int64) string {
return fmt.Sprintf("<%s>; rel=\"%s\"", u.String(), rel)
}

type formatter func(location *url.URL, rel string, itemsPerPage int64, offset int64) string

// HeaderWithFormatter adds an HTTP header for pagination which uses a custom formatter for generating the URL links.
func HeaderWithFormatter(w http.ResponseWriter, u *url.URL, total int64, page, itemsPerPage int, formatter func(*url.URL, string, int64, int64) string) {
func HeaderWithFormatter(w http.ResponseWriter, u *url.URL, total int64, page, itemsPerPage int, f formatter) {
if itemsPerPage <= 0 {
itemsPerPage = 1
}
Expand All @@ -44,38 +46,38 @@ func HeaderWithFormatter(w http.ResponseWriter, u *url.URL, total int64, page, i
if offset >= lastOffset {
if total == 0 {
w.Header().Set("Link", strings.Join([]string{
formatter(u, "first", itemsPerPage64, 0),
formatter(u, "next", itemsPerPage64, ((offset/itemsPerPage64)+1)*itemsPerPage64),
formatter(u, "prev", itemsPerPage64, ((offset/itemsPerPage64)-1)*itemsPerPage64),
f(u, "first", itemsPerPage64, 0),
f(u, "next", itemsPerPage64, ((offset/itemsPerPage64)+1)*itemsPerPage64),
f(u, "prev", itemsPerPage64, ((offset/itemsPerPage64)-1)*itemsPerPage64),
}, ","))
return
}

if total <= itemsPerPage64 {
w.Header().Set("link", formatter(u, "first", total, 0))
w.Header().Set("link", f(u, "first", total, 0))
return
}

w.Header().Set("Link", strings.Join([]string{
formatter(u, "first", itemsPerPage64, 0),
formatter(u, "prev", itemsPerPage64, lastOffset-itemsPerPage64),
f(u, "first", itemsPerPage64, 0),
f(u, "prev", itemsPerPage64, lastOffset-itemsPerPage64),
}, ","))
return
}

if offset < itemsPerPage64 {
w.Header().Set("Link", strings.Join([]string{
formatter(u, "next", itemsPerPage64, itemsPerPage64),
formatter(u, "last", itemsPerPage64, lastOffset),
f(u, "next", itemsPerPage64, itemsPerPage64),
f(u, "last", itemsPerPage64, lastOffset),
}, ","))
return
}

w.Header().Set("Link", strings.Join([]string{
formatter(u, "first", itemsPerPage64, 0),
formatter(u, "next", itemsPerPage64, ((offset/itemsPerPage64)+1)*itemsPerPage64),
formatter(u, "prev", itemsPerPage64, ((offset/itemsPerPage64)-1)*itemsPerPage64),
formatter(u, "last", itemsPerPage64, lastOffset),
f(u, "first", itemsPerPage64, 0),
f(u, "next", itemsPerPage64, ((offset/itemsPerPage64)+1)*itemsPerPage64),
f(u, "prev", itemsPerPage64, ((offset/itemsPerPage64)-1)*itemsPerPage64),
f(u, "last", itemsPerPage64, lastOffset),
}, ","))
}

Expand Down
4 changes: 2 additions & 2 deletions ...PaginationHeader-Create_next_and_last,_but_not_previous_or_first_if_at_the_beginning.json
View file
@@ -1,5 +1,5 @@
[
"\u003chttp://example.com?page=1\u0026page_size=50\u0026page_token=eyJwYWdlIjoiNTAiLCJ2IjoxfQ\u0026per_page=50\u003e",
"rel=\"next\",\u003chttp://example.com?page=2\u0026page_size=50\u0026page_token=eyJwYWdlIjoiMTAwIiwidiI6MX0\u0026per_page=50\u003e",
"\u003chttp://example.com?page=1\u0026page_size=50\u0026page_token=eyJvZmZzZXQiOiI1MCIsInYiOjJ9\u0026per_page=50\u003e",
"rel=\"next\",\u003chttp://example.com?page=2\u0026page_size=50\u0026page_token=eyJvZmZzZXQiOiIxMDAiLCJ2IjoyfQ\u0026per_page=50\u003e",
"rel=\"last\""
]
2 changes: 1 addition & 1 deletion ...nHeader-Create_only_first_if_the_limits_provided_exceeds_the_number_of_clients_found.json
View file
@@ -1,4 +1,4 @@
[
"\u003chttp://example.com?page=0\u0026page_size=5\u0026page_token=eyJwYWdlIjoiMCIsInYiOjF9\u0026per_page=5\u003e",
"\u003chttp://example.com?page=0\u0026page_size=5\u0026page_token=eyJvZmZzZXQiOiIwIiwidiI6Mn0\u0026per_page=5\u003e",
"rel=\"first\""
]
8 changes: 4 additions & 4 deletions ...apshots/TestPaginationHeader-Create_previous,_next,_first,_and_last_if_in_the_middle.json
View file
@@ -1,7 +1,7 @@
[
"\u003chttp://example.com?page=0\u0026page_size=50\u0026page_token=eyJwYWdlIjoiMCIsInYiOjF9\u0026per_page=50\u003e",
"rel=\"first\",\u003chttp://example.com?page=4\u0026page_size=50\u0026page_token=eyJwYWdlIjoiMjAwIiwidiI6MX0\u0026per_page=50\u003e",
"rel=\"next\",\u003chttp://example.com?page=2\u0026page_size=50\u0026page_token=eyJwYWdlIjoiMTAwIiwidiI6MX0\u0026per_page=50\u003e",
"rel=\"prev\",\u003chttp://example.com?page=5\u0026page_size=50\u0026page_token=eyJwYWdlIjoiMjUwIiwidiI6MX0\u0026per_page=50\u003e",
"\u003chttp://example.com?page=0\u0026page_size=50\u0026page_token=eyJvZmZzZXQiOiIwIiwidiI6Mn0\u0026per_page=50\u003e",
"rel=\"first\",\u003chttp://example.com?page=4\u0026page_size=50\u0026page_token=eyJvZmZzZXQiOiIyMDAiLCJ2IjoyfQ\u0026per_page=50\u003e",
"rel=\"next\",\u003chttp://example.com?page=2\u0026page_size=50\u0026page_token=eyJvZmZzZXQiOiIxMDAiLCJ2IjoyfQ\u0026per_page=50\u003e",
"rel=\"prev\",\u003chttp://example.com?page=5\u0026page_size=50\u0026page_token=eyJvZmZzZXQiOiIyNTAiLCJ2IjoyfQ\u0026per_page=50\u003e",
"rel=\"last\""
]
6 changes: 3 additions & 3 deletions ...reate_previous,_next,_first,_but_not_last_if_in_the_middle_and_no_total_was_provided.json
View file
@@ -1,6 +1,6 @@
[
"\u003chttp://example.com?page=0\u0026page_size=50\u0026page_token=eyJwYWdlIjoiMCIsInYiOjF9\u0026per_page=50\u003e",
"rel=\"first\",\u003chttp://example.com?page=4\u0026page_size=50\u0026page_token=eyJwYWdlIjoiMjAwIiwidiI6MX0\u0026per_page=50\u003e",
"rel=\"next\",\u003chttp://example.com?page=2\u0026page_size=50\u0026page_token=eyJwYWdlIjoiMTAwIiwidiI6MX0\u0026per_page=50\u003e",
"\u003chttp://example.com?page=0\u0026page_size=50\u0026page_token=eyJvZmZzZXQiOiIwIiwidiI6Mn0\u0026per_page=50\u003e",
"rel=\"first\",\u003chttp://example.com?page=4\u0026page_size=50\u0026page_token=eyJvZmZzZXQiOiIyMDAiLCJ2IjoyfQ\u0026per_page=50\u003e",
"rel=\"next\",\u003chttp://example.com?page=2\u0026page_size=50\u0026page_token=eyJvZmZzZXQiOiIxMDAiLCJ2IjoyfQ\u0026per_page=50\u003e",
"rel=\"prev\""
]
4 changes: 2 additions & 2 deletions ...ts/TestPaginationHeader-Create_previous_and_first_but_not_next_or_last_if_at_the_end.json
View file
@@ -1,5 +1,5 @@
[
"\u003chttp://example.com?page=0\u0026page_size=50\u0026page_token=eyJwYWdlIjoiMCIsInYiOjF9\u0026per_page=50\u003e",
"rel=\"first\",\u003chttp://example.com?page=1\u0026page_size=50\u0026page_token=eyJwYWdlIjoiNTAiLCJ2IjoxfQ\u0026per_page=50\u003e",
"\u003chttp://example.com?page=0\u0026page_size=50\u0026page_token=eyJvZmZzZXQiOiIwIiwidiI6Mn0\u0026per_page=50\u003e",
"rel=\"first\",\u003chttp://example.com?page=1\u0026page_size=50\u0026page_token=eyJvZmZzZXQiOiI1MCIsInYiOjJ9\u0026per_page=50\u003e",
"rel=\"prev\""
]
8 changes: 4 additions & 4 deletions ...napshots/TestPaginationHeader-Header_should_default_limit_to_1_no_limit_was_provided.json
View file
@@ -1,7 +1,7 @@
[
"\u003chttp://example.com?page=0\u0026page_size=1\u0026page_token=eyJwYWdlIjoiMCIsInYiOjF9\u0026per_page=1\u003e",
"rel=\"first\",\u003chttp://example.com?page=21\u0026page_size=1\u0026page_token=eyJwYWdlIjoiMjEiLCJ2IjoxfQ\u0026per_page=1\u003e",
"rel=\"next\",\u003chttp://example.com?page=19\u0026page_size=1\u0026page_token=eyJwYWdlIjoiMTkiLCJ2IjoxfQ\u0026per_page=1\u003e",
"rel=\"prev\",\u003chttp://example.com?page=99\u0026page_size=1\u0026page_token=eyJwYWdlIjoiOTkiLCJ2IjoxfQ\u0026per_page=1\u003e",
"\u003chttp://example.com?page=0\u0026page_size=1\u0026page_token=eyJvZmZzZXQiOiIwIiwidiI6Mn0\u0026per_page=1\u003e",
"rel=\"first\",\u003chttp://example.com?page=21\u0026page_size=1\u0026page_token=eyJvZmZzZXQiOiIyMSIsInYiOjJ9\u0026per_page=1\u003e",
"rel=\"next\",\u003chttp://example.com?page=19\u0026page_size=1\u0026page_token=eyJvZmZzZXQiOiIxOSIsInYiOjJ9\u0026per_page=1\u003e",
"rel=\"prev\",\u003chttp://example.com?page=99\u0026page_size=1\u0026page_token=eyJvZmZzZXQiOiI5OSIsInYiOjJ9\u0026per_page=1\u003e",
"rel=\"last\""
]
6 changes: 3 additions & 3 deletions pagination/migrationpagination/pagination.go
View file
Expand Up @@ -33,12 +33,12 @@ func (p *Paginator) ParsePagination(r *http.Request) (page, itemsPerPage int) {
return p.p.ParsePagination(r)
}

func header(u *url.URL, rel string, itemsPerPage, page int64) string {
func header(u *url.URL, rel string, itemsPerPage, offset int64) string {
q := u.Query()
q.Set("page_size", fmt.Sprintf("%d", itemsPerPage))
q.Set("page_token", tokenpagination.Encode(page))
q.Set("page_token", tokenpagination.Encode(offset))
q.Set("per_page", fmt.Sprintf("%d", itemsPerPage))
q.Set("page", fmt.Sprintf("%d", page/itemsPerPage))
q.Set("page", fmt.Sprintf("%d", offset/itemsPerPage))
u.RawQuery = q.Encode()
return fmt.Sprintf("<%s>; rel=\"%s\"", u.String(), rel)
}
Expand Down
6 changes: 3 additions & 3 deletions pagination/migrationpagination/pagination_test.go
View file
Expand Up @@ -84,11 +84,11 @@ func TestParsePagination(t *testing.T) {
expectedItemsPerPage int
expectedPage int
}{
{"normal", "http://localhost/foo?page_size=10&page_token=eyJwYWdlIjoxMH0", 10, 10},
{"normal-encoded", fmt.Sprintf("http://localhost/foo?page_size=10&page_token=%s", tokenpagination.Encode(10)), 10, 10},
{"normal", "http://localhost/foo?page_size=10&page_token=eyJvZmZzZXQiOjEwfQ", 10, 1},
{"normal-encoded", fmt.Sprintf("http://localhost/foo?page_size=10&page_token=%s", tokenpagination.Encode(10)), 10, 1},
{"defaults", "http://localhost/foo", 250, 0},
{"limits", "http://localhost/foo?page_size=2000", 1000, 0},
{"negatives", "http://localhost/foo?page_size=-1&page=eyJwYWdlIjotMX0", 1, 0},
{"negatives", "http://localhost/foo?page_size=-1&page=eyJvZmZzZXQiOi0xfQ", 1, 0},
{"negatives-encoded", fmt.Sprintf("http://localhost/foo?page_size=-1&page=%s", tokenpagination.Encode(-1)), 1, 0},
{"invalid_params", "http://localhost/foo?page_size=a&page=b", 250, 0},
{"legacy-normal", "http://localhost/foo?per_page=10&page=10", 10, 10},
Expand Down
4 changes: 2 additions & 2 deletions pagination/pagepagination/pagination.go
View file
Expand Up @@ -66,10 +66,10 @@ func (p *PagePaginator) ParsePagination(r *http.Request) (page, itemsPerPage int
return
}

func header(u *url.URL, rel string, limit, page int64) string {
func header(u *url.URL, rel string, limit, offset int64) string {
q := u.Query()
q.Set("per_page", fmt.Sprintf("%d", limit))
q.Set("page", fmt.Sprintf("%d", page/limit))
q.Set("page", fmt.Sprintf("%d", offset/limit))
u.RawQuery = q.Encode()
return fmt.Sprintf("<%s>; rel=\"%s\"", u.String(), rel)
}
Expand Down
4 changes: 2 additions & 2 deletions ...PaginationHeader-Create_next_and_last,_but_not_previous_or_first_if_at_the_beginning.json
View file
@@ -1,5 +1,5 @@
[
"\u003chttp://example.com?page_size=50\u0026page_token=eyJwYWdlIjoiNTAiLCJ2IjoxfQ\u003e",
"rel=\"next\",\u003chttp://example.com?page_size=50\u0026page_token=eyJwYWdlIjoiMTAwIiwidiI6MX0\u003e",
"\u003chttp://example.com?page_size=50\u0026page_token=eyJvZmZzZXQiOiI1MCIsInYiOjJ9\u003e",
"rel=\"next\",\u003chttp://example.com?page_size=50\u0026page_token=eyJvZmZzZXQiOiIxMDAiLCJ2IjoyfQ\u003e",
"rel=\"last\""
]
2 changes: 1 addition & 1 deletion ...nHeader-Create_only_first_if_the_limits_provided_exceeds_the_number_of_clients_found.json
View file
@@ -1,4 +1,4 @@
[
"\u003chttp://example.com?page_size=5\u0026page_token=eyJwYWdlIjoiMCIsInYiOjF9\u003e",
"\u003chttp://example.com?page_size=5\u0026page_token=eyJvZmZzZXQiOiIwIiwidiI6Mn0\u003e",
"rel=\"first\""
]
8 changes: 4 additions & 4 deletions ...apshots/TestPaginationHeader-Create_previous,_next,_first,_and_last_if_in_the_middle.json
View file
@@ -1,7 +1,7 @@
[
"\u003chttp://example.com?page_size=50\u0026page_token=eyJwYWdlIjoiMCIsInYiOjF9\u003e",
"rel=\"first\",\u003chttp://example.com?page_size=50\u0026page_token=eyJwYWdlIjoiMjAwIiwidiI6MX0\u003e",
"rel=\"next\",\u003chttp://example.com?page_size=50\u0026page_token=eyJwYWdlIjoiMTAwIiwidiI6MX0\u003e",
"rel=\"prev\",\u003chttp://example.com?page_size=50\u0026page_token=eyJwYWdlIjoiMjUwIiwidiI6MX0\u003e",
"\u003chttp://example.com?page_size=50\u0026page_token=eyJvZmZzZXQiOiIwIiwidiI6Mn0\u003e",
"rel=\"first\",\u003chttp://example.com?page_size=50\u0026page_token=eyJvZmZzZXQiOiIyMDAiLCJ2IjoyfQ\u003e",
"rel=\"next\",\u003chttp://example.com?page_size=50\u0026page_token=eyJvZmZzZXQiOiIxMDAiLCJ2IjoyfQ\u003e",
"rel=\"prev\",\u003chttp://example.com?page_size=50\u0026page_token=eyJvZmZzZXQiOiIyNTAiLCJ2IjoyfQ\u003e",
"rel=\"last\""
]
6 changes: 3 additions & 3 deletions ...reate_previous,_next,_first,_but_not_last_if_in_the_middle_and_no_total_was_provided.json
View file
@@ -1,6 +1,6 @@
[
"\u003chttp://example.com?page_size=50\u0026page_token=eyJwYWdlIjoiMCIsInYiOjF9\u003e",
"rel=\"first\",\u003chttp://example.com?page_size=50\u0026page_token=eyJwYWdlIjoiMjAwIiwidiI6MX0\u003e",
"rel=\"next\",\u003chttp://example.com?page_size=50\u0026page_token=eyJwYWdlIjoiMTAwIiwidiI6MX0\u003e",
"\u003chttp://example.com?page_size=50\u0026page_token=eyJvZmZzZXQiOiIwIiwidiI6Mn0\u003e",
"rel=\"first\",\u003chttp://example.com?page_size=50\u0026page_token=eyJvZmZzZXQiOiIyMDAiLCJ2IjoyfQ\u003e",
"rel=\"next\",\u003chttp://example.com?page_size=50\u0026page_token=eyJvZmZzZXQiOiIxMDAiLCJ2IjoyfQ\u003e",
"rel=\"prev\""
]
4 changes: 2 additions & 2 deletions ...ts/TestPaginationHeader-Create_previous_and_first_but_not_next_or_last_if_at_the_end.json
View file
@@ -1,5 +1,5 @@
[
"\u003chttp://example.com?page_size=50\u0026page_token=eyJwYWdlIjoiMCIsInYiOjF9\u003e",
"rel=\"first\",\u003chttp://example.com?page_size=50\u0026page_token=eyJwYWdlIjoiNTAiLCJ2IjoxfQ\u003e",
"\u003chttp://example.com?page_size=50\u0026page_token=eyJvZmZzZXQiOiIwIiwidiI6Mn0\u003e",
"rel=\"first\",\u003chttp://example.com?page_size=50\u0026page_token=eyJvZmZzZXQiOiI1MCIsInYiOjJ9\u003e",
"rel=\"prev\""
]
8 changes: 4 additions & 4 deletions ...napshots/TestPaginationHeader-Header_should_default_limit_to_1_no_limit_was_provided.json
View file
@@ -1,7 +1,7 @@
[
"\u003chttp://example.com?page_size=1\u0026page_token=eyJwYWdlIjoiMCIsInYiOjF9\u003e",
"rel=\"first\",\u003chttp://example.com?page_size=1\u0026page_token=eyJwYWdlIjoiMjEiLCJ2IjoxfQ\u003e",
"rel=\"next\",\u003chttp://example.com?page_size=1\u0026page_token=eyJwYWdlIjoiMTkiLCJ2IjoxfQ\u003e",
"rel=\"prev\",\u003chttp://example.com?page_size=1\u0026page_token=eyJwYWdlIjoiOTkiLCJ2IjoxfQ\u003e",
"\u003chttp://example.com?page_size=1\u0026page_token=eyJvZmZzZXQiOiIwIiwidiI6Mn0\u003e",
"rel=\"first\",\u003chttp://example.com?page_size=1\u0026page_token=eyJvZmZzZXQiOiIyMSIsInYiOjJ9\u003e",
"rel=\"next\",\u003chttp://example.com?page_size=1\u0026page_token=eyJvZmZzZXQiOiIxOSIsInYiOjJ9\u003e",
"rel=\"prev\",\u003chttp://example.com?page_size=1\u0026page_token=eyJvZmZzZXQiOiI5OSIsInYiOjJ9\u003e",
"rel=\"last\""
]
15 changes: 10 additions & 5 deletions pagination/tokenpagination/pagination.go
View file
Expand Up @@ -19,7 +19,7 @@ import (
)

func Encode(offset int64) string {
return base64.RawURLEncoding.EncodeToString([]byte(fmt.Sprintf(`{"page":"%d","v":1}`, offset)))
return base64.RawURLEncoding.EncodeToString([]byte(fmt.Sprintf(`{"offset":"%d","v":2}`, offset)))
}

func decode(s string) (int, error) {
Expand All @@ -28,7 +28,7 @@ func decode(s string) (int, error) {
return 0, errors.WithStack(herodot.ErrBadRequest.WithWrap(err).WithReasonf("Unable to parse pagination token: %s", err))
}

return int(gjson.Get(string(b), "page").Int()), nil
return int(gjson.Get(string(b), "offset").Int()), nil
}

type TokenPaginator struct {
Expand All @@ -50,8 +50,9 @@ func (p *TokenPaginator) defaults() {
func (p *TokenPaginator) ParsePagination(r *http.Request) (page, itemsPerPage int) {
p.defaults()

var offset int
if offsetParam := r.URL.Query().Get("page_token"); len(offsetParam) > 0 {
page, _ = decode(offsetParam)
offset, _ = decode(offsetParam)
}

if gotLimit, err := strconv.ParseInt(r.URL.Query().Get("page_size"), 10, 0); err == nil {
Expand All @@ -68,17 +69,21 @@ func (p *TokenPaginator) ParsePagination(r *http.Request) (page, itemsPerPage in
itemsPerPage = 1
}

if offset > 0 {
page = offset / itemsPerPage
}

if page < 0 {
page = 0
}

return
}

func header(u *url.URL, rel string, itemsPerPage, page int64) string {
func header(u *url.URL, rel string, itemsPerPage, offset int64) string {
q := u.Query()
q.Set("page_size", fmt.Sprintf("%d", itemsPerPage))
q.Set("page_token", Encode(page))
q.Set("page_token", Encode(offset))
u.RawQuery = q.Encode()
return fmt.Sprintf("<%s>; rel=\"%s\"", u.String(), rel)
}
Expand Down
6 changes: 3 additions & 3 deletions pagination/tokenpagination/pagination_test.go
View file
Expand Up @@ -81,11 +81,11 @@ func TestParsePagination(t *testing.T) {
expectedItemsPerPage int
expectedPage int
}{
{"normal", "http://localhost/foo?page_size=10&page_token=eyJwYWdlIjoxMH0", 10, 10},
{"normal-encoded", "http://localhost/foo?page_size=10&page_token=" + Encode(10), 10, 10},
{"normal", "http://localhost/foo?page_size=10&page_token=eyJvZmZzZXQiOjEwfQ", 10, 1},
{"normal-encoded", "http://localhost/foo?page_size=10&page_token=" + Encode(10), 10, 1},
{"defaults", "http://localhost/foo", 250, 0},
{"limits", "http://localhost/foo?page_size=2000", 1000, 0},
{"negatives", "http://localhost/foo?page_size=-1&page=eyJwYWdlIjotMX0", 1, 0},
{"negatives", "http://localhost/foo?page_size=-1&page=eyJvZmZzZXQiOi0xfQ", 1, 0},
{"negatives-encoded", "http://localhost/foo?page_size=-1&page=" + Encode(-1), 1, 0},
{"invalid_params", "http://localhost/foo?page_size=a&page=b", 250, 0},
} {
Expand Down

0 comments on commit 6e01212

Please sign in to comment.