Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat(firestore): implement limitToLast #2420

Merged
merged 14 commits into from Jul 27, 2020
88 changes: 86 additions & 2 deletions firestore/integration_test.go
Expand Up @@ -632,7 +632,7 @@ func TestIntegration_WriteBatch(t *testing.T) {
// TODO(jba): test verify when it is supported.
}

func TestIntegration_Query(t *testing.T) {
func TestIntegration_QueryDocuments(t *testing.T) {
ctx := context.Background()
coll := integrationColl(t)
h := testHelper{t}
Expand Down Expand Up @@ -681,7 +681,91 @@ func TestIntegration_Query(t *testing.T) {
if err != nil {
t.Fatal(err)
}
seen := map[int64]bool{} // "q" values we see
seen := map[int64]bool{} // "q" values we see.
for _, d := range allDocs {
data := d.Data()
q, ok := data["q"]
if !ok {
// A document from another test.
continue
}
if seen[q.(int64)] {
t.Errorf("%v: duplicate doc", data)
}
seen[q.(int64)] = true
if data["x"] != int64(1) {
t.Errorf("%v: wrong or missing 'x'", data)
}
if len(data) != 2 {
t.Errorf("%v: want two keys", data)
}
}
if got, want := len(seen), len(wants); got != want {
t.Errorf("got %d docs with 'q', want %d", len(seen), len(wants))
}
}
IlyaFaer marked this conversation as resolved.
Show resolved Hide resolved

func TestIntegration_QueryDocuments_LimitToLast_Fail(t *testing.T) {
ctx := context.Background()
coll := integrationColl(t)
q := coll.Select("q").OrderBy("q", Asc).LimitToLast(1)
gotDocs, err := q.Documents(ctx).GetAll()
if err == nil {
t.Errorf("got %v docs, want error", len(gotDocs))
}
}

func TestIntegration_QueryGet(t *testing.T) {
IlyaFaer marked this conversation as resolved.
Show resolved Hide resolved
ctx := context.Background()
coll := integrationColl(t)
h := testHelper{t}
var wants []map[string]interface{}
for i := 0; i < 3; i++ {
doc := coll.NewDoc()
// To support running this test in parallel with the others, use a field name
// that we don't use anywhere else.
h.mustCreate(doc, map[string]interface{}{"q": i, "x": 1})
wants = append(wants, map[string]interface{}{"q": int64(i)})
}
q := coll.Select("q").OrderBy("q", Asc)
for i, test := range []struct {
q Query
want []map[string]interface{}
}{
{q, wants},
{q.Where("q", ">", 1), wants[2:]},
{q.WherePath([]string{"q"}, ">", 1), wants[2:]},
{q.Offset(1).Limit(1), wants[1:2]},
{q.StartAt(1), wants[1:]},
{q.StartAfter(1), wants[2:]},
{q.EndAt(1), wants[:2]},
{q.EndBefore(1), wants[:1]},
{q.LimitToLast(2), wants[1:]},
} {
gotDocs, err := test.q.Get(ctx)
if err != nil {
t.Errorf("#%d: %+v: %v", i, test.q, err)
continue
}
if len(gotDocs) != len(test.want) {
t.Errorf("#%d: %+v: got %d docs, want %d", i, test.q, len(gotDocs), len(test.want))
continue
}
for j, g := range gotDocs {
if got, want := g.Data(), test.want[j]; !testEqual(got, want) {
t.Errorf("#%d: %+v, #%d: got\n%+v\nwant\n%+v", i, test.q, j, got, want)
}
}
}
_, err := coll.Select("q").Where("x", "==", 1).OrderBy("q", Asc).Get(ctx)
codeEq(t, "Where and OrderBy on different fields without an index", codes.FailedPrecondition, err)

// Using the collection itself as the query should return the full documents.
allDocs, err := coll.Get(ctx)
if err != nil {
t.Fatal(err)
}
seen := map[int64]bool{} // "q" values we see.
for _, d := range allDocs {
data := d.Data()
q, ok := data["q"]
Expand Down
54 changes: 52 additions & 2 deletions firestore/query.go
Expand Up @@ -43,6 +43,7 @@ type Query struct {
orders []order
offset int32
limit *wrappers.Int32Value
limitToLast bool
IlyaFaer marked this conversation as resolved.
Show resolved Hide resolved
startVals, endVals []interface{}
startDoc, endDoc *DocumentSnapshot
startBefore, endBefore bool
Expand Down Expand Up @@ -162,10 +163,19 @@ func (q Query) Offset(n int) Query {
return q
}

// Limit returns a new Query that specifies the maximum number of results to return.
// It must not be negative.
// Limit returns a new Query that specifies the maximum number of first results
// to return. It must not be negative.
func (q Query) Limit(n int) Query {
q.limit = &wrappers.Int32Value{Value: trunc32(n)}
q.limitToLast = false
IlyaFaer marked this conversation as resolved.
Show resolved Hide resolved
return q
}

// LimitToLast returns a new Query that specifies the maximum number of last
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Will this the meaning of this first / last results distinction be obvious to users of Firestore? @BenWhitehead

// results to return. It must not be negative.
func (q Query) LimitToLast(n int) Query {
q.limit = &wrappers.Int32Value{Value: trunc32(n)}
q.limitToLast = true
return q
}

Expand Down Expand Up @@ -569,8 +579,48 @@ func trunc32(i int) int32 {
return int32(i)
}

// Get returns an array of query's resulting documents.
func (q Query) Get(ctx context.Context) ([]*DocumentSnapshot, error) {
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

To be consistent with Query.Documents , I wonder if this should be called GetAll

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I thought about that. The thing is that in other languages it's called just get(). And GetAll() is a method of an iterator, so it gives all the docs once instead of streaming/iterating them - that makes sense for an iterator, but I'm not sure about Query. Anyway, I'm just explaining, if you think it should be renamed, I'll rename.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Looks like pretty much all other get methods in cloud.google.com/go/firestore which return a slice are called GetAll, whereas Get() is used for a single Transaction or DocumentRef. From that perspective I think GetAll makes more sense.

Also, I feel like the fact that this method is being added to the surface for the library is a bigger change than what's implied by the title of the PR. Could you change the title to reflect this? I also think it would be good to add an example of how this would be used to examples_test.go for the docs.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

One more thing-- looking at this a little more closely I see that the following are now equivalent for a non-limitToLast query q:

q.Get()
q.Documents().GetAll()

Do we have an opinion about which is preferred?

And really, is it necessary to have this duplication? (I guess the answer is yes with the current design because DocumentIterator doesn't have access to the Query fields. But in theory we could just add an unexported field there and pass it along I think-- this would mean that to do a limitToLast query you would also call q.Documents().GetAll(), though it would mean that other DocumentIterator methods would then need to error if limitToLast were set to true).

Would be curious on your thoughts @BenWhitehead @codyoss

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If there isn't a semantic difference to how those methods execute, I suspect it's fine to have a single method. And if there is a uniform way for users to keep doing what they're used to doing to read results it's probably better to stick with that, than it is to try and have this implementation be uniform with the other languages.

The complexity with limitToLast is that it has to buffer the up to n results and flip the order in memory before return to the application. So, as long as that property can be ensured whichever way is fine with me.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Discussed this with @codyoss offline, he also thinks that removing Query.Get and enabling this feature through the existing Query.DocumentIterator.GetAll method instead seems preferable. @IlyaFaer could you modify this change to do that? Feel free to reach out if there are questions that come up.

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

NP. As you mentioned before, I've added an unexported Query field into DocumentIterator and moved all the implementation into DocumentIterator.GetAll(). Seems to me we're no more in need of an example (as there are no new methods added), and a separate test.

limitedToLast := q.limitToLast

if q.limitToLast {
// Flip order statements before posting a request.
for i := range q.orders {
if q.orders[i].dir == Asc {
q.orders[i].dir = Desc
} else {
q.orders[i].dir = Asc
}
}
// Swap cursors.
q.startVals, q.endVals = q.endVals, q.startVals
q.startDoc, q.endDoc = q.endDoc, q.startDoc
q.startBefore, q.endBefore = q.endBefore, q.startBefore

q.limitToLast = false
IlyaFaer marked this conversation as resolved.
Show resolved Hide resolved
}
docs, err := q.Documents(ctx).GetAll()
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Should all this logic just happen within this method? @tritone Thoughts?

If we did this the err returned from calling Documents() could instead be used to Documents.Next.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If I'm reading this correctly-- we will have flipped q.limitToLast to false before calling q.Documents, so there shouldn't be an error from q.Documents unless one is set by newQueryDocumentIterator, correct? It would just be equivalent to having L624-626 up here. Seems clearer in some ways to decouple these methods but I'm not super opinionated on this.

if err != nil {
return nil, err
}
if limitedToLast {
// Flip docs order before return.
for i, j := 0, len(docs)-1; i < j; {
docs[i], docs[j] = docs[j], docs[i]
codyoss marked this conversation as resolved.
Show resolved Hide resolved
i++
j--
}
}
return docs, nil
}

// Documents returns an iterator over the query's resulting documents.
func (q Query) Documents(ctx context.Context) *DocumentIterator {
if q.limitToLast {
return &DocumentIterator{
err: errors.New("firestore: queries that include limitToLast constraints cannot be streamed. Use Query.Get() instead"),
}
}
return &DocumentIterator{
iter: newQueryDocumentIterator(withResourceHeader(ctx, q.c.path()), &q, nil),
}
Expand Down