Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: add aggregation query APIs #1765

Merged
merged 12 commits into from Sep 16, 2022
Merged
7,744 changes: 4,288 additions & 3,456 deletions dev/protos/firestore_admin_v1_proto_api.d.ts

Large diffs are not rendered by default.

21,679 changes: 11,837 additions & 9,842 deletions dev/protos/firestore_admin_v1_proto_api.js

Large diffs are not rendered by default.

4,215 changes: 2,816 additions & 1,399 deletions dev/protos/firestore_v1_proto_api.d.ts

Large diffs are not rendered by default.

12,115 changes: 7,692 additions & 4,423 deletions dev/protos/firestore_v1_proto_api.js

Large diffs are not rendered by default.

3,961 changes: 2,484 additions & 1,477 deletions dev/protos/firestore_v1beta1_proto_api.d.ts

Large diffs are not rendered by default.

11,120 changes: 6,724 additions & 4,396 deletions dev/protos/firestore_v1beta1_proto_api.js

Large diffs are not rendered by default.

42 changes: 42 additions & 0 deletions dev/protos/google/firestore/v1/aggregation_result.proto
@@ -0,0 +1,42 @@
// Copyright 2022 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

syntax = "proto3";

package google.firestore.v1;

import "google/firestore/v1/document.proto";

option csharp_namespace = "Google.Cloud.Firestore.V1";
option go_package = "google.golang.org/genproto/googleapis/firestore/v1;firestore";
option java_multiple_files = true;
option java_outer_classname = "AggregationResultProto";
option java_package = "com.google.firestore.v1";
option objc_class_prefix = "GCFS";
option php_namespace = "Google\\Cloud\\Firestore\\V1";
option ruby_package = "Google::Cloud::Firestore::V1";

// The result of a single bucket from a Firestore aggregation query.
//
// The keys of `aggregate_fields` are the same for all results in an aggregation
// query, unlike document queries which can have different fields present for
// each result.
message AggregationResult {
// The result of the aggregation functions, ex: `COUNT(*) AS total_docs`.
//
// The key is the [alias][google.firestore.v1.StructuredAggregationQuery.Aggregation.alias]
// assigned to the aggregation function on input and the size of this map
// equals the number of aggregation functions in the query.
map<string, Value> aggregate_fields = 2;
}
79 changes: 79 additions & 0 deletions dev/protos/google/firestore/v1/firestore.proto
Expand Up @@ -19,6 +19,7 @@ package google.firestore.v1;
import "google/api/annotations.proto";
import "google/api/client.proto";
import "google/api/field_behavior.proto";
import "google/firestore/v1/aggregation_result.proto";
import "google/firestore/v1/common.proto";
import "google/firestore/v1/document.proto";
import "google/firestore/v1/query.proto";
Expand Down Expand Up @@ -135,6 +136,29 @@ service Firestore {
};
}

// Runs an aggregation query.
//
// Rather than producing [Document][google.firestore.v1.Document] results like [Firestore.RunQuery][google.firestore.v1.Firestore.RunQuery],
// this API allows running an aggregation to produce a series of
// [AggregationResult][google.firestore.v1.AggregationResult] server-side.
//
// High-Level Example:
//
// ```
// -- Return the number of documents in table given a filter.
// SELECT COUNT(*) FROM ( SELECT * FROM k where a = true );
// ```
rpc RunAggregationQuery(RunAggregationQueryRequest) returns (stream RunAggregationQueryResponse) {
option (google.api.http) = {
post: "/v1/{parent=projects/*/databases/*/documents}:runAggregationQuery"
body: "*"
additional_bindings {
post: "/v1/{parent=projects/*/databases/*/documents/*/**}:runAggregationQuery"
body: "*"
}
};
}

// Partitions a query by returning partition cursors that can be used to run
// the query in parallel. The returned partition cursors are split points that
// can be used by RunQuery as starting/end points for the query results.
Expand Down Expand Up @@ -534,6 +558,61 @@ message RunQueryResponse {
}
}

// The request for [Firestore.RunAggregationQuery][google.firestore.v1.Firestore.RunAggregationQuery].
message RunAggregationQueryRequest {
// Required. The parent resource name. In the format:
// `projects/{project_id}/databases/{database_id}/documents` or
// `projects/{project_id}/databases/{database_id}/documents/{document_path}`.
// For example:
// `projects/my-project/databases/my-database/documents` or
// `projects/my-project/databases/my-database/documents/chatrooms/my-chatroom`
string parent = 1 [(google.api.field_behavior) = REQUIRED];

// The query to run.
oneof query_type {
// An aggregation query.
StructuredAggregationQuery structured_aggregation_query = 2;
}

// The consistency mode for the query, defaults to strong consistency.
oneof consistency_selector {
// Run the aggregation within an already active transaction.
//
// The value here is the opaque transaction ID to execute the query in.
bytes transaction = 4;

// Starts a new transaction as part of the query, defaulting to read-only.
//
// The new transaction ID will be returned as the first response in the
// stream.
TransactionOptions new_transaction = 5;

// Executes the query at the given timestamp.
//
// Requires:
//
// * Cannot be more than 270 seconds in the past.
google.protobuf.Timestamp read_time = 6;
}
}

// The response for [Firestore.RunAggregationQuery][google.firestore.v1.Firestore.RunAggregationQuery].
message RunAggregationQueryResponse {
// A single aggregation result.
//
// Not present when reporting partial progress.
AggregationResult result = 1;

// The transaction that was started as part of this request.
//
// Only present on the first response when the request requested to start
// a new transaction.
bytes transaction = 2;

// The time at which the aggregate value is valid for.
google.protobuf.Timestamp read_time = 3;
}

// The request for [Firestore.PartitionQuery][google.firestore.v1.Firestore.PartitionQuery].
message PartitionQueryRequest {
// Required. The parent resource name. In the format:
Expand Down
148 changes: 141 additions & 7 deletions dev/protos/google/firestore/v1/query.proto
Expand Up @@ -228,8 +228,13 @@ message StructuredQuery {
DESCENDING = 2;
}

// A reference to a field, such as `max(messages.time) as max_time`.
// A reference to a field in a document, ex: `stats.operations`.
message FieldReference {
// The relative path of the document being referenced.
//
// Requires:
//
// * Conform to [document field name][google.firestore.v1.Document.fields] limitations.
string field_path = 2;
}

Expand Down Expand Up @@ -273,25 +278,154 @@ message StructuredQuery {
// `WHERE __name__ > ... AND a > 1 ORDER BY a ASC, __name__ ASC`
repeated Order order_by = 4;

// A starting point for the query results.
// A potential prefix of a position in the result set to start the query at.
//
// The ordering of the result set is based on the `ORDER BY` clause of the
// original query.
//
// ```
// SELECT * FROM k WHERE a = 1 AND b > 2 ORDER BY b ASC, __name__ ASC;
// ```
//
// This query's results are ordered by `(b ASC, __name__ ASC)`.
//
// Cursors can reference either the full ordering or a prefix of the location,
// though it cannot reference more fields than what are in the provided
// `ORDER BY`.
//
// Continuing off the example above, attaching the following start cursors
// will have varying impact:
//
// - `START BEFORE (2, /k/123)`: start the query right before `a = 1 AND
// b > 2 AND __name__ > /k/123`.
// - `START AFTER (10)`: start the query right after `a = 1 AND b > 10`.
//
// Unlike `OFFSET` which requires scanning over the first N results to skip,
// a start cursor allows the query to begin at a logical position. This
// position is not required to match an actual result, it will scan forward
// from this position to find the next document.
//
// Requires:
//
// * The number of values cannot be greater than the number of fields
// specified in the `ORDER BY` clause.
Cursor start_at = 7;

// A end point for the query results.
// A potential prefix of a position in the result set to end the query at.
//
// This is similar to `START_AT` but with it controlling the end position
// rather than the start position.
//
// Requires:
//
// * The number of values cannot be greater than the number of fields
// specified in the `ORDER BY` clause.
Cursor end_at = 8;

// The number of results to skip.
// The number of documents to skip before returning the first result.
//
// Applies before limit, but after all other constraints. Must be >= 0 if
// specified.
// This applies after the constraints specified by the `WHERE`, `START AT`, &
// `END AT` but before the `LIMIT` clause.
//
// Requires:
//
// * The value must be greater than or equal to zero if specified.
int32 offset = 6;

// The maximum number of results to return.
//
// Applies after all other constraints.
// Must be >= 0 if specified.
//
// Requires:
//
// * The value must be greater than or equal to zero if specified.
google.protobuf.Int32Value limit = 5;
}

// Firestore query for running an aggregation over a [StructuredQuery][google.firestore.v1.StructuredQuery].
message StructuredAggregationQuery {
// Defines a aggregation that produces a single result.
message Aggregation {
// Count of documents that match the query.
//
// The `COUNT(*)` aggregation function operates on the entire document
// so it does not require a field reference.
message Count {
// Optional. Optional constraint on the maximum number of documents to count.
//
// This provides a way to set an upper bound on the number of documents
// to scan, limiting latency and cost.
//
// Unspecified is interpreted as no bound.
//
// High-Level Example:
//
// ```
// AGGREGATE COUNT_UP_TO(1000) OVER ( SELECT * FROM k );
// ```
//
// Requires:
//
// * Must be greater than zero when present.
google.protobuf.Int64Value up_to = 1 [(google.api.field_behavior) = OPTIONAL];
}

// The type of aggregation to perform, required.
oneof operator {
// Count aggregator.
Count count = 1;
}

// Optional. Optional name of the field to store the result of the aggregation into.
//
// If not provided, Firestore will pick a default name following the format
// `field_<incremental_id++>`. For example:
//
// ```
// AGGREGATE
// COUNT_UP_TO(1) AS count_up_to_1,
// COUNT_UP_TO(2),
// COUNT_UP_TO(3) AS count_up_to_3,
// COUNT_UP_TO(4)
// OVER (
// ...
// );
// ```
//
// becomes:
//
// ```
// AGGREGATE
// COUNT_UP_TO(1) AS count_up_to_1,
// COUNT_UP_TO(2) AS field_1,
// COUNT_UP_TO(3) AS count_up_to_3,
// COUNT_UP_TO(4) AS field_2
// OVER (
// ...
// );
// ```
//
// Requires:
//
// * Must be unique across all aggregation aliases.
// * Conform to [document field name][google.firestore.v1.Document.fields] limitations.
string alias = 7 [(google.api.field_behavior) = OPTIONAL];
}

// The base query to aggregate over.
oneof query_type {
// Nested structured query.
StructuredQuery structured_query = 1;
}

// Optional. Series of aggregations to apply over the results of the `structured_query`.
//
// Requires:
//
// * A minimum of one and maximum of five aggregations per query.
repeated Aggregation aggregations = 3 [(google.api.field_behavior) = OPTIONAL];
}

// A position in a query result set.
message Cursor {
// The values that represent a position, in the order they appear in
Expand Down