Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Resolves #160 - Split larger SQL parameters into multiple queries
- Loading branch information
1 parent
c21cfa4
commit 9623b11
Showing
7 changed files
with
299 additions
and
21 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
95 changes: 95 additions & 0 deletions
95
app/src/main/scala/org/alephium/explorer/persistence/queries/QueryUtil.scala
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,95 @@ | ||
| // Copyright 2018 The Alephium Authors | ||
| // This file is part of the alephium project. | ||
| // | ||
| // The library is free software: you can redistribute it and/or modify | ||
| // it under the terms of the GNU Lesser General Public License as published by | ||
| // the Free Software Foundation, either version 3 of the License, or | ||
| // (at your option) any later version. | ||
| // | ||
| // The library is distributed in the hope that it will be useful, | ||
| // but WITHOUT ANY WARRANTY; without even the implied warranty of | ||
| // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | ||
| // GNU Lesser General Public License for more details. | ||
| // | ||
| // You should have received a copy of the GNU Lesser General Public License | ||
| // along with the library. If not, see <http://www.gnu.org/licenses/>. | ||
|
|
||
| package org.alephium.explorer.persistence.queries | ||
|
|
||
| import scala.annotation.tailrec | ||
|
|
||
| import slick.dbio.DBIOAction | ||
|
|
||
| import org.alephium.explorer.persistence.DBActionW | ||
| import org.alephium.explorer.persistence.schema.CustomSetParameter.paramPlaceholder | ||
|
|
||
| object QueryUtil { | ||
|
|
||
| /** Maximum number of parameters allowed by Postgres per query. | ||
| * | ||
| * TODO: This should be passed as a configuration instead of hardcoding it | ||
| * and should be passed as parameter to the function using this value. | ||
| * */ | ||
| val postgresDefaultParameterLimit: Short = Short.MaxValue | ||
|
|
||
| private val emptyUpdates = DBIOAction.successful(0) | ||
|
|
||
| /** Splits update queries into batches limited by the total number parameters allowed per query */ | ||
| @SuppressWarnings(Array("org.wartremover.warts.DefaultArguments")) | ||
| def splitUpdates[R](rows: Iterable[R], | ||
| queryRowParams: Int, | ||
| queryMaxParams: Short = postgresDefaultParameterLimit)( | ||
| queryBuilder: (Iterable[R], String) => DBActionW[Int]): DBActionW[Int] = | ||
| splitFoldLeft[R, DBActionW[Int]](initialQuery = emptyUpdates, | ||
| queryRowParams = queryRowParams, | ||
| queryMaxParams = queryMaxParams, | ||
| rows = rows) { (rows, placeholder, action) => | ||
| action andThen queryBuilder(rows, placeholder) | ||
| } | ||
|
|
||
| /** | ||
| * Splits queries into batches limited by the total number parameters allowed per query. | ||
| * | ||
| * @param rows All query parameters | ||
| * @param initialQuery Returned when params are empty | ||
| * @param queryRowParams Max number of parameters in a single row of a query. Or the number of '?' in a single row. | ||
| * @param queryMaxParams Maximum number of parameters for each query. Eg: [[postgresDefaultParameterLimit]] | ||
| * @param foldLeft Given a set of following inputs (Tuple3) returns the next query. | ||
| * Similar to foldLeft in a collection type. | ||
| * - _1 = Parameter split for current query | ||
| * - _2 = Placeholder for current query | ||
| * - _3 = Previous query. Used to join with next query. | ||
| * @tparam R type of rows | ||
| * @tparam Q type of query | ||
| * @return A single query of type [[Q]] | ||
| */ | ||
| def splitFoldLeft[R, Q](rows: Iterable[R], | ||
| initialQuery: Q, | ||
| queryRowParams: Int, | ||
| queryMaxParams: Short)(foldLeft: (Iterable[R], String, Q) => Q): Q = { | ||
|
|
||
| //maximum number of rows per query | ||
| val maxRows = queryMaxParams / queryRowParams | ||
|
|
||
| @tailrec | ||
| def build(rowsRemaining: Iterable[R], previousQuery: Q): Q = | ||
| if (rowsRemaining.isEmpty) { | ||
| previousQuery | ||
| } else { | ||
| //number of rows for this query | ||
| val queryRows = rowsRemaining.size min maxRows | ||
| //generate placeholder string | ||
| val placeholder = paramPlaceholder(rows = queryRows, columns = queryRowParams) | ||
|
|
||
| //thisBatch = rows for this query | ||
| //remaining = rows not processed in this query | ||
| val (thisBatch, remaining) = rowsRemaining.splitAt(queryRows) | ||
| val nextResult = foldLeft(thisBatch, placeholder, previousQuery) | ||
|
|
||
| //process remaining | ||
| build(rowsRemaining = remaining, previousQuery = nextResult) | ||
| } | ||
|
|
||
| build(rowsRemaining = rows, previousQuery = initialQuery) | ||
| } | ||
| } |
Large diffs are not rendered by default.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.