From 6c0c5da4075960e1dffa91480fb8b3a4932f8e1b Mon Sep 17 00:00:00 2001 From: Vladimir Sitnikov Date: Tue, 31 May 2022 11:13:24 +0300 Subject: [PATCH] fix: support queries with up to 65535 (inclusive) parameters Previously the execution failed with "Tried to send an out-of-range integer as a 2-byte value" when the user attempted executing a query with more than 32767 parameters. Technically speaking, the wire protocol limit is 2-byte-unsigned-int, so we should support 65535 parameters. In practice, simple mode (preferQueryMode=simple) allows executing queries with an arbitrary number of parameters, however, that escape hatch is not recommended as it still has limits on the SQL length, and it would likely be slow. fixes #1311 --- CHANGELOG.md | 1 + .../java/org/postgresql/core/PGStream.java | 4 +- .../postgresql/jdbc/PgPreparedStatement.java | 16 ++- .../BatchedInsertReWriteEnabledTest.java | 32 ++++-- .../jdbc42/PreparedStatement64KBindsTest.java | 101 ++++++++++++++++++ 5 files changed, 142 insertions(+), 12 deletions(-) create mode 100644 pgjdbc/src/test/java/org/postgresql/test/jdbc42/PreparedStatement64KBindsTest.java diff --git a/CHANGELOG.md b/CHANGELOG.md index 250af8d380..6816045ae3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/). ### Added ### Fixed +- fix: queries with up to 65535 (inclusive) parameters are supported now (previous limit was 32767) [PR #2525](https://github.com/pgjdbc/pgjdbc/pull/2525) ## [42.3.6] (2022-05-24 08:52:27 -0400) ### Changed diff --git a/pgjdbc/src/main/java/org/postgresql/core/PGStream.java b/pgjdbc/src/main/java/org/postgresql/core/PGStream.java index ee91f6d79f..53ead54d44 100644 --- a/pgjdbc/src/main/java/org/postgresql/core/PGStream.java +++ b/pgjdbc/src/main/java/org/postgresql/core/PGStream.java @@ -355,8 +355,8 @@ public void sendInteger4(int val) throws IOException { * @throws IOException if an I/O error occurs or {@code val} cannot be encoded in 2 bytes */ public void sendInteger2(int val) throws IOException { - if (val < Short.MIN_VALUE || val > Short.MAX_VALUE) { - throw new IOException("Tried to send an out-of-range integer as a 2-byte value: " + val); + if (val < 0 || val > 65535) { + throw new IllegalArgumentException("Tried to send an out-of-range integer as a 2-byte unsigned int value: " + val); } int2Buf[0] = (byte) (val >>> 8); int2Buf[1] = (byte) val; diff --git a/pgjdbc/src/main/java/org/postgresql/jdbc/PgPreparedStatement.java b/pgjdbc/src/main/java/org/postgresql/jdbc/PgPreparedStatement.java index 8990f450ba..41bc152d61 100644 --- a/pgjdbc/src/main/java/org/postgresql/jdbc/PgPreparedStatement.java +++ b/pgjdbc/src/main/java/org/postgresql/jdbc/PgPreparedStatement.java @@ -95,11 +95,25 @@ class PgPreparedStatement extends PgStatement implements PreparedStatement { this.preparedQuery = query; this.preparedParameters = this.preparedQuery.query.createParameterList(); + int parameterCount = preparedParameters.getParameterCount(); + int maxSupportedParameters = maximumNumberOfParameters(); + if (parameterCount > maxSupportedParameters) { + throw new PSQLException( + GT.tr("PreparedStatement can have at most {0} parameters. Please consider using arrays, or splitting the query in several ones, or using COPY. Given query has {0} parameters", + maxSupportedParameters, + parameterCount), + PSQLState.INVALID_PARAMETER_VALUE); + } + // TODO: this.wantsGeneratedKeysAlways = true; setPoolable(true); // As per JDBC spec: prepared and callable statements are poolable by } + final int maximumNumberOfParameters() { + return connection.getPreferQueryMode() == PreferQueryMode.SIMPLE ? Integer.MAX_VALUE : 65535; + } + @Override public ResultSet executeQuery(String sql) throws SQLException { throw new PSQLException( @@ -1691,7 +1705,7 @@ protected void transformQueriesAndParameters() throws SQLException { final int highestBlockCount = 128; final int maxValueBlocks = bindCount == 0 ? 1024 /* if no binds, use 1024 rows */ : Integer.highestOneBit( // deriveForMultiBatch supports powers of two only - Math.min(Math.max(1, (Short.MAX_VALUE - 1) / bindCount), highestBlockCount)); + Math.min(Math.max(1, maximumNumberOfParameters() / bindCount), highestBlockCount)); int unprocessedBatchCount = batchParameters.size(); final int fullValueBlocksCount = unprocessedBatchCount / maxValueBlocks; final int partialValueBlocksCount = Integer.bitCount(unprocessedBatchCount % maxValueBlocks); diff --git a/pgjdbc/src/test/java/org/postgresql/test/jdbc2/BatchedInsertReWriteEnabledTest.java b/pgjdbc/src/test/java/org/postgresql/test/jdbc2/BatchedInsertReWriteEnabledTest.java index 19aa1dd849..486ad91522 100644 --- a/pgjdbc/src/test/java/org/postgresql/test/jdbc2/BatchedInsertReWriteEnabledTest.java +++ b/pgjdbc/src/test/java/org/postgresql/test/jdbc2/BatchedInsertReWriteEnabledTest.java @@ -386,13 +386,18 @@ public void testReWriteDisabledForPlainBatch() throws Exception { } @Test - public void test32000Binds() throws Exception { - testNBinds(32000); + public void test32767Binds() throws Exception { + testNBinds(32767); } @Test - public void test17000Binds() throws Exception { - testNBinds(17000); + public void test32768Binds() throws Exception { + testNBinds(32768); + } + + @Test + public void test65535Binds() throws Exception { + testNBinds(65535); } public void testNBinds(int nBinds) throws Exception { @@ -411,11 +416,20 @@ public void testNBinds(int nBinds) throws Exception { } pstmt.addBatch(); } - Assert.assertEquals( - "Statement with " + nBinds - + " binds should not be batched => two executions with exactly one row inserted each", - Arrays.toString(new int[] { 1, 1 }), - Arrays.toString(pstmt.executeBatch())); + if (nBinds * 2 <= 65535) { + Assert.assertEquals( + "Insert with " + nBinds + " binds should be rewritten into multi-value insert" + + ", so expecting Statement.SUCCESS_NO_INFO == -2", + Arrays.toString(new int[]{Statement.SUCCESS_NO_INFO, Statement.SUCCESS_NO_INFO}), + Arrays.toString(pstmt.executeBatch())); + } else { + Assert.assertEquals( + "Insert with " + nBinds + " binds can't be rewritten into multi-value insert" + + " since write format allows 65535 binds maximum" + + ", so expecting batch to be executed as individual statements", + Arrays.toString(new int[]{1, 1}), + Arrays.toString(pstmt.executeBatch())); + } } catch (BatchUpdateException be) { SQLException e = be; while (true) { diff --git a/pgjdbc/src/test/java/org/postgresql/test/jdbc42/PreparedStatement64KBindsTest.java b/pgjdbc/src/test/java/org/postgresql/test/jdbc42/PreparedStatement64KBindsTest.java new file mode 100644 index 0000000000..9343d8e919 --- /dev/null +++ b/pgjdbc/src/test/java/org/postgresql/test/jdbc42/PreparedStatement64KBindsTest.java @@ -0,0 +1,101 @@ +/* + * Copyright (c) 2022, PostgreSQL Global Development Group + * See the LICENSE file in the project root for more information. + */ + +package org.postgresql.test.jdbc42; + +import org.postgresql.PGProperty; +import org.postgresql.jdbc.PreferQueryMode; +import org.postgresql.test.jdbc2.BaseTest4; +import org.postgresql.util.PSQLState; + +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; + +import java.sql.Array; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.Properties; +import java.util.stream.Collectors; +import java.util.stream.IntStream; + +@RunWith(Parameterized.class) +public class PreparedStatement64KBindsTest extends BaseTest4 { + private final int numBinds; + private final PreferQueryMode preferQueryMode; + private final BinaryMode binaryMode; + + public PreparedStatement64KBindsTest(int numBinds, PreferQueryMode preferQueryMode, + BinaryMode binaryMode) { + this.numBinds = numBinds; + this.preferQueryMode = preferQueryMode; + this.binaryMode = binaryMode; + } + + @Parameterized.Parameters(name = "numBinds={0}, preferQueryMode={1}, binaryMode={2}}") + public static Iterable data() { + Collection ids = new ArrayList(); + for (PreferQueryMode preferQueryMode : PreferQueryMode.values()) { + for (BinaryMode binaryMode : BinaryMode.values()) { + for (int numBinds : new int[]{32766, 32767, 32768, 65534, 65535, 65536}) { + ids.add(new Object[]{numBinds, preferQueryMode, binaryMode}); + } + } + } + return ids; + } + + @Override + protected void updateProperties(Properties props) { + super.updateProperties(props); + PGProperty.PREFER_QUERY_MODE.set(props, preferQueryMode.value()); + setBinaryMode(binaryMode); + } + + @Test + public void executeWith65535BindsWorks() throws SQLException { + String sql = Collections.nCopies(numBinds, "?").stream() + .collect(Collectors.joining(",", "select ARRAY[", "]")); + + try (PreparedStatement ps = con.prepareStatement(sql)) { + for (int i = 1; i <= numBinds; i++) { + ps.setString(i, "v" + i); + } + String expected = Arrays.toString( + IntStream.rangeClosed(1, numBinds) + .mapToObj(i -> "v" + i).toArray() + ); + + try (ResultSet rs = ps.executeQuery()) { + rs.next(); + Array res = rs.getArray(1); + Object[] elements = (Object[]) res.getArray(); + String actual = Arrays.toString(elements); + + if (preferQueryMode == PreferQueryMode.SIMPLE || numBinds <= 65535) { + Assert.assertEquals("SELECT query with " + numBinds + " should work", actual, expected); + } else { + Assert.fail("con.prepareStatement(..." + numBinds + " binds) should fail since the wire protocol allows only 65535 parameters"); + } + } + } catch (SQLException e) { + if (preferQueryMode != PreferQueryMode.SIMPLE && numBinds > 65535) { + Assert.assertEquals( + "con.prepareStatement(..." + numBinds + " binds) should fail since the wire protocol allows only 65535 parameters. SQL State is ", + PSQLState.INVALID_PARAMETER_VALUE.getState(), + e.getSQLState() + ); + } else { + throw e; + } + } + } +}