From 18819b1e58e1fede8ae0f69c73ed22eab5b7bb3b Mon Sep 17 00:00:00 2001 From: Fabien Renaud Date: Tue, 25 Feb 2020 19:37:39 -0800 Subject: [PATCH] Backport #603 fix, update release notes --- release-notes/CREDITS-2.x | 5 +- release-notes/VERSION-2.x | 2 + .../core/json/ReaderBasedJsonParser.java | 9 +- .../core/json/UTF8StreamJsonParser.java | 11 +- .../core/read/LocationOffsetsTest.java | 113 ++++++++++++++++++ 5 files changed, 129 insertions(+), 11 deletions(-) diff --git a/release-notes/CREDITS-2.x b/release-notes/CREDITS-2.x index a154633c8d..a89973bad1 100644 --- a/release-notes/CREDITS-2.x +++ b/release-notes/CREDITS-2.x @@ -196,9 +196,12 @@ David Nault (dnault@github) (2.10.0) Fabien Renaud (fabienrenaud@github) - * Reported, contributed fix fir #533: UTF-8 BOM not accounted for in + * Reported, contributed fix for #533: UTF-8 BOM not accounted for in `JsonLocation.getByteOffset()` (2.10.0) + * Reported, contributed fix for #603: 'JsonParser.getCurrentLocation()` + byte/char offset update incorrectly for big payloads + (2.10.3) Todd O'Bryan (toddobryan@github) * Contributed fix fox #455: Jackson reports wrong locations for JsonEOFException diff --git a/release-notes/VERSION-2.x b/release-notes/VERSION-2.x index 93b6b78440..8fa92f8eb0 100644 --- a/release-notes/VERSION-2.x +++ b/release-notes/VERSION-2.x @@ -18,6 +18,8 @@ JSON library. #592: DataFormatMatcher#getMatchedFormatName throws NPE when no match exists (reported by Scott L) +#603: 'JsonParser.getCurrentLocation()` byte/char offset update incorrectly for big payloads + (reported, fix contributed by Fabien R) 2.10.2 (05-Jan-2020) diff --git a/src/main/java/com/fasterxml/jackson/core/json/ReaderBasedJsonParser.java b/src/main/java/com/fasterxml/jackson/core/json/ReaderBasedJsonParser.java index 5a27a99280..7ee6493e0b 100644 --- a/src/main/java/com/fasterxml/jackson/core/json/ReaderBasedJsonParser.java +++ b/src/main/java/com/fasterxml/jackson/core/json/ReaderBasedJsonParser.java @@ -244,14 +244,10 @@ protected void _loadMoreGuaranteed() throws IOException { protected boolean _loadMore() throws IOException { - final int bufSize = _inputEnd; - if (_reader != null) { int count = _reader.read(_inputBuffer, 0, _inputBuffer.length); if (count > 0) { - _inputPtr = 0; - _inputEnd = count; - + final int bufSize = _inputEnd; _currInputProcessed += bufSize; _currInputRowStart -= bufSize; @@ -260,6 +256,9 @@ protected boolean _loadMore() throws IOException // in negative value, which is fine as combine value remains unchanged. _nameStartOffset -= bufSize; + _inputPtr = 0; + _inputEnd = count; + return true; } // End of input diff --git a/src/main/java/com/fasterxml/jackson/core/json/UTF8StreamJsonParser.java b/src/main/java/com/fasterxml/jackson/core/json/UTF8StreamJsonParser.java index 0f988380ea..cfd98b0b53 100644 --- a/src/main/java/com/fasterxml/jackson/core/json/UTF8StreamJsonParser.java +++ b/src/main/java/com/fasterxml/jackson/core/json/UTF8StreamJsonParser.java @@ -210,7 +210,6 @@ public Object getInputSource() { protected final boolean _loadMore() throws IOException { - final int bufSize = _inputEnd; if (_inputStream != null) { int space = _inputBuffer.length; if (space == 0) { // only occurs when we've been closed @@ -219,17 +218,19 @@ protected final boolean _loadMore() throws IOException int count = _inputStream.read(_inputBuffer, 0, space); if (count > 0) { - _inputPtr = 0; - _inputEnd = count; + final int bufSize = _inputEnd; - _currInputProcessed += _inputEnd; - _currInputRowStart -= _inputEnd; + _currInputProcessed += bufSize; + _currInputRowStart -= bufSize; // 26-Nov-2015, tatu: Since name-offset requires it too, must offset // this increase to avoid "moving" name-offset, resulting most likely // in negative value, which is fine as combine value remains unchanged. _nameStartOffset -= bufSize; + _inputPtr = 0; + _inputEnd = count; + return true; } // End of input diff --git a/src/test/java/com/fasterxml/jackson/core/read/LocationOffsetsTest.java b/src/test/java/com/fasterxml/jackson/core/read/LocationOffsetsTest.java index a33cb24b1a..07116763de 100644 --- a/src/test/java/com/fasterxml/jackson/core/read/LocationOffsetsTest.java +++ b/src/test/java/com/fasterxml/jackson/core/read/LocationOffsetsTest.java @@ -2,6 +2,9 @@ import com.fasterxml.jackson.core.*; +import java.io.IOException; +import java.util.Random; + public class LocationOffsetsTest extends com.fasterxml.jackson.core.BaseTest { final JsonFactory JSON_F = new JsonFactory(); @@ -104,6 +107,45 @@ public void testOffsetWithoutInputOffset() throws Exception p.close(); } + public void testWithLazyStringReadStreaming() throws Exception + { + _testWithLazyStringRead(MODE_READER); + _testWithLazyStringRead(MODE_INPUT_STREAM); + } + + public void testWithLazyStringReadDataInput() throws Exception + { + // DataInput-backed reader does not track column, so can not + // verify much; but force finishToken() regardless + JsonParser p = createParser(JSON_F, MODE_DATA_INPUT, "[\"text\"]"); + assertToken(JsonToken.START_ARRAY, p.nextToken()); + assertToken(JsonToken.VALUE_STRING, p.nextToken()); + assertEquals(1, p.getCurrentLocation().getLineNr()); + p.finishToken(); + assertEquals("text", p.getText()); + p.close(); + } + + private void _testWithLazyStringRead(int readMode) throws Exception + { + JsonParser p = createParser(JSON_F, readMode, "[\"text\"]"); + assertToken(JsonToken.START_ARRAY, p.nextToken()); + assertToken(JsonToken.VALUE_STRING, p.nextToken()); + // initially location pointing to first character + assertEquals(3, p.getCurrentLocation().getColumnNr()); + p.finishToken(); + // but will move once we force reading + assertEquals(8, p.getCurrentLocation().getColumnNr()); + // and no change if we call again (but is ok to call) + p.finishToken(); + assertEquals(8, p.getCurrentLocation().getColumnNr()); + + // also just for fun, verify content + assertEquals("text", p.getText()); + assertEquals(8, p.getCurrentLocation().getColumnNr()); + p.close(); + } + // for [core#533] public void testUtf8Bom() throws Exception { @@ -192,4 +234,75 @@ private byte[] withUtf8Bom(byte[] bytes) { System.arraycopy(bytes, 0, arr, 3, bytes.length); return arr; } + + // [core#603] + public void testBigPayload() throws IOException { + JsonLocation loc; + JsonParser p; + + String doc = "{\"key\":\"" + generateRandomAlpha(50000) + "\"}"; + + p = createParserUsingStream(JSON_F, doc, "UTF-8"); + + assertToken(JsonToken.START_OBJECT, p.nextToken()); + loc = p.getTokenLocation(); + assertEquals(0, loc.getByteOffset()); + assertEquals(-1L, loc.getCharOffset()); + assertEquals(1, loc.getLineNr()); + assertEquals(1, loc.getColumnNr()); + loc = p.getCurrentLocation(); + assertEquals(1, loc.getByteOffset()); + assertEquals(-1L, loc.getCharOffset()); + assertEquals(1, loc.getLineNr()); + assertEquals(2, loc.getColumnNr()); + + assertToken(JsonToken.FIELD_NAME, p.nextToken()); + loc = p.getTokenLocation(); + assertEquals(1, loc.getByteOffset()); + assertEquals(-1L, loc.getCharOffset()); + assertEquals(1, loc.getLineNr()); + assertEquals(2, loc.getColumnNr()); + loc = p.getCurrentLocation(); + assertEquals(8, loc.getByteOffset()); + assertEquals(-1L, loc.getCharOffset()); + assertEquals(1, loc.getLineNr()); + assertEquals(9, loc.getColumnNr()); + + assertToken(JsonToken.VALUE_STRING, p.nextToken()); + loc = p.getTokenLocation(); + assertEquals(7, loc.getByteOffset()); + assertEquals(-1L, loc.getCharOffset()); + assertEquals(1, loc.getLineNr()); + assertEquals(8, loc.getColumnNr()); + loc = p.getCurrentLocation(); + assertEquals(8, loc.getByteOffset()); + assertEquals(-1L, loc.getCharOffset()); + assertEquals(1, loc.getLineNr()); + assertEquals(9, loc.getColumnNr()); + + p.getTextCharacters(); + loc = p.getTokenLocation(); + assertEquals(7, loc.getByteOffset()); + assertEquals(-1L, loc.getCharOffset()); + assertEquals(1, loc.getLineNr()); + assertEquals(8, loc.getColumnNr()); + loc = p.getCurrentLocation(); + assertEquals(doc.length() - 1, loc.getByteOffset()); + assertEquals(-1L, loc.getCharOffset()); + assertEquals(1, loc.getLineNr()); + assertEquals(doc.length(), loc.getColumnNr()); + + p.close(); + } + + private String generateRandomAlpha(int length) { + StringBuilder sb = new StringBuilder(length); + Random rnd = new Random(length); + for (int i = 0; i < length; ++i) { + // let's limit it not to include surrogate pairs: + char ch = (char) ('A' + rnd.nextInt(26)); + sb.append(ch); + } + return sb.toString(); + } }