From 10a992be45018721f6d41b57e39125942b47c44b Mon Sep 17 00:00:00 2001 From: Tyler Gregg Date: Fri, 30 Jun 2023 16:43:03 -0700 Subject: [PATCH] Plumbs the new continuable binary reader through the IonReaderBuilder and all IonReader tests. --- src/com/amazon/ion/IonCursor.java | 3 + src/com/amazon/ion/Timestamp.java | 44 ++- .../impl/IonReaderContinuableApplication.java | 3 + .../ion/impl/IonReaderContinuableCore.java | 3 + .../ion/impl/_Private_IonReaderBuilder.java | 184 ++++++++---- .../ion/impl/_Private_IonReaderFactory.java | 270 ++++++------------ .../amazon/ion/impl/lite/IonLoaderLite.java | 21 +- .../amazon/ion/impl/lite/IonSystemLite.java | 12 +- ...ByteArrayIteratorSystemProcessingTest.java | 2 +- ...IonReaderIteratorSystemProcessingTest.java | 2 +- .../ion/BinaryReaderSystemProcessingTest.java | 163 +++++++++++ ...aryStreamIteratorSystemProcessingTest.java | 2 +- ...atagramTreeReaderSystemProcessingTest.java | 6 +- test/com/amazon/ion/IonSystemTest.java | 10 - .../LoadBinaryBytesSystemProcessingTest.java | 2 +- ...adBinaryIonReaderSystemProcessingTest.java | 2 +- .../LoadBinaryStreamSystemProcessingTest.java | 2 +- .../ion/RawValueSpanReaderBasicTest.java | 8 - test/com/amazon/ion/ReaderMaker.java | 29 -- test/com/amazon/ion/TestUtils.java | 2 + test/com/amazon/ion/TimestampTest.java | 28 +- ...va => IonReaderBinaryLargeStreamTest.java} | 73 +++-- .../ion/impl/RawValueSpanReaderTest.java | 23 +- .../streaming/OffsetSpanBinaryReaderTest.java | 10 - .../ion/streaming/OffsetSpanReaderTest.java | 10 - .../ion/streaming/ReaderFacetTestCase.java | 17 +- .../ion/streaming/SeekableReaderTest.java | 106 ++++--- .../amazon/ion/streaming/SpanReaderTest.java | 60 ---- .../ion/system/IonReaderBuilderTest.java | 22 +- 29 files changed, 615 insertions(+), 504 deletions(-) rename test/com/amazon/ion/impl/{IonReaderBinaryRawLargeStreamTest.java => IonReaderBinaryLargeStreamTest.java} (88%) diff --git a/src/com/amazon/ion/IonCursor.java b/src/com/amazon/ion/IonCursor.java index ae3180ad4e..c0f0a02eae 100644 --- a/src/com/amazon/ion/IonCursor.java +++ b/src/com/amazon/ion/IonCursor.java @@ -1,3 +1,6 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + package com.amazon.ion; import java.io.Closeable; diff --git a/src/com/amazon/ion/Timestamp.java b/src/com/amazon/ion/Timestamp.java index 6c8955a8a5..3ecb91baa3 100644 --- a/src/com/amazon/ion/Timestamp.java +++ b/src/com/amazon/ion/Timestamp.java @@ -76,6 +76,8 @@ public final class Timestamp { private static final boolean APPLY_OFFSET_YES = true; private static final boolean APPLY_OFFSET_NO = false; + private static final boolean CHECK_FRACTION_YES = true; + private static final boolean CHECK_FRACTION_NO = false; private static final int NO_MONTH = 0; private static final int NO_DAY = 0; @@ -444,7 +446,7 @@ private void set_fields_from_calendar(Calendar cal, */ private Timestamp(int zyear) { - this(Precision.YEAR, zyear, NO_MONTH, NO_DAY, NO_HOURS, NO_MINUTES, NO_SECONDS, NO_FRACTIONAL_SECONDS, UNKNOWN_OFFSET, APPLY_OFFSET_NO); + this(Precision.YEAR, zyear, NO_MONTH, NO_DAY, NO_HOURS, NO_MINUTES, NO_SECONDS, NO_FRACTIONAL_SECONDS, UNKNOWN_OFFSET, APPLY_OFFSET_NO, CHECK_FRACTION_NO); } /** @@ -454,7 +456,7 @@ private Timestamp(int zyear) */ private Timestamp(int zyear, int zmonth) { - this(Precision.MONTH, zyear, zmonth, NO_DAY, NO_HOURS, NO_MINUTES, NO_SECONDS, NO_FRACTIONAL_SECONDS, UNKNOWN_OFFSET, APPLY_OFFSET_NO); + this(Precision.MONTH, zyear, zmonth, NO_DAY, NO_HOURS, NO_MINUTES, NO_SECONDS, NO_FRACTIONAL_SECONDS, UNKNOWN_OFFSET, APPLY_OFFSET_NO, CHECK_FRACTION_NO); } /** @@ -467,7 +469,7 @@ private Timestamp(int zyear, int zmonth) @Deprecated public Timestamp(int zyear, int zmonth, int zday) { - this(Precision.DAY, zyear, zmonth, zday, NO_HOURS, NO_MINUTES, NO_SECONDS, NO_FRACTIONAL_SECONDS, UNKNOWN_OFFSET, APPLY_OFFSET_NO); + this(Precision.DAY, zyear, zmonth, zday, NO_HOURS, NO_MINUTES, NO_SECONDS, NO_FRACTIONAL_SECONDS, UNKNOWN_OFFSET, APPLY_OFFSET_NO, CHECK_FRACTION_NO); } @@ -490,7 +492,7 @@ public Timestamp(int year, int month, int day, int hour, int minute, Integer offset) { - this(Precision.MINUTE, year, month, day, hour, minute, NO_SECONDS, NO_FRACTIONAL_SECONDS, offset, APPLY_OFFSET_YES); + this(Precision.MINUTE, year, month, day, hour, minute, NO_SECONDS, NO_FRACTIONAL_SECONDS, offset, APPLY_OFFSET_YES, CHECK_FRACTION_NO); } /** @@ -512,7 +514,7 @@ public Timestamp(int year, int month, int day, int hour, int minute, int second, Integer offset) { - this(Precision.SECOND, year, month, day, hour, minute, second, NO_FRACTIONAL_SECONDS, offset, APPLY_OFFSET_YES); + this(Precision.SECOND, year, month, day, hour, minute, second, NO_FRACTIONAL_SECONDS, offset, APPLY_OFFSET_YES, CHECK_FRACTION_NO); } /** @@ -541,7 +543,7 @@ public Timestamp(int year, int month, int day, int hour, int minute, int second, BigDecimal frac, Integer offset) { - this(Precision.SECOND, year, month, day, hour, minute, second, frac, offset, APPLY_OFFSET_YES); + this(Precision.SECOND, year, month, day, hour, minute, second, frac, offset, APPLY_OFFSET_YES, CHECK_FRACTION_YES); } /** @@ -564,7 +566,7 @@ public Timestamp(int year, int month, int day, */ private Timestamp(Precision p, int zyear, int zmonth, int zday, int zhour, int zminute, int zsecond, BigDecimal frac, - Integer offset, boolean shouldApplyOffset) + Integer offset, boolean shouldApplyOffset, boolean shouldCheckFraction) { boolean dayPrecision = false; @@ -573,13 +575,22 @@ private Timestamp(Precision p, int zyear, int zmonth, int zday, throw new IllegalArgumentException("invalid Precision passed to constructor"); case FRACTION: case SECOND: - if (frac == null || frac.equals(BigDecimal.ZERO)) + if (frac == null) { _fraction = null; } + else if (shouldCheckFraction) + { + if (frac.equals(BigDecimal.ZERO)) { + _fraction = null; + } else { + checkFraction(p, frac); + _fraction = frac; + } + } else { - _fraction = frac.abs(); + _fraction = frac; } _second = checkAndCastSecond(zsecond); case MINUTE: @@ -599,7 +610,7 @@ private Timestamp(Precision p, int zyear, int zmonth, int zday, _day = checkAndCastDay(zday, zyear, zmonth); } - _precision = checkFraction(p, _fraction); + _precision = p; if (shouldApplyOffset && offset != null) { apply_offset(offset); @@ -662,7 +673,7 @@ private Timestamp(Precision p, int zyear, int zmonth, int zday, { return new Timestamp(p, zyear, zmonth, zday, zhour, zminute, zsecond, frac, - offset, APPLY_OFFSET_NO); + offset, APPLY_OFFSET_NO, CHECK_FRACTION_YES); } /** @@ -1111,7 +1122,7 @@ else if (timezone_start == '+' || timezone_start == '-') Timestamp ts = new Timestamp(precision, year, month, day, - hour, minute, seconds, fraction, offset, APPLY_OFFSET_YES); + hour, minute, seconds, fraction, offset, APPLY_OFFSET_YES, CHECK_FRACTION_NO); return ts; } @@ -1198,7 +1209,8 @@ public Timestamp clone() _second, _fraction, _offset, - APPLY_OFFSET_NO); + APPLY_OFFSET_NO, + CHECK_FRACTION_NO); } /** @@ -1229,7 +1241,8 @@ private Timestamp make_localtime() _second, _fraction, _offset, - APPLY_OFFSET_NO); + APPLY_OFFSET_NO, + CHECK_FRACTION_NO); // explicitly apply the local offset to the time field values localtime.apply_offset(-offset); @@ -1344,7 +1357,8 @@ public static Timestamp forSecond(int year, int month, int day, // Storing them separately is silly. int s = second.intValue(); BigDecimal frac = second.subtract(BigDecimal.valueOf(s)); - return new Timestamp(Precision.SECOND, year, month, day, hour, minute, s, frac, offset, APPLY_OFFSET_YES); + return new Timestamp(Precision.SECOND, year, month, day, hour, minute, s, frac, offset, APPLY_OFFSET_YES, + CHECK_FRACTION_YES); } diff --git a/src/com/amazon/ion/impl/IonReaderContinuableApplication.java b/src/com/amazon/ion/impl/IonReaderContinuableApplication.java index 84b4c0d951..f087733d94 100644 --- a/src/com/amazon/ion/impl/IonReaderContinuableApplication.java +++ b/src/com/amazon/ion/impl/IonReaderContinuableApplication.java @@ -1,3 +1,6 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + package com.amazon.ion.impl; import com.amazon.ion.IonType; diff --git a/src/com/amazon/ion/impl/IonReaderContinuableCore.java b/src/com/amazon/ion/impl/IonReaderContinuableCore.java index 35a22108c4..c6866e449f 100644 --- a/src/com/amazon/ion/impl/IonReaderContinuableCore.java +++ b/src/com/amazon/ion/impl/IonReaderContinuableCore.java @@ -1,3 +1,6 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + package com.amazon.ion.impl; import com.amazon.ion.Decimal; diff --git a/src/com/amazon/ion/impl/_Private_IonReaderBuilder.java b/src/com/amazon/ion/impl/_Private_IonReaderBuilder.java index 6e4f29263c..436d26ed77 100644 --- a/src/com/amazon/ion/impl/_Private_IonReaderBuilder.java +++ b/src/com/amazon/ion/impl/_Private_IonReaderBuilder.java @@ -1,5 +1,9 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + package com.amazon.ion.impl; +import com.amazon.ion.IonCatalog; import com.amazon.ion.IonException; import com.amazon.ion.IonReader; import com.amazon.ion.IonTextReader; @@ -7,15 +11,16 @@ import com.amazon.ion.system.IonReaderBuilder; import com.amazon.ion.util.IonStreamUtils; -import java.io.BufferedInputStream; import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.InputStream; import java.io.Reader; +import java.io.SequenceInputStream; +import java.util.zip.GZIPInputStream; import static com.amazon.ion.impl.LocalSymbolTable.DEFAULT_LST_FACTORY; -import static com.amazon.ion.impl._Private_IonReaderFactory.makeIncrementalReader; import static com.amazon.ion.impl._Private_IonReaderFactory.makeReader; +import static com.amazon.ion.impl._Private_IonReaderFactory.makeReaderText; /** * {@link IonReaderBuilder} extension for internal use only. @@ -87,19 +92,53 @@ protected void mutationCheck() { } + @FunctionalInterface + interface IonReaderFromBytesFactoryText { + IonReader makeReader(IonCatalog catalog, byte[] ionData, int offset, int length, _Private_LocalSymbolTableFactory lstFactory); + } + + @FunctionalInterface + interface IonReaderFromBytesFactoryBinary { + IonReader makeReader(_Private_IonReaderBuilder builder, byte[] ionData, int offset, int length); + } + + static IonReader buildReader( + _Private_IonReaderBuilder builder, + byte[] ionData, + int offset, + int length, + IonReaderFromBytesFactoryBinary binary, + IonReaderFromBytesFactoryText text + ) { + if (IonStreamUtils.isGzip(ionData, offset, length)) { + try { + return buildReader( + builder, + new GZIPInputStream(new ByteArrayInputStream(ionData, offset, length)), + _Private_IonReaderFactory::makeReaderBinary, + _Private_IonReaderFactory::makeReaderText + ); + } catch (IOException e) { + throw new IonException(e); + } + } + if (IonStreamUtils.isIonBinary(ionData, offset, length)) { + return binary.makeReader(builder, ionData, offset, length); + } + return text.makeReader(builder.validateCatalog(), ionData, offset, length, builder.lstFactory); + } + @Override public IonReader build(byte[] ionData, int offset, int length) { - if (isIncrementalReadingEnabled()) { - if (IonStreamUtils.isGzip(ionData, offset, length)) { - throw new IllegalArgumentException("Automatic GZIP detection is not supported with incremental" + - "support enabled. Wrap the bytes with a GZIPInputStream and call build(InputStream)."); - } - if (IonStreamUtils.isIonBinary(ionData, offset, length)) { - return makeIncrementalReader(this, new ByteArrayInputStream(ionData, offset, length)); - } - } - return makeReader(validateCatalog(), ionData, offset, length, lstFactory); + return buildReader( + this, + ionData, + offset, + length, + _Private_IonReaderFactory::makeReaderBinary, + _Private_IonReaderFactory::makeReaderText + ); } /** @@ -109,54 +148,103 @@ public IonReader build(byte[] ionData, int offset, int length) * @return true if the first 'length' bytes in 'buffer' match the first 'length' bytes in the binary IVM. */ private static boolean startsWithIvm(byte[] buffer, int length) { - for (int i = 0; i < length; i++) { - if (_Private_IonConstants.BINARY_VERSION_MARKER_1_0[i] != buffer[i]) { - return false; - } + if (length >= _Private_IonConstants.BINARY_VERSION_MARKER_SIZE) { + return buffer[0] == (byte) 0xE0 + && buffer[3] == (byte) 0xEA; + } else if (length >= 1) { + return buffer[0] == (byte) 0xE0; } return true; } - @Override - public IonReader build(InputStream ionData) - { - InputStream wrapper = ionData; - if (isIncrementalReadingEnabled()) { - if (!ionData.markSupported()) { - wrapper = new BufferedInputStream(ionData); - } - wrapper.mark(_Private_IonConstants.BINARY_VERSION_MARKER_SIZE); - byte[] possibleIVM = new byte[_Private_IonConstants.BINARY_VERSION_MARKER_SIZE]; - int bytesRead; + static final byte[] GZIP_HEADER = {0x1F, (byte) 0x8B}; + + private static boolean startsWithGzipHeader(byte[] buffer, int length) { + if (length >= GZIP_HEADER.length) { + return buffer[0] == GZIP_HEADER[0] && buffer[1] == GZIP_HEADER[1]; + } + return false; + } + + @FunctionalInterface + interface IonReaderFromInputStreamFactoryText { + IonReader makeReader(IonCatalog catalog, InputStream source, _Private_LocalSymbolTableFactory lstFactory); + } + + @FunctionalInterface + interface IonReaderFromInputStreamFactoryBinary { + IonReader makeReader(_Private_IonReaderBuilder builder, InputStream source, byte[] alreadyRead, int alreadyReadOff, int alreadyReadLen); + } + + static IonReader buildReader( + _Private_IonReaderBuilder builder, + InputStream source, + IonReaderFromInputStreamFactoryBinary binary, + IonReaderFromInputStreamFactoryText text + ) { + if (source == null) { + throw new NullPointerException("Cannot build a reader from a null InputStream."); + } + // Note: this can create a lot of layers of InputStream wrappers. For example, if this method is called + // from build(byte[]) and the bytes contain GZIP, the chain will be SequenceInputStream(ByteArrayInputStream, + // GZIPInputStream -> PushbackInputStream -> ByteArrayInputStream). If this creates a drag on efficiency, + // alternatives should be evaluated. + byte[] possibleIVM = new byte[_Private_IonConstants.BINARY_VERSION_MARKER_SIZE]; + InputStream ionData = source; + int bytesRead; + try { + bytesRead = ionData.read(possibleIVM); + } catch (IOException e) { + throw new IonException(e); + } + // If the input stream is growing, it is possible that fewer than BINARY_VERSION_MARKER_SIZE bytes are + // available yet. Simply check whether the stream *could* contain binary Ion based on the available bytes. + // If it can't, fall back to text. + // NOTE: if incremental text reading is added, there will need to be logic that handles the case where + // the reader is created with 0 bytes available, as it is impossible to determine text vs. binary without + // reading at least one byte. Currently, in that case, just create a binary incremental reader. Either the + // stream will always be empty (in which case it doesn't matter whether a text or binary reader is used) + // or it's a binary stream (in which case the correct reader was created) or it's a growing text stream + // (which has always been unsupported). + if (startsWithGzipHeader(possibleIVM, bytesRead)) { try { - bytesRead = wrapper.read(possibleIVM); - wrapper.reset(); + ionData = new GZIPInputStream( + new SequenceInputStream(new ByteArrayInputStream(possibleIVM, 0, bytesRead), ionData) + ); + bytesRead = ionData.read(possibleIVM); } catch (IOException e) { throw new IonException(e); } - if (IonStreamUtils.isGzip(possibleIVM, 0, possibleIVM.length)) { - throw new IllegalArgumentException("Automatic GZIP detection is not supported with incremental" + - "support enabled. Wrap the bytes with a GZIPInputStream and call build(InputStream)."); - } - // If the input stream is growing, it is possible that fewer than BINARY_VERSION_MARKER_SIZE bytes are - // available yet. Simply check whether the stream *could* contain binary Ion based on the available bytes. - // If it can't, fall back to text. - // NOTE: if incremental text reading is added, there will need to be logic that handles the case where - // the reader is created with 0 bytes available, as it is impossible to determine text vs. binary without - // reading at least one byte. Currently, in that case, just create a binary incremental reader. Either the - // stream will always be empty (in which case it doesn't matter whether a text or binary reader is used) - // or it's a binary stream (in which case the correct reader was created) or it's a growing text stream - // (which has always been unsupported). - if (startsWithIvm(possibleIVM, bytesRead)) { - return makeIncrementalReader(this, wrapper); - } } - return makeReader(validateCatalog(), wrapper, lstFactory); + if (startsWithIvm(possibleIVM, bytesRead)) { + return binary.makeReader(builder, ionData, possibleIVM, 0, bytesRead); + } + InputStream wrapper; + if (bytesRead > 0) { + wrapper = new SequenceInputStream( + new ByteArrayInputStream(possibleIVM, 0, bytesRead), + ionData + ); + } else { + wrapper = ionData; + } + return text.makeReader(builder.validateCatalog(), wrapper, builder.lstFactory); + } + + @Override + public IonReader build(InputStream source) + { + return buildReader( + this, + source, + _Private_IonReaderFactory::makeReaderBinary, + _Private_IonReaderFactory::makeReaderText + ); } @Override public IonReader build(Reader ionText) { - return makeReader(validateCatalog(), ionText, lstFactory); + return makeReaderText(validateCatalog(), ionText, lstFactory); } @Override @@ -166,7 +254,7 @@ public IonReader build(IonValue value) { @Override public IonTextReader build(String ionText) { - return makeReader(validateCatalog(), ionText, lstFactory); + return makeReaderText(validateCatalog(), ionText, lstFactory); } } diff --git a/src/com/amazon/ion/impl/_Private_IonReaderFactory.java b/src/com/amazon/ion/impl/_Private_IonReaderFactory.java index be09960dc4..1f0e83ce57 100644 --- a/src/com/amazon/ion/impl/_Private_IonReaderFactory.java +++ b/src/com/amazon/ion/impl/_Private_IonReaderFactory.java @@ -16,10 +16,7 @@ package com.amazon.ion.impl; import static com.amazon.ion.impl.UnifiedInputStreamX.makeStream; -import static com.amazon.ion.impl._Private_IonConstants.BINARY_VERSION_MARKER_SIZE; -import static com.amazon.ion.util.IonStreamUtils.isIonBinary; -import com.amazon.ion.IonBufferConfiguration; import com.amazon.ion.IonCatalog; import com.amazon.ion.IonException; import com.amazon.ion.IonReader; @@ -41,184 +38,117 @@ public final class _Private_IonReaderFactory { - public static final IonReader makeReader(IonCatalog catalog, - byte[] bytes) - { - return makeReader(catalog, bytes, 0, bytes.length); - } - - public static final IonReader makeReader(IonCatalog catalog, - byte[] bytes, - _Private_LocalSymbolTableFactory lstFactory) - { - return makeReader(catalog, bytes, 0, bytes.length, lstFactory); - } - public static IonReader makeSystemReader(byte[] bytes) { return makeSystemReader(bytes, 0, bytes.length); } - public static final IonReader makeReader(IonCatalog catalog, - byte[] bytes, - int offset, - int length) - { - try - { - UnifiedInputStreamX uis = makeUnifiedStream(bytes, offset, length); - return makeReader(catalog, uis, offset, LocalSymbolTable.DEFAULT_LST_FACTORY); - } - catch (IOException e) - { - throw new IonException(e); - } - } - - public static final IonReader makeReader(IonCatalog catalog, - byte[] bytes, - int offset, - int length, - _Private_LocalSymbolTableFactory lstFactory) + public static final IonReader makeReaderText(IonCatalog catalog, + byte[] bytes, + int offset, + int length, + _Private_LocalSymbolTableFactory lstFactory) { + UnifiedInputStreamX uis; try { - UnifiedInputStreamX uis = makeUnifiedStream(bytes, offset, length); - return makeReader(catalog, uis, offset, lstFactory); + uis = makeUnifiedStream(bytes, offset, length); } catch (IOException e) { throw new IonException(e); } + return new IonReaderTextUserX(catalog, lstFactory, uis, offset); } public static IonReader makeSystemReader(byte[] bytes, int offset, int length) { - try - { - UnifiedInputStreamX uis = makeUnifiedStream(bytes, offset, length); - return makeSystemReader(uis); - } - catch (IOException e) - { - throw new IonException(e); - } - } - - - public static final IonTextReader makeReader(IonCatalog catalog, - char[] chars) - { - return makeReader(catalog, chars, 0, chars.length); - } - - public static final IonReader makeSystemReader(char[] chars) - { - UnifiedInputStreamX in = makeStream(chars); - return new IonReaderTextSystemX(in); - } - - - public static final IonTextReader makeReader(IonCatalog catalog, - char[] chars, - int offset, - int length) - { - UnifiedInputStreamX in = makeStream(chars, offset, length); - return new IonReaderTextUserX(catalog, LocalSymbolTable.DEFAULT_LST_FACTORY, in, offset); - } - - public static final IonReader makeSystemReader(char[] chars, - int offset, - int length) - { - UnifiedInputStreamX in = makeStream(chars, offset, length); - return new IonReaderTextSystemX(in); - } - - - public static final IonTextReader makeReader(IonCatalog catalog, - CharSequence chars) - { - return makeReader(catalog, chars, LocalSymbolTable.DEFAULT_LST_FACTORY); + return _Private_IonReaderBuilder.buildReader( + (_Private_IonReaderBuilder) _Private_IonReaderBuilder.standard(), + bytes, + offset, + length, + _Private_IonReaderFactory::makeSystemReaderBinary, + _Private_IonReaderFactory::makeSystemReaderText + ); } - public static final IonTextReader makeReader(IonCatalog catalog, - CharSequence chars, - _Private_LocalSymbolTableFactory lstFactory) + public static final IonTextReader makeReaderText(IonCatalog catalog, + CharSequence chars, + _Private_LocalSymbolTableFactory lstFactory) { UnifiedInputStreamX in = makeStream(chars); return new IonReaderTextUserX(catalog, lstFactory, in); } - public static final IonReader makeSystemReader(CharSequence chars) + public static final IonReader makeSystemReaderText(CharSequence chars) { UnifiedInputStreamX in = makeStream(chars); return new IonReaderTextSystemX(in); } - - public static final IonTextReader makeReader(IonCatalog catalog, - CharSequence chars, - int offset, - int length) + public static final IonReader makeReaderText(IonCatalog catalog, + InputStream is, + _Private_LocalSymbolTableFactory lstFactory) { - UnifiedInputStreamX in = makeStream(chars, offset, length); - return new IonReaderTextUserX(catalog, LocalSymbolTable.DEFAULT_LST_FACTORY, in, offset); + UnifiedInputStreamX uis; + try { + uis = makeUnifiedStream(is); + } catch (IOException e) { + throw new IonException(e); + } + return new IonReaderTextUserX(catalog, lstFactory, uis, 0); } - public static final IonReader makeSystemReader(CharSequence chars, - int offset, - int length) + public static IonReader makeSystemReaderText(InputStream is) { - UnifiedInputStreamX in = makeStream(chars, offset, length); - return new IonReaderTextSystemX(in); + return _Private_IonReaderBuilder.buildReader( + (_Private_IonReaderBuilder) _Private_IonReaderBuilder.standard(), + is, + _Private_IonReaderFactory::makeSystemReaderBinary, + _Private_IonReaderFactory::makeSystemReaderText + ); } - - public static final IonReader makeReader(IonCatalog catalog, - InputStream is) + private static IonReader makeSystemReaderText(IonCatalog catalog, + InputStream is, + _Private_LocalSymbolTableFactory lstFactory) { - return makeReader(catalog, is, LocalSymbolTable.DEFAULT_LST_FACTORY); - } - - public static final IonReader makeReader(IonCatalog catalog, - InputStream is, - _Private_LocalSymbolTableFactory lstFactory) - { - try { - UnifiedInputStreamX uis = makeUnifiedStream(is); - return makeReader(catalog, uis, 0, lstFactory); + UnifiedInputStreamX uis; + try + { + uis = makeUnifiedStream(is); } - catch (IOException e) { + catch (IOException e) + { throw new IonException(e); } + return new IonReaderTextSystemX(uis); } - public static IonReader makeSystemReader(InputStream is) - { - try { - UnifiedInputStreamX uis = makeUnifiedStream(is); - return makeSystemReader(uis); + private static IonReader makeSystemReaderText(IonCatalog catalog, + byte[] bytes, + int offset, + int length, + _Private_LocalSymbolTableFactory lstFactory) { + UnifiedInputStreamX uis; + try + { + uis = makeUnifiedStream(bytes, offset, length); } - catch (IOException e) { + catch (IOException e) + { throw new IonException(e); } + return new IonReaderTextSystemX(uis); } - - public static final IonTextReader makeReader(IonCatalog catalog, - Reader chars) - { - return makeReader(catalog, chars, LocalSymbolTable.DEFAULT_LST_FACTORY); - } - - public static final IonTextReader makeReader(IonCatalog catalog, - Reader chars, - _Private_LocalSymbolTableFactory lstFactory) + public static final IonTextReader makeReaderText(IonCatalog catalog, + Reader chars, + _Private_LocalSymbolTableFactory lstFactory) { try { UnifiedInputStreamX in = makeStream(chars); @@ -229,7 +159,7 @@ public static final IonTextReader makeReader(IonCatalog catalog, } } - public static final IonReader makeSystemReader(Reader chars) + public static final IonReader makeSystemReaderText(Reader chars) { try { UnifiedInputStreamX in = makeStream(chars); @@ -247,8 +177,8 @@ public static final IonReader makeReader(IonCatalog catalog, return new IonReaderTreeUserX(value, catalog, lstFactory); } - public static final IonReader makeSystemReader(IonSystem system, - IonValue value) + public static final IonReader makeSystemReaderText(IonSystem system, + IonValue value) { if (system != null && system != value.getSystem()) { throw new IonException("you can't mix values from different systems"); @@ -256,47 +186,32 @@ public static final IonReader makeSystemReader(IonSystem system, return new IonReaderTreeSystem(value); } - public static final IonReader makeIncrementalReader(IonReaderBuilder builder, InputStream is) + public static final IonReader makeReaderBinary(IonReaderBuilder builder, InputStream is, byte[] alreadyRead, int alreadyReadOff, int alreadyReadLen) { - return new IonReaderBinaryIncremental(builder, is); + return new IonReaderContinuableTopLevelBinary(builder, is, alreadyRead, alreadyReadOff, alreadyReadLen); } + public static final IonReader makeSystemReaderBinary(IonReaderBuilder builder, InputStream is, byte[] alreadyRead, int alreadyReadOff, int alreadyReadLen) + { + return new IonReaderNonContinuableSystem( + new IonReaderContinuableCoreBinary(builder.getBufferConfiguration(), is, alreadyRead, alreadyReadOff, alreadyReadLen) + ); + } - //========================================================================= - - - - private static IonReader makeReader(IonCatalog catalog, - UnifiedInputStreamX uis, - int offset, - _Private_LocalSymbolTableFactory lstFactory) - throws IOException + public static final IonReader makeReaderBinary(IonReaderBuilder builder, byte[] buffer, int off, int len) { - IonReader r; - if (has_binary_cookie(uis)) { - r = new IonReaderBinaryUserX(catalog, lstFactory, uis, offset); - } - else { - r = new IonReaderTextUserX(catalog, lstFactory, uis, offset); - } - return r; + return new IonReaderContinuableTopLevelBinary(builder, buffer, off, len); } - private static IonReader makeSystemReader(UnifiedInputStreamX uis) - throws IOException + public static final IonReader makeSystemReaderBinary(IonReaderBuilder builder, byte[] buffer, int off, int len) { - IonReader r; - if (has_binary_cookie(uis)) { - // TODO pass offset, or spans will be incorrect - r = new IonReaderBinarySystemX(uis); - } - else { - // TODO pass offset, or spans will be incorrect - r = new IonReaderTextSystemX(uis); - } - return r; + return new IonReaderNonContinuableSystem( + new IonReaderContinuableCoreBinary(builder.getBufferConfiguration(), buffer, off, len) + ); } + + //========================================================================= // // helper functions // @@ -331,27 +246,4 @@ private static UnifiedInputStreamX makeUnifiedStream(InputStream in) UnifiedInputStreamX uis = UnifiedInputStreamX.makeStream(in); return uis; } - - private static final boolean has_binary_cookie(UnifiedInputStreamX uis) - throws IOException - { - byte[] bytes = new byte[BINARY_VERSION_MARKER_SIZE]; - - // try to read the first 4 bytes and unread them (we want - // the data stream undisturbed by our peeking ahead) - int len; - for (len = 0; len < BINARY_VERSION_MARKER_SIZE; len++) { - int c = uis.read(); - if (c == UnifiedInputStreamX.EOF) { - break; - } - bytes[len] = (byte)c; - } - for (int ii=len; ii>0; ) { - ii--; - uis.unread(bytes[ii] & 0xff); - } - boolean is_cookie = isIonBinary(bytes, 0, len); - return is_cookie; - } } diff --git a/src/com/amazon/ion/impl/lite/IonLoaderLite.java b/src/com/amazon/ion/impl/lite/IonLoaderLite.java index e0a2aff94e..91f97d3393 100644 --- a/src/com/amazon/ion/impl/lite/IonLoaderLite.java +++ b/src/com/amazon/ion/impl/lite/IonLoaderLite.java @@ -16,13 +16,13 @@ package com.amazon.ion.impl.lite; import com.amazon.ion.IonCatalog; +import com.amazon.ion.IonCursor; import com.amazon.ion.IonDatagram; import com.amazon.ion.IonException; import com.amazon.ion.IonLoader; import com.amazon.ion.IonReader; import com.amazon.ion.IonSystem; import com.amazon.ion.IonWriter; -import com.amazon.ion.impl._Private_IncrementalReader; import com.amazon.ion.impl._Private_IonWriterFactory; import com.amazon.ion.system.IonReaderBuilder; @@ -85,6 +85,14 @@ private IonDatagramLite load_helper(IonReader reader) IonDatagramLite datagram = new IonDatagramLite(_system, _catalog); IonWriter writer = _Private_IonWriterFactory.makeWriter(datagram); writer.writeValues(reader); + if (_readerBuilder.isIncrementalReadingEnabled() && reader instanceof IonCursor) { + // Force incremental readers to either disambiguate an incomplete value or raise an error. + if (((IonCursor) reader).endStream() != IonCursor.Event.NEEDS_DATA) { + // Only text readers can reach this case, because it indicates that completion of an ambiguous token + // was forced. Add the resulting value to the datagram. + writer.writeValue(reader); + } + } return datagram; } @@ -148,22 +156,15 @@ public IonDatagram load(byte[] ionData) throws IonException public IonDatagram load(InputStream ionData) throws IonException, IOException { - IonReader reader = null; try { - reader = _readerBuilder.build(ionData); - return load(reader); + return load(_readerBuilder.build(ionData)); } catch (IonException e) { IOException io = e.causeOfType(IOException.class); if (io != null) throw io; throw e; - } finally { - // If a value was incomplete, incremental readers will not yet have raised an error. Force an error - // to be raised in this case. - if (_readerBuilder.isIncrementalReadingEnabled() && reader instanceof _Private_IncrementalReader) { - ((_Private_IncrementalReader) reader).requireCompleteValue(); - } } + // Note: the reader cannot be closed, as this would close the InputStream, which was provided by the user. } public IonDatagram load(IonReader reader) throws IonException diff --git a/src/com/amazon/ion/impl/lite/IonSystemLite.java b/src/com/amazon/ion/impl/lite/IonSystemLite.java index 2bc8cec3bd..671f822451 100644 --- a/src/com/amazon/ion/impl/lite/IonSystemLite.java +++ b/src/com/amazon/ion/impl/lite/IonSystemLite.java @@ -19,6 +19,7 @@ import static com.amazon.ion.SystemSymbols.ION_1_0; import static com.amazon.ion.SystemSymbols.ION_SYMBOL_TABLE; import static com.amazon.ion.impl._Private_IonReaderFactory.makeSystemReader; +import static com.amazon.ion.impl._Private_IonReaderFactory.makeSystemReaderText; import static com.amazon.ion.impl._Private_Utils.addAllNonNull; import static com.amazon.ion.impl._Private_Utils.initialSymtab; import static com.amazon.ion.impl._Private_Utils.newSymbolToken; @@ -45,7 +46,6 @@ import com.amazon.ion.impl._Private_IonReaderBuilder; import com.amazon.ion.impl._Private_IonSystem; import com.amazon.ion.impl._Private_IonWriterFactory; -import com.amazon.ion.impl._Private_ScalarConversions.CantConvertException; import com.amazon.ion.impl._Private_Utils; import com.amazon.ion.system.IonReaderBuilder; import com.amazon.ion.system.IonTextWriterBuilder; @@ -128,7 +128,7 @@ public T clone(T value) throws IonException { IonDatagram datagram = newDatagram(); IonWriter writer = _Private_IonWriterFactory.makeWriter(datagram); - IonReader reader = makeSystemReader(value.getSystem(), value); + IonReader reader = makeSystemReaderText(value.getSystem(), value); try { writer.writeValues(reader); @@ -730,7 +730,7 @@ public IonTextReader newReader(String ionText) public IonReader newSystemReader(String ionText) { - return makeSystemReader(ionText); + return makeSystemReaderText(ionText); } @@ -741,7 +741,7 @@ public IonReader newReader(InputStream ionData) public IonReader newSystemReader(InputStream ionData) { - return makeSystemReader(ionData); + return makeSystemReaderText(ionData); } @@ -756,7 +756,7 @@ public IonReader newReader(Reader ionText) public IonReader newSystemReader(Reader ionText) { - return makeSystemReader(ionText); + return makeSystemReaderText(ionText); } @@ -767,7 +767,7 @@ public IonReader newReader(IonValue value) public IonReader newSystemReader(IonValue value) { - return makeSystemReader(this, value); + return makeSystemReaderText(this, value); } diff --git a/test/com/amazon/ion/BinaryByteArrayIteratorSystemProcessingTest.java b/test/com/amazon/ion/BinaryByteArrayIteratorSystemProcessingTest.java index cfa28c8f3c..8ce34efe35 100644 --- a/test/com/amazon/ion/BinaryByteArrayIteratorSystemProcessingTest.java +++ b/test/com/amazon/ion/BinaryByteArrayIteratorSystemProcessingTest.java @@ -26,7 +26,7 @@ public class BinaryByteArrayIteratorSystemProcessingTest @Override protected int expectedLocalNullSlotSymbolId() { - return getStreamingMode() == StreamingMode.NEW_STREAMING_INCREMENTAL ? 0 : 10; + return 0; } @Override diff --git a/test/com/amazon/ion/BinaryIonReaderIteratorSystemProcessingTest.java b/test/com/amazon/ion/BinaryIonReaderIteratorSystemProcessingTest.java index f363b7c6b0..0e529939da 100644 --- a/test/com/amazon/ion/BinaryIonReaderIteratorSystemProcessingTest.java +++ b/test/com/amazon/ion/BinaryIonReaderIteratorSystemProcessingTest.java @@ -30,6 +30,6 @@ protected Iterator systemIterate() protected int expectedLocalNullSlotSymbolId() { // The spec allows for implementations to treat these malformed symbols as "null slots", and all "null slots" // in local symbol tables as equivalent to symbol zero. - return getStreamingMode() == StreamingMode.NEW_STREAMING_INCREMENTAL ? 0 : 10; + return 0; } } diff --git a/test/com/amazon/ion/BinaryReaderSystemProcessingTest.java b/test/com/amazon/ion/BinaryReaderSystemProcessingTest.java index 3373712a93..a332d0462b 100644 --- a/test/com/amazon/ion/BinaryReaderSystemProcessingTest.java +++ b/test/com/amazon/ion/BinaryReaderSystemProcessingTest.java @@ -16,7 +16,14 @@ package com.amazon.ion; +import org.junit.Test; +import java.math.BigDecimal; +import java.math.BigInteger; +import java.util.Date; + +import static com.amazon.ion.SystemSymbols.ION_1_0; +import static com.amazon.ion.SystemSymbols.ION_1_0_SID; public class BinaryReaderSystemProcessingTest extends ReaderSystemProcessingTestCase @@ -46,4 +53,160 @@ public IonReader systemRead() throws Exception { return system().newSystemReader(myBytes); } + + private void prepare(int... binary) { + myMissingSymbolTokensHaveText = false; + myBytes = BitUtils.bytes(binary); + } + + @Test + public void testMultipleIvmsBetweenValues() + throws Exception + { + prepare( + 0xE0, 0x01, 0x00, 0xEA, // IVM + 0xE0, 0x01, 0x00, 0xEA, // IVM + 0x21, 0x01, // 1 + 0xE0, 0x01, 0x00, 0xEA, // IVM + 0x21, 0x02, // 2 + 0xE0, 0x01, 0x00, 0xEA, // IVM + 0xE0, 0x01, 0x00, 0xEA, // IVM + 0xE0, 0x01, 0x00, 0xEA, // IVM + 0x21, 0x03, // 3 + 0xE0, 0x01, 0x00, 0xEA, // IVM + 0xE0, 0x01, 0x00, 0xEA // IVM + ); + startSystemIteration(); + + nextValue(); + checkSymbol(ION_1_0, ION_1_0_SID); + nextValue(); + checkSymbol(ION_1_0, ION_1_0_SID); + nextValue(); + checkInt(1); + + nextValue(); + checkSymbol(ION_1_0, ION_1_0_SID); + nextValue(); + checkInt(2); + + nextValue(); + checkSymbol(ION_1_0, ION_1_0_SID); + nextValue(); + checkSymbol(ION_1_0, ION_1_0_SID); + nextValue(); + checkSymbol(ION_1_0, ION_1_0_SID); + nextValue(); + checkInt(3); + + nextValue(); + checkSymbol(ION_1_0, ION_1_0_SID); + nextValue(); + checkSymbol(ION_1_0, ION_1_0_SID); + } + + @Test + public void testSymbolTokensInSystemSymbolTable() + throws Exception + { + prepare("{name: imports::max_id}"); + startSystemIteration(); + + nextValue(); + checkSymbol(ION_1_0, ION_1_0_SID); + nextValue(); + checkType(IonType.STRUCT); + myReader.stepIn(); + nextValue(); + checkFieldName("name", SystemSymbols.NAME_SID); + checkAnnotation("imports", SystemSymbols.IMPORTS_SID); + checkSymbol("max_id", SystemSymbols.MAX_ID_SID); + myReader.stepOut(); + checkTopEof(); + } + + @Test + public void testSymbolTokensInLocalSymbolTable() + throws Exception + { + prepare("{foo: bar::baz}"); + startSystemIteration(); + + nextValue(); + checkSymbol(ION_1_0, ION_1_0_SID); + + nextValue(); + checkType(IonType.STRUCT); + checkAnnotation(SystemSymbols.ION_SYMBOL_TABLE, SystemSymbols.ION_SYMBOL_TABLE_SID); + myReader.stepIn(); + nextValue(); + checkType(IonType.LIST); + checkFieldName(SystemSymbols.SYMBOLS, SystemSymbols.SYMBOLS_SID); + myReader.stepIn(); + nextValue(); + checkString("baz"); + nextValue(); + checkString("bar"); + nextValue(); + checkString("foo"); + myReader.stepOut(); + myReader.stepOut(); + + nextValue(); + checkType(IonType.STRUCT); + myReader.stepIn(); + nextValue(); + checkFieldName(null, SystemSymbols.ION_1_0_MAX_ID + 3); + checkAnnotation(null, SystemSymbols.ION_1_0_MAX_ID + 2); + checkSymbol(null, SystemSymbols.ION_1_0_MAX_ID + 1); + myReader.stepOut(); + checkTopEof(); + } + + @Test + public void testSystemReaderReadsUserValues() + throws Exception + { + prepare("true 123 1.23e0 1.23d0 2023T {{ YWJj }}"); + + startSystemIteration(); + + nextValue(); + checkSymbol(ION_1_0, ION_1_0_SID); + + nextValue(); + checkType(IonType.BOOL); + assertTrue(myReader.booleanValue()); + + nextValue(); + checkType(IonType.INT); + assertEquals(IntegerSize.INT, myReader.getIntegerSize()); + assertEquals(123, myReader.intValue()); + assertEquals(123, myReader.longValue()); + assertEquals(BigInteger.valueOf(123), myReader.bigIntegerValue()); + + nextValue(); + checkFloat(1.23e0); + + nextValue(); + checkType(IonType.DECIMAL); + assertEquals(BigDecimal.valueOf(123, 2), myReader.bigDecimalValue()); + assertEquals(Decimal.valueOf(123, 2), myReader.decimalValue()); + + nextValue(); + checkType(IonType.TIMESTAMP); + assertEquals(Timestamp.valueOf("2023T"), myReader.timestampValue()); + assertEquals(Timestamp.valueOf("2023T").dateValue(), myReader.dateValue()); + + nextValue(); + checkType(IonType.BLOB); + assertEquals(3, myReader.byteSize()); + byte[] expected = new byte[]{'a', 'b', 'c'}; + assertArrayEquals(expected, myReader.newBytes()); + byte[] result = new byte[3]; + assertEquals(3, myReader.getBytes(result, 0, 3)); + assertArrayEquals(expected, result); + + checkTopEof(); + } } diff --git a/test/com/amazon/ion/BinaryStreamIteratorSystemProcessingTest.java b/test/com/amazon/ion/BinaryStreamIteratorSystemProcessingTest.java index e814e2982b..02e3be87fd 100644 --- a/test/com/amazon/ion/BinaryStreamIteratorSystemProcessingTest.java +++ b/test/com/amazon/ion/BinaryStreamIteratorSystemProcessingTest.java @@ -29,7 +29,7 @@ public class BinaryStreamIteratorSystemProcessingTest @Override protected int expectedLocalNullSlotSymbolId() { - return getStreamingMode() == StreamingMode.NEW_STREAMING_INCREMENTAL ? 0 : 10; + return 0; } @Override diff --git a/test/com/amazon/ion/DatagramTreeReaderSystemProcessingTest.java b/test/com/amazon/ion/DatagramTreeReaderSystemProcessingTest.java index 9c328e8043..d689528a98 100644 --- a/test/com/amazon/ion/DatagramTreeReaderSystemProcessingTest.java +++ b/test/com/amazon/ion/DatagramTreeReaderSystemProcessingTest.java @@ -43,9 +43,9 @@ public void setDatagramMaker(DatagramMaker maker) @Override protected int expectedLocalNullSlotSymbolId() { - return getStreamingMode() == StreamingMode.NEW_STREAMING_INCREMENTAL && myDatagramMaker.sourceIsBinary() - ? 0 - : 10; + // The spec allows for implementations to treat these malformed symbols as "null slots", and all "null slots" + // in local symbol tables as equivalent to symbol zero. + return myDatagramMaker.sourceIsBinary() ? 0 : 10; } /** diff --git a/test/com/amazon/ion/IonSystemTest.java b/test/com/amazon/ion/IonSystemTest.java index ff1f20d486..fdd0578100 100644 --- a/test/com/amazon/ion/IonSystemTest.java +++ b/test/com/amazon/ion/IonSystemTest.java @@ -319,12 +319,6 @@ private void checkGzipDetection(byte[] bytes) checkInt(1234, dg.get(0)); } - private void expectIncrementalReaderToFail() { - if (getStreamingMode() == StreamingMode.NEW_STREAMING_INCREMENTAL) { - thrown.expect(IllegalArgumentException.class); - } - } - @Test public void testGzipDetection() throws Exception @@ -334,14 +328,12 @@ public void testGzipDetection() byte[] gzipTextBytes = gzip(textBytes); checkGzipDetection(textBytes); - expectIncrementalReaderToFail(); checkGzipDetection(gzipTextBytes); byte[] binaryBytes = loader().load(ionText).getBytes(); byte[] gzipBinaryBytes = gzip(binaryBytes); checkGzipDetection(binaryBytes); - expectIncrementalReaderToFail(); checkGzipDetection(gzipBinaryBytes); } @@ -362,7 +354,6 @@ public void singleValueTextGzip() throws IOException String ionText = "1234"; byte[] textBytes = _Private_Utils.utf8(ionText); byte[] gzipTextBytes = gzip(textBytes); - expectIncrementalReaderToFail(); IonInt ionValue = (IonInt) system().singleValue(gzipTextBytes); assertEquals(1234, ionValue.intValue()); @@ -374,7 +365,6 @@ public void singleValueBinaryGzip() throws IOException String ionText = "1234"; byte[] binaryIon = toBinaryIon(ionText); byte[] gzipBytes = gzip(binaryIon); - expectIncrementalReaderToFail(); IonInt ionValue = (IonInt) system().singleValue(gzipBytes); assertEquals(1234, ionValue.intValue()); diff --git a/test/com/amazon/ion/LoadBinaryBytesSystemProcessingTest.java b/test/com/amazon/ion/LoadBinaryBytesSystemProcessingTest.java index 5e46a99545..848da71f56 100644 --- a/test/com/amazon/ion/LoadBinaryBytesSystemProcessingTest.java +++ b/test/com/amazon/ion/LoadBinaryBytesSystemProcessingTest.java @@ -26,7 +26,7 @@ public class LoadBinaryBytesSystemProcessingTest @Override protected int expectedLocalNullSlotSymbolId() { - return getStreamingMode() == StreamingMode.NEW_STREAMING_INCREMENTAL ? 0 : 10; + return 0; } @Override diff --git a/test/com/amazon/ion/LoadBinaryIonReaderSystemProcessingTest.java b/test/com/amazon/ion/LoadBinaryIonReaderSystemProcessingTest.java index 90ec7b306f..c46d5f1227 100644 --- a/test/com/amazon/ion/LoadBinaryIonReaderSystemProcessingTest.java +++ b/test/com/amazon/ion/LoadBinaryIonReaderSystemProcessingTest.java @@ -26,6 +26,6 @@ protected IonDatagram load() throws Exception protected int expectedLocalNullSlotSymbolId() { // The spec allows for implementations to treat these malformed symbols as "null slots", and all "null slots" // in local symbol tables as equivalent to symbol zero. - return getStreamingMode() == StreamingMode.NEW_STREAMING_INCREMENTAL ? 0 : 10; + return 0; } } diff --git a/test/com/amazon/ion/LoadBinaryStreamSystemProcessingTest.java b/test/com/amazon/ion/LoadBinaryStreamSystemProcessingTest.java index 487c0203bd..08904d3212 100644 --- a/test/com/amazon/ion/LoadBinaryStreamSystemProcessingTest.java +++ b/test/com/amazon/ion/LoadBinaryStreamSystemProcessingTest.java @@ -27,7 +27,7 @@ public class LoadBinaryStreamSystemProcessingTest @Override protected int expectedLocalNullSlotSymbolId() { - return getStreamingMode() == StreamingMode.NEW_STREAMING_INCREMENTAL ? 0 : 10; + return 0; } @Override diff --git a/test/com/amazon/ion/RawValueSpanReaderBasicTest.java b/test/com/amazon/ion/RawValueSpanReaderBasicTest.java index 4da4cafe95..a6d30b6114 100644 --- a/test/com/amazon/ion/RawValueSpanReaderBasicTest.java +++ b/test/com/amazon/ion/RawValueSpanReaderBasicTest.java @@ -52,14 +52,6 @@ public void testTextReaderReturnsNullFacet() assertNull(reader.asFacet(RawValueSpanProvider.class)); } - @Test - public void incrementalBinaryReaderReturnsNullFacet() - { - IonReader reader = IonReaderBuilder.standard().withIncrementalReadingEnabled(true).build(dummyData); - assertNull(reader.asFacet(RawValueSpanProvider.class)); - assertNull(reader.asFacet(SeekableReader.class)); - } - @Test public void testNonByteBackedReaderNotSupported() { diff --git a/test/com/amazon/ion/ReaderMaker.java b/test/com/amazon/ion/ReaderMaker.java index b4a3e9a0ca..6e182da06f 100644 --- a/test/com/amazon/ion/ReaderMaker.java +++ b/test/com/amazon/ion/ReaderMaker.java @@ -19,7 +19,6 @@ import static com.amazon.ion.TestUtils.ensureText; import com.amazon.ion.impl._Private_Utils; -import com.amazon.ion.system.IonReaderBuilder; import java.io.ByteArrayInputStream; import java.io.IOException; @@ -140,34 +139,6 @@ public IonReader newReader(IonSystem system, byte[] ionData) } }, - FROM_INPUT_STREAM_BINARY_INCREMENTAL(Feature.BINARY, Feature.STREAM) - { - @Override - public IonReader newReader(IonSystem system, byte[] ionData, - InputStreamWrapper wrapper) - throws IOException - { - ionData = ensureBinary(system, ionData); - InputStream in = new ByteArrayInputStream(ionData); - InputStream wrapped = wrapper.wrap(in); - return newReader(system, wrapped); - } - - @Override - public IonReader newReader(IonSystem system, byte[] ionData) - { - ionData = ensureBinary(system, ionData); - InputStream in = new ByteArrayInputStream(ionData); - return newReader(system, in); - } - - @Override - public IonReader newReader(IonSystem system, InputStream inputStream) { - IonCatalog catalog = system.getCatalog(); - return IonReaderBuilder.standard().withIncrementalReadingEnabled(true).withCatalog(catalog).build(inputStream); - } - }, - /** * Invokes {@link IonSystem#newReader(InputStream)} with Ion text. */ diff --git a/test/com/amazon/ion/TestUtils.java b/test/com/amazon/ion/TestUtils.java index c8177b0e42..a1fb71c414 100644 --- a/test/com/amazon/ion/TestUtils.java +++ b/test/com/amazon/ion/TestUtils.java @@ -150,6 +150,8 @@ public boolean accept(File dir, String name) ,"bad/typecodes/type_6_length_0.10n" // TODO amazon-ion/ion-java#272 ,"good/typecodes/T7-large.10n" // TODO amazon-ion/ion-java#273 ,"good/equivs/clobNewlines.ion" // TODO amazon-ion/ion-java#274 + ,"bad/minLongWithLenTooSmall.10n" // Note: The long itself is fine. The data ends with 0x01, a 2-byte NOP pad header. It is not worth adding the logic to detect this as unexpected EOF. + ,"bad/nopPadTooShort.10n" // Note: There are fewer bytes than the NOP pad header declares. It is not worth adding the logic to detect this as unexpected EOF. ) ); diff --git a/test/com/amazon/ion/TimestampTest.java b/test/com/amazon/ion/TimestampTest.java index ede7f1d847..1709fe8da8 100644 --- a/test/com/amazon/ion/TimestampTest.java +++ b/test/com/amazon/ion/TimestampTest.java @@ -1288,7 +1288,7 @@ public void testNewTimestampFromClearCalendar() @Test - public void testTimestampWithNegativeFraction() + public void testTimestampWithNegativeZeroFraction() { BigDecimal frac = Decimal.negativeZero(3); @@ -1296,10 +1296,27 @@ public void testTimestampWithNegativeFraction() Timestamp expected = Timestamp.valueOf("2000-11-14T17:30:12.000Z"); assertEquals(expected, ts); assertEquals(expected.hashCode(), ts.hashCode()); + } + + @Test + public void testTimestampWithNegativeFractionDecimalFromConstructorFails() { + thrown.expect(IllegalArgumentException.class); + new Timestamp(2000, 11, 14, 17, 30, 12, new BigDecimal("-0.123"), 0); + } + + @Test + public void testTimestampWithNegativeFractionDecimalFromUtcFieldsFails() { + thrown.expect(IllegalArgumentException.class); + createFromUtcFields(FRACTION, 2000, 11, 14, 17, 30, 12, Decimal.valueOf("-0.123"), + UTC_OFFSET); + } - frac = new BigDecimal("-0.123"); - ts = new Timestamp(2000, 11, 14, 17, 30, 12, frac, 0); - assertEquals("2000-11-14T17:30:12.123Z", ts.toString()); + @Test + public void testTimestampWithNegativeFractionBigDecimalFromUtcFieldsFails() { + thrown.expect(IllegalArgumentException.class); + checkFraction(".123", new BigDecimal ("-0.123")); + createFromUtcFields(FRACTION, 2000, 11, 14, 17, 30, 12, new BigDecimal("-0.123"), + UTC_OFFSET); } @@ -1326,9 +1343,6 @@ public void testTimestampWithDecimalFraction() checkFraction(".345", Decimal.valueOf(".345")); checkFraction(".345", new BigDecimal (".345")); - - checkFraction(".123", Decimal.valueOf("-0.123")); - checkFraction(".123", new BigDecimal ("-0.123")); } @Test diff --git a/test/com/amazon/ion/impl/IonReaderBinaryRawLargeStreamTest.java b/test/com/amazon/ion/impl/IonReaderBinaryLargeStreamTest.java similarity index 88% rename from test/com/amazon/ion/impl/IonReaderBinaryRawLargeStreamTest.java rename to test/com/amazon/ion/impl/IonReaderBinaryLargeStreamTest.java index cd94276dc9..50123ac277 100644 --- a/test/com/amazon/ion/impl/IonReaderBinaryRawLargeStreamTest.java +++ b/test/com/amazon/ion/impl/IonReaderBinaryLargeStreamTest.java @@ -1,5 +1,10 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + package com.amazon.ion.impl; +import com.amazon.ion.BufferConfiguration; +import com.amazon.ion.IonBufferConfiguration; import com.amazon.ion.IonException; import com.amazon.ion.IonReader; import com.amazon.ion.IonType; @@ -24,7 +29,7 @@ import static org.junit.Assert.assertEquals; // NOTE: these tests each take several seconds to complete. -public class IonReaderBinaryRawLargeStreamTest { +public class IonReaderBinaryLargeStreamTest { private byte[] testData(Timestamp timestamp) throws IOException { ByteArrayOutputStream out = new ByteArrayOutputStream(); @@ -40,22 +45,24 @@ private byte[] testData(Timestamp timestamp) throws IOException { return data; } + private static IonReaderBuilder newReaderBuilderThatThrowsOnOversizedValues(boolean enableIncremental) { + return IonReaderBuilder.standard() + .withIncrementalReadingEnabled(enableIncremental) + .withBufferConfiguration( + IonBufferConfiguration.Builder.standard() + .onOversizedValue(new BufferConfiguration.OversizedValueHandler() { + @Override + public void onOversizedValue() { + throw new IonException("oversized"); + } + }) + .build() + ); + } + public void readLargeScalarStream(IonReaderBuilder readerBuilder) throws Exception { final Timestamp timestamp = Timestamp.forDay(2000, 1, 1); byte[] data = testData(timestamp); - // The binary reader uses Integer.MIN_VALUE to mean NO_LIMIT for its _local_remaining value, which keeps track - // of the remaining number of bytes in the current value. Between values at the top level, this should always be - // NO_LIMIT. No arithmetic should ever be performed on the value when it is set to NO_LIMIT. If bugs exist that - // violate this, then between top level values _local_remaining will never again be NO_LIMIT, meaning that - // arithmetic will continue to be performed on it. Eventually, due to integer overflow, the value will roll over - // into a small enough positive value that the reader will erroneously determine that there are fewer bytes - // remaining than are needed to complete the current value. The reader will then finish early before reading the - // entire stream. The bug that prompted this test to be written involved an unconditional subtraction of the - // current value's length as declared in its header from the current value of _local_remaining within - // stringValue(), decimalValue(), and timestampValue(). This caused _local_remaining to overflow to a very - // large value immediately. For every top level value subsequently read, the length of that value would be - // subtracted from _local_remaining until eventually _local_remaining prematurely reached 0 around the time - // the stream reached Integer.MAX_VALUE in length. // Repeat the batch a sufficient number of times to exceed a total stream length of Integer.MAX_VALUE, plus // a few more to make sure batches continue to be read correctly. final int totalNumberOfBatches = (Integer.MAX_VALUE / data.length) + 7; // 7 makes the value exceed Integer.MAX_VALUE by an arbitrary amount. @@ -82,12 +89,12 @@ public void readLargeScalarStream(IonReaderBuilder readerBuilder) throws Excepti @Test public void readLargeScalarStreamNonIncremental() throws Exception { - readLargeScalarStream(IonReaderBuilder.standard()); + readLargeScalarStream(newReaderBuilderThatThrowsOnOversizedValues(false)); } @Test public void readLargeScalarStreamIncremental() throws Exception { - readLargeScalarStream(IonReaderBuilder.standard().withIncrementalReadingEnabled(true)); + readLargeScalarStream(newReaderBuilderThatThrowsOnOversizedValues(true)); } @Test @@ -303,20 +310,26 @@ private void cleanlyFailsOnLargeScalar(IonReaderBuilder readerBuilder) throws Ex new RepeatInputStream(data, totalNumberOfBatches - 1) // This will provide the data 'totalNumberOfBatches' times ); IonReader reader = readerBuilder.build(inputStream); - // If support for large scalars is added, the following line will be deleted and the rest of the test + // If support for large scalars is added, the following will be deleted and the rest of the test // completed to assert the correctness of the value. - thrown.expect(IonException.class); - reader.next(); + if (readerBuilder.isIncrementalReadingEnabled()) { + thrown.expect(IonException.class); + reader.next(); + } else { + assertEquals(IonType.STRING, reader.next()); + thrown.expect(IonException.class); + reader.stringValue(); + } } @Test public void cleanlyFailsOnLargeScalarNonIncremental() throws Exception { - cleanlyFailsOnLargeScalar(IonReaderBuilder.standard()); + cleanlyFailsOnLargeScalar(newReaderBuilderThatThrowsOnOversizedValues(false)); } @Test public void cleanlyFailsOnLargeScalarIncremental() throws Exception { - cleanlyFailsOnLargeScalar(IonReaderBuilder.standard().withIncrementalReadingEnabled(true)); + cleanlyFailsOnLargeScalar(newReaderBuilderThatThrowsOnOversizedValues(true)); } private void cleanlyFailsOnLargeAnnotatedScalar(IonReaderBuilder readerBuilder) throws Exception { @@ -336,20 +349,26 @@ private void cleanlyFailsOnLargeAnnotatedScalar(IonReaderBuilder readerBuilder) new RepeatInputStream(data, totalNumberOfBatches - 1) // This will provide the data 'totalNumberOfBatches' times ); IonReader reader = readerBuilder.build(inputStream); - // If support for large scalars is added, the following line will be deleted and the rest of the test + // If support for large scalars is added, the following will be deleted and the rest of the test // completed to assert the correctness of the value. - thrown.expect(IonException.class); - reader.next(); + if (readerBuilder.isIncrementalReadingEnabled()) { + thrown.expect(IonException.class); + reader.next(); + } else { + assertEquals(IonType.STRING, reader.next()); + thrown.expect(IonException.class); + reader.stringValue(); + } } @Test public void cleanlyFailsOnLargeAnnotatedScalarNonIncremental() throws Exception { - cleanlyFailsOnLargeAnnotatedScalar(IonReaderBuilder.standard()); + cleanlyFailsOnLargeAnnotatedScalar(newReaderBuilderThatThrowsOnOversizedValues(false)); } @Test public void cleanlyFailsOnLargeAnnotatedScalarIncremental() throws Exception { - cleanlyFailsOnLargeAnnotatedScalar(IonReaderBuilder.standard().withIncrementalReadingEnabled(true)); + cleanlyFailsOnLargeAnnotatedScalar(newReaderBuilderThatThrowsOnOversizedValues(true)); } @Test @@ -368,7 +387,7 @@ public void cleanlyFailsOnLargeContainerIncremental() throws Exception { new RepeatInputStream(data, totalNumberOfBatches - 1) // This will provide the data 'totalNumberOfBatches' times ); - IonReader reader = IonReaderBuilder.standard().withIncrementalReadingEnabled(true).build(inputStream); + IonReader reader = newReaderBuilderThatThrowsOnOversizedValues(true).build(inputStream); thrown.expect(IonException.class); reader.next(); } diff --git a/test/com/amazon/ion/impl/RawValueSpanReaderTest.java b/test/com/amazon/ion/impl/RawValueSpanReaderTest.java index e99ee79569..8e06606ad2 100644 --- a/test/com/amazon/ion/impl/RawValueSpanReaderTest.java +++ b/test/com/amazon/ion/impl/RawValueSpanReaderTest.java @@ -40,6 +40,7 @@ import java.util.ArrayList; import java.util.Collections; import java.util.List; + import org.junit.Test; import org.junit.runner.RunWith; @@ -76,7 +77,8 @@ public void setTestFile(File file) myTestFile = file; } - private IonReaderBinaryUserX reader; + private IonReader reader; + private byte[] inputBytes; private RawValueSpanProvider spanProvider; private SeekableReader seekableReader; @@ -262,7 +264,17 @@ private void assertSpan(SpanTester tester) throws Exception } } - private int countNopPad(int start) throws IOException + private int readVarUInt(int start) { + int currentByte = 0; + int result = 0; + while ((currentByte & 0x80) == 0) { + currentByte = inputBytes[start++]; + result = (result << 7) | (currentByte & 0x7F); + } + return result; + } + + private int countNopPad(int start) { int index = start; int len = 0; @@ -274,13 +286,13 @@ private int countNopPad(int start) throws IOException len += index - start; } - int td = reader.getCurrentBuffer()[index] & 0xFF; + int td = inputBytes[index++] & 0xFF; int tid = _Private_IonConstants.getTypeCode(td); int typeLen = _Private_IonConstants.getLowNibble(td); if(tid == _Private_IonConstants.tidNull && typeLen != _Private_IonConstants.lnIsNull){ if(typeLen == _Private_IonConstants.lnIsVarLen) { - len += reader.readVarUInt(); + len += readVarUInt(index); } else { len += 1; @@ -310,7 +322,8 @@ private static byte[] readFileAsBytes(File file) throws IOException public void testSpans() throws Exception { // Seeking currently only works over byte-backed IonReaders -- not InputStreams - reader = (IonReaderBinaryUserX) IonReaderBuilder.standard().build(readFileAsBytes(myTestFile)); + inputBytes = readFileAsBytes(myTestFile); + reader = IonReaderBuilder.standard().build(inputBytes); spanProvider = reader.asFacet(RawValueSpanProvider.class); seekableReader = reader.asFacet(SeekableReader.class); diff --git a/test/com/amazon/ion/streaming/OffsetSpanBinaryReaderTest.java b/test/com/amazon/ion/streaming/OffsetSpanBinaryReaderTest.java index 5720b15701..665eb853d2 100644 --- a/test/com/amazon/ion/streaming/OffsetSpanBinaryReaderTest.java +++ b/test/com/amazon/ion/streaming/OffsetSpanBinaryReaderTest.java @@ -65,11 +65,6 @@ private void readBeyondMaxInt(byte[] data, IonType valueType) @Test public void testCurrentSpanBeyondMaxInt() { - if (getStreamingMode() == StreamingMode.NEW_STREAMING_INCREMENTAL) { - // TODO the incremental reader does not currently support the SpanProvider or SeekableReader facets. - // See ion-java/issues/382 and ion-java/issues/383. - return; - } IonDatagram dg = system().newDatagram(); dg.add().newBlob(new byte[2000]); byte[] binary = dg.getBytes(); @@ -85,11 +80,6 @@ public void testCurrentSpanBeyondMaxInt() @Test public void testCurrentSpanBeyondMaxIntForOrderedStruct() { - if (getStreamingMode() == StreamingMode.NEW_STREAMING_INCREMENTAL) { - // TODO the incremental reader does not currently support the SpanProvider or SeekableReader facets. - // See ion-java/issues/382 and ion-java/issues/383. - return; - } // Value is ordered-struct { name:{{ /* 1024 bytes */}} } byte[] data = hexToBytes("E0 01 00 EA " diff --git a/test/com/amazon/ion/streaming/OffsetSpanReaderTest.java b/test/com/amazon/ion/streaming/OffsetSpanReaderTest.java index 5c6fa61776..9c62e0312f 100644 --- a/test/com/amazon/ion/streaming/OffsetSpanReaderTest.java +++ b/test/com/amazon/ion/streaming/OffsetSpanReaderTest.java @@ -55,11 +55,6 @@ public void checkCurrentSpan(int binaryStart, int binaryFinish, int textStart) @Test public void testCurrentSpan() { - if (getStreamingMode() == StreamingMode.NEW_STREAMING_INCREMENTAL) { - // TODO the incremental reader does not currently support the SpanProvider or SeekableReader facets. - // See ion-java/issues/382 and ion-java/issues/383. - return; - } read("'''hello''' 1 2 3 4 5 6 7 8 9 10 '''Kumo the fluffy dog! He is so fluffy and yet so happy!'''"); assertSame(IonType.STRING, in.next()); checkCurrentSpan(4, 10, 0); @@ -77,11 +72,6 @@ public void testCurrentSpan() @Test public void testCurrentSpanFromStreamMed() throws IOException { - if (getStreamingMode() == StreamingMode.NEW_STREAMING_INCREMENTAL) { - // TODO the incremental reader does not currently support the SpanProvider or SeekableReader facets. - // See ion-java/issues/382 and ion-java/issues/383. - return; - } final ByteArrayOutputStream buf = new ByteArrayOutputStream(); final int count = 8000; for (int i = 0; i < count; i++) { diff --git a/test/com/amazon/ion/streaming/ReaderFacetTestCase.java b/test/com/amazon/ion/streaming/ReaderFacetTestCase.java index 83ba276449..4c1baf6417 100644 --- a/test/com/amazon/ion/streaming/ReaderFacetTestCase.java +++ b/test/com/amazon/ion/streaming/ReaderFacetTestCase.java @@ -36,21 +36,13 @@ public abstract class ReaderFacetTestCase * * @see NonSpanReaderTest */ - public static final ReaderMaker[] NON_SPAN_READERS = - { - ReaderMaker.FROM_INPUT_STREAM_BINARY_INCREMENTAL - }; + public static final ReaderMaker[] NON_SPAN_READERS = {}; /** * These are the readers that don't support {@link OffsetSpan}s. */ - public static final ReaderMaker[] NON_OFFSET_SPAN_READERS; - static { - ReaderMaker[] domReaders = ReaderMaker.valuesWith(ReaderMaker.Feature.DOM); - NON_OFFSET_SPAN_READERS = new ReaderMaker[domReaders.length + 1]; - System.arraycopy(domReaders, 0, NON_OFFSET_SPAN_READERS, 0, domReaders.length); - NON_OFFSET_SPAN_READERS[domReaders.length] = ReaderMaker.FROM_INPUT_STREAM_BINARY_INCREMENTAL; - } + public static final ReaderMaker[] NON_OFFSET_SPAN_READERS = + ReaderMaker.valuesWith(ReaderMaker.Feature.DOM); /** * These are the readers that don't support {@link TextSpan}s. @@ -67,8 +59,7 @@ public abstract class ReaderFacetTestCase { ReaderMaker.FROM_INPUT_STREAM_BINARY, ReaderMaker.FROM_INPUT_STREAM_TEXT, - ReaderMaker.FROM_READER, - ReaderMaker.FROM_INPUT_STREAM_BINARY_INCREMENTAL + ReaderMaker.FROM_READER }; diff --git a/test/com/amazon/ion/streaming/SeekableReaderTest.java b/test/com/amazon/ion/streaming/SeekableReaderTest.java index b31c2ad9aa..5510068edc 100644 --- a/test/com/amazon/ion/streaming/SeekableReaderTest.java +++ b/test/com/amazon/ion/streaming/SeekableReaderTest.java @@ -17,14 +17,20 @@ import com.amazon.ion.IonDatagram; import com.amazon.ion.IonType; +import com.amazon.ion.IonWriter; import com.amazon.ion.ReaderMaker; import com.amazon.ion.Span; import com.amazon.ion.TestUtils; import com.amazon.ion.impl._Private_Utils; import com.amazon.ion.junit.Injected.Inject; import com.amazon.ion.junit.IonAssert; + +import java.io.ByteArrayOutputStream; import java.io.File; import java.io.IOException; + +import com.amazon.ion.system.IonBinaryWriterBuilder; +import com.amazon.ion.system.SimpleCatalog; import org.junit.Assert; import org.junit.Test; @@ -58,11 +64,6 @@ private void checkSpans(IonDatagram dg, Span[] positions) @Test public void testTrivialSpan() { - if (getStreamingMode() == StreamingMode.NEW_STREAMING_INCREMENTAL) { - // TODO the incremental reader does not currently support the SpanProvider or SeekableReader facets. - // See ion-java/issues/382 and ion-java/issues/383. - return; - } String text = "null"; read(text); in.next(); @@ -78,11 +79,6 @@ public void testTrivialSpan() @Test public void testWalkingBackwards() { - if (getStreamingMode() == StreamingMode.NEW_STREAMING_INCREMENTAL) { - // TODO the incremental reader does not currently support the SpanProvider or SeekableReader facets. - // See ion-java/issues/382 and ion-java/issues/383. - return; - } String text = "null true 3 4e0 5.0 6666-06-06T '7' \"8\" {{\"\"}} {{}} [] () {}"; @@ -123,11 +119,6 @@ public void testWalkingBackwards() @Test public void testHoistingWithinContainers() { - if (getStreamingMode() == StreamingMode.NEW_STREAMING_INCREMENTAL) { - // TODO the incremental reader does not currently support the SpanProvider or SeekableReader facets. - // See ion-java/issues/382 and ion-java/issues/383. - return; - } read("{f:v,g:[c, (d), e], /* h */ $0:null} s"); in.next(); @@ -203,11 +194,6 @@ public void testHoistingWithinContainers() @Test public void testHoistingLongValue() { - if (getStreamingMode() == StreamingMode.NEW_STREAMING_INCREMENTAL) { - // TODO the incremental reader does not currently support the SpanProvider or SeekableReader facets. - // See ion-java/issues/382 and ion-java/issues/383. - return; - } // This value is "long" in that it has a length subfield in the prefix. String text = " \"123456789012345\" "; read(text); @@ -225,11 +211,6 @@ public void testHoistingLongValue() public void testHoistingOrderedStruct() throws IOException { - if (getStreamingMode() == StreamingMode.NEW_STREAMING_INCREMENTAL) { - // TODO the incremental reader does not currently support the SpanProvider or SeekableReader facets. - // See ion-java/issues/382 and ion-java/issues/383. - return; - } File file = getTestdataFile("good/structOrdered.10n"); byte[] binary = _Private_Utils.loadFileBytes(file); @@ -249,11 +230,6 @@ public void testHoistingOrderedStruct() public void testHoistingAnnotatedTopLevelValue() throws IOException { - if (getStreamingMode() == StreamingMode.NEW_STREAMING_INCREMENTAL) { - // TODO the incremental reader does not currently support the SpanProvider or SeekableReader facets. - // See ion-java/issues/382 and ion-java/issues/383. - return; - } read("a::v"); in.next(); Span span = sr.currentSpan(); @@ -271,11 +247,6 @@ public void testHoistingAnnotatedTopLevelValue() public void testHoistingAnnotatedContainedValue() throws IOException { - if (getStreamingMode() == StreamingMode.NEW_STREAMING_INCREMENTAL) { - // TODO the incremental reader does not currently support the SpanProvider or SeekableReader facets. - // See ion-java/issues/382 and ion-java/issues/383. - return; - } read("[a::v]"); in.next(); in.stepIn(); @@ -291,6 +262,71 @@ public void testHoistingAnnotatedContainedValue() expectTopEof(); } + @Test + public void testHoistingAcrossSymbolTableBoundary() + throws IOException + { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + try (IonWriter writer = IonBinaryWriterBuilder.standard().withLocalSymbolTableAppendEnabled().build(out)) { + writer.writeInt(123); + writer.finish(); + writer.writeSymbol("abc"); + writer.finish(); + writer.writeSymbol("def"); + writer.flush(); + writer.writeSymbol("ghi"); + writer.writeSymbol("jkl"); + writer.finish(); + writer.writeSymbol("mno"); + } + if (myReaderMaker.sourceIsText()) { + read(out.toByteArray()); + } else { + in = getStreamingMode().newIonReader(new SimpleCatalog(), out.toByteArray()); + initFacets(); + } + + in.next(); + Span integer = sp.currentSpan(); + in.next(); + Span abc = sp.currentSpan(); + in.next(); + Span def = sp.currentSpan(); + in.next(); + Span ghi = sp.currentSpan(); + in.next(); + Span jkl = sp.currentSpan(); + in.next(); + Span mno = sp.currentSpan(); + in.next(); + + hoist(jkl); + assertSame(IonType.SYMBOL, in.next()); + assertEquals("jkl", in.stringValue()); + + hoist(ghi); + assertSame(IonType.SYMBOL, in.next()); + assertEquals("ghi", in.stringValue()); + + hoist(integer); + assertSame(IonType.INT, in.next()); + assertEquals(123, in.intValue()); + + hoist(def); + assertSame(IonType.SYMBOL, in.next()); + assertEquals("def", in.stringValue()); + + hoist(mno); + assertSame(IonType.SYMBOL, in.next()); + assertEquals("mno", in.stringValue()); + + hoist(abc); + assertSame(IonType.SYMBOL, in.next()); + assertEquals("abc", in.stringValue()); + + expectTopEof(); + } + //======================================================================== // Failure cases diff --git a/test/com/amazon/ion/streaming/SpanReaderTest.java b/test/com/amazon/ion/streaming/SpanReaderTest.java index d582049b36..646906caca 100644 --- a/test/com/amazon/ion/streaming/SpanReaderTest.java +++ b/test/com/amazon/ion/streaming/SpanReaderTest.java @@ -43,11 +43,6 @@ public SpanReaderTest() @Test public void testCallingCurrentSpan() { - if (getStreamingMode() == StreamingMode.NEW_STREAMING_INCREMENTAL) { - // TODO the incremental reader does not currently support the SpanProvider or SeekableReader facets. - // See ion-java/issues/382 and ion-java/issues/383. - return; - } String text = "null true 3 4e0 5.0 6666-06-06T '7' \"8\" {{\"\"}} {{}} [] () {}"; @@ -85,11 +80,6 @@ public void testCallingCurrentSpan() @Test public void testCurrentSpanWithinContainers() { - if (getStreamingMode() == StreamingMode.NEW_STREAMING_INCREMENTAL) { - // TODO the incremental reader does not currently support the SpanProvider or SeekableReader facets. - // See ion-java/issues/382 and ion-java/issues/383. - return; - } read("{f:v,g:[c]} s"); in.next(); @@ -120,11 +110,6 @@ public void testCurrentSpanWithinContainers() @Test(expected=IllegalStateException.class) public void testCurrentSpanBeforeFirstTopLevel() { - if (getStreamingMode() == StreamingMode.NEW_STREAMING_INCREMENTAL) { - // TODO the incremental reader does not currently support the SpanProvider or SeekableReader facets. - // See ion-java/issues/382 and ion-java/issues/383. - throw new IllegalStateException(); - } read("foo"); sp.currentSpan(); } @@ -132,11 +117,6 @@ public void testCurrentSpanBeforeFirstTopLevel() private void callCurrentSpanBeforeFirstChild(String ionText) { - if (getStreamingMode() == StreamingMode.NEW_STREAMING_INCREMENTAL) { - // TODO the incremental reader does not currently support the SpanProvider or SeekableReader facets. - // See ion-java/issues/382 and ion-java/issues/383. - return; - } read(ionText); in.next(); in.stepIn(); @@ -146,44 +126,24 @@ private void callCurrentSpanBeforeFirstChild(String ionText) @Test(expected=IllegalStateException.class) public void testCurrentSpanBeforeFirstListChild() { - if (getStreamingMode() == StreamingMode.NEW_STREAMING_INCREMENTAL) { - // TODO the incremental reader does not currently support the SpanProvider or SeekableReader facets. - // See ion-java/issues/382 and ion-java/issues/383. - throw new IllegalStateException(); - } callCurrentSpanBeforeFirstChild("[v]"); } @Test(expected=IllegalStateException.class) public void testCurrentSpanBeforeFirstSexpChild() { - if (getStreamingMode() == StreamingMode.NEW_STREAMING_INCREMENTAL) { - // TODO the incremental reader does not currently support the SpanProvider or SeekableReader facets. - // See ion-java/issues/382 and ion-java/issues/383. - throw new IllegalStateException(); - } callCurrentSpanBeforeFirstChild("(v)"); } @Test(expected=IllegalStateException.class) public void testCurrentSpanBeforeFirstStructChild() { - if (getStreamingMode() == StreamingMode.NEW_STREAMING_INCREMENTAL) { - // TODO the incremental reader does not currently support the SpanProvider or SeekableReader facets. - // See ion-java/issues/382 and ion-java/issues/383. - throw new IllegalStateException(); - } callCurrentSpanBeforeFirstChild("{f:v}"); } private void callCurrentSpanAfterLastChild(String ionText) { - if (getStreamingMode() == StreamingMode.NEW_STREAMING_INCREMENTAL) { - // TODO the incremental reader does not currently support the SpanProvider or SeekableReader facets. - // See ion-java/issues/382 and ion-java/issues/383. - return; - } read(ionText); in.next(); in.stepIn(); @@ -195,33 +155,18 @@ private void callCurrentSpanAfterLastChild(String ionText) @Test(expected=IllegalStateException.class) public void testCurrentSpanAfterLastListChild() { - if (getStreamingMode() == StreamingMode.NEW_STREAMING_INCREMENTAL) { - // TODO the incremental reader does not currently support the SpanProvider or SeekableReader facets. - // See ion-java/issues/382 and ion-java/issues/383. - throw new IllegalStateException(); - } callCurrentSpanAfterLastChild("[v]"); } @Test(expected=IllegalStateException.class) public void testCurrentSpanAfterLastSexpChild() { - if (getStreamingMode() == StreamingMode.NEW_STREAMING_INCREMENTAL) { - // TODO the incremental reader does not currently support the SpanProvider or SeekableReader facets. - // See ion-java/issues/382 and ion-java/issues/383. - throw new IllegalStateException(); - } callCurrentSpanAfterLastChild("(v)"); } @Test(expected=IllegalStateException.class) public void testCurrentSpanAfterLastStructChild() { - if (getStreamingMode() == StreamingMode.NEW_STREAMING_INCREMENTAL) { - // TODO the incremental reader does not currently support the SpanProvider or SeekableReader facets. - // See ion-java/issues/382 and ion-java/issues/383. - throw new IllegalStateException(); - } callCurrentSpanAfterLastChild("{f:v}"); } @@ -229,11 +174,6 @@ public void testCurrentSpanAfterLastStructChild() @Test(expected=IllegalStateException.class) public void testCurrentSpanAtEndOfStream() { - if (getStreamingMode() == StreamingMode.NEW_STREAMING_INCREMENTAL) { - // TODO the incremental reader does not currently support the SpanProvider or SeekableReader facets. - // See ion-java/issues/382 and ion-java/issues/383. - throw new IllegalStateException(); - } read("foo"); in.next(); assertEquals(null, in.next()); diff --git a/test/com/amazon/ion/system/IonReaderBuilderTest.java b/test/com/amazon/ion/system/IonReaderBuilderTest.java index f4433f2136..a79d6a7482 100644 --- a/test/com/amazon/ion/system/IonReaderBuilderTest.java +++ b/test/com/amazon/ion/system/IonReaderBuilderTest.java @@ -168,7 +168,7 @@ public void testBufferConfiguration() } @Test - public void testIncrementalReadingDoesNotSupportAutoGzip() throws IOException + public void testIncrementalReadingSupportsAutoGzip() throws IOException { IonReaderBuilder builder = IonReaderBuilder.standard(); builder.withIncrementalReadingEnabled(true); @@ -177,18 +177,14 @@ public void testIncrementalReadingDoesNotSupportAutoGzip() throws IOException gzip.write(_Private_IonConstants.BINARY_VERSION_MARKER_1_0); gzip.write(0x20); // int 0. gzip.close(); - try { - builder.build(data.toByteArray()); - fail(); - } catch (IllegalArgumentException e) { - // Expected; non-incremental readers do not support auto-deflate of GZIP. - } - try { - builder.build(new ByteArrayInputStream(data.toByteArray())); - fail(); - } catch (IllegalArgumentException e) { - // Expected; non-incremental readers do not support auto-deflate of GZIP. - } + IonReader reader1 = builder.build(data.toByteArray()); + assertEquals(IonType.INT, reader1.next()); + assertEquals(0, reader1.intValue()); + reader1.close(); + IonReader reader2 = builder.build(new ByteArrayInputStream(data.toByteArray())); + assertEquals(IonType.INT, reader2.next()); + assertEquals(0, reader2.intValue()); + reader2.close(); } }