diff --git a/cdm/radial/src/main/java/ucar/nc2/iosp/nids/Nidsheader.java b/cdm/radial/src/main/java/ucar/nc2/iosp/nids/Nidsheader.java index 260d855d39..3b85416c34 100644 --- a/cdm/radial/src/main/java/ucar/nc2/iosp/nids/Nidsheader.java +++ b/cdm/radial/src/main/java/ucar/nc2/iosp/nids/Nidsheader.java @@ -255,7 +255,6 @@ void read(ucar.unidata.io.RandomAccessFile raf, ucar.nc2.NetcdfFile ncfile) thro int rc; /* function return status */ int hoff = 0; int type; - int zlibed; boolean isZ = false; int encrypt; long actualSize; @@ -295,8 +294,8 @@ void read(ucar.unidata.io.RandomAccessFile raf, ucar.nc2.NetcdfFile ncfile) thro // byte[] b4 = new byte[4]; System.arraycopy(b, hoff, b2, 0, 2); - zlibed = isZlibHed(b2); - if (zlibed == 0) { + boolean zlibed = isZlibHed(b2); + if (!zlibed) { encrypt = IsEncrypt(b2); if (encrypt == 1) { log.error("error reading encryted product " + raf.getLocation()); @@ -331,7 +330,7 @@ void read(ucar.unidata.io.RandomAccessFile raf, ucar.nc2.NetcdfFile ncfile) thro break; } - if (zlibed == 1) { + if (zlibed) { isZ = true; uncompdata = GetZlibedNexr(b, readLen, hoff); // uncompdata = Nidsiosp.readCompData(hoff, 160) ; @@ -3282,23 +3281,23 @@ public void close() throws IOException { /** * Name: IsZlibed - * + *

* Purpose: Check a two-byte sequence to see if it indicates the start of a zlib-compressed * buffer */ - int isZlibHed(byte[] buf) { + boolean isZlibHed(byte[] buf) { short b0 = convertunsignedByte2Short(buf[0]); short b1 = convertunsignedByte2Short(buf[1]); if ((b0 & 0xf) == Z_DEFLATED) { if ((b0 >> 4) + 8 <= DEF_WBITS) { if ((((b0 << 8) + b1) % 31) == 0) { - return 1; + return true; } } } - return 0; + return false; } @@ -3363,56 +3362,52 @@ byte[] GetZlibedNexr(byte[] buf, int buflen, int hoff) throws IOException { // decompress the bytes int resultLength; int result = 0; - // byte[] inflateData = null; - byte[] tmp; - int uncompLen = 24500; /* length of decompress space */ - byte[] uncomp = new byte[uncompLen]; - Inflater inflater = new Inflater(false); + byte[] chunk; + List chunks = new ArrayList<>(1); + List sizes = new ArrayList<>(1); - inflater.setInput(buf, hoff, numin - 4); - int offset = 0; - int limit = 20000; + Inflater inf = new Inflater(false); + inf.setInput(buf, hoff, numin - 4); - while (inflater.getRemaining() > 0) { + while (inf.getRemaining() > 0) { try { - resultLength = inflater.inflate(uncomp, offset, 4000); + chunk = new byte[4000]; + resultLength = inf.inflate(chunk, 0, 4000); + chunks.add(chunk); + sizes.add(resultLength); } catch (DataFormatException ex) { - log.error("nids Inflater", ex); - throw new IOException(ex.getMessage(), ex); - } - offset = offset + resultLength; - result = result + resultLength; - if (result > limit) { - // when uncomp data larger then limit, the uncomp need to increase size - tmp = new byte[result]; - System.arraycopy(uncomp, 0, tmp, 0, result); - uncompLen = uncompLen + 10000; - uncomp = new byte[uncompLen]; - System.arraycopy(tmp, 0, uncomp, 0, result); + log.error("ERROR on inflation ", ex); + throw new IOException(ex.getMessage()); } + + result += resultLength; if (resultLength == 0) { - int tt = inflater.getRemaining(); + int tt = inf.getRemaining(); byte[] b2 = new byte[2]; System.arraycopy(buf, hoff + numin - 4 - tt, b2, 0, 2); - if (result + tt > uncompLen) { - tmp = new byte[result]; - System.arraycopy(uncomp, 0, tmp, 0, result); - uncompLen = uncompLen + 10000; - uncomp = new byte[uncompLen]; - System.arraycopy(tmp, 0, uncomp, 0, result); - } - if (isZlibHed(b2) == 0) { - System.arraycopy(buf, hoff + numin - 4 - tt, uncomp, result, tt); - result = result + tt; + if (!isZlibHed(b2)) { + chunk = new byte[tt]; + System.arraycopy(buf, hoff + numin - 4 - tt, chunk, 0, tt); + chunks.add(chunk); + sizes.add(tt); + result += tt; break; + } else { + inf.reset(); + inf.setInput(buf, hoff + numin - 4 - tt, tt); } - inflater.reset(); - inflater.setInput(buf, hoff + numin - 4 - tt, tt); } + } + inf.end(); + // Combine the chunks into one buffer + byte[] uncomp = new byte[result]; + int pos = 0; + for (int i = 0; i < chunks.size(); ++i) { + System.arraycopy(chunks.get(i), 0, uncomp, pos, sizes.get(i)); + pos += sizes.get(i); } - inflater.end(); /* ** Find out how long CCB is. This is done by using the lower order ** 6 bits from the first uncompressed byte and all 8 bits of the diff --git a/cdm/radial/src/main/java/ucar/nc2/iosp/nids/Nidsiosp.java b/cdm/radial/src/main/java/ucar/nc2/iosp/nids/Nidsiosp.java index f7e21f956b..140dcbba2d 100644 --- a/cdm/radial/src/main/java/ucar/nc2/iosp/nids/Nidsiosp.java +++ b/cdm/radial/src/main/java/ucar/nc2/iosp/nids/Nidsiosp.java @@ -1490,9 +1490,8 @@ public byte[] readCompData1(byte[] uncomp, long hoff, long doff) { */ public byte[] readCompData(long hoff, long doff) throws IOException { int numin; /* # input bytes processed */ - long pos = 0; long len = raf.length(); - raf.seek(pos); + raf.seek(0); numin = (int) (len - hoff); // Read in the contents of the NEXRAD Level III product header @@ -1506,53 +1505,55 @@ public byte[] readCompData(long hoff, long doff) throws IOException { // System.arraycopy( b, (int)hoff, comp, 0, numin -4 ); // decompress the bytes - Inflater inf = new Inflater(false); - int resultLength; int result = 0; - // byte[] inflateData = null; - byte[] tmp; - int uncompLen = 24500; /* length of decompress space */ - byte[] uncomp = new byte[uncompLen]; + byte[] chunk; + List chunks = new ArrayList<>(1); + List sizes = new ArrayList<>(1); + Inflater inf = new Inflater(false); inf.setInput(b, (int) hoff, numin - 4); - int limit = 20000; while (inf.getRemaining() > 0) { try { - resultLength = inf.inflate(uncomp, result, 4000); + chunk = new byte[4000]; + resultLength = inf.inflate(chunk, 0, 4000); + chunks.add(chunk); + sizes.add(resultLength); } catch (DataFormatException ex) { logger.error("ERROR on inflation ", ex); throw new IOException(ex.getMessage()); } - result = result + resultLength; - if (result > limit) { - // when uncomp data larger then limit, the uncomp need to increase size - tmp = new byte[result]; - System.arraycopy(uncomp, 0, tmp, 0, result); - uncompLen = uncompLen + 10000; - uncomp = new byte[uncompLen]; - System.arraycopy(tmp, 0, uncomp, 0, result); - } + result += resultLength; if (resultLength == 0) { int tt = inf.getRemaining(); byte[] b2 = new byte[2]; System.arraycopy(b, (int) hoff + numin - 4 - tt, b2, 0, 2); - if (headerParser.isZlibHed(b2) == 0) { - System.arraycopy(b, (int) hoff + numin - 4 - tt, uncomp, result, tt); - result = result + tt; + if (!headerParser.isZlibHed(b2)) { + chunk = new byte[tt]; + System.arraycopy(b, (int) hoff + numin - 4 - tt, chunk, 0, tt); + chunks.add(chunk); + sizes.add(tt); + result += tt; break; + } else { + inf.reset(); + inf.setInput(b, (int) hoff + numin - 4 - tt, tt); } - inf.reset(); - inf.setInput(b, (int) hoff + numin - 4 - tt, tt); } - } inf.end(); - int off; - off = 2 * (((uncomp[0] & 0x3F) << 8) | (uncomp[1] & 0xFF)); + // Combine the chunks into one buffer + byte[] uncomp = new byte[result]; + int pos = 0; + for (int i = 0; i < chunks.size(); ++i) { + System.arraycopy(chunks.get(i), 0, uncomp, pos, sizes.get(i)); + pos += sizes.get(i); + } + + int off = 2 * (((uncomp[0] & 0x3F) << 8) | (uncomp[1] & 0xFF)); /* eat WMO and PIL */ for (int i = 0; i < 2; i++) { while ((off < result) && (uncomp[off] != '\n')) diff --git a/cdm/radial/src/test/data/nids/Level3_CYS_N0B_20241108_1948.nids b/cdm/radial/src/test/data/nids/Level3_CYS_N0B_20241108_1948.nids new file mode 100644 index 0000000000..311804a45e Binary files /dev/null and b/cdm/radial/src/test/data/nids/Level3_CYS_N0B_20241108_1948.nids differ diff --git a/cdm/radial/src/test/java/ucar/nc2/iosp/nids/TestNids.java b/cdm/radial/src/test/java/ucar/nc2/iosp/nids/TestNids.java index 97f78546ec..accdc654c0 100644 --- a/cdm/radial/src/test/java/ucar/nc2/iosp/nids/TestNids.java +++ b/cdm/radial/src/test/java/ucar/nc2/iosp/nids/TestNids.java @@ -40,6 +40,7 @@ public class TestNids { private static String digitPrecipArrayFile = TestDir.localTestDataDir + "nids/DPA_20041123_1709"; private static String vertIntegLiquidFile = TestDir.localTestDataDir + "nids/NVL_20041130_1946"; private static String vadWindProfileFile = TestDir.localTestDataDir + "nids/NVW_20041117_1657"; + private static String compressedBlockFile = TestDir.localTestDataDir + "nids/Level3_CYS_N0B_20241108_1948.nids"; @Test public void testNidsReadRadial() throws IOException { @@ -503,6 +504,21 @@ public void testRadialImageMessagePcode167() throws IOException { } } + @Test + public void testNidsCompressedBlock() throws IOException { + NetcdfFile ncfile = null; + try { + System.out.println("**** Open " + compressedBlockFile); + ncfile = NetcdfFile.open(compressedBlockFile); + } catch (java.io.IOException e) { + System.out.println(" fail = " + e); + e.printStackTrace(); + assert (false); + } + ncfile.close(); + } + + private void testReadData(Variable v) { Array a = null; assert (null != v);