Skip to content

Commit

Permalink
BUG: Fix decompressing all zlib data for NIDS
Browse files Browse the repository at this point in the history 
Code was only expanding the buffer by 10000 bytes as it read through.
This resulted in blowing the array bounds when the last, noncompressed
chunk of data was larger than this. This was easily broken by products
with a single compressed chunk, followed by all uncompressed data. A
representative data file and smoke test is included.
  • Loading branch information
@dopplershift
dopplershift committed Nov 8, 2024
1 parent c46f966 commit a96a0cc
Show file tree
Hide file tree
Showing 4 changed files with 83 additions and 71 deletions.
83 changes: 39 additions & 44 deletions cdm/radial/src/main/java/ucar/nc2/iosp/nids/Nidsheader.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -255,7 +255,6 @@ void read(ucar.unidata.io.RandomAccessFile raf, ucar.nc2.NetcdfFile ncfile) thro
int rc; /* function return status */
int hoff = 0;
int type;
int zlibed;
boolean isZ = false;
int encrypt;
long actualSize;
Expand Down Expand Up @@ -295,8 +294,8 @@ void read(ucar.unidata.io.RandomAccessFile raf, ucar.nc2.NetcdfFile ncfile) thro
// byte[] b4 = new byte[4];
System.arraycopy(b, hoff, b2, 0, 2);

zlibed = isZlibHed(b2);
if (zlibed == 0) {
boolean zlibed = isZlibHed(b2);
if (!zlibed) {
encrypt = IsEncrypt(b2);
if (encrypt == 1) {
log.error("error reading encryted product " + raf.getLocation());
Expand Down Expand Up @@ -331,7 +330,7 @@ void read(ucar.unidata.io.RandomAccessFile raf, ucar.nc2.NetcdfFile ncfile) thro
break;
}

if (zlibed == 1) {
if (zlibed) {
isZ = true;
uncompdata = GetZlibedNexr(b, readLen, hoff);
// uncompdata = Nidsiosp.readCompData(hoff, 160) ;
Expand Down Expand Up @@ -3282,23 +3281,23 @@ public void close() throws IOException {

/**
* Name: IsZlibed
*
* <p>
* Purpose: Check a two-byte sequence to see if it indicates the start of a zlib-compressed
* buffer
*/
int isZlibHed(byte[] buf) {
boolean isZlibHed(byte[] buf) {
short b0 = convertunsignedByte2Short(buf[0]);
short b1 = convertunsignedByte2Short(buf[1]);

if ((b0 & 0xf) == Z_DEFLATED) {
if ((b0 >> 4) + 8 <= DEF_WBITS) {
if ((((b0 << 8) + b1) % 31) == 0) {
return 1;
return true;
}
}
}

return 0;
return false;

}

Expand Down Expand Up @@ -3363,56 +3362,52 @@ byte[] GetZlibedNexr(byte[] buf, int buflen, int hoff) throws IOException {
// decompress the bytes
int resultLength;
int result = 0;
// byte[] inflateData = null;
byte[] tmp;
int uncompLen = 24500; /* length of decompress space */
byte[] uncomp = new byte[uncompLen];
Inflater inflater = new Inflater(false);
byte[] chunk;
List<byte[]> chunks = new ArrayList<>(1);
List<Integer> sizes = new ArrayList<>(1);

inflater.setInput(buf, hoff, numin - 4);
int offset = 0;
int limit = 20000;
Inflater inf = new Inflater(false);
inf.setInput(buf, hoff, numin - 4);

while (inflater.getRemaining() > 0) {
while (inf.getRemaining() > 0) {
try {
resultLength = inflater.inflate(uncomp, offset, 4000);
chunk = new byte[4000];
resultLength = inf.inflate(chunk, 0, 4000);
chunks.add(chunk);
sizes.add(resultLength);
} catch (DataFormatException ex) {
log.error("nids Inflater", ex);
throw new IOException(ex.getMessage(), ex);
}
offset = offset + resultLength;
result = result + resultLength;
if (result > limit) {
// when uncomp data larger then limit, the uncomp need to increase size
tmp = new byte[result];
System.arraycopy(uncomp, 0, tmp, 0, result);
uncompLen = uncompLen + 10000;
uncomp = new byte[uncompLen];
System.arraycopy(tmp, 0, uncomp, 0, result);
log.error("ERROR on inflation ", ex);
throw new IOException(ex.getMessage());
}

result += resultLength;
if (resultLength == 0) {
int tt = inflater.getRemaining();
int tt = inf.getRemaining();
byte[] b2 = new byte[2];
System.arraycopy(buf, hoff + numin - 4 - tt, b2, 0, 2);
if (result + tt > uncompLen) {
tmp = new byte[result];
System.arraycopy(uncomp, 0, tmp, 0, result);
uncompLen = uncompLen + 10000;
uncomp = new byte[uncompLen];
System.arraycopy(tmp, 0, uncomp, 0, result);
}
if (isZlibHed(b2) == 0) {
System.arraycopy(buf, hoff + numin - 4 - tt, uncomp, result, tt);
result = result + tt;
if (!isZlibHed(b2)) {
chunk = new byte[tt];
System.arraycopy(buf, hoff + numin - 4 - tt, chunk, 0, tt);
chunks.add(chunk);
sizes.add(tt);
result += tt;
break;
} else {
inf.reset();
inf.setInput(buf, hoff + numin - 4 - tt, tt);
}
inflater.reset();
inflater.setInput(buf, hoff + numin - 4 - tt, tt);
}
}
inf.end();

// Combine the chunks into one buffer
byte[] uncomp = new byte[result];
int pos = 0;
for (int i = 0; i < chunks.size(); ++i) {
System.arraycopy(chunks.get(i), 0, uncomp, pos, sizes.get(i));
pos += sizes.get(i);
}

inflater.end();
/*
** Find out how long CCB is. This is done by using the lower order
** 6 bits from the first uncompressed byte and all 8 bits of the
Expand Down
55 changes: 28 additions & 27 deletions cdm/radial/src/main/java/ucar/nc2/iosp/nids/Nidsiosp.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1490,9 +1490,8 @@ public byte[] readCompData1(byte[] uncomp, long hoff, long doff) {
*/
public byte[] readCompData(long hoff, long doff) throws IOException {
int numin; /* # input bytes processed */
long pos = 0;
long len = raf.length();
raf.seek(pos);
raf.seek(0);
numin = (int) (len - hoff);
// Read in the contents of the NEXRAD Level III product header

Expand All @@ -1506,53 +1505,55 @@ public byte[] readCompData(long hoff, long doff) throws IOException {
// System.arraycopy( b, (int)hoff, comp, 0, numin -4 );

// decompress the bytes
Inflater inf = new Inflater(false);

int resultLength;
int result = 0;
// byte[] inflateData = null;
byte[] tmp;
int uncompLen = 24500; /* length of decompress space */
byte[] uncomp = new byte[uncompLen];
byte[] chunk;
List<byte[]> chunks = new ArrayList<>(1);
List<Integer> sizes = new ArrayList<>(1);

Inflater inf = new Inflater(false);
inf.setInput(b, (int) hoff, numin - 4);
int limit = 20000;

while (inf.getRemaining() > 0) {
try {
resultLength = inf.inflate(uncomp, result, 4000);
chunk = new byte[4000];
resultLength = inf.inflate(chunk, 0, 4000);
chunks.add(chunk);
sizes.add(resultLength);
} catch (DataFormatException ex) {
logger.error("ERROR on inflation ", ex);
throw new IOException(ex.getMessage());
}

result = result + resultLength;
if (result > limit) {
// when uncomp data larger then limit, the uncomp need to increase size
tmp = new byte[result];
System.arraycopy(uncomp, 0, tmp, 0, result);
uncompLen = uncompLen + 10000;
uncomp = new byte[uncompLen];
System.arraycopy(tmp, 0, uncomp, 0, result);
}
result += resultLength;
if (resultLength == 0) {
int tt = inf.getRemaining();
byte[] b2 = new byte[2];
System.arraycopy(b, (int) hoff + numin - 4 - tt, b2, 0, 2);
if (headerParser.isZlibHed(b2) == 0) {
System.arraycopy(b, (int) hoff + numin - 4 - tt, uncomp, result, tt);
result = result + tt;
if (!headerParser.isZlibHed(b2)) {
chunk = new byte[tt];
System.arraycopy(b, (int) hoff + numin - 4 - tt, chunk, 0, tt);
chunks.add(chunk);
sizes.add(tt);
result += tt;
break;
} else {
inf.reset();
inf.setInput(b, (int) hoff + numin - 4 - tt, tt);
}
inf.reset();
inf.setInput(b, (int) hoff + numin - 4 - tt, tt);
}

}
inf.end();

int off;
off = 2 * (((uncomp[0] & 0x3F) << 8) | (uncomp[1] & 0xFF));
// Combine the chunks into one buffer
byte[] uncomp = new byte[result];
int pos = 0;
for (int i = 0; i < chunks.size(); ++i) {
System.arraycopy(chunks.get(i), 0, uncomp, pos, sizes.get(i));
pos += sizes.get(i);
}

int off = 2 * (((uncomp[0] & 0x3F) << 8) | (uncomp[1] & 0xFF));
/* eat WMO and PIL */
for (int i = 0; i < 2; i++) {
while ((off < result) && (uncomp[off] != '\n'))
Expand Down
Binary file added cdm/radial/src/test/data/nids/Level3_CYS_N0B_20241108_1948.nids
View file
Binary file not shown.
16 changes: 16 additions & 0 deletions cdm/radial/src/test/java/ucar/nc2/iosp/nids/TestNids.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,7 @@ public class TestNids {
private static String digitPrecipArrayFile = TestDir.localTestDataDir + "nids/DPA_20041123_1709";
private static String vertIntegLiquidFile = TestDir.localTestDataDir + "nids/NVL_20041130_1946";
private static String vadWindProfileFile = TestDir.localTestDataDir + "nids/NVW_20041117_1657";
private static String compressedBlockFile = TestDir.localTestDataDir + "nids/Level3_CYS_N0B_20241108_1948.nids";

@Test
public void testNidsReadRadial() throws IOException {
Expand Down Expand Up @@ -503,6 +504,21 @@ public void testRadialImageMessagePcode167() throws IOException {
}
}

@Test
public void testNidsCompressedBlock() throws IOException {
NetcdfFile ncfile = null;
try {
System.out.println("**** Open " + compressedBlockFile);
ncfile = NetcdfFile.open(compressedBlockFile);
} catch (java.io.IOException e) {
System.out.println(" fail = " + e);
e.printStackTrace();
assert (false);
}
ncfile.close();
}


private void testReadData(Variable v) {
Array a = null;
assert (null != v);
Expand Down

0 comments on commit a96a0cc

Please sign in to comment.