Skip to content

Commit

Permalink
Merge pull request #1394 from dopplershift/fix-nids-deflate
Browse files Browse the repository at this point in the history 
BUG: Fix decompressing all zlib data for NIDS
  • Loading branch information
@lesserwhirls
lesserwhirls authored Nov 11, 2024
2 parents c46f966 + a96a0cc commit 7f24638
Show file tree
Hide file tree
Showing 4 changed files with 83 additions and 71 deletions.
83 changes: 39 additions & 44 deletions cdm/radial/src/main/java/ucar/nc2/iosp/nids/Nidsheader.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -255,7 +255,6 @@ void read(ucar.unidata.io.RandomAccessFile raf, ucar.nc2.NetcdfFile ncfile) thro
int rc; /* function return status */
int hoff = 0;
int type;
int zlibed;
boolean isZ = false;
int encrypt;
long actualSize;
Expand Down Expand Up @@ -295,8 +294,8 @@ void read(ucar.unidata.io.RandomAccessFile raf, ucar.nc2.NetcdfFile ncfile) thro
// byte[] b4 = new byte[4];
System.arraycopy(b, hoff, b2, 0, 2);

zlibed = isZlibHed(b2);
if (zlibed == 0) {
boolean zlibed = isZlibHed(b2);
if (!zlibed) {
encrypt = IsEncrypt(b2);
if (encrypt == 1) {
log.error("error reading encryted product " + raf.getLocation());
Expand Down Expand Up @@ -331,7 +330,7 @@ void read(ucar.unidata.io.RandomAccessFile raf, ucar.nc2.NetcdfFile ncfile) thro
break;
}

if (zlibed == 1) {
if (zlibed) {
isZ = true;
uncompdata = GetZlibedNexr(b, readLen, hoff);
// uncompdata = Nidsiosp.readCompData(hoff, 160) ;
Expand Down Expand Up @@ -3282,23 +3281,23 @@ public void close() throws IOException {

/**
* Name: IsZlibed
*
* <p>
* Purpose: Check a two-byte sequence to see if it indicates the start of a zlib-compressed
* buffer
*/
int isZlibHed(byte[] buf) {
boolean isZlibHed(byte[] buf) {
short b0 = convertunsignedByte2Short(buf[0]);
short b1 = convertunsignedByte2Short(buf[1]);

if ((b0 & 0xf) == Z_DEFLATED) {
if ((b0 >> 4) + 8 <= DEF_WBITS) {
if ((((b0 << 8) + b1) % 31) == 0) {
return 1;
return true;
}
}
}

return 0;
return false;

}

Expand Down Expand Up @@ -3363,56 +3362,52 @@ byte[] GetZlibedNexr(byte[] buf, int buflen, int hoff) throws IOException {
// decompress the bytes
int resultLength;
int result = 0;
// byte[] inflateData = null;
byte[] tmp;
int uncompLen = 24500; /* length of decompress space */
byte[] uncomp = new byte[uncompLen];
Inflater inflater = new Inflater(false);
byte[] chunk;
List<byte[]> chunks = new ArrayList<>(1);
List<Integer> sizes = new ArrayList<>(1);

inflater.setInput(buf, hoff, numin - 4);
int offset = 0;
int limit = 20000;
Inflater inf = new Inflater(false);
inf.setInput(buf, hoff, numin - 4);

while (inflater.getRemaining() > 0) {
while (inf.getRemaining() > 0) {
try {
resultLength = inflater.inflate(uncomp, offset, 4000);
chunk = new byte[4000];
resultLength = inf.inflate(chunk, 0, 4000);
chunks.add(chunk);
sizes.add(resultLength);
} catch (DataFormatException ex) {
log.error("nids Inflater", ex);
throw new IOException(ex.getMessage(), ex);
}
offset = offset + resultLength;
result = result + resultLength;
if (result > limit) {
// when uncomp data larger then limit, the uncomp need to increase size
tmp = new byte[result];
System.arraycopy(uncomp, 0, tmp, 0, result);
uncompLen = uncompLen + 10000;
uncomp = new byte[uncompLen];
System.arraycopy(tmp, 0, uncomp, 0, result);
log.error("ERROR on inflation ", ex);
throw new IOException(ex.getMessage());
}

result += resultLength;
if (resultLength == 0) {
int tt = inflater.getRemaining();
int tt = inf.getRemaining();
byte[] b2 = new byte[2];
System.arraycopy(buf, hoff + numin - 4 - tt, b2, 0, 2);
if (result + tt > uncompLen) {
tmp = new byte[result];
System.arraycopy(uncomp, 0, tmp, 0, result);
uncompLen = uncompLen + 10000;
uncomp = new byte[uncompLen];
System.arraycopy(tmp, 0, uncomp, 0, result);
}
if (isZlibHed(b2) == 0) {
System.arraycopy(buf, hoff + numin - 4 - tt, uncomp, result, tt);
result = result + tt;
if (!isZlibHed(b2)) {
chunk = new byte[tt];
System.arraycopy(buf, hoff + numin - 4 - tt, chunk, 0, tt);
chunks.add(chunk);
sizes.add(tt);
result += tt;
break;
} else {
inf.reset();
inf.setInput(buf, hoff + numin - 4 - tt, tt);
}
inflater.reset();
inflater.setInput(buf, hoff + numin - 4 - tt, tt);
}
}
inf.end();

// Combine the chunks into one buffer
byte[] uncomp = new byte[result];
int pos = 0;
for (int i = 0; i < chunks.size(); ++i) {
System.arraycopy(chunks.get(i), 0, uncomp, pos, sizes.get(i));
pos += sizes.get(i);
}

inflater.end();
/*
** Find out how long CCB is. This is done by using the lower order
** 6 bits from the first uncompressed byte and all 8 bits of the
Expand Down
55 changes: 28 additions & 27 deletions cdm/radial/src/main/java/ucar/nc2/iosp/nids/Nidsiosp.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1490,9 +1490,8 @@ public byte[] readCompData1(byte[] uncomp, long hoff, long doff) {
*/
public byte[] readCompData(long hoff, long doff) throws IOException {
int numin; /* # input bytes processed */
long pos = 0;
long len = raf.length();
raf.seek(pos);
raf.seek(0);
numin = (int) (len - hoff);
// Read in the contents of the NEXRAD Level III product header

Expand All @@ -1506,53 +1505,55 @@ public byte[] readCompData(long hoff, long doff) throws IOException {
// System.arraycopy( b, (int)hoff, comp, 0, numin -4 );

// decompress the bytes
Inflater inf = new Inflater(false);

int resultLength;
int result = 0;
// byte[] inflateData = null;
byte[] tmp;
int uncompLen = 24500; /* length of decompress space */
byte[] uncomp = new byte[uncompLen];
byte[] chunk;
List<byte[]> chunks = new ArrayList<>(1);
List<Integer> sizes = new ArrayList<>(1);

Inflater inf = new Inflater(false);
inf.setInput(b, (int) hoff, numin - 4);
int limit = 20000;

while (inf.getRemaining() > 0) {
try {
resultLength = inf.inflate(uncomp, result, 4000);
chunk = new byte[4000];
resultLength = inf.inflate(chunk, 0, 4000);
chunks.add(chunk);
sizes.add(resultLength);
} catch (DataFormatException ex) {
logger.error("ERROR on inflation ", ex);
throw new IOException(ex.getMessage());
}

result = result + resultLength;
if (result > limit) {
// when uncomp data larger then limit, the uncomp need to increase size
tmp = new byte[result];
System.arraycopy(uncomp, 0, tmp, 0, result);
uncompLen = uncompLen + 10000;
uncomp = new byte[uncompLen];
System.arraycopy(tmp, 0, uncomp, 0, result);
}
result += resultLength;
if (resultLength == 0) {
int tt = inf.getRemaining();
byte[] b2 = new byte[2];
System.arraycopy(b, (int) hoff + numin - 4 - tt, b2, 0, 2);
if (headerParser.isZlibHed(b2) == 0) {
System.arraycopy(b, (int) hoff + numin - 4 - tt, uncomp, result, tt);
result = result + tt;
if (!headerParser.isZlibHed(b2)) {
chunk = new byte[tt];
System.arraycopy(b, (int) hoff + numin - 4 - tt, chunk, 0, tt);
chunks.add(chunk);
sizes.add(tt);
result += tt;
break;
} else {
inf.reset();
inf.setInput(b, (int) hoff + numin - 4 - tt, tt);
}
inf.reset();
inf.setInput(b, (int) hoff + numin - 4 - tt, tt);
}

}
inf.end();

int off;
off = 2 * (((uncomp[0] & 0x3F) << 8) | (uncomp[1] & 0xFF));
// Combine the chunks into one buffer
byte[] uncomp = new byte[result];
int pos = 0;
for (int i = 0; i < chunks.size(); ++i) {
System.arraycopy(chunks.get(i), 0, uncomp, pos, sizes.get(i));
pos += sizes.get(i);
}

int off = 2 * (((uncomp[0] & 0x3F) << 8) | (uncomp[1] & 0xFF));
/* eat WMO and PIL */
for (int i = 0; i < 2; i++) {
while ((off < result) && (uncomp[off] != '\n'))
Expand Down
Binary file added cdm/radial/src/test/data/nids/Level3_CYS_N0B_20241108_1948.nids
View file
Binary file not shown.
16 changes: 16 additions & 0 deletions cdm/radial/src/test/java/ucar/nc2/iosp/nids/TestNids.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,7 @@ public class TestNids {
private static String digitPrecipArrayFile = TestDir.localTestDataDir + "nids/DPA_20041123_1709";
private static String vertIntegLiquidFile = TestDir.localTestDataDir + "nids/NVL_20041130_1946";
private static String vadWindProfileFile = TestDir.localTestDataDir + "nids/NVW_20041117_1657";
private static String compressedBlockFile = TestDir.localTestDataDir + "nids/Level3_CYS_N0B_20241108_1948.nids";

@Test
public void testNidsReadRadial() throws IOException {
Expand Down Expand Up @@ -503,6 +504,21 @@ public void testRadialImageMessagePcode167() throws IOException {
}
}

@Test
public void testNidsCompressedBlock() throws IOException {
NetcdfFile ncfile = null;
try {
System.out.println("**** Open " + compressedBlockFile);
ncfile = NetcdfFile.open(compressedBlockFile);
} catch (java.io.IOException e) {
System.out.println(" fail = " + e);
e.printStackTrace();
assert (false);
}
ncfile.close();
}


private void testReadData(Variable v) {
Array a = null;
assert (null != v);
Expand Down

0 comments on commit 7f24638

Please sign in to comment.