Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

BUG: Fix decompressing all zlib data for NIDS #1394

Merged
merged 1 commit into from
Nov 11, 2024
Merged

BUG: Fix decompressing all zlib data for NIDS #1394

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
83 changes: 39 additions & 44 deletions cdm/radial/src/main/java/ucar/nc2/iosp/nids/Nidsheader.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -255,7 +255,6 @@ void read(ucar.unidata.io.RandomAccessFile raf, ucar.nc2.NetcdfFile ncfile) thro
int rc; /* function return status */
int hoff = 0;
int type;
int zlibed;
boolean isZ = false;
int encrypt;
long actualSize;
Expand Down Expand Up @@ -295,8 +294,8 @@ void read(ucar.unidata.io.RandomAccessFile raf, ucar.nc2.NetcdfFile ncfile) thro
// byte[] b4 = new byte[4];
System.arraycopy(b, hoff, b2, 0, 2);

zlibed = isZlibHed(b2);
if (zlibed == 0) {
boolean zlibed = isZlibHed(b2);
if (!zlibed) {
encrypt = IsEncrypt(b2);
if (encrypt == 1) {
log.error("error reading encryted product " + raf.getLocation());
Expand Down Expand Up @@ -331,7 +330,7 @@ void read(ucar.unidata.io.RandomAccessFile raf, ucar.nc2.NetcdfFile ncfile) thro
break;
}

if (zlibed == 1) {
if (zlibed) {
isZ = true;
uncompdata = GetZlibedNexr(b, readLen, hoff);
// uncompdata = Nidsiosp.readCompData(hoff, 160) ;
Expand Down Expand Up @@ -3282,23 +3281,23 @@ public void close() throws IOException {

/**
* Name: IsZlibed
*
* <p>
* Purpose: Check a two-byte sequence to see if it indicates the start of a zlib-compressed
* buffer
*/
int isZlibHed(byte[] buf) {
boolean isZlibHed(byte[] buf) {
short b0 = convertunsignedByte2Short(buf[0]);
short b1 = convertunsignedByte2Short(buf[1]);

if ((b0 & 0xf) == Z_DEFLATED) {
if ((b0 >> 4) + 8 <= DEF_WBITS) {
if ((((b0 << 8) + b1) % 31) == 0) {
return 1;
return true;
}
}
}

return 0;
return false;

}

Expand Down Expand Up @@ -3363,56 +3362,52 @@ byte[] GetZlibedNexr(byte[] buf, int buflen, int hoff) throws IOException {
// decompress the bytes
int resultLength;
int result = 0;
// byte[] inflateData = null;
byte[] tmp;
int uncompLen = 24500; /* length of decompress space */
byte[] uncomp = new byte[uncompLen];
Inflater inflater = new Inflater(false);
byte[] chunk;
List<byte[]> chunks = new ArrayList<>(1);
List<Integer> sizes = new ArrayList<>(1);

inflater.setInput(buf, hoff, numin - 4);
int offset = 0;
int limit = 20000;
Inflater inf = new Inflater(false);
inf.setInput(buf, hoff, numin - 4);

while (inflater.getRemaining() > 0) {
while (inf.getRemaining() > 0) {
try {
resultLength = inflater.inflate(uncomp, offset, 4000);
chunk = new byte[4000];
resultLength = inf.inflate(chunk, 0, 4000);
chunks.add(chunk);
sizes.add(resultLength);
} catch (DataFormatException ex) {
log.error("nids Inflater", ex);
throw new IOException(ex.getMessage(), ex);
}
offset = offset + resultLength;
result = result + resultLength;
if (result > limit) {
// when uncomp data larger then limit, the uncomp need to increase size
tmp = new byte[result];
System.arraycopy(uncomp, 0, tmp, 0, result);
uncompLen = uncompLen + 10000;
uncomp = new byte[uncompLen];
System.arraycopy(tmp, 0, uncomp, 0, result);
log.error("ERROR on inflation ", ex);
throw new IOException(ex.getMessage());
}

result += resultLength;
if (resultLength == 0) {
int tt = inflater.getRemaining();
int tt = inf.getRemaining();
byte[] b2 = new byte[2];
System.arraycopy(buf, hoff + numin - 4 - tt, b2, 0, 2);
if (result + tt > uncompLen) {
tmp = new byte[result];
System.arraycopy(uncomp, 0, tmp, 0, result);
uncompLen = uncompLen + 10000;
uncomp = new byte[uncompLen];
System.arraycopy(tmp, 0, uncomp, 0, result);
}
if (isZlibHed(b2) == 0) {
System.arraycopy(buf, hoff + numin - 4 - tt, uncomp, result, tt);
result = result + tt;
if (!isZlibHed(b2)) {
chunk = new byte[tt];
System.arraycopy(buf, hoff + numin - 4 - tt, chunk, 0, tt);
chunks.add(chunk);
sizes.add(tt);
result += tt;
break;
} else {
inf.reset();
inf.setInput(buf, hoff + numin - 4 - tt, tt);
}
inflater.reset();
inflater.setInput(buf, hoff + numin - 4 - tt, tt);
}
}
inf.end();

// Combine the chunks into one buffer
byte[] uncomp = new byte[result];
int pos = 0;
for (int i = 0; i < chunks.size(); ++i) {
System.arraycopy(chunks.get(i), 0, uncomp, pos, sizes.get(i));
pos += sizes.get(i);
}

inflater.end();
/*
** Find out how long CCB is. This is done by using the lower order
** 6 bits from the first uncompressed byte and all 8 bits of the
Expand Down
55 changes: 28 additions & 27 deletions cdm/radial/src/main/java/ucar/nc2/iosp/nids/Nidsiosp.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1490,9 +1490,8 @@ public byte[] readCompData1(byte[] uncomp, long hoff, long doff) {
*/
public byte[] readCompData(long hoff, long doff) throws IOException {
int numin; /* # input bytes processed */
long pos = 0;
long len = raf.length();
raf.seek(pos);
raf.seek(0);
numin = (int) (len - hoff);
// Read in the contents of the NEXRAD Level III product header

Expand All @@ -1506,53 +1505,55 @@ public byte[] readCompData(long hoff, long doff) throws IOException {
// System.arraycopy( b, (int)hoff, comp, 0, numin -4 );

// decompress the bytes
Inflater inf = new Inflater(false);

int resultLength;
int result = 0;
// byte[] inflateData = null;
byte[] tmp;
int uncompLen = 24500; /* length of decompress space */
byte[] uncomp = new byte[uncompLen];
byte[] chunk;
List<byte[]> chunks = new ArrayList<>(1);
List<Integer> sizes = new ArrayList<>(1);

Inflater inf = new Inflater(false);
inf.setInput(b, (int) hoff, numin - 4);
int limit = 20000;

while (inf.getRemaining() > 0) {
try {
resultLength = inf.inflate(uncomp, result, 4000);
chunk = new byte[4000];
resultLength = inf.inflate(chunk, 0, 4000);
chunks.add(chunk);
sizes.add(resultLength);
} catch (DataFormatException ex) {
logger.error("ERROR on inflation ", ex);
throw new IOException(ex.getMessage());
}

result = result + resultLength;
if (result > limit) {
// when uncomp data larger then limit, the uncomp need to increase size
tmp = new byte[result];
System.arraycopy(uncomp, 0, tmp, 0, result);
uncompLen = uncompLen + 10000;
uncomp = new byte[uncompLen];
System.arraycopy(tmp, 0, uncomp, 0, result);
}
result += resultLength;
if (resultLength == 0) {
int tt = inf.getRemaining();
byte[] b2 = new byte[2];
System.arraycopy(b, (int) hoff + numin - 4 - tt, b2, 0, 2);
if (headerParser.isZlibHed(b2) == 0) {
System.arraycopy(b, (int) hoff + numin - 4 - tt, uncomp, result, tt);
result = result + tt;
if (!headerParser.isZlibHed(b2)) {
chunk = new byte[tt];
System.arraycopy(b, (int) hoff + numin - 4 - tt, chunk, 0, tt);
chunks.add(chunk);
sizes.add(tt);
result += tt;
break;
} else {
inf.reset();
inf.setInput(b, (int) hoff + numin - 4 - tt, tt);
}
inf.reset();
inf.setInput(b, (int) hoff + numin - 4 - tt, tt);
}

}
inf.end();

int off;
off = 2 * (((uncomp[0] & 0x3F) << 8) | (uncomp[1] & 0xFF));
// Combine the chunks into one buffer
byte[] uncomp = new byte[result];
int pos = 0;
for (int i = 0; i < chunks.size(); ++i) {
System.arraycopy(chunks.get(i), 0, uncomp, pos, sizes.get(i));
pos += sizes.get(i);
}

int off = 2 * (((uncomp[0] & 0x3F) << 8) | (uncomp[1] & 0xFF));
/* eat WMO and PIL */
for (int i = 0; i < 2; i++) {
while ((off < result) && (uncomp[off] != '\n'))
Expand Down
Binary file added cdm/radial/src/test/data/nids/Level3_CYS_N0B_20241108_1948.nids
View file
Open in desktop
Binary file not shown.
16 changes: 16 additions & 0 deletions cdm/radial/src/test/java/ucar/nc2/iosp/nids/TestNids.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,7 @@ public class TestNids {
private static String digitPrecipArrayFile = TestDir.localTestDataDir + "nids/DPA_20041123_1709";
private static String vertIntegLiquidFile = TestDir.localTestDataDir + "nids/NVL_20041130_1946";
private static String vadWindProfileFile = TestDir.localTestDataDir + "nids/NVW_20041117_1657";
private static String compressedBlockFile = TestDir.localTestDataDir + "nids/Level3_CYS_N0B_20241108_1948.nids";

@Test
public void testNidsReadRadial() throws IOException {
Expand Down Expand Up @@ -503,6 +504,21 @@ public void testRadialImageMessagePcode167() throws IOException {
}
}

@Test
public void testNidsCompressedBlock() throws IOException {
NetcdfFile ncfile = null;
try {
System.out.println("**** Open " + compressedBlockFile);
ncfile = NetcdfFile.open(compressedBlockFile);
} catch (java.io.IOException e) {
System.out.println(" fail = " + e);
e.printStackTrace();
assert (false);
}
ncfile.close();
}


private void testReadData(Variable v) {
Array a = null;
assert (null != v);
Expand Down
Loading