diff --git a/commons-hpcc/src/main/java/org/hpccsystems/commons/ecl/FieldDef.java b/commons-hpcc/src/main/java/org/hpccsystems/commons/ecl/FieldDef.java index b8f735bcc..70be3a698 100644 --- a/commons-hpcc/src/main/java/org/hpccsystems/commons/ecl/FieldDef.java +++ b/commons-hpcc/src/main/java/org/hpccsystems/commons/ecl/FieldDef.java @@ -35,7 +35,6 @@ public class FieldDef implements Serializable private long len = 0; private boolean fixedLength = false; private boolean isUnsigned = false; - private boolean isBiased = false; private int additionalFlags = 0; /** @@ -54,7 +53,6 @@ public FieldDef(FieldDef rhs) this.len = rhs.len; this.fixedLength = rhs.fixedLength; this.isUnsigned = rhs.isUnsigned; - this.isBiased = rhs.isBiased; this.additionalFlags = rhs.additionalFlags; } @@ -313,15 +311,24 @@ public boolean isUnsigned() * Is the underlying value biased? * * @return true when biased + * + * @deprecated */ public boolean isBiased() { - return this.isBiased; + return isNonStandardInt(); } - void setIsBiased(boolean biased) + /** + * + * + * @return true when biased + */ + public boolean isNonStandardInt() { - this.isBiased = biased; + return this.srcType == HpccSrcType.KEYED_INTEGER + || this.srcType == HpccSrcType.SWAPPED_INTEGER + || this.srcType == HpccSrcType.BIAS_SWAPPED_INTEGER; } /** diff --git a/commons-hpcc/src/main/java/org/hpccsystems/commons/ecl/RecordDefinitionTranslator.java b/commons-hpcc/src/main/java/org/hpccsystems/commons/ecl/RecordDefinitionTranslator.java index 80e8001d8..e63cef0c4 100644 --- a/commons-hpcc/src/main/java/org/hpccsystems/commons/ecl/RecordDefinitionTranslator.java +++ b/commons-hpcc/src/main/java/org/hpccsystems/commons/ecl/RecordDefinitionTranslator.java @@ -474,7 +474,7 @@ public static JSONObject toJsonRecord(FieldDef field) throws Exception */ private static int getTypeID(FieldDef field) throws Exception { - int typeID = 0; + int typeID = -1; switch (field.getFieldType()) { case SET: @@ -753,9 +753,10 @@ private static int getJsonTypeDefinition(FieldDef field, HashMap 0) + + int flags = childTypeID | childField.getAdditionalFlags(); + if (flags > 0) { - int flags = childTypeID | childField.getAdditionalFlags(); childJson.put("flags", flags); } @@ -783,11 +784,10 @@ else if (childField.getFieldType() == FieldType.SET) } } - if (field.isBiased()) + if (field.isNonStandardInt()) { FieldDef nonKeyedField = new FieldDef(field); nonKeyedField.setSourceType(HpccSrcType.LITTLE_ENDIAN); - nonKeyedField.setIsBiased(false); int childTypeHash = getJsonTypeDefinition(nonKeyedField, typeDefinitionMap, typeDefinitions); int childTypeIndex = typeDefinitionMap.get(childTypeHash); @@ -972,10 +972,6 @@ private static FieldDef parseJsonTypeDefinition(JSONObject jsonTypeDefinitions, { FieldDef fd = new FieldDef("", fieldType, fieldType.description(), length, isFixedLength(typeID), isUnsigned(typeID), getSourceType(typeID), new FieldDef[0]); - if ((typeID & TYPE_ID_MASK) == type_keyedint) - { - fd.setIsBiased(true); - } return fd; } } diff --git a/commons-hpcc/src/main/java/org/hpccsystems/commons/ecl/TestFieldDefinitions.java b/commons-hpcc/src/main/java/org/hpccsystems/commons/ecl/TestFieldDefinitions.java index fa89580fd..761d02736 100644 --- a/commons-hpcc/src/main/java/org/hpccsystems/commons/ecl/TestFieldDefinitions.java +++ b/commons-hpcc/src/main/java/org/hpccsystems/commons/ecl/TestFieldDefinitions.java @@ -55,13 +55,13 @@ public class TestFieldDefinitions + "\"ty9\"\r\n },\r\n \"ty11\": {\r\n \"fieldType\": 257,\r\n \"length\": 2\r\n },\r\n \"ty12\": {\r\n \"fieldType\": 266,\r\n \"length\": " + "2,\r\n \"child\": \"ty11\"\r\n },\r\n \"ty13\": {\r\n \"fieldType\": 259,\r\n \"length\": 524304\r\n },\r\n \"ty14\": {\r\n \"fieldType\":" + " 4,\r\n \"length\": 8\r\n },\r\n \"ty15\": {\r\n \"fieldType\": 0,\r\n \"length\": 1\r\n },\r\n \"ty16\": {\r\n \"fieldType\": 285,\r\n " - + "\"length\": 8\r\n },\r\n \"fieldType\": 13,\r\n \"length\": 61,\r\n \"fields\": [\r\n {\r\n \"name\": \"int8\",\r\n \"type\": \"ty2\",\r\n" + + "\"vinit\": 2,\r\n \"length\": 8\r\n },\r\n \"fieldType\": 13,\r\n \"length\": 61,\r\n \"fields\": [\r\n {\r\n \"name\": \"int8\",\r\n \"type\": \"ty2\",\r\n" + " \"flags\": 10\r\n },\r\n {\r\n \"name\": \"uint8\",\r\n \"type\": \"ty4\",\r\n \"flags\": 266\r\n },\r\n {\r\n \"name\": \"int4\"," + "\r\n \"type\": \"ty6\",\r\n \"flags\": 10\r\n },\r\n {\r\n \"name\": \"uint4\",\r\n \"type\": \"ty8\",\r\n \"flags\": 266\r\n },\r\n" + " {\r\n \"name\": \"int2\",\r\n \"type\": \"ty10\",\r\n \"flags\": 10\r\n },\r\n {\r\n \"name\": \"uint2\",\r\n \"type\": \"ty12\",\r\n" + " \"flags\": 266\r\n },\r\n {\r\n \"name\": \"udec16\",\r\n \"type\": \"ty13\",\r\n \"flags\": 259\r\n },\r\n {\r\n \"name\": \"fixstr8\",\r\n" + " \"type\": \"ty14\",\r\n \"flags\": 4\r\n },\r\n {\r\n \"name\": \"recptr\",\r\n \"type\": \"ty4\",\r\n \"flags\": 266\r\n },\r\n {\r\n" - + " \"name\": \"isactive\",\r\n \"type\": \"ty15\",\r\n \"flags\": 65536\r\n },\r\n {\r\n \"name\": \"__internal_fpos__\",\r\n \"type\": \"ty16\",\r\n" + + " \"name\": \"isactive\",\r\n \"type\": \"ty15\",\r\n \"flags\": 65536\r\n },\r\n {\r\n \"name\": \"__internal_fpos__\",\r\n \"type\": \"ty16\",\r\n" + " \"flags\": 65821\r\n }\r\n ]\r\n}"; /** diff --git a/dfsclient/src/main/java/org/hpccsystems/dfs/client/BinaryRecordReader.java b/dfsclient/src/main/java/org/hpccsystems/dfs/client/BinaryRecordReader.java index 565a59454..fa6cfcb74 100644 --- a/dfsclient/src/main/java/org/hpccsystems/dfs/client/BinaryRecordReader.java +++ b/dfsclient/src/main/java/org/hpccsystems/dfs/client/BinaryRecordReader.java @@ -401,7 +401,7 @@ private Object parseFlatField(FieldDef fd, boolean isLittleEndian) throws Unpars } else { - intValue = getInt((int) fd.getDataLen(), fd.getSourceType() == HpccSrcType.LITTLE_ENDIAN, fd.isBiased()); + intValue = getInt((int) fd.getDataLen(), fd.getSourceType() == HpccSrcType.LITTLE_ENDIAN); fieldValue = Long.valueOf(intValue); } break; @@ -456,7 +456,7 @@ private Object parseFlatField(FieldDef fd, boolean isLittleEndian) throws Unpars break; case BOOLEAN: // fixed length for each boolean value specified by type def - long value = getInt((int) fd.getDataLen(), fd.getSourceType() == HpccSrcType.LITTLE_ENDIAN, fd.isBiased()); + long value = getInt((int) fd.getDataLen(), fd.getSourceType() == HpccSrcType.LITTLE_ENDIAN); fieldValue = Boolean.valueOf(value != 0); break; case CHAR: @@ -719,13 +719,11 @@ private void readIntoScratchBuffer(int offset, int dataLen) throws IOException * the length, 1 to 8 bytes * @param little_endian * true if the value is little endian - * @param shouldCorrectBias - * true if the value should be corrected for index bias * @return the integer extracted as a long * @throws IOException * Signals that an I/O exception has occurred. */ - private long getInt(int len, boolean little_endian, boolean shouldCorrectBias) throws IOException + private long getInt(int len, boolean little_endian) throws IOException { long v = getUnsigned(len, little_endian); @@ -739,12 +737,6 @@ private long getInt(int len, boolean little_endian, boolean shouldCorrectBias) t } } - if (isIndex && shouldCorrectBias) - { - // Roxie indexes are biased to allow for easier comparison. This corrects the bias - v += negMask; - } - return v; } diff --git a/dfsclient/src/main/java/org/hpccsystems/dfs/client/FileUtility.java b/dfsclient/src/main/java/org/hpccsystems/dfs/client/FileUtility.java index 3da02c805..e6f1c7b15 100644 --- a/dfsclient/src/main/java/org/hpccsystems/dfs/client/FileUtility.java +++ b/dfsclient/src/main/java/org/hpccsystems/dfs/client/FileUtility.java @@ -786,7 +786,7 @@ private static Runnable[] createNonRedistributingCopyTasks(HPCCFile file, DFUCre for (int j = 0; j < numIncomingParts; j++) { DataPartition inFilePart = inFileParts[incomingFilePartIndex + j]; - filePartReaders[j] = new HpccRemoteFileReader(inFilePart, recordDef, new HPCCRecordBuilder(recordDef)); + filePartReaders[j] = new HpccRemoteFileReader(inFilePart, recordDef, new HPCCRecordBuilder(file.getProjectedRecordDefinition())); } incomingFilePartIndex += numIncomingParts; diff --git a/dfsclient/src/main/java/org/hpccsystems/dfs/client/HPCCFile.java b/dfsclient/src/main/java/org/hpccsystems/dfs/client/HPCCFile.java index afec67413..a46c1f76a 100644 --- a/dfsclient/src/main/java/org/hpccsystems/dfs/client/HPCCFile.java +++ b/dfsclient/src/main/java/org/hpccsystems/dfs/client/HPCCFile.java @@ -26,6 +26,7 @@ import org.apache.logging.log4j.LogManager; import org.hpccsystems.commons.ecl.FieldDef; import org.hpccsystems.commons.ecl.FileFilter; +import org.hpccsystems.commons.ecl.HpccSrcType; import org.hpccsystems.commons.ecl.RecordDefinitionTranslator; import org.hpccsystems.commons.errors.HpccFileException; import org.hpccsystems.dfs.cluster.ClusterRemapper; @@ -163,12 +164,12 @@ public static int getFilePartFromFPos(long fpos) } /** - * Extracts the offset in the file part from a fileposition value. + * Extracts the offset in the file part from a fileposition value. * * @param fpos file position * @return the project list */ - public static long getOffsetFromFPos(long fpos) + public static long getOffsetFromFPos(long fpos) { // First 48 bits store the offset return fpos & 0xffffffffffffL; @@ -198,11 +199,27 @@ public HPCCFile setProjectList(String projectList) throws Exception this.columnPruner = new ColumnPruner(projectList); if (this.recordDefinition != null) { - this.projectedRecordDefinition = this.columnPruner.pruneRecordDefinition(this.recordDefinition); + updateProjectedRecordDef(); } return this; } + private void updateProjectedRecordDef() throws Exception + { + this.projectedRecordDefinition = this.columnPruner.pruneRecordDefinition(this.recordDefinition); + + // By default project all sub-integer types to standard integers + for (int i = 0; i < this.projectedRecordDefinition.getNumDefs(); i++) + { + FieldDef field = this.projectedRecordDefinition.getDef(i); + if (field.isNonStandardInt()) + { + field.setSourceType(HpccSrcType.LITTLE_ENDIAN); + } + + } + } + /** * Gets the file access expiry secs. * @@ -434,7 +451,7 @@ private void createDataParts() throws HpccFileException this.partitionProcessor = new PartitionProcessor(this.recordDefinition, this.dataParts, null); } - this.projectedRecordDefinition = this.columnPruner.pruneRecordDefinition(this.recordDefinition); + updateProjectedRecordDef(); } else throw new HpccFileException("Could not fetch metadata for file: '" + fileName + "'"); @@ -622,13 +639,13 @@ private static String acquireFileAccess(String fileName, HPCCWsDFUClient hpcc, i String uniqueID = "HPCC-FILE: " + UUID.randomUUID().toString(); return hpcc.getFileAccessBlob(fileName, clusterName, expirySeconds, uniqueID); } - + /** * @return the file metadata information for this HPCCFile (if it exists) */ - public DFUFileDetailWrapper getOriginalFileMetadata() + public DFUFileDetailWrapper getOriginalFileMetadata() { - if (originalFileMetadata==null) + if (originalFileMetadata==null) { HPCCWsDFUClient dfuClient = HPCCWsDFUClient.get(espConnInfo); if (dfuClient.hasInitError())