Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

HPCC4J-650 DFSClient should retain integer subtypes #764

Merged
merged 1 commit into from
Oct 3, 2024
Merged

HPCC4J-650 DFSClient should retain integer subtypes #764

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
39 changes: 31 additions & 8 deletions commons-hpcc/src/main/java/org/hpccsystems/commons/ecl/FieldDef.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,6 @@ public class FieldDef implements Serializable
private long len = 0;
private boolean fixedLength = false;
private boolean isUnsigned = false;
private boolean isBiased = false;
private int additionalFlags = 0;

/**
Expand All @@ -49,12 +48,17 @@ public FieldDef(FieldDef rhs)
this.fieldName = rhs.fieldName;
this.fieldType = rhs.fieldType;
this.typeName = rhs.typeName;
this.defs = rhs.defs;

this.defs = new FieldDef[rhs.defs.length];
for (int i = 0; i < rhs.defs.length; i++)
{
this.defs[i] = new FieldDef(rhs.defs[i]);
rpastrana marked this conversation as resolved.
Show resolved Hide resolved
}

this.srcType = rhs.srcType;
this.len = rhs.len;
this.fixedLength = rhs.fixedLength;
this.isUnsigned = rhs.isUnsigned;
this.isBiased = rhs.isBiased;
this.additionalFlags = rhs.additionalFlags;
}

Expand Down Expand Up @@ -172,6 +176,16 @@ public HpccSrcType getSourceType()
return this.srcType;
}

/**
* Sets data type on the HPCC cluster.
*
* @param srcType the new source type
*/
public void setSourceType(HpccSrcType srcType)
{
this.srcType = srcType;
}

/**
* Length of data or minimum length if variable.
*
Expand Down Expand Up @@ -305,18 +319,27 @@ public boolean isUnsigned()
}

/**
* Is the underlying value biased?
* Is the underlying value biased? Deprecated in favor of isNonStandardInt.
*
* @return true when biased
* @return true when biased
*
* @deprecated
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

let's add a comment indicating which method to use instead of this deprecated method

*/
public boolean isBiased()
{
return this.isBiased;
return isNonStandardInt();
}

void setIsBiased(boolean biased)
/**
*
*
* @return true when biased
*/
public boolean isNonStandardInt()
{
this.isBiased = biased;
return this.srcType == HpccSrcType.KEYED_INTEGER
|| this.srcType == HpccSrcType.SWAPPED_INTEGER
|| this.srcType == HpccSrcType.BIAS_SWAPPED_INTEGER;
}

/**
Expand Down
6 changes: 6 additions & 0 deletions commons-hpcc/src/main/java/org/hpccsystems/commons/ecl/HpccSrcType.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,12 @@ public enum HpccSrcType
"little endian", false
), BINARY_CODED_DECIMAL (
"Binary coded decimal", false
), KEYED_INTEGER (
"Non-payload integer field within a key", false
), SWAPPED_INTEGER (
"Byte swapped integer, used within keys", false
), BIAS_SWAPPED_INTEGER (
"Byte swapped integer, deprecated", false
), UNKNOWN (
"Unkown", false
);
Expand Down
70 changes: 53 additions & 17 deletions commons-hpcc/src/main/java/org/hpccsystems/commons/ecl/RecordDefinitionTranslator.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,8 @@ public class RecordDefinitionTranslator
private static final String CHILD_KEY = "child";
private static final String FLAGS_KEY = "flags";

private static final String ESP_TYPE_NAME_PREFIX = "ty";

private static final int FLAG_UNSIGNED = 256;
private static final int FLAG_UNKNOWN_SIZE = 1024;
private static final int TYPE_ID_MASK = 0xff; // 0x7fff & ~FLAG_UNKNOWN_SIZE & ~FLAG_UNSIGNED;
Expand All @@ -55,6 +57,7 @@ public class RecordDefinitionTranslator
final private static int type_varunicode = 33;
final private static int type_utf8 = 41;


// FNoInitializer, // 0 means no initialiser - not a special virtual initialiser
// FVirtualFilePosition,
// FVirtualLocalFilePosition,
Expand All @@ -68,6 +71,8 @@ public class RecordDefinitionTranslator
final private static int type_char = 11; // Convert to string
final private static int type_qstring = 30; // Convert to string

final private static char XPATH_DELIMITER = 0x0001;

// Additional retained flags
final private static int FLAG_IS_PAYLOAD_FIELD = 0x00010000;

Expand Down Expand Up @@ -134,9 +139,12 @@ private static HpccSrcType getSourceType(int typeID)
case type_int:
case type_real:
return HpccSrcType.LITTLE_ENDIAN;
case type_swapint:
case type_biasedswapint:
return HpccSrcType.BIAS_SWAPPED_INTEGER;
case type_swapint:
return HpccSrcType.SWAPPED_INTEGER;
case type_keyedint:
return HpccSrcType.KEYED_INTEGER;
case type_filepos:
return HpccSrcType.BIG_ENDIAN;
case type_utf8:
Expand Down Expand Up @@ -272,7 +280,7 @@ private static String getEClTypeDefinition(FieldDef field, HashMap<String, Strin
{
return "DATA" + field.getDataLen();
}

return "DATA";
}
case BOOLEAN:
Expand All @@ -298,7 +306,7 @@ private static String getEClTypeDefinition(FieldDef field, HashMap<String, Strin
{
if (field.isUnsigned() == false)
{
throw new Exception("Error: Filepos must be unsigned");
throw new Exception("Error: Filepos must be unsigned");
}

if (field.getDataLen() != 8)
Expand Down Expand Up @@ -454,7 +462,7 @@ public static JSONObject toJsonRecord(FieldDef field) throws Exception
continue;
}

rootDefinition.put("ty" + (i + 1), typeDefinition);
rootDefinition.put(ESP_TYPE_NAME_PREFIX + (i + 1), typeDefinition);
}

return rootDefinition;
Expand All @@ -471,7 +479,7 @@ public static JSONObject toJsonRecord(FieldDef field) throws Exception
*/
private static int getTypeID(FieldDef field) throws Exception
{
int typeID = 0;
int typeID = -1;
switch (field.getFieldType())
{
case SET:
Expand Down Expand Up @@ -514,10 +522,25 @@ private static int getTypeID(FieldDef field) throws Exception
case INTEGER:
{
typeID = type_int;
HpccSrcType srcType = field.getSourceType();
if (srcType == HpccSrcType.KEYED_INTEGER)
{
typeID = type_keyedint;
}
else if (srcType == HpccSrcType.SWAPPED_INTEGER)
{
typeID = type_swapint;
}
else if (srcType == HpccSrcType.BIAS_SWAPPED_INTEGER)
{
typeID = type_biasedswapint;
}

if (field.isUnsigned())
{
typeID |= FLAG_UNSIGNED;
}

break;
}
case DECIMAL:
Expand Down Expand Up @@ -625,7 +648,7 @@ else if (srcType == HpccSrcType.UTF16LE || srcType == HpccSrcType.UTF16BE)
*/
private static int getTypeHash(FieldDef field) throws Exception
{
int numHashComponents = 2 + field.getNumDefs();
int numHashComponents = 4 + field.getNumDefs();
if (field.getFieldType() == FieldType.DECIMAL)
{
numHashComponents += 2;
Expand All @@ -634,8 +657,10 @@ private static int getTypeHash(FieldDef field) throws Exception
long[] hashComponents = new long[numHashComponents];
hashComponents[0] = getTypeID(field);
hashComponents[1] = field.getDataLen();
hashComponents[2] = field.getSourceType().ordinal();
hashComponents[3] = field.getAdditionalFlags();

int hashCompIndex = 2;
int hashCompIndex = 4;
for (int i = 0; i < field.getNumDefs(); i++, hashCompIndex++)
{
hashComponents[hashCompIndex] = getTypeHash(field.getDef(i));
Expand Down Expand Up @@ -685,7 +710,7 @@ private static int getJsonTypeDefinition(FieldDef field, HashMap<Integer, Intege

int childTypeHash = getJsonTypeDefinition(field.getDef(0), typeDefinitionMap, typeDefinitions);
int childTypeIndex = typeDefinitionMap.get(childTypeHash);
String childTypeName = "ty" + (childTypeIndex + 1);
String childTypeName = ESP_TYPE_NAME_PREFIX + (childTypeIndex + 1);
typeDef.put("child", childTypeName);
break;
}
Expand Down Expand Up @@ -727,22 +752,26 @@ private static int getJsonTypeDefinition(FieldDef field, HashMap<Integer, Intege

int childTypeHash = getJsonTypeDefinition(childField, typeDefinitionMap, typeDefinitions);
int childTypeIndex = typeDefinitionMap.get(childTypeHash);
String childTypeName = "ty" + (childTypeIndex + 1);
String childTypeName = ESP_TYPE_NAME_PREFIX + (childTypeIndex + 1);
int childTypeID = getTypeID(childField);

JSONObject childJson = new JSONObject();
childJson.put("name", childField.getFieldName());
childJson.put("type", childTypeName);
if (childTypeID > 0)

int flags = childTypeID | childField.getAdditionalFlags();
if (flags > 0)
{
int flags = childTypeID | childField.getAdditionalFlags();
childJson.put("flags", flags);
}

if (childField.getFieldType() == FieldType.DATASET)
{
char delim = 0x0001;
childJson.put("xpath", childField.getFieldName() + delim + "Row");
childJson.put("xpath", childField.getFieldName() + XPATH_DELIMITER + "Row");
}
else if (childField.getFieldType() == FieldType.SET)
{
childJson.put("xpath", childField.getFieldName() + XPATH_DELIMITER + "Item");
}

fields.put(childJson);
Expand All @@ -758,6 +787,17 @@ private static int getJsonTypeDefinition(FieldDef field, HashMap<Integer, Intege
}
}

if (field.isNonStandardInt())
{
FieldDef nonKeyedField = new FieldDef(field);
nonKeyedField.setSourceType(HpccSrcType.LITTLE_ENDIAN);

int childTypeHash = getJsonTypeDefinition(nonKeyedField, typeDefinitionMap, typeDefinitions);
int childTypeIndex = typeDefinitionMap.get(childTypeHash);
String childTypeName = ESP_TYPE_NAME_PREFIX + (childTypeIndex + 1);
typeDef.put("child", childTypeName);
}

int newTypeIndex = typeDefinitions.size();
typeDefinitions.add(typeDef);
typeDefinitionMap.put(typeHash, newTypeIndex);
Expand Down Expand Up @@ -935,10 +975,6 @@ private static FieldDef parseJsonTypeDefinition(JSONObject jsonTypeDefinitions,
{
FieldDef fd = new FieldDef("", fieldType, fieldType.description(), length,
isFixedLength(typeID), isUnsigned(typeID), getSourceType(typeID), new FieldDef[0]);
if ((typeID & TYPE_ID_MASK) == type_keyedint)
{
fd.setIsBiased(true);
}
return fd;
}
}
Expand Down
23 changes: 23 additions & 0 deletions commons-hpcc/src/main/java/org/hpccsystems/commons/ecl/TestFieldDefinitions.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,24 @@ public class TestFieldDefinitions
+ "\"type\": \"ty17\",\r\n \"xpath\": \"childdataset\\u0001Row\",\r\n \"flags\": 1044\r\n },\r\n {\r\n \"name\": \"int2set\",\r\n "
+ "\"type\": \"ty19\",\r\n \"xpath\": \"int2set\\u0001Item\",\r\n \"flags\": 1045\r\n }\r\n ]\r\n}";

private static final String allTypesIndexRecordDefinitionStr = "{\r\n \"ty1\": {\r\n \"fieldType\": 1,\r\n \"length\": 8\r\n },\r\n \"ty2\":"
+ " {\r\n \"fieldType\": 10,\r\n \"length\": 8,\r\n \"child\": \"ty1\"\r\n },\r\n \"ty3\": {\r\n \"fieldType\": 257,\r\n \"length\":"
+ " 8\r\n },\r\n \"ty4\": {\r\n \"fieldType\": 266,\r\n \"length\": 8,\r\n \"child\": \"ty3\"\r\n },\r\n \"ty5\": {\r\n \"fieldType\": "
+ "1,\r\n \"length\": 4\r\n },\r\n \"ty6\": {\r\n \"fieldType\": 10,\r\n \"length\": 4,\r\n \"child\": \"ty5\"\r\n },\r\n \"ty7\": {\r\n "
+ " \"fieldType\": 257,\r\n \"length\": 4\r\n },\r\n \"ty8\": {\r\n \"fieldType\": 266,\r\n \"length\": 4,\r\n \"child\": \"ty7\"\r\n },\r\n"
+ " \"ty9\": {\r\n \"fieldType\": 1,\r\n \"length\": 2\r\n },\r\n \"ty10\": {\r\n \"fieldType\": 10,\r\n \"length\": 2,\r\n \"child\": "
+ "\"ty9\"\r\n },\r\n \"ty11\": {\r\n \"fieldType\": 257,\r\n \"length\": 2\r\n },\r\n \"ty12\": {\r\n \"fieldType\": 266,\r\n \"length\": "
+ "2,\r\n \"child\": \"ty11\"\r\n },\r\n \"ty13\": {\r\n \"fieldType\": 259,\r\n \"length\": 524304\r\n },\r\n \"ty14\": {\r\n \"fieldType\":"
+ " 4,\r\n \"length\": 8\r\n },\r\n \"ty15\": {\r\n \"fieldType\": 0,\r\n \"length\": 1\r\n },\r\n \"ty16\": {\r\n \"fieldType\": 285,\r\n "
+ "\"vinit\": 2,\r\n \"length\": 8\r\n },\r\n \"fieldType\": 13,\r\n \"length\": 61,\r\n \"fields\": [\r\n {\r\n \"name\": \"int8\",\r\n \"type\": \"ty2\",\r\n"
+ " \"flags\": 10\r\n },\r\n {\r\n \"name\": \"uint8\",\r\n \"type\": \"ty4\",\r\n \"flags\": 266\r\n },\r\n {\r\n \"name\": \"int4\","
+ "\r\n \"type\": \"ty6\",\r\n \"flags\": 10\r\n },\r\n {\r\n \"name\": \"uint4\",\r\n \"type\": \"ty8\",\r\n \"flags\": 266\r\n },\r\n"
+ " {\r\n \"name\": \"int2\",\r\n \"type\": \"ty10\",\r\n \"flags\": 10\r\n },\r\n {\r\n \"name\": \"uint2\",\r\n \"type\": \"ty12\",\r\n"
+ " \"flags\": 266\r\n },\r\n {\r\n \"name\": \"udec16\",\r\n \"type\": \"ty13\",\r\n \"flags\": 259\r\n },\r\n {\r\n \"name\": \"fixstr8\",\r\n"
+ " \"type\": \"ty14\",\r\n \"flags\": 4\r\n },\r\n {\r\n \"name\": \"recptr\",\r\n \"type\": \"ty4\",\r\n \"flags\": 266\r\n },\r\n {\r\n"
+ " \"name\": \"isactive\",\r\n \"type\": \"ty15\",\r\n \"flags\": 65536\r\n },\r\n {\r\n \"name\": \"__internal_fpos__\",\r\n \"type\": \"ty16\",\r\n"
+ " \"flags\": 65821\r\n }\r\n ]\r\n}";

/**
* Gets the complex record definition json.
*
Expand All @@ -56,6 +74,11 @@ public static String getComplexRecordDefinitionJson()
return complexRecordDefinitionStr;
}

public static String getAllTypesIndexRecordDefinitionJson()
{
return allTypesIndexRecordDefinitionStr;
}

/**
* Gets the complex record definition.
*
Expand Down
42 changes: 23 additions & 19 deletions commons-hpcc/src/test/java/org/hpccsystems/commons/ecl/RecordDefinitionTests.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,30 +38,34 @@ public void setup()
@Test
public void testJsonRecordParsing() throws Exception
{
String jsonRecDefStr = TestFieldDefinitions.getComplexRecordDefinitionJson();
try
String[] recordDefStrs = new String[] { TestFieldDefinitions.getComplexRecordDefinitionJson(),
TestFieldDefinitions.getAllTypesIndexRecordDefinitionJson() };
for (String recordDefStr : recordDefStrs)
{
JSONObject jsonRecDef = new JSONObject(jsonRecDefStr);
FieldDef recDef = RecordDefinitionTranslator.parseJsonRecordDefinition(jsonRecDef);
try
{
JSONObject expectedRefDef = new JSONObject(recordDefStr);

JSONObject tJsonRecDef = RecordDefinitionTranslator.toJsonRecord(recDef);
FieldDef recDef = RecordDefinitionTranslator.parseJsonRecordDefinition(expectedRefDef);
JSONObject actualRecDef = RecordDefinitionTranslator.toJsonRecord(recDef);

if (tJsonRecDef.equals(jsonRecDef))
// Use similar instead of equals because the order of fields in the JSON object may differ
if (!actualRecDef.similar(expectedRefDef))
{
System.out.println("Expected Record Def: " + expectedRefDef + "\n");
System.out.println("Actual Record Def: " + actualRecDef + "\n");
Assert.fail("Translated JSON record definition differs from original");
}
}
catch (UnparsableContentException e)
{
System.out.println(jsonRecDef + "\n");
System.out.println(tJsonRecDef + "\n");
Assert.fail("Translated JSON record definition differs from original");
Assert.fail("Encountered invalid record definition: '" + recordDefStr + "'");
}
catch (Exception e)
{
System.out.println(e.getMessage());
Assert.fail("Exception occurred: '" + e.getMessage() + "'");
}
}
catch (UnparsableContentException e)
{
Assert.fail("Encountered invalid record definition: '" + jsonRecDefStr + "'");
}
catch (Exception e)
{
System.out.println(e.getMessage());
Assert.fail("Exception occurred: '" + e.getMessage() + "'");
}

}
Expand Down
Loading
Loading