Skip to content

GH-3404: Canonicalize decimals during inlining for TDB2 #3428

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
wants to merge 3 commits into
base: main
Choose a base branch
from
Draft

GH-3404: Canonicalize decimals during inlining for TDB2 #3428

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
0bc4a1c
GH-3404: Canonicalize decimals during inlining for TDB2.
Aklakan Aug 29, 2025
1e41a02
GH-3404: Tidy up NodeId and DecimalNode56
afs Sep 8, 2025
cf91b67
GH-3404: Disable BigDecimal normalization
afs Sep 10, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 16 additions & 0 deletions jena-arq/src/main/java/org/apache/jena/sparql/util/XSDNumUtils.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -166,4 +166,20 @@ public static String canonicalDecimalStrWithDot(BigDecimal decimal) {
str = str + ".0";
return str;
}

/**
* Return a canonical decimal with a trailing ".0".
* This is the BigDecimal form used to encode NodeIds in TDB2.
* It has a trailing ".0" so it is Turtle compatible.
*/
public static BigDecimal canonicalDecimal(BigDecimal decimal) {
BigDecimal result = decimal;
if (result.scale() > 1) {
result = decimal.stripTrailingZeros();
}
if (result.scale() <= 0) {
result = result.setScale(1);
}
return result;
}
}
24 changes: 18 additions & 6 deletions jena-tdb2/src/main/java/org/apache/jena/tdb2/store/NodeId.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -104,15 +104,24 @@ private final void check(NodeIdType type, int v1, long v2) {
// public long getPtrLo() { return value2; }
// public int getPtrHi() { return value1 & 0x00FFFFFF; }

// 64 bit
/** The pointer part of a NodeId reference. */
public long getPtrLocation() { return value2; }
// Long.

// 96 bit
// public long getPtrLo() { return value2; }
// public int getPtrHi() { return value1; }

public int getTypeValue() { return type.type(); }
// 64 bit.
public int getTypeValue() {
return type.type();
}

/** The value (encoding) part of an inline literal (56 bits) */
public long getValue56() {
return value2;
}

public boolean isInline() {
public boolean isInline() {
return isInline(this);
}

Expand Down Expand Up @@ -141,8 +150,11 @@ public boolean isValue() {
// public static boolean isDefined(NodeId nodeId) { return nodeId == NodeIdDefined; }
// public static boolean isUndefined(NodeId nodeId) { return nodeId == NodeIdUndefined; }

/** Create from a long-encoded value */
/*package*/ static NodeId createRaw(NodeIdType type, long value) {
/**
* Create from a long-encoded value.
* Caution: an illegal value for the long argument will cause serious problems.
*/
public static NodeId createRaw(NodeIdType type, long value) {
return new NodeId(type, 0, value);
}

Expand Down
2 changes: 1 addition & 1 deletion jena-tdb2/src/main/java/org/apache/jena/tdb2/store/NodeIdFactory.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -68,7 +68,7 @@ public static NodeId createPtr(long lo) {
return createNew(PTR, 0, lo);
}

/*package*/ /*long*/ static NodeId createPtrLong(int hi, long lo) {
/*package*/ static NodeId createPtrLong(int hi, long lo) {
return create(PTR, hi, lo);
}

Expand Down
4 changes: 4 additions & 0 deletions jena-tdb2/src/main/java/org/apache/jena/tdb2/store/NodeIdInline.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -164,6 +164,10 @@ public static boolean hasInlineDatatype(Node node) {
// chopping by .trim is safe.
BigDecimal decimal = new BigDecimal(lit.getLexicalForm().trim());

// [Jena6] - enable for Jena6
if ( false )
decimal = XSDNumUtils.canonicalDecimal(decimal);

// Does range checking.
DecimalNode56 dn = DecimalNode56.valueOf(decimal);
// null is "does not fit"
Expand Down
44 changes: 21 additions & 23 deletions jena-tdb2/src/main/java/org/apache/jena/tdb2/store/value/DecimalNode56.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,41 +23,39 @@

import org.apache.jena.atlas.lib.BitsLong;


// Decimal packed into 56 bits.
public class DecimalNode56
{
//private static Logger log = LoggerFactory.getLogger(DecimalNode.class);

BigDecimal decimal = null;

// signed 8 bits of scale, signed 48 bits of value.
// Decimal precision is 47 bits (it's signed) or around 14 places.
// Not finance industry accuracy nor XSD (18 places minimum) but still useful.
// 56 bits signed 8 bits of scale, signed 48 bits of value.
// Decimal precision is 47 bits (it's signed) or around 13 places (canonical form).
// This is not finance industry accuracy nor XSD (18 places minimum) but still useful.

static final int SCALE_LEN = 8;
static final int VALUE_LEN = 48;
static final int ENC_LEN = 48 + SCALE_LEN;
private static final int SCALE_LEN = 8;
private static final int VALUE_LEN = 48;
private static final int ENC_LEN = 48 + SCALE_LEN;

static final long MAX_VALUE = (1L << (VALUE_LEN - 1)) - 1;
static final long MIN_VALUE = -(1L << (VALUE_LEN - 1));
private static final long MAX_VALUE = (1L << (VALUE_LEN - 1)) - 1;
private static final long MIN_VALUE = -(1L << (VALUE_LEN - 1));

static final int MAX_SCALE = (1 << (SCALE_LEN - 1)) - 1;
static final int MIN_SCALE = -(1 << (SCALE_LEN - 1));
private static final int MAX_SCALE = (1 << (SCALE_LEN - 1)) - 1;
private static final int MIN_SCALE = -(1 << (SCALE_LEN - 1));

static final BigInteger MAX_I = BigInteger.valueOf(MAX_VALUE);
static final BigInteger MIN_I = BigInteger.valueOf(MIN_VALUE);
private static final BigInteger MAX_I = BigInteger.valueOf(MAX_VALUE);
private static final BigInteger MIN_I = BigInteger.valueOf(MIN_VALUE);

// Bits counts
static private int SCALE_LO = 56 - SCALE_LEN;
static private int SCALE_HI = 56; // Exclusive
// index
// Bits positions [LO, HI)
private static final int SCALE_LO = ENC_LEN - SCALE_LEN;
private static final int SCALE_HI = ENC_LEN;

static private int VALUE_LO = 0;
static private int VALUE_HI = VALUE_LO + VALUE_LEN;
private static final int VALUE_LO = 0;
private static final int VALUE_HI = VALUE_LO + VALUE_LEN;

private int scale;
private long value;
// Object fields.
private BigDecimal decimal = null;
private int scale;
private long value;

public static DecimalNode56 valueOf(BigDecimal decimal) {
int scale = decimal.scale();
Expand Down
67 changes: 59 additions & 8 deletions jena-tdb2/src/test/java/org/apache/jena/tdb2/store/value/TestNodeIdInline.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -91,35 +91,68 @@ public class TestNodeIdInline
@Test public void nodeId_int_21()
{ testNoInline("'300'^^xsd:byte"); }

@Test public void nodeId_decimal_1()
@Test public void nodeId_decimal_01()
{ test("3.14", "3.14"); }

@Test public void nodeId_decimal_2()
@Test public void nodeId_decimal_02()
{ testNoInline("123456789.123456789"); }

// Just this once, directly create the Node.
@Test public void nodeId_decimal_3()
@Test public void nodeId_decimal_03()
{ test("12.89", NodeFactory.createLiteralDT("12.89", XSDDatatype.XSDdecimal)); }

@Test public void nodeId_decimal_4()
@Test public void nodeId_decimal_04()
{ test("-1.0", "-1.0"); }

// This number has > 47 bits of value : 2412.80478192688
@Test public void nodeId_decimal_5()
@Test public void nodeId_decimal_05()
{ testNoInline("2412.80478192688"); }

// This number has > 47 bits of value : -2412.80478192688
@Test public void nodeId_decimal_6()
@Test public void nodeId_decimal_06()
{ testNoInline("-2412.80478192688"); }

@Test public void nodeId_decimal_7()
@Test public void nodeId_decimal_07()
{ test("'0.00000001'^^xsd:decimal",
NodeFactory.createLiteralDT("0.00000001", XSDDatatype.XSDdecimal));
}

@Test public void nodeId_decimal_8()
@Test public void nodeId_decimal_08()
{ test("0.00000001", NodeFactory.createLiteralDT("0.00000001", XSDDatatype.XSDdecimal)); }

@Test public void nodeId_decimal_09()
{ testNodeIdRoundtripDecimal("0"); }

@Test public void nodeId_decimal_10()
{ testNodeIdRoundtripDecimal("-0"); }

@Test public void nodeId_decimal_11()
{ testNodeIdRoundtripDecimal("0.0"); }

@Test public void nodeId_decimal_12()
{ testNodeIdRoundtripDecimal("-0.0"); }

@Test public void nodeId_decimal_13()
{ testNodeIdRoundtripDecimal(".0"); }

@Test public void nodeId_decimal_14()
{ testNodeIdRoundtripDecimal("-.0"); }

@Test public void nodeId_decimal_15()
{ testNodeIdRoundtripDecimal("18"); }

@Test public void nodeId_decimal_16()
{ testNodeIdRoundtripDecimal("18."); }

@Test public void nodeId_decimal_17()
{ testNodeIdRoundtripDecimal("18.0"); }

@Test public void nodeId_decimal_18()
{ testNodeIdRoundtripDecimal("18.00"); }

@Test public void nodeId_decimal_19()
{ testNodeIdRoundtripDecimal("18.000"); }

@Test public void nodeId_dateTime_01()
{ test("'2008-04-28T15:36:15+01:00'^^xsd:dateTime"); }

Expand Down Expand Up @@ -306,4 +339,22 @@ private void test(String x, Node correct) {
// Term equality.
assertEquals(correct, n2, ()->"Not same term");
}

private static void testNodeIdRoundtripDecimal(String decimalStr) {
Node node = NodeFactory.createLiteralDT(decimalStr, XSDDatatype.XSDdecimal);
testNodeIdRoundtrip(node);
}

/** For a Node n assert: nodeId(n) == nodeId(extract(nodeId(n)) */
private static void testNodeIdRoundtrip(Node node) {
NodeId nodeId = NodeId.inline(node);
testNodeIdRoundtrip(nodeId);
}

/** For a NodeId n assert: n == nodeId(extract(n)) */
private static void testNodeIdRoundtrip(NodeId expected) {
Node extractedNode = NodeId.extract(expected);
NodeId actual = NodeId.inline(extractedNode);
assertEquals(expected, actual);
}
}