fix(sdk): remove hex encoding

cmdline/src/main/java/io/opentdf/platform/Command.java

@@ -52,10 +52,16 @@
 
 import javax.net.ssl.TrustManager;
 
-
-@CommandLine.Command(name = "tdf", subcommands = {HelpCommand.class})
+@CommandLine.Command(
+    name = "tdf",
+    subcommands = {HelpCommand.class},
+    version = "{\"version\":\"0.7.5\",\"tdfSpecVersion\":\"4.3.0\"}"
+)
 class Command {
 
+    @Option(names = {"-V", "--version"}, versionHelp = true, description = "display version info")
+    boolean versionInfoRequested;
+
     private static final String PRIVATE_KEY_HEADER = "-----BEGIN PRIVATE KEY-----";
     private static final String PRIVATE_KEY_FOOTER = "-----END PRIVATE KEY-----";
     private static final String PEM_HEADER = "-----BEGIN (.*)-----";
@@ -150,7 +156,7 @@ void encrypt(
             @Option(names = { "--mime-type" }, defaultValue = Option.NULL_VALUE) Optional<String> mimeType,
             @Option(names = { "--with-assertions" }, defaultValue = Option.NULL_VALUE) Optional<String> assertion)
 
-            throws IOException, JOSEException, AutoConfigureException, InterruptedException, ExecutionException {
+            throws IOException, JOSEException, AutoConfigureException, InterruptedException, ExecutionException, DecoderException {
 
         var sdk = buildSDK();
         var kasInfos = kas.stream().map(k -> {

examples/src/main/java/io/opentdf/platform/EncryptExample.java

@@ -6,11 +6,13 @@
 import java.io.FileOutputStream;
 
 import com.nimbusds.jose.JOSEException;
+import org.apache.commons.codec.DecoderException;
+
 import java.io.IOException;
 import java.util.concurrent.ExecutionException;
 
 public class EncryptExample {
-    public static void main(String[] args) throws IOException, JOSEException, AutoConfigureException, InterruptedException, ExecutionException {
+    public static void main(String[] args) throws IOException, JOSEException, AutoConfigureException, InterruptedException, ExecutionException, DecoderException {
         String clientId = "opentdf";
         String clientSecret = "secret";
         String platformEndpoint = "localhost:8080";

sdk/pom.xml

@@ -44,6 +44,11 @@
             <artifactId>gson</artifactId>
             <version>2.11.0</version>
         </dependency>
+        <dependency>
+            <groupId>com.vdurmont</groupId>
+            <artifactId>semver4j</artifactId>
+            <version>3.1.0</version>
+        </dependency>
         <dependency>
             <groupId>commons-codec</groupId>
             <artifactId>commons-codec</artifactId>

sdk/src/main/java/io/opentdf/platform/sdk/Manifest.java

@@ -1,18 +1,28 @@
 package io.opentdf.platform.sdk;
 
-import com.google.gson.*;
+import com.google.gson.Gson;
+import com.google.gson.GsonBuilder;
+import com.google.gson.JsonDeserializationContext;
+import com.google.gson.JsonDeserializer;
+import com.google.gson.JsonElement;
+import com.google.gson.JsonParseException;
+import com.google.gson.JsonSerializationContext;
+import com.google.gson.JsonSerializer;
 import com.google.gson.annotations.JsonAdapter;
 import com.google.gson.annotations.SerializedName;
-import com.nimbusds.jose.*;
+import com.nimbusds.jose.JOSEException;
+import com.nimbusds.jose.JWSAlgorithm;
+import com.nimbusds.jose.JWSHeader;
+import com.nimbusds.jose.JWSSigner;
+import com.nimbusds.jose.JWSVerifier;
+import com.nimbusds.jose.KeyLengthException;
 import com.nimbusds.jose.crypto.MACSigner;
 import com.nimbusds.jose.crypto.MACVerifier;
 import com.nimbusds.jose.crypto.RSASSASigner;
 import com.nimbusds.jose.crypto.RSASSAVerifier;
 import com.nimbusds.jwt.JWTClaimsSet;
 import com.nimbusds.jwt.SignedJWT;
-
 import io.opentdf.platform.sdk.TDF.AssertionException;
-
 import org.apache.commons.codec.binary.Hex;
 import org.erdtman.jcs.JsonCanonicalizer;
 
@@ -44,21 +54,20 @@ public class Manifest {
     private static final Gson gson = new GsonBuilder()
                                         .registerTypeAdapter(Manifest.class, new ManifestDeserializer())
                                         .create();
+    @SerializedName(value = "schemaVersion")
+    String tdfVersion;
 
     @Override
     public boolean equals(Object o) {
-        if (this == o)
-            return true;
-        if (o == null || getClass() != o.getClass())
-            return false;
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
         Manifest manifest = (Manifest) o;
-        return Objects.equals(encryptionInformation, manifest.encryptionInformation)
-                && Objects.equals(payload, manifest.payload) && Objects.equals(assertions, manifest.assertions);
+        return Objects.equals(tdfVersion, manifest.tdfVersion) && Objects.equals(encryptionInformation, manifest.encryptionInformation) && Objects.equals(payload, manifest.payload) && Objects.equals(assertions, manifest.assertions);
     }
 
     @Override
     public int hashCode() {
-        return Objects.hash(encryptionInformation, payload, assertions);
+        return Objects.hash(tdfVersion, encryptionInformation, payload, assertions);
     }
 
     private static class PolicyBindingSerializer implements JsonDeserializer<Object>, JsonSerializer<Object> {
@@ -183,6 +192,7 @@ static public class KeyAccess {
         public String encryptedMetadata;
         public String kid;
         public String sid;
+        public String schemaVersion;
 
         @Override
         public boolean equals(Object o) {
@@ -195,12 +205,13 @@ public boolean equals(Object o) {
                     && Objects.equals(protocol, keyAccess.protocol) && Objects.equals(wrappedKey, keyAccess.wrappedKey)
                     && Objects.equals(policyBinding, keyAccess.policyBinding)
                     && Objects.equals(encryptedMetadata, keyAccess.encryptedMetadata)
-                    && Objects.equals(kid, keyAccess.kid);
+                    && Objects.equals(kid, keyAccess.kid)
+                    && Objects.equals(schemaVersion, keyAccess.schemaVersion);
         }
 
         @Override
         public int hashCode() {
-            return Objects.hash(keyType, url, protocol, wrappedKey, policyBinding, encryptedMetadata, kid);
+            return Objects.hash(keyType, url, protocol, wrappedKey, policyBinding, encryptedMetadata, kid, schemaVersion);
         }
     }
 

sdk/src/main/java/io/opentdf/platform/sdk/TDF.java

@@ -17,11 +17,9 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import java.io.BufferedReader;
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
 import java.io.InputStream;
-import java.io.InputStreamReader;
 import java.io.OutputStream;
 import java.io.StringReader;
 import java.nio.channels.SeekableByteChannel;
@@ -39,6 +37,8 @@
  */
 public class TDF {
 
+    private static final String TDF_VERSION = "4.3.0";
+    private static final String KEY_ACCESS_SECHMA_VERSION = "4.3.0";
     private final long maximumSize;
 
     /**
@@ -203,6 +203,7 @@ PolicyObject createPolicyObject(List<Autoconfigure.AttributeValueFQN> attributes
         private static final Base64.Encoder encoder = Base64.getEncoder();
 
         private void prepareManifest(Config.TDFConfig tdfConfig, SDK.KAS kas) {
+            manifest.tdfVersion = TDF_VERSION;
             manifest.encryptionInformation.keyAccessType = kSplitKeyType;
             manifest.encryptionInformation.keyAccessObj = new ArrayList<>();
 
@@ -307,6 +308,7 @@ private void prepareManifest(Config.TDFConfig tdfConfig, SDK.KAS kas) {
                     keyAccess.wrappedKey = encoder.encodeToString(wrappedKey);
                     keyAccess.encryptedMetadata = encryptedMetadata;
                     keyAccess.sid = splitID;
+                    keyAccess.schemaVersion = KEY_ACCESS_SECHMA_VERSION;
 
                     manifest.encryptionInformation.keyAccessObj.add(keyAccess);
                 }
@@ -370,6 +372,8 @@ public void readPayload(OutputStream outputStream) throws TDFReadFailed,
                     throw new TDFReadFailed("failed to read payload");
                 }
 
+                var isLegacyTdf = manifest.tdfVersion == null || manifest.tdfVersion.isEmpty();
+
                 if (manifest.payload.isEncrypted) {
                     String segHashAlg = manifest.encryptionInformation.integrityInformation.segmentHashAlg;
                     Config.IntegrityAlgorithm sigAlg = Config.IntegrityAlgorithm.HS256;
@@ -378,9 +382,11 @@ public void readPayload(OutputStream outputStream) throws TDFReadFailed,
                     }
 
                     var payloadSig = calculateSignature(readBuf, payloadKey, sigAlg);
-                    byte[] payloadSigAsBytes = payloadSig.getBytes(StandardCharsets.UTF_8);
+                    if (isLegacyTdf) {
+                        payloadSig = Hex.encodeHexString(payloadSig).getBytes(StandardCharsets.UTF_8);
+                    }
 
-                    if (segment.hash.compareTo(Base64.getEncoder().encodeToString(payloadSigAsBytes)) != 0) {
+                    if (segment.hash.compareTo(Base64.getEncoder().encodeToString(payloadSig)) != 0) {
                         throw new SegmentSignatureMismatch("segment signature miss match");
                     }
 
@@ -403,24 +409,22 @@ public PolicyObject readPolicyObject() {
         }
     }
 
-    private static String calculateSignature(byte[] data, byte[] secret, Config.IntegrityAlgorithm algorithm) {
+    private static byte[] calculateSignature(byte[] data, byte[] secret, Config.IntegrityAlgorithm algorithm) {
         if (algorithm == Config.IntegrityAlgorithm.HS256) {
-            byte[] hmac = CryptoUtils.CalculateSHA256Hmac(secret, data);
-            return Hex.encodeHexString(hmac);
+            return CryptoUtils.CalculateSHA256Hmac(secret, data);
         }
 
         if (kGMACPayloadLength > data.length) {
             throw new FailedToCreateGMAC("fail to create gmac signature");
         }
 
-        byte[] gmacPayload = Arrays.copyOfRange(data, data.length - kGMACPayloadLength, data.length);
-        return Hex.encodeHexString(gmacPayload);
+        return Arrays.copyOfRange(data, data.length - kGMACPayloadLength, data.length);
     }
 
     public TDFObject createTDF(InputStream payload,
             OutputStream outputStream,
             Config.TDFConfig tdfConfig, SDK.KAS kas, AttributesServiceFutureStub attrService)
-            throws IOException, JOSEException, AutoConfigureException, InterruptedException, ExecutionException {
+            throws IOException, JOSEException, AutoConfigureException, InterruptedException, ExecutionException, DecoderException {
 
         if (tdfConfig.autoconfigure) {
             Autoconfigure.Granter granter = new Autoconfigure.Granter(new ArrayList<>());
@@ -454,7 +458,7 @@ public TDFObject createTDF(InputStream payload,
         long encryptedSegmentSize = tdfConfig.defaultSegmentSize + kGcmIvSize + kAesBlockSize;
         TDFWriter tdfWriter = new TDFWriter(outputStream);
 
-        StringBuilder aggregateHash = new StringBuilder();
+        ByteArrayOutputStream aggregateHash = new ByteArrayOutputStream();
         byte[] readBuf = new byte[tdfConfig.defaultSegmentSize];
 
         tdfObject.manifest.encryptionInformation.integrityInformation.segments = new ArrayList<>();
@@ -476,17 +480,17 @@ public TDFObject createTDF(InputStream payload,
                 }
 
                 byte[] cipherData;
-                String segmentSig;
+                byte[] segmentSig;
                 Manifest.Segment segmentInfo = new Manifest.Segment();
 
                 // encrypt
                 cipherData = tdfObject.aesGcm.encrypt(readBuf, 0, readThisLoop).asBytes();
                 payloadOutput.write(cipherData);
 
                 segmentSig = calculateSignature(cipherData, tdfObject.payloadKey, tdfConfig.segmentIntegrityAlgorithm);
-                segmentInfo.hash = Base64.getEncoder().encodeToString(segmentSig.getBytes(StandardCharsets.UTF_8));
+                segmentInfo.hash = Base64.getEncoder().encodeToString(segmentSig);
 
-                aggregateHash.append(segmentSig);
+                aggregateHash.write(segmentSig);
                 segmentInfo.segmentSize = readThisLoop;
                 segmentInfo.encryptedSegmentSize = cipherData.length;
 
@@ -496,9 +500,9 @@ public TDFObject createTDF(InputStream payload,
 
         Manifest.RootSignature rootSignature = new Manifest.RootSignature();
 
-        String rootSig = calculateSignature(aggregateHash.toString().getBytes(),
+        byte[] rootSig = calculateSignature(aggregateHash.toByteArray(),
                 tdfObject.payloadKey, tdfConfig.integrityAlgorithm);
-        rootSignature.signature = Base64.getEncoder().encodeToString(rootSig.getBytes(StandardCharsets.UTF_8));
+        rootSignature.signature = Base64.getEncoder().encodeToString(rootSig);
 
         String alg = kGmacIntegrityAlgorithm;
         if (tdfConfig.integrityAlgorithm == Config.IntegrityAlgorithm.HS256) {
@@ -534,19 +538,24 @@ public TDFObject createTDF(InputStream payload,
             assertion.statement = assertionConfig.statement;
             assertion.appliesToState = assertionConfig.appliesToState.toString();
 
-            var assertionHash = assertion.hash();
-            var completeHashBuilder = new StringBuilder(aggregateHash);
-            completeHashBuilder.append(assertionHash);
+            var assertionHashAsHex = assertion.hash();
+            var assertionHash = Hex.decodeHex(assertionHashAsHex);
+            byte[] completeHash = new byte[aggregateHash.size() + assertionHash.length];
+            System.arraycopy(aggregateHash.toByteArray(), 0, completeHash, 0, aggregateHash.size());
+            System.arraycopy(assertionHash, 0, completeHash, aggregateHash.size(), assertionHash.length);
 
-            var encodedHash = Base64.getEncoder().encodeToString(completeHashBuilder.toString().getBytes());
+            var encodedHash = Base64.getEncoder().encodeToString(completeHash);
 
             var assertionSigningKey = new AssertionConfig.AssertionKey(AssertionConfig.AssertionKeyAlg.HS256,
                     tdfObject.aesGcm.getKey());
             if (assertionConfig.signingKey != null && assertionConfig.signingKey.isDefined()) {
                 assertionSigningKey = assertionConfig.signingKey;
             }
-
-            assertion.sign(new Manifest.Assertion.HashValues(assertionHash, encodedHash), assertionSigningKey);
+            var hashValues = new Manifest.Assertion.HashValues(
+                    assertionHashAsHex,
+                    encodedHash
+            );
+            assertion.sign(hashValues, assertionSigningKey);
             signedAssertions.add(assertion);
         }
 
@@ -594,7 +603,7 @@ public Reader loadTDF(SeekableByteChannel tdf, SDK.KAS kas,
         byte[] payloadKey = new byte[GCM_KEY_SIZE];
         String unencryptedMetadata = null;
 
-        Set<String> knownSplits = new HashSet<String>();
+        Set<String> knownSplits = new HashSet<>();
         Set<String> foundSplits = new HashSet<>();
 
         Map<Autoconfigure.KeySplitStep, Exception> skippedSplits = new HashMap<>();
@@ -632,7 +641,7 @@ public Reader loadTDF(SeekableByteChannel tdf, SDK.KAS kas,
                     AesGcm aesGcm = new AesGcm(unwrappedKey);
 
                     String decodedMetadata = new String(Base64.getDecoder().decode(keyAccess.encryptedMetadata),
-                            "UTF-8");
+                            StandardCharsets.UTF_8);
                     EncryptedMetadata encryptedMetadata = gson.fromJson(decodedMetadata, EncryptedMetadata.class);
 
                     var encryptedData = new AesGcm.Encrypted(
@@ -679,15 +688,18 @@ public Reader loadTDF(SeekableByteChannel tdf, SDK.KAS kas,
         }
 
         String rootSigValue;
+        boolean isLegacyTdf = manifest.tdfVersion == null || manifest.tdfVersion.isEmpty();
         if (manifest.payload.isEncrypted) {
             Config.IntegrityAlgorithm sigAlg = Config.IntegrityAlgorithm.HS256;
             if (rootAlgorithm.compareToIgnoreCase(kGmacIntegrityAlgorithm) == 0) {
                 sigAlg = Config.IntegrityAlgorithm.GMAC;
             }
 
             var sig = calculateSignature(aggregateHash.toByteArray(), payloadKey, sigAlg);
-            rootSigValue = Base64.getEncoder().encodeToString(sig.getBytes(StandardCharsets.UTF_8));
-
+            if (isLegacyTdf) {
+                sig = Hex.encodeHexString(sig).getBytes();
+            }
+            rootSigValue = Base64.getEncoder().encodeToString(sig);
         } else {
             rootSigValue = Base64.getEncoder().encodeToString(digest.digest(aggregateHash.toString().getBytes()));
         }
@@ -703,6 +715,7 @@ public Reader loadTDF(SeekableByteChannel tdf, SDK.KAS kas,
             throw new SegmentSizeMismatch("mismatch encrypted segment size in manifest");
         }
 
+        var aggregateHashByteArrayBytes = aggregateHash.toByteArray();
         // Validate assertions
         for (var assertion : manifest.assertions) {
             // Skip assertion verification if disabled
@@ -721,16 +734,18 @@ public Reader loadTDF(SeekableByteChannel tdf, SDK.KAS kas,
             }
 
             var hashValues = assertion.verify(assertionKey);
-            var assertionAsJson = gson.toJson(assertion);
-            JsonCanonicalizer jc = new JsonCanonicalizer(assertionAsJson);
-            var hashOfAssertion = Hex.encodeHexString(digest.digest(jc.getEncodedUTF8()));
-            var signature = aggregateHash + hashOfAssertion;
-            var encodeSignature = Base64.getEncoder().encodeToString(signature.getBytes());
+            var hashOfAssertionAsHex = assertion.hash();
 
-            if (!Objects.equals(hashOfAssertion, hashValues.getAssertionHash())) {
+            if (!Objects.equals(hashOfAssertionAsHex, hashValues.getAssertionHash())) {
                 throw new AssertionException("assertion hash mismatch", assertion.id);
             }
 
+            byte[] hashOfAssertion = isLegacyTdf ? hashOfAssertionAsHex.getBytes(StandardCharsets.UTF_8) : Hex.decodeHex(hashOfAssertionAsHex);
+            var signature = new byte[aggregateHashByteArrayBytes.length + hashOfAssertion.length];
+            System.arraycopy(aggregateHashByteArrayBytes, 0, signature, 0, aggregateHashByteArrayBytes.length);
+            System.arraycopy(hashOfAssertion, 0, signature, aggregateHashByteArrayBytes.length, hashOfAssertion.length);
+            var encodeSignature = Base64.getEncoder().encodeToString(signature);
+
             if (!Objects.equals(encodeSignature, hashValues.getSignature())) {
                 throw new AssertionException("failed integrity check on assertion signature", assertion.id);
             }