test(test_vectors): Support reading manifests that specify a hierarchy keyring

.github/workflows/library_interop_tests.yml

@@ -76,6 +76,14 @@ jobs:
           # This works because `node` is installed by default on GHA runners
           CORES=$(node -e 'console.log(os.cpus().length)')
           make transpile_net CORES=$CORES
+
+      - name: Compile MPL TestVectors implementation
+        shell: bash
+        working-directory: ./mpl/TestVectorsAwsCryptographicMaterialProviders
+        run: |
+          # This works because `node` is installed by default on GHA runners
+          CORES=$(node -e 'console.log(os.cpus().length)')
+          make transpile_net CORES=$CORES
 
       - name: Fetch Python 2.3.0 Test Vectors
         working-directory: ./
@@ -166,6 +174,15 @@ jobs:
           # This works because `node` is installed by default on GHA runners
           CORES=$(node -e 'console.log(os.cpus().length)')
           make transpile_net CORES=$CORES
+
+
+      - name: Compile MPL TestVectors implementation
+        shell: bash
+        working-directory: ./mpl/TestVectorsAwsCryptographicMaterialProviders
+        run: |
+          # This works because `node` is installed by default on GHA runners
+          CORES=$(node -e 'console.log(os.cpus().length)')
+          make transpile_net CORES=$CORES
 
 
       # # TODO: Fix Zip file creation on Windows
@@ -177,6 +194,16 @@ jobs:
       # #     Set-Location -Path "$env:GITHUB_WORKSPACE\net41\vectors"
       # #     Compress-Archive -Path "$env:GITHUB_WORKSPACE\net41\vectors\*" -DestinationPath "$env:GITHUB_WORKSPACE\net41\vectors\net41.zip"        
 
+      - name: Fetch Python 2.3.0 Test Vectors
+        working-directory: ./
+        shell: bash
+        run: |
+          PYTHON_23_VECTOR_PATH=$GITHUB_WORKSPACE/python23/vectors
+          mkdir -p $PYTHON_23_VECTOR_PATH
+          DOWNLOAD_NAME=python23.zip
+          curl --no-progress-meter --output $DOWNLOAD_NAME --location $VECTORS_URL
+          unzip -o -qq $DOWNLOAD_NAME  -d $PYTHON_23_VECTOR_PATH
+          rm  $DOWNLOAD_NAME
 
       - name: Generate Test Vectors with .NET Framework net6.0
         # TODO Post-#619: Fix Zip file creation on Windows
@@ -187,6 +214,8 @@ jobs:
           NET_41_VECTOR_PATH=net41/vectors
           mkdir -p $NET_41_VECTOR_PATH
           GEN_PATH=runtimes/net/TestVectorsNative/TestVectorGenerator
+          PYTHON_23_VECTOR_PATH=$GITHUB_WORKSPACE/python23/vectors
+          DAFNY_AWS_ESDK_TEST_VECTOR_MANIFEST_PATH="$PYTHON_23_VECTOR_PATH/manifest.json" \
           dotnet run --project $GEN_PATH --framework net6.0 -- \
             --encrypt-manifest $GEN_PATH/resources/0006-awses-message-decryption-generation.v2.json \
             --output-dir $NET_41_VECTOR_PATH

.github/workflows/library_net_tests.yml

@@ -92,6 +92,15 @@ jobs:
           CORES=$(node -e 'console.log(os.cpus().length)')
           make transpile_net CORES=$CORES
 
+
+      - name: Compile MPL TestVectors implementation
+        shell: bash
+        working-directory: ./mpl/TestVectorsAwsCryptographicMaterialProviders
+        run: |
+          # This works because `node` is installed by default on GHA runners
+          CORES=$(node -e 'console.log(os.cpus().length)')
+          make transpile_net CORES=$CORES
+
       - name: Test .NET Framework net48
         working-directory: ./AwsEncryptionSDK
         if: matrix.os == 'windows-latest' 

AwsEncryptionSDK/runtimes/net/TestVectorsNative/TestVectorLib/AWSEncryptionSDKTestVectorLib.csproj

@@ -10,6 +10,9 @@
     <ItemGroup>
         <PackageReference Include="Newtonsoft.Json" Version="13.0.1" />
         <ProjectReference Include="../../ESDK.csproj" />
+
+        <!-- TODO: Reference published MPL TestVectors project -->
+        <ProjectReference Include="../../../../../mpl/TestVectorsAwsCryptographicMaterialProviders/runtimes/net/TestVectors.csproj" />
     </ItemGroup>
 
 </Project>

AwsEncryptionSDK/runtimes/net/TestVectorsNative/TestVectorLib/MaterialProviderFactory.cs

@@ -2,9 +2,13 @@
 // SPDX-License-Identifier: Apache-2.0
 
 using System.Diagnostics;
+using Newtonsoft.Json;
 using Amazon;
+using Amazon.DynamoDBv2;
 using Amazon.KeyManagementService;
+using AWS.Cryptography.KeyStore;
 using AWS.Cryptography.MaterialProviders;
+using AWS.Cryptography.MaterialProvidersTestVectorKeys;
 
 using RSAEncryption;
 
@@ -18,6 +22,14 @@
     public static class MaterialProviderFactory
     {
         private static readonly MaterialProviders materialProviders = new(new MaterialProvidersConfig());
+        // TODO: Get this from CLI or something?
+        private static string manifestPath = Utils.GetEnvironmentVariableOrError("DAFNY_AWS_ESDK_TEST_VECTOR_MANIFEST_PATH");
+        private static DecryptManifest manifest = Utils.LoadObjectFromPath<DecryptManifest>(manifestPath);
+        private static readonly KeyVectorsConfig keyVectorsConfig = new KeyVectorsConfig
+        {
+            KeyManifestPath = Utils.ManifestUriToPath(manifest.KeysUri, manifestPath)
+        };
+        private static KeyVectors keyVectors = new(keyVectorsConfig);
 
         public static ICryptographicMaterialsManager CreateDecryptCmm(
             DecryptVector vector,
@@ -55,11 +67,11 @@
         private static IKeyring CreateDecryptKeyring(DecryptVector vector, Dictionary<string, Key> keys) {
             List<IKeyring> children = new List<IKeyring>();
             Debug.Assert(vector.MasterKeys != null, "vector.MasterKeys != null");
             foreach (MasterKey keyInfo in vector.MasterKeys)
             {
                 // Some keyrings, like discovery KMS keyrings, do not specify keys
                 Key key = keyInfo.Key == null ? null : keys[keyInfo.Key];
                 children.Add(CreateKeyring(keyInfo, key, CryptoOperation.DECRYPT));
             }
             CreateMultiKeyringInput createMultiKeyringInput = new CreateMultiKeyringInput
             {
@@ -102,14 +114,14 @@
             IList<MasterKey> masterKeys = vector.Scenario.MasterKeys;
             Debug.Assert(masterKeys.Count >= 1);
 
             Key generatorKey = keys[masterKeys[0].Key];
             IKeyring generatorKeyring = CreateKeyring(masterKeys[0], generatorKey, CryptoOperation.ENCRYPT);
 
             List<IKeyring> children = masterKeys
                 .Skip(1)
                 .Select(masterKey =>
                 {
                     Key key = keys[masterKey.Key];
                     return CreateKeyring(masterKey, key, CryptoOperation.ENCRYPT);
                 })
                 .ToList();
@@ -141,12 +153,12 @@
             }
 
             if (keyInfo.Type == "aws-kms-mrk-aware-discovery" && operation == CryptoOperation.DECRYPT) {
                 AWS.Cryptography.MaterialProviders.DiscoveryFilter filter = null;
                 if (keyInfo.AwsKmsDiscoveryFilter != null)
                 {
                     filter = new AWS.Cryptography.MaterialProviders.DiscoveryFilter
                     {
                         AccountIds = (List<string>)keyInfo.AwsKmsDiscoveryFilter.AccountIds,
                         Partition = keyInfo.AwsKmsDiscoveryFilter.Partition,
                     };
                 }
@@ -160,6 +172,36 @@
                 return materialProviders.CreateAwsKmsMrkDiscoveryKeyring(createKeyringInput);
             }
 
+            if (keyInfo.Type == "aws-kms-hierarchy") {
+                // Convert JSON to bytes for KeyVectors input
+                string jsonString = JsonConvert.SerializeObject(keyInfo);
+
+                var stream = new MemoryStream();
+                var writer = new StreamWriter(stream);
+                writer.Write(jsonString);
+                writer.Flush();
+                stream.Position = 0;
+
+                // Create KeyVectors keyring
+                var getKeyDescriptionInput = new GetKeyDescriptionInput
+                {
+                    Json = stream
+                };
+
+                var desc = keyVectors.GetKeyDescription(getKeyDescriptionInput);
+
+                var testVectorKeyringInput = new TestVectorKeyringInput
+                {
+                    KeyDescription = desc.KeyDescription
+                };
+
+                var keyring = keyVectors.CreateTestVectorKeyring(
+                    testVectorKeyringInput
+                );
+
+                return keyring!;
+            }
+
             if (keyInfo.Type == "raw" && keyInfo.EncryptionAlgorithm == "aes") {
                 CreateRawAesKeyringInput createKeyringInput = new CreateRawAesKeyringInput
                 {
@@ -173,7 +215,7 @@
             }
 
             if (keyInfo.Type == "raw" && keyInfo.EncryptionAlgorithm == "rsa" && key.Type == "private") {
                 PaddingScheme padding = RSAPaddingFromStrings(keyInfo.PaddingAlgorithm, keyInfo.PaddingHash);
                 byte[] privateKey = RSA.ParsePEMString(key.Material);
                 CreateRawRsaKeyringInput createKeyringInput = new CreateRawRsaKeyringInput
                 {
@@ -209,7 +251,7 @@
             // string operationStr = operation == CryptoOperation.ENCRYPT
             //     ? "encryption"
             //     : "decryption";
-            throw new Exception($"Unsupported keyring type for {operation}");
+            throw new Exception($"Unsupported keyring {keyInfo.Type} type for {operation}");
         }
 
         private static AesWrappingAlg AesAlgorithmFromBits(ushort bits) {

AwsEncryptionSDK/runtimes/net/TestVectorsNative/TestVectorLib/TestVectorTypes.cs

@@ -25,6 +25,12 @@ public class Key {
         public string? Encoding { get; set; }
         [JsonProperty("material")]
         public string? Material { get; set; }
+        [JsonProperty("branchKeyVersion")]
+        public string? BranchKeyVersion { get; set; }
+        [JsonProperty("branchKey")]
+        public string? BranchKey { get; set; }
+        [JsonProperty("beaconKey")]
+        public string? BeaconKey { get; set; }
     }
 
     public class KeyManifest