From ebb94d1ac6a664bae6eb2100d64d58f0cacbff1f Mon Sep 17 00:00:00 2001 From: Fraser Waters Date: Wed, 29 Jan 2025 12:28:11 +0000 Subject: [PATCH 1/7] Add InputList tests --- .../Serialization/MarshalOutputTests.cs | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/sdk/Pulumi.Tests/Serialization/MarshalOutputTests.cs b/sdk/Pulumi.Tests/Serialization/MarshalOutputTests.cs index 9f7e70eb..9c315748 100644 --- a/sdk/Pulumi.Tests/Serialization/MarshalOutputTests.cs +++ b/sdk/Pulumi.Tests/Serialization/MarshalOutputTests.cs @@ -90,6 +90,21 @@ public sealed class BarArgs : ResourceArgs ImmutableArray.Empty.Add(CreateOutputValue("hello", isSecret: true)) }, new object[] + { + new InputList { "hello" }, + ImmutableArray.Empty.Add("hello") + }, + new object[] + { + new InputList { Output.Create("hello") }, + ImmutableArray.Empty.Add("hello") + }, + new object[] + { + new InputList { Output.CreateSecret("hello") }, + ImmutableArray.Empty.Add(CreateOutputValue("hello", isSecret: true)) + }, + new object[] { new Dictionary> { { "foo", "hello" } }, ImmutableDictionary.Empty.Add("foo", "hello") From c5e6278cfb1469e2219e7dce5d511c2c1659db1a Mon Sep 17 00:00:00 2001 From: Fraser Waters Date: Wed, 29 Jan 2025 13:34:09 +0000 Subject: [PATCH 2/7] Fix InputList to not flatten --- .../Serialization/MarshalOutputTests.cs | 10 ++++- sdk/Pulumi/Core/InputList.cs | 43 ++++++++++++++----- sdk/Pulumi/Serialization/Serializer.cs | 10 ++++- 3 files changed, 51 insertions(+), 12 deletions(-) diff --git a/sdk/Pulumi.Tests/Serialization/MarshalOutputTests.cs b/sdk/Pulumi.Tests/Serialization/MarshalOutputTests.cs index 9c315748..c0bd8f8b 100644 --- a/sdk/Pulumi.Tests/Serialization/MarshalOutputTests.cs +++ b/sdk/Pulumi.Tests/Serialization/MarshalOutputTests.cs @@ -102,7 +102,7 @@ public sealed class BarArgs : ResourceArgs new object[] { new InputList { Output.CreateSecret("hello") }, - ImmutableArray.Empty.Add(CreateOutputValue("hello", isSecret: true)) + ImmutableArray.Empty.Add(CreateSecretValue("hello")) }, new object[] { @@ -171,5 +171,13 @@ public static Task TestSerialize(object input, object expected) => RunInNormal(a if (deps.Length > 0) b.Add(Constants.DependenciesName, deps.ToImmutableArray()); return b.ToImmutableDictionary(); } + + private static ImmutableDictionary CreateSecretValue(object? value) + { + var b = ImmutableDictionary.CreateBuilder(); + b.Add(Constants.SpecialSigKey, Constants.SpecialSecretSig); + b.Add(Constants.ValueName, value); + return b.ToImmutableDictionary(); + } } } diff --git a/sdk/Pulumi/Core/InputList.cs b/sdk/Pulumi/Core/InputList.cs index f402728b..8ae1ce01 100644 --- a/sdk/Pulumi/Core/InputList.cs +++ b/sdk/Pulumi/Core/InputList.cs @@ -46,18 +46,28 @@ namespace Pulumi /// public sealed class InputList : Input>, IEnumerable, IAsyncEnumerable> { - public InputList() : this(Output.Create(ImmutableArray.Empty)) + Input>> _inputValue; + Input>> Value { + get => _inputValue; + set { + _inputValue = value; + _outputValue = _inputValue.Apply(inputs => Output.All(inputs)); + } + } + + public InputList() : this(ImmutableArray>.Empty) { } - private InputList(Output> values) - : base(values) + private InputList(Input>> values) + : base(values.Apply(values => Output.All(values))) { + _inputValue = values; } public void Add(params Input[] inputs) { - _outputValue = Concat(inputs); + Value = Concat(inputs).Value; } /// @@ -70,18 +80,24 @@ public void Add(InputList inputs) public void AddRange(InputList inputs) { - _outputValue = Concat(inputs); + Value = Concat(inputs).Value; } /// /// Concatenates the values in this list with the values in , /// returning the concatenated sequence in a new . /// - public InputList Concat(InputList other) - => Output.Concat(_outputValue, other._outputValue); + public InputList Concat(InputList other) { + var list = new InputList(); + list.Value = Output.Tuple(Value, other.Value).Apply(t => { + var (first, second) = t; + return first.AddRange(second); + }); + return list; + } internal InputList Clone() - => new InputList(_outputValue); + => new InputList(Value); #region construct from unary @@ -131,7 +147,7 @@ public static implicit operator InputList(ImmutableArray> values) => values.SelectAsArray(v => (Input)v); public static implicit operator InputList(ImmutableArray> values) - => Output.All(values); + => new InputList(values); #endregion @@ -147,7 +163,14 @@ public static implicit operator InputList(Output> values) => values.Apply(ImmutableArray.CreateRange); public static implicit operator InputList(Output> values) - => new InputList(values); + => new InputList(values.Apply(values => { + var builder = ImmutableArray.CreateBuilder>(values.Length); + foreach (var value in values) + { + builder.Add(value); + } + return builder.MoveToImmutable(); + })); #endregion diff --git a/sdk/Pulumi/Serialization/Serializer.cs b/sdk/Pulumi/Serialization/Serializer.cs index 2aa7b4c1..96236aab 100644 --- a/sdk/Pulumi/Serialization/Serializer.cs +++ b/sdk/Pulumi/Serialization/Serializer.cs @@ -103,6 +103,15 @@ prop is double || $"Tasks are not allowed inside ResourceArgs. Please wrap your Task in an Output:\n\t{ctx}"); } + // if prop is an InputList + var propType = prop.GetType(); + if (propType.IsGenericType && propType.GetGenericTypeDefinition() == typeof(InputList<>)) + { + // pull off the Value property from the InputList + var inputList = propType.GetProperty("Value", BindingFlags.NonPublic | BindingFlags.Instance)?.GetValue(prop); + return await SerializeAsync(ctx, inputList, keepResources, keepOutputValues).ConfigureAwait(false); + } + if (prop is IInput input) { if (_excessiveDebugOutput) @@ -289,7 +298,6 @@ prop is double || return null; } - var propType = prop.GetType(); if (propType.IsValueType && propType.GetCustomAttribute() != null) { var mi = propType.GetMethod("op_Explicit", BindingFlags.Public | BindingFlags.Static, null, new[] { propType }, null); From de4c3b46dd028230356ee4014a13733ad34a998c Mon Sep 17 00:00:00 2001 From: Fraser Waters Date: Wed, 29 Jan 2025 13:34:42 +0000 Subject: [PATCH 3/7] Add InputMap tests --- .../Serialization/MarshalOutputTests.cs | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/sdk/Pulumi.Tests/Serialization/MarshalOutputTests.cs b/sdk/Pulumi.Tests/Serialization/MarshalOutputTests.cs index c0bd8f8b..787cabf1 100644 --- a/sdk/Pulumi.Tests/Serialization/MarshalOutputTests.cs +++ b/sdk/Pulumi.Tests/Serialization/MarshalOutputTests.cs @@ -120,6 +120,21 @@ public sealed class BarArgs : ResourceArgs ImmutableDictionary.Empty.Add("foo", CreateOutputValue("hello", isSecret: true)) }, new object[] + { + new InputMap { { "foo", "hello" } }, + ImmutableDictionary.Empty.Add("foo", "hello") + }, + new object[] + { + new InputMap { { "foo", Output.Create("hello") } }, + ImmutableDictionary.Empty.Add("foo", "hello") + }, + new object[] + { + new InputMap { { "foo", Output.CreateSecret("hello") } }, + ImmutableDictionary.Empty.Add("foo", CreateOutputValue("hello", isSecret: true)) + }, + new object[] { new BarArgs { Foo = new FooArgs { Foo = "hello" } }, ImmutableDictionary.Empty.Add("foo", From d2476b5c3a4eb8d11c2d7de2522888213401122a Mon Sep 17 00:00:00 2001 From: Fraser Waters Date: Wed, 29 Jan 2025 14:12:29 +0000 Subject: [PATCH 4/7] Fix InputMap to not flatten --- .../Serialization/MarshalOutputTests.cs | 13 +++- sdk/Pulumi/Core/InputMap.cs | 63 ++++++++++++++----- sdk/Pulumi/Serialization/Serializer.cs | 9 ++- 3 files changed, 67 insertions(+), 18 deletions(-) diff --git a/sdk/Pulumi.Tests/Serialization/MarshalOutputTests.cs b/sdk/Pulumi.Tests/Serialization/MarshalOutputTests.cs index 787cabf1..67240cc0 100644 --- a/sdk/Pulumi.Tests/Serialization/MarshalOutputTests.cs +++ b/sdk/Pulumi.Tests/Serialization/MarshalOutputTests.cs @@ -6,6 +6,7 @@ using System.Linq; using System.Threading.Tasks; using Pulumi.Serialization; +using Pulumi.Utilities; using Xunit; namespace Pulumi.Tests.Serialization @@ -105,6 +106,11 @@ public sealed class BarArgs : ResourceArgs ImmutableArray.Empty.Add(CreateSecretValue("hello")) }, new object[] + { + new InputList { OutputUtilities.CreateUnknown("") }, + ImmutableArray.Empty.Add(Constants.UnknownValue) + }, + new object[] { new Dictionary> { { "foo", "hello" } }, ImmutableDictionary.Empty.Add("foo", "hello") @@ -132,7 +138,12 @@ public sealed class BarArgs : ResourceArgs new object[] { new InputMap { { "foo", Output.CreateSecret("hello") } }, - ImmutableDictionary.Empty.Add("foo", CreateOutputValue("hello", isSecret: true)) + ImmutableDictionary.Empty.Add("foo", CreateSecretValue("hello")) + }, + new object[] + { + new InputMap { { "foo", OutputUtilities.CreateUnknown("") } }, + ImmutableDictionary.Empty.Add("foo", Constants.UnknownValue) }, new object[] { diff --git a/sdk/Pulumi/Core/InputMap.cs b/sdk/Pulumi/Core/InputMap.cs index d5f07931..c865b4e4 100644 --- a/sdk/Pulumi/Core/InputMap.cs +++ b/sdk/Pulumi/Core/InputMap.cs @@ -42,20 +42,43 @@ namespace Pulumi /// public sealed class InputMap : Input>, IEnumerable, IAsyncEnumerable>> { - public InputMap() : this(Output.Create(ImmutableDictionary.Empty)) + private static Input> Flatten(Input>> inputs) { + return inputs.Apply(inputs => { + var list = inputs.Select(kv => kv.Value.Apply(value => KeyValuePair.Create(kv.Key, value))); + return Output.All(list).Apply(kvs => { + var result = ImmutableDictionary.CreateBuilder(); + foreach (var (k, v) in kvs) + { + result[k] = v; + } + return result.ToImmutable(); + }); + }); } - private InputMap(Output> values) - : base(values) + Input>> _inputValue; + Input>> Value { + get => _inputValue; + set { + _inputValue = value; + _outputValue = Flatten(_inputValue); + } + } + + public InputMap() : this(ImmutableDictionary>.Empty) + { + } + + private InputMap(Input>> values) + : base(Flatten(values)) { + _inputValue = values; } public void Add(string key, Input value) { - var inputDictionary = (Input>)_outputValue; - _outputValue = Output.Tuple(inputDictionary, value) - .Apply(x => x.Item1.Add(key, x.Item2)); + Value = Value.Apply(self => self.Add(key, value)); } /// @@ -68,8 +91,7 @@ public void Add(InputMap values) public void AddRange(InputMap values) { - var inputDictionary = (Input>)_outputValue; - _outputValue = Output.Tuple(inputDictionary, values) + Value = Output.Tuple(Value, values.Value) .Apply(x => x.Item1.AddRange(x.Item2)); } @@ -91,16 +113,18 @@ public Input this[string key] /// both input maps. public static InputMap Merge(InputMap first, InputMap second) { - var output = Output.Tuple(first._outputValue, second._outputValue) + var output = Output.Tuple(first.Value, second.Value) .Apply(dicts => { - var result = new Dictionary(dicts.Item1); - // Overwrite keys if duplicates are found - foreach (var (k, v) in dicts.Item2) - result[k] = v; - return result; + var builder = ImmutableDictionary.CreateBuilder>(); + foreach (var (k, v) in dicts.Item1) + builder[k] = v; + // Overwrite keys if duplicates are found + foreach (var (k, v) in dicts.Item2) + builder[k] = v; + return builder.ToImmutable(); }); - return output; + return new InputMap(output); } #region construct from dictionary types @@ -118,7 +142,14 @@ public static implicit operator InputMap(Output> value => values.Apply(ImmutableDictionary.CreateRange); public static implicit operator InputMap(Output> values) - => new InputMap(values); + => new InputMap(values.Apply(values => { + var builder = ImmutableDictionary.CreateBuilder>(); + foreach (var value in values) + { + builder.Add(value.Key, value.Value); + } + return builder.ToImmutable(); + })); #endregion diff --git a/sdk/Pulumi/Serialization/Serializer.cs b/sdk/Pulumi/Serialization/Serializer.cs index 96236aab..d21bc8a0 100644 --- a/sdk/Pulumi/Serialization/Serializer.cs +++ b/sdk/Pulumi/Serialization/Serializer.cs @@ -103,14 +103,21 @@ prop is double || $"Tasks are not allowed inside ResourceArgs. Please wrap your Task in an Output:\n\t{ctx}"); } - // if prop is an InputList var propType = prop.GetType(); + // if prop is an InputList if (propType.IsGenericType && propType.GetGenericTypeDefinition() == typeof(InputList<>)) { // pull off the Value property from the InputList var inputList = propType.GetProperty("Value", BindingFlags.NonPublic | BindingFlags.Instance)?.GetValue(prop); return await SerializeAsync(ctx, inputList, keepResources, keepOutputValues).ConfigureAwait(false); } + // if prop is an InputMap + if (propType.IsGenericType && propType.GetGenericTypeDefinition() == typeof(InputMap<>)) + { + // pull off the Value property from the InputMap + var inputList = propType.GetProperty("Value", BindingFlags.NonPublic | BindingFlags.Instance)?.GetValue(prop); + return await SerializeAsync(ctx, inputList, keepResources, keepOutputValues).ConfigureAwait(false); + } if (prop is IInput input) { From 4fe12abaae7553206c87cb33783ae5773645f6e3 Mon Sep 17 00:00:00 2001 From: Fraser Waters Date: Wed, 29 Jan 2025 14:14:50 +0000 Subject: [PATCH 5/7] Add changelog --- .changes/unreleased/Improvements-449.yaml | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 .changes/unreleased/Improvements-449.yaml diff --git a/.changes/unreleased/Improvements-449.yaml b/.changes/unreleased/Improvements-449.yaml new file mode 100644 index 00000000..12b8357c --- /dev/null +++ b/.changes/unreleased/Improvements-449.yaml @@ -0,0 +1,7 @@ +component: sdk +kind: Improvements +body: InputMap and InputList no longer flatten nested unknowns/secrets to apply to + the whole object. +time: 2025-01-29T14:14:44.986465684Z +custom: + PR: "449" From fe2d32f69e21b805b6a74500e87a41268ffb7d5c Mon Sep 17 00:00:00 2001 From: Fraser Waters Date: Wed, 29 Jan 2025 14:16:38 +0000 Subject: [PATCH 6/7] format --- sdk/Pulumi/Core/InputList.cs | 15 ++++++++++----- sdk/Pulumi/Core/InputMap.cs | 29 +++++++++++++++++------------ 2 files changed, 27 insertions(+), 17 deletions(-) diff --git a/sdk/Pulumi/Core/InputList.cs b/sdk/Pulumi/Core/InputList.cs index 8ae1ce01..b407c530 100644 --- a/sdk/Pulumi/Core/InputList.cs +++ b/sdk/Pulumi/Core/InputList.cs @@ -47,9 +47,11 @@ namespace Pulumi public sealed class InputList : Input>, IEnumerable, IAsyncEnumerable> { Input>> _inputValue; - Input>> Value { + Input>> Value + { get => _inputValue; - set { + set + { _inputValue = value; _outputValue = _inputValue.Apply(inputs => Output.All(inputs)); } @@ -87,9 +89,11 @@ public void AddRange(InputList inputs) /// Concatenates the values in this list with the values in , /// returning the concatenated sequence in a new . /// - public InputList Concat(InputList other) { + public InputList Concat(InputList other) + { var list = new InputList(); - list.Value = Output.Tuple(Value, other.Value).Apply(t => { + list.Value = Output.Tuple(Value, other.Value).Apply(t => + { var (first, second) = t; return first.AddRange(second); }); @@ -163,7 +167,8 @@ public static implicit operator InputList(Output> values) => values.Apply(ImmutableArray.CreateRange); public static implicit operator InputList(Output> values) - => new InputList(values.Apply(values => { + => new InputList(values.Apply(values => + { var builder = ImmutableArray.CreateBuilder>(values.Length); foreach (var value in values) { diff --git a/sdk/Pulumi/Core/InputMap.cs b/sdk/Pulumi/Core/InputMap.cs index c865b4e4..1f63d160 100644 --- a/sdk/Pulumi/Core/InputMap.cs +++ b/sdk/Pulumi/Core/InputMap.cs @@ -44,9 +44,11 @@ public sealed class InputMap : Input>, IEnumer { private static Input> Flatten(Input>> inputs) { - return inputs.Apply(inputs => { + return inputs.Apply(inputs => + { var list = inputs.Select(kv => kv.Value.Apply(value => KeyValuePair.Create(kv.Key, value))); - return Output.All(list).Apply(kvs => { + return Output.All(list).Apply(kvs => + { var result = ImmutableDictionary.CreateBuilder(); foreach (var (k, v) in kvs) { @@ -58,9 +60,11 @@ private static Input> Flatten(Input>> _inputValue; - Input>> Value { + Input>> Value + { get => _inputValue; - set { + set + { _inputValue = value; _outputValue = Flatten(_inputValue); } @@ -116,13 +120,13 @@ public static InputMap Merge(InputMap first, InputMap second) var output = Output.Tuple(first.Value, second.Value) .Apply(dicts => { - var builder = ImmutableDictionary.CreateBuilder>(); - foreach (var (k, v) in dicts.Item1) - builder[k] = v; - // Overwrite keys if duplicates are found - foreach (var (k, v) in dicts.Item2) - builder[k] = v; - return builder.ToImmutable(); + var builder = ImmutableDictionary.CreateBuilder>(); + foreach (var (k, v) in dicts.Item1) + builder[k] = v; + // Overwrite keys if duplicates are found + foreach (var (k, v) in dicts.Item2) + builder[k] = v; + return builder.ToImmutable(); }); return new InputMap(output); } @@ -142,7 +146,8 @@ public static implicit operator InputMap(Output> value => values.Apply(ImmutableDictionary.CreateRange); public static implicit operator InputMap(Output> values) - => new InputMap(values.Apply(values => { + => new InputMap(values.Apply(values => + { var builder = ImmutableDictionary.CreateBuilder>(); foreach (var value in values) { From e82f9b79eef0022268c434d8b8d47ec6ce66889b Mon Sep 17 00:00:00 2001 From: Fraser Waters Date: Fri, 31 Jan 2025 11:40:00 +0000 Subject: [PATCH 7/7] Add comments --- sdk/Pulumi/Core/InputList.cs | 8 ++++++++ sdk/Pulumi/Core/InputMap.cs | 9 +++++++++ sdk/Pulumi/Pulumi.xml | 21 +++++++++++++++++++++ 3 files changed, 38 insertions(+) diff --git a/sdk/Pulumi/Core/InputList.cs b/sdk/Pulumi/Core/InputList.cs index b407c530..726c3fff 100644 --- a/sdk/Pulumi/Core/InputList.cs +++ b/sdk/Pulumi/Core/InputList.cs @@ -47,6 +47,14 @@ namespace Pulumi public sealed class InputList : Input>, IEnumerable, IAsyncEnumerable> { Input>> _inputValue; + /// + /// InputList externally has to behave as an Input{ImmutableArray{T}}, but we actually want to + /// keep nested Input/Output values separate, so that we can serialise the overall list shape even if one of the + /// inner elements is an unknown value. + /// + /// To do that we keep a separate value of the form Input{ImmutableArray{Input{T}}}/> which each + /// time we set syncs the flattened value to the base Input{ImmutableArray{T}}. + /// Input>> Value { get => _inputValue; diff --git a/sdk/Pulumi/Core/InputMap.cs b/sdk/Pulumi/Core/InputMap.cs index 1f63d160..f2e842c5 100644 --- a/sdk/Pulumi/Core/InputMap.cs +++ b/sdk/Pulumi/Core/InputMap.cs @@ -60,6 +60,15 @@ private static Input> Flatten(Input>> _inputValue; + /// + /// InputMap externally has to behave as an Input{ImmutableDictionary{string, T}}, but we actually + /// want to keep nested Input/Output values separate, so that we can serialise the overall map shape even if + /// one of the inner elements is an unknown value. + /// + /// To do that we keep a separate value of the form Input{ImmutableDictionary{string, Input{T}}} + /// which each time we set syncs the flattened value to the base Input{ImmutableDictionary{string, + /// T}}. + /// Input>> Value { get => _inputValue; diff --git a/sdk/Pulumi/Pulumi.xml b/sdk/Pulumi/Pulumi.xml index 68a723e9..48e207d5 100644 --- a/sdk/Pulumi/Pulumi.xml +++ b/sdk/Pulumi/Pulumi.xml @@ -513,6 +513,16 @@ + + + InputList externally has to behave as an Input{ImmutableArray{T}}, but we actually want to + keep nested Input/Output values separate, so that we can serialise the overall list shape even if one of the + inner elements is an unknown value. + + To do that we keep a separate value of the form Input{ImmutableArray{Input{T}}}/> which each + time we set syncs the flattened value to the base Input{ImmutableArray{T}}. + + Note: this is non-standard convenience for use with collection initializers. @@ -557,6 +567,17 @@ + + + InputMap externally has to behave as an Input{ImmutableDictionary{string, T}}, but we actually + want to keep nested Input/Output values separate, so that we can serialise the overall map shape even if + one of the inner elements is an unknown value. + + To do that we keep a separate value of the form Input{ImmutableDictionary{string, Input{T}}} + which each time we set syncs the flattened value to the base Input{ImmutableDictionary{string, + T}}. + + Note: this is non-standard convenience for use with collection initializers.