Fix Use security and twin tests (Azure#2239)

docs/opc-publisher/commandline.md

@@ -280,6 +280,7 @@ Messaging configuration
                                Allowed values:
                                    `IoTHub`
                                    `Mqtt`
+                                   `EventHub`
                                    `Dapr`
                                    `Http`
                                    `FileSystem`
@@ -323,6 +324,15 @@ Transport settings
                                    `Any`
                                Default: `Mqtt` if device or edge hub connection
                                string is provided, ignored otherwise.
+      --eh, --eventhubnamespaceconnectionstring, --EventHubNamespaceConnectionString=VALUE
+                             The connection string of an existing event hub
+                               namespace to use for the Azure EventHub
+                               transport.
+                               Default: `not set`.
+      --sg, --schemagroup, --SchemaGroupName=VALUE
+                             The schema group in an event hub namespace to
+                               publish message schemas to.
+                               Default: `not set`.
   -d, --dcs, --daprconnectionstring, --DaprConnectionString=VALUE
                              Connect the OPC Publisher to a dapr pub sub
                                component using a connection string.
@@ -331,7 +341,8 @@ Transport settings
                                side car connection if needed.
                                Use the format 'PubSubComponent=<PubSubComponent>
                                [;GrpcPort=<GrpcPort>;HttpPort=<HttpPort>[;
-                               Scheme=<'https'|'http'>][;Host=<IPorDnsName>]][;ApiKey=<ApiKey>]'.
+                               Scheme=<'https'|'http'>][;Host=<IPorDnsName>]][;
+                               ApiKey=<ApiKey>]'.
                                To publish through dapr by default specify `-t=
                                Dapr`.
                                Default: `not set`.

e2e-tests/IIoTPlatform-E2E-Tests/TestExtensions/IIoTPlatformTestContext.cs

@@ -239,6 +239,9 @@ private static IConfigurationRoot GetConfiguration()
         public string ImagesTag => GetStringOrDefault(TestConstants.EnvironmentVariablesNames.PCS_IMAGES_TAG,
             () => "latest");
 
+        public string Token { get; internal set; }
+        public DateTime TokenExpiration { get; internal set; }
+
         public void LogEnvironment(ITestOutputHelper output)
         {
             if (output == null || _logged || output is DummyOutput)

e2e-tests/IIoTPlatform-E2E-Tests/TestHelper.cs

@@ -6,6 +6,7 @@
 namespace IIoTPlatformE2ETests
 {
     using Azure.Messaging.EventHubs.Consumer;
+    using Microsoft.Azure.Devices.Common.Exceptions;
     using Microsoft.Azure.Devices;
     using Newtonsoft.Json;
     using IIoTPlatformE2ETests.Config;
@@ -30,7 +31,6 @@ namespace IIoTPlatformE2ETests
     using Azure.IIoT.OpcUa.Publisher.Models;
     using Newtonsoft.Json.Converters;
     using Xunit.Sdk;
-    using Microsoft.Azure.Devices.Common.Exceptions;
 
     public record class MethodResultModel(string JsonPayload, int Status);
     public record class MethodParameterModel
@@ -52,13 +52,21 @@ public static async Task<string> GetTokenAsync(
             CancellationToken ct = default
         )
         {
-            return await GetTokenAsync(
+            if (context.Token != null && (DateTime.UtcNow + TimeSpan.FromSeconds(10) < context.TokenExpiration))
+            {
+                return context.Token;
+            }
+            var (expiration, token) = await GetTokenAsync(
                 context.IIoTPlatformConfigHubConfig.AuthTenant,
                 context.IIoTPlatformConfigHubConfig.AuthClientId,
                 context.IIoTPlatformConfigHubConfig.AuthClientSecret,
                 context.IIoTPlatformConfigHubConfig.AuthServiceId,
+                context.OutputHelper,
                 ct
             ).ConfigureAwait(false);
+            context.Token = token;
+            context.TokenExpiration = expiration;
+            return token;
         }
 
         /// <summary>
@@ -70,11 +78,12 @@ public static async Task<string> GetTokenAsync(
         /// <param name="serviceId">service id of deployed Industrial IoT</param>
         /// <param name="ct">Cancellation token</param>
         /// <returns>Return content of request token or empty string</returns>
-        public static async Task<string> GetTokenAsync(
+        public static async Task<(DateTime Expiration, string Token)> GetTokenAsync(
             string tenantId,
             string clientId,
             string clientSecret,
             string serviceId,
+            ITestOutputHelper outputHelper,
             CancellationToken ct = default
         )
         {
@@ -84,7 +93,7 @@ public static async Task<string> GetTokenAsync(
             Assert.False(string.IsNullOrWhiteSpace(serviceId));
 
             Exception saved = new UnauthorizedAccessException();
-            for (var i = 0; i < 3; i++)
+            for (var i = 0; i < 10; i++)
             {
                 try
                 {
@@ -104,12 +113,21 @@ public static async Task<string> GetTokenAsync(
                     dynamic json = JsonConvert.DeserializeObject(response.Content);
                     Assert.NotNull(json);
                     Assert.NotEmpty(json);
-                    return $"{json.token_type} {json.access_token}";
+
+                    var expiration = DateTime.UtcNow.AddMinutes(1);
+                    try
+                    {
+                        var seconds = (int)json.expires_in;
+                        expiration = DateTime.UtcNow.AddSeconds(seconds);
+                        outputHelper?.WriteLine($"Retrieved access token, token expires in {seconds} sec.");
+                    }
+                    catch { }
+                    return (DateTime.UtcNow.AddMinutes(5), $"{json.token_type} {json.access_token}");
                 }
                 catch (Exception ex)
                 {
                     saved = ex;
-                    Console.WriteLine($"Error occurred while requesting token: {ex.Message}");
+                    outputHelper?.WriteLine($"Error occurred while requesting token: {ex.Message}");
                     await Task.Delay(1000, ct).ConfigureAwait(false);
                 }
             }
@@ -175,7 +193,7 @@ public static async Task<IDictionary<string, PublishedNodesEntryModel>> GetSimul
 
         private static async Task<PublishedNodesEntryModel[]> GetPublishedNodesEntryModel(string host, CancellationToken ct)
         {
-            for (var attempt = 0; ;  attempt++)
+            for (var attempt = 0; ; attempt++)
             {
                 try
                 {

e2e-tests/IIoTPlatform-E2E-Tests/Twin/TwinBrowseTestTheory.cs

@@ -8,6 +8,7 @@ namespace IIoTPlatformE2ETests.Twin
     using IIoTPlatformE2ETests.TestExtensions;
     using System;
     using System.Collections.Generic;
+    using System.Linq;
     using System.Threading;
     using System.Threading.Tasks;
     using Xunit;
@@ -117,11 +118,18 @@ public async Task BrowseAllNodesOfTypeObject()
 
                 Assert.True(nodes.Count > 150);
 
-                Assert.Contains(nodes, n => string.Equals("i=85", n.NodeId, StringComparison.Ordinal));
-                Assert.Contains(nodes, n => string.Equals("i=2253", n.NodeId, StringComparison.Ordinal));
-                Assert.Contains(nodes, n => string.Equals("nsu=http://microsoft.com/Opc/OpcPlc/Boiler;i=5", n.NodeId, StringComparison.Ordinal));
-                Assert.Contains(nodes, n => string.Equals("nsu=http://microsoft.com/Opc/OpcPlc/;s=OpcPlc", n.NodeId, StringComparison.Ordinal));
-                Assert.Contains(nodes, n => string.Equals("i=15668", n.NodeId, StringComparison.Ordinal));
+
+                var set = new HashSet<string>
+                {
+                    "i=2253",
+                    "nsu=http://microsoft.com/Opc/OpcPlc/Boiler;i=5",
+                    "nsu=http://microsoft.com/Opc/OpcPlc/;s=OpcPlc",
+                    "i=15668",
+                    "nsu=http://microsoft.com/Opc/OpcPlc/ReferenceTest;s=Scalar_Static_Mass_Boolean",
+                };
+                var counted = nodes.Count(n => set.Contains(n.NodeId));
+                _context.OutputHelper.WriteLine($"Found count is {counted}");
+                Assert.Equal(set.Count, counted);
             }
             catch
             {
@@ -144,11 +152,18 @@ public async Task BrowseAllNodesOfTypeVariable()
                 Assert.NotNull(nodes);
                 Assert.NotEmpty(nodes);
 
-                Assert.True(nodes.Count > 150);
-
-                Assert.Contains(nodes, n => string.Equals("i=2254", n.NodeId, StringComparison.Ordinal));
-                Assert.Contains(nodes, n => string.Equals("nsu=http://microsoft.com/Opc/OpcPlc/;s=LongString1", n.NodeId, StringComparison.Ordinal));
-                Assert.Contains(nodes, n => string.Equals("nsu=http://microsoft.com/Opc/OpcPlc/;s=SlowUInt1", n.NodeId, StringComparison.Ordinal));
+                Assert.True(nodes.Count > 5000);
+
+                var set = new HashSet<string>
+                {
+                    "i=2254",
+                    "nsu=http://microsoft.com/Opc/OpcPlc/;s=LongString10kB",
+                    "nsu=http://microsoft.com/Opc/OpcPlc/;s=SlowUInt1",
+                    "nsu=http://microsoft.com/Opc/OpcPlc/ReferenceTest;s=Scalar_Simulation_Mass_Boolean_Boolean_22"
+                };
+                var counted = nodes.Count(n => set.Contains(n.NodeId));
+                _context.OutputHelper.WriteLine($"Found count is {counted}");
+                Assert.Equal(set.Count, counted);
             }
             catch
             {

src/Azure.IIoT.OpcUa.Publisher.Models/src/PublishedNodesEntryModel.cs

@@ -161,7 +161,7 @@ public sealed record class PublishedNodesEntryModel
         /// endpoint. Overrides <see cref="UseSecurity"/> setting.
         /// If the security mode is not available with any
         /// configured security policy connectivity will fail.
-        /// Default: <see cref="SecurityMode.SignAndEncrypt"/> if
+        /// Default: <see cref="SecurityMode.NotNone"/> if
         /// <see cref="UseSecurity"/> is <c>true</c>,
         /// otherwise <see cref="SecurityMode.None"/>
         /// </summary>
@@ -264,7 +264,7 @@ public sealed record class PublishedNodesEntryModel
         /// the opc server.
         /// </summary>
         [DataMember(Name = "UseSecurity", Order = 30)]
-        public bool UseSecurity { get; set; }
+        public bool? UseSecurity { get; set; }
 
         /// <summary>
         /// authentication mode

src/Azure.IIoT.OpcUa.Publisher.Models/src/SecurityMode.cs

@@ -5,12 +5,14 @@
 
 namespace Azure.IIoT.OpcUa.Publisher.Models
 {
+    using System;
     using System.Runtime.Serialization;
 
     /// <summary>
     /// Security mode of endpoint
     /// </summary>
     [DataContract]
+    [Flags]
     public enum SecurityMode
     {
         /// <summary>
@@ -35,6 +37,12 @@ public enum SecurityMode
         /// No security
         /// </summary>
         [EnumMember(Value = "None")]
-        None
+        None,
+
+        /// <summary>
+        /// Use sign or sign and encrypt
+        /// </summary>
+        [EnumMember(Value = "NotNone")]
+        NotNone,
     }
 }

src/Azure.IIoT.OpcUa.Publisher.Models/tests/PublishNodesEndpointApiModelTests.cs

@@ -29,7 +29,7 @@ public void UseSecurityDeserializationTest()
 
 """;
             var model = newtonSoftJsonSerializer.Deserialize<PublishedNodesEntryModel>(modelJson);
-            Assert.False(model.UseSecurity);
+            Assert.Null(model.UseSecurity);
 
             modelJson = """
 
@@ -78,7 +78,7 @@ public void UseSecuritySerializationTest()
             };
 
             var modeJson = newtonSoftJsonSerializer.SerializeToString(model);
-            Assert.Contains("\"UseSecurity\":false", modeJson, StringComparison.Ordinal);
+            Assert.DoesNotContain("\"UseSecurity\":false", modeJson, StringComparison.Ordinal);
 
             model = new PublishedNodesEntryModel
             {

src/Azure.IIoT.OpcUa.Publisher.Module/cli/Program.cs

@@ -532,7 +532,6 @@ static async Task RunAsync(ILoggerFactory loggerFactory, string publishProfile,
                     {
                         "-c",
                         "--ps",
-                        $"--ssf={outputFolder}",
                         $"--pf={publishedNodesFilePath}",
                         $"--me={messageProfile.MessageEncoding}",
                         $"--mm={messageProfile.MessagingMode}",

src/Azure.IIoT.OpcUa.Publisher.Module/src/Runtime/CommandLine.cs

@@ -248,7 +248,7 @@ public CommandLine(string[] args, CommandLineLogger? logger = null)
                     t => this[PublisherConfig.DiagnosticsTopicTemplateKey] = t },
                 { $"mdt|metadatatopictemplate:|{PublisherConfig.DataSetMetaDataTopicTemplateKey}:",
                     "The topic that metadata should be sent to.\nIn case of MQTT the message will be sent as RETAIN message with a TTL of either metadata send interval or infinite if metadata send interval is not configured.\nOnly valid if metadata is supported and/or explicitely enabled.\nThe template variables\n    `{{RootTopic}}`\n    `{{SiteId}}`\n    `{{TelemetryTopic}}`\n    `{{Encoding}}`\n    `{{PublisherId}}`\n    `{{DataSetClassId}}`\n    `{{DataSetWriter}}` and\n    `{{WriterGroup}}`\ncan be used as dynamic parts in the template. \nDefault: `{{TelemetryTopic}}` which means metadata is sent to the same output as regular messages. If specified without value, the default output is `{{TelemetryTopic}}/metadata`.\n",
-                    s => this[PublisherConfig.DataSetMetaDataTopicTemplateKey] = !string.IsNullOrEmpty(s) ? s : "{TelemetryTopic}/metadata" },
+                    s => this[PublisherConfig.DataSetMetaDataTopicTemplateKey] = !string.IsNullOrEmpty(s) ? s : PublisherConfig.MetadataTopicTemplateDefault },
                 { $"uns|datasetrouting=|{PublisherConfig.DefaultDataSetRoutingKey}=",
                     $"Configures whether messages should automatically be routed using the browse path of the monitored item inside the address space starting from the RootFolder.\nThe browse path is appended as topic structure to the telemetry topic root which can be configured using `--ttt`. Reserved characters in browse names are escaped with their hex ASCII code.\nAllowed values:\n    `{string.Join("`\n    `", Enum.GetNames(typeof(DataSetRoutingMode)))}`\nDefault: `{nameof(DataSetRoutingMode.None)}` (Topics must be configured).\n",
                     (DataSetRoutingMode m) => this[PublisherConfig.DefaultDataSetRoutingKey] = m.ToString() },

src/Azure.IIoT.OpcUa.Publisher.Module/tests/Controller/DmApiPublisherControllerTests.cs

@@ -262,7 +262,7 @@ await FluentActions
                 writer => Assert.Equal(publishNodesRequests[0].EndpointUrl,
                     writer.DataSet.DataSetSource.Connection.Endpoint.Url));
             Assert.Equal(publishNodesRequests
-                .Select(n => n.UseSecurity ? SecurityMode.SignAndEncrypt : SecurityMode.None)
+                .Select(n => (n.UseSecurity ?? false) ? SecurityMode.SignAndEncrypt : SecurityMode.None)
                 .ToHashSet(),
                 writerGroup.DataSetWriters
                 .Select(w => w.DataSet.DataSetSource.Connection.Endpoint.SecurityMode.Value)

src/Azure.IIoT.OpcUa.Publisher.Service/src/Services/Extensions/EndpointRegistrationEx.cs

@@ -320,7 +320,7 @@ public static DeviceTwinModel Patch(this EndpointRegistration? existing,
                         Url = string.IsNullOrEmpty(registration.EndpointUrl) ?
                             registration.EndpointUrlLC : registration.EndpointUrl,
                         AlternativeUrls = registration.AlternativeUrls?.DecodeAsList().ToHashSetSafe(),
-                        SecurityMode = registration.SecurityMode == SecurityMode.Best ?
+                        SecurityMode = registration.SecurityMode == SecurityMode.NotNone ?
                             null : registration.SecurityMode,
                         SecurityPolicy = string.IsNullOrEmpty(registration.SecurityPolicy) ?
                             null : registration.SecurityPolicy,
@@ -360,7 +360,7 @@ public static EndpointRegistration ToEndpointRegistration(this EndpointInfoModel
                 AuthenticationMethods = model.Registration?.AuthenticationMethods?
                     .EncodeAsDictionary(),
                 SecurityMode = model.Registration?.Endpoint?.SecurityMode ??
-                    SecurityMode.Best,
+                    SecurityMode.NotNone,
                 SecurityPolicy = model.Registration?.Endpoint?.SecurityPolicy,
                 Thumbprint = model.Registration?.Endpoint?.Certificate
             };
@@ -454,7 +454,7 @@ public int GetHashCode(EndpointRegistration obj)
                     EqualityComparer<string>.Default.GetHashCode(obj.ApplicationId ?? string.Empty);
                 hashCode = (hashCode * -1521134295) +
                     EqualityComparer<SecurityMode>.Default.GetHashCode(
-                        obj.SecurityMode ?? SecurityMode.Best);
+                        obj.SecurityMode ?? SecurityMode.NotNone);
                 hashCode = (hashCode * -1521134295) +
                     EqualityComparer<string>.Default.GetHashCode(obj.SecurityPolicy ?? string.Empty);
                 return hashCode;

src/Azure.IIoT.OpcUa.Publisher.Service/src/Services/Models/EndpointRegistration.cs

@@ -186,7 +186,7 @@ public override int GetHashCode()
                 EqualityComparer<int?>.Default.GetHashCode(SecurityLevel ?? 0);
             hashCode = (hashCode * -1521134295) +
                 EqualityComparer<SecurityMode>.Default.GetHashCode(
-                    SecurityMode ?? Publisher.Models.SecurityMode.Best);
+                    SecurityMode ?? Publisher.Models.SecurityMode.NotNone);
             hashCode = (hashCode * -1521134295) +
                 EqualityComparer<string>.Default.GetHashCode(SecurityPolicy ?? string.Empty);
             return hashCode;

src/Azure.IIoT.OpcUa.Publisher/src/Runtime/PublisherConfig.cs

@@ -99,6 +99,8 @@ public sealed class PublisherConfig : PostConfigureOptionBase<PublisherOptions>
             $"{{{RootTopicVariableName}}}/methods";
         public const string EventsTopicTemplateDefault =
             $"{{{RootTopicVariableName}}}/events";
+        public const string MetadataTopicTemplateDefault =
+            $"{{{TelemetryTopicVariableName}}}/metadata";
         public const string DiagnosticsTopicTemplateDefault =
             $"{{{RootTopicVariableName}}}/diagnostics/{{{WriterGroupVariableName}}}";
         public const string RootTopicTemplateDefault =

src/Azure.IIoT.OpcUa.Publisher/src/Stack/Extensions/EndpointDescriptionEx.cs

@@ -22,8 +22,7 @@ public static class EndpointDescriptionEx
         public static bool IsSameAs(this EndpointDescription endpoint,
             EndpointModel model)
         {
-            if (endpoint.SecurityMode !=
-                (model.SecurityMode ?? SecurityMode.SignAndEncrypt).ToStackType())
+            if (endpoint.SecurityMode.IsSame(model.SecurityMode ?? SecurityMode.SignAndEncrypt))
             {
                 return false;
             }

src/Azure.IIoT.OpcUa.Publisher/src/Stack/Extensions/StackTypesEx.cs

@@ -167,21 +167,27 @@ public static UaBrowseDirection ToStackType(this BrowseDirection mode)
         }
 
         /// <summary>
-        /// Convert security mode
+        /// Match security model
         /// </summary>
         /// <param name="mode"></param>
+        /// <param name="securityMode"></param>
         /// <returns></returns>
-        public static UaSecurityMode ToStackType(this SecurityMode mode)
+        public static bool IsSame(this UaSecurityMode mode, SecurityMode securityMode)
         {
-            switch (mode)
+            switch (securityMode)
             {
+                case SecurityMode.Best:
+                    return true; // Any security mode
                 case SecurityMode.Sign:
-                    return UaSecurityMode.Sign;
+                    return mode == UaSecurityMode.Sign;
                 case SecurityMode.SignAndEncrypt:
-                case SecurityMode.Best:
-                    return UaSecurityMode.SignAndEncrypt;
+                    return mode == UaSecurityMode.SignAndEncrypt;
+                case SecurityMode.None:
+                    return mode == UaSecurityMode.None;
+                case SecurityMode.NotNone:
+                    return mode != UaSecurityMode.None;
                 default:
-                    return UaSecurityMode.None;
+                    return false;
             }
         }