Skip to content

Commit

Permalink
Using physical file provider to overcome inotify limits and preview n…
Browse files Browse the repository at this point in the history
…uget packages of OPC UA stack (#2220)
  • Loading branch information
marcschier authored May 5, 2024
1 parent 11c37fa commit 6b5f13d
Show file tree
Hide file tree
Showing 17 changed files with 92 additions and 148 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,9 @@ public CommandLine(string[] args, CommandLineLogger? logger = null)
{ $"cf|createifnotexist:|{PublisherConfig.CreatePublishFileIfNotExistKey}:",
"Permit publisher to create the the specified publish file if it does not exist. The new file will be created under the access rights of the publisher module.\nThe default file 'publishednodes.json' is always created when no file name was provided on the command line and this option is ignored.\nIf a file was specified but does not exist and should not be created the module exits.\nDefault: `false`\n",
(bool? b) => this[PublisherConfig.CreatePublishFileIfNotExistKey] = b?.ToString() ?? "True" },
{ $"pol|usepolling:|{PublisherConfig.UseFileChangePollingKey}:",
"Poll for file changes instead of using a file system watcher.\nUse this setting when the underlying file system does not support file system notifications such as in some docker container setups.\nDefault: `false`\n",
(bool? b) => this[PublisherConfig.UseFileChangePollingKey] = b?.ToString() ?? "True" },
{ $"fe|forceencryptedcredentials:|{PublisherConfig.ForceCredentialEncryptionKey}:",
"If set to true the publisher will never write plain text credentials into the published nodes configuration file.\nIf a credential cannot be written to the file using the IoT Edge workload API crypto provider the publisher will exit with an error.\nDefault: `false` (write secrets as plain text into the configuration file which should be properly ACL'ed)\n",
(bool? b) => this[PublisherConfig.ForceCredentialEncryptionKey] = b?.ToString() ?? "True" },
Expand Down
17 changes: 8 additions & 9 deletions src/Azure.IIoT.OpcUa.Publisher.Module/src/Startup.cs
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,6 @@ namespace Azure.IIoT.OpcUa.Publisher.Module
using Autofac;
using Autofac.Extensions.DependencyInjection;
using Microsoft.AspNetCore.Builder;
using Microsoft.AspNetCore.Hosting;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Hosting;
Expand All @@ -31,24 +30,24 @@ public class Startup
/// </summary>
public IConfigurationRoot Configuration { get; }

/// <summary>
/// Current hosting environment
/// </summary>
public IWebHostEnvironment Environment { get; }

/// <summary>
/// Create startup
/// </summary>
/// <param name="env"></param>
/// <param name="configuration"></param>
public Startup(IWebHostEnvironment env, IConfiguration configuration)
public Startup(IConfiguration configuration)
{
Environment = env;
Configuration = new ConfigurationBuilder()
.AddConfiguration(configuration)
.AddFromDotEnvFile()
.AddEnvironmentVariables()
.Build();

// Set polling mode on file watcher if configured
if (Configuration.GetValue<string>(PublisherConfig.UseFileChangePollingKey)?
.Equals("True", StringComparison.OrdinalIgnoreCase) ?? false)
{
Environment.SetEnvironmentVariable("DOTNET_USE_POLLING_FILE_WATCHER", "1");
}
}

/// <summary>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,7 @@ public DmApiPublisherControllerTests(ITestOutputHelper output)

_options = new PublisherConfig(new ConfigurationBuilder().Build()).ToOptions();
_options.Value.PublishedNodesFile = _tempFile;
_options.Value.UseFileChangePolling = true;
_options.Value.MessagingProfile = MessagingProfile.Get(
MessagingMode.PubSub, MessageEncoding.Json);

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -312,6 +312,11 @@ protected void StartPublisher(string test, string publishedNodesFile = null,
$"--pf={_publishedNodesFilePath}"
}).ToArray();

if (OperatingSystem.IsLinux())
{
arguments = arguments.Append("--pol").ToArray();
}

if (reverseConnectPort != null)
{
arguments = arguments.Concat(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -155,7 +155,10 @@ public PublisherModule(IMessageSink messageSink, IEnumerable<DeviceTwinModel> de
"--ki=90",
"--aa"
}).ToArray();

if (OperatingSystem.IsLinux())
{
arguments = arguments.Append("--pol").ToArray();
}
if (_useMqtt)
{
arguments = arguments.Append("-t=Mqtt").ToArray();
Expand Down
9 changes: 1 addition & 8 deletions src/Azure.IIoT.OpcUa.Publisher.Service.WebApi/src/Startup.cs
Original file line number Diff line number Diff line change
Expand Up @@ -47,19 +47,12 @@ public class Startup
/// </summary>
public ServiceInfo ServiceInfo { get; } = new ServiceInfo();

/// <summary>
/// Current hosting environment - Initialized in constructor
/// </summary>
public IWebHostEnvironment Environment { get; }

/// <summary>
/// Create startup
/// </summary>
/// <param name="env"></param>
/// <param name="configuration"></param>
public Startup(IWebHostEnvironment env, IConfiguration configuration)
public Startup(IConfiguration configuration)
{
Environment = env;
Configuration = new ConfigurationBuilder()
.AddConfiguration(configuration)
.AddFromDotEnvFile()
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -89,7 +89,10 @@ public PublisherModule(ILifetimeScope serviceContainer)
"--ki=90",
"--aa"
}).ToArray();

if (OperatingSystem.IsLinux())
{
arguments = arguments.Append("--pol").ToArray();
}
var configBuilder = new ConfigurationBuilder()
.AddInMemoryCollection(new Dictionary<string, string>
{
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,6 @@
<ItemGroup>
<PackageReference Include="Furly.Extensions" Version="1.0.40" />
<PackageReference Include="System.Linq.Async" Version="6.0.1" />
<PackageReference Include="OPCFoundation.NetStandard.Opc.Ua.Server" Version="1.5.374.36" />
<PackageReference Include="OPCFoundation.NetStandard.Opc.Ua.Server" Version="1.5.374.50-preview" />
</ItemGroup>
</Project>
Original file line number Diff line number Diff line change
Expand Up @@ -5,12 +5,13 @@
<Nullable>enable</Nullable>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Microsoft.Extensions.FileProviders.Physical" Version="8.0.0" />
<PackageReference Include="Microsoft.Extensions.Hosting.Abstractions" Version="8.0.0" />
<PackageReference Include="Nito.AsyncEx" Version="5.1.2" />
<PackageReference Include="Furly.Azure.IoT.Edge" Version="1.0.40" />
<PackageReference Include="Irony" Version="1.5.1" />
<PackageReference Include="OPCFoundation.NetStandard.Opc.Ua.Client.ComplexTypes" Version="1.5.374.36" />
<PackageReference Include="OPCFoundation.NetStandard.Opc.Ua.Client" Version="1.5.374.36" />
<PackageReference Include="OPCFoundation.NetStandard.Opc.Ua.Client.ComplexTypes" Version="1.5.374.50-preview" />
<PackageReference Include="OPCFoundation.NetStandard.Opc.Ua.Client" Version="1.5.374.50-preview" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\..\Azure.IIoT.OpcUa\src\Azure.IIoT.OpcUa.csproj" />
Expand Down
20 changes: 0 additions & 20 deletions src/Azure.IIoT.OpcUa.Publisher/src/IStorageProvider.cs
Original file line number Diff line number Diff line change
Expand Up @@ -13,31 +13,11 @@ namespace Azure.IIoT.OpcUa.Publisher
/// </summary>
public interface IStorageProvider
{
/// <summary>
/// Occurs when published nodes file is deleted.
/// </summary>
event EventHandler<FileSystemEventArgs> Deleted;

/// <summary>
/// Occurs when published nodes file is created.
/// </summary>
event EventHandler<FileSystemEventArgs> Created;

/// <summary>
/// Occurs when published nodes file is changed.
/// </summary>
event EventHandler<FileSystemEventArgs> Changed;

/// <summary>
/// Occurs when published nodes file is renamed.
/// </summary>
event EventHandler<RenamedEventArgs> Renamed;

/// <summary>
/// Gets or sets a value indicating whether triggering of events is enabled.
/// </summary>
bool EnableRaisingEvents { get; set; }

/// <summary>
/// Get last write time of published nodes file.
/// </summary>
Expand Down
2 changes: 2 additions & 0 deletions src/Azure.IIoT.OpcUa.Publisher/src/Runtime/PublisherConfig.cs
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@ public sealed class PublisherConfig : PostConfigureOptionBase<PublisherOptions>
#pragma warning disable CS1591 // Missing XML comment for publicly visible type or member
public const string SiteIdKey = "SiteId";
public const string PublishedNodesFileKey = "PublishedNodesFile";
public const string UseFileChangePollingKey = "UseFileChangePolling";
public const string CreatePublishFileIfNotExistKey = "CreatePublishFileIfNotExistKey";
public const string MessagingModeKey = "MessagingMode";
public const string MessageEncodingKey = "MessageEncoding";
Expand Down Expand Up @@ -124,6 +125,7 @@ public override void PostConfigure(string? name, PublisherOptions options)
options.SiteId ??= GetStringOrDefault(SiteIdKey);

options.PublishedNodesFile ??= GetStringOrDefault(PublishedNodesFileKey);
options.UseFileChangePolling ??= GetBoolOrNull(UseFileChangePollingKey);

if (options.DefaultTransport == null && Enum.TryParse<WriterGroupTransport>(
GetStringOrDefault(DefaultTransportKey), out var transport))
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,11 @@ public sealed class PublisherOptions
/// </summary>
public string? PublishedNodesFile { get; set; }

/// <summary>
/// Poll changes instead of using file watcher
/// </summary>
public bool? UseFileChangePolling { get; set; }

/// <summary>
/// Create the configuration file if it does not exist
/// </summary>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -871,45 +871,9 @@ public void Dispose()
/// <param name="e"></param>
private void OnChanged(object? sender, FileSystemEventArgs e)
{
_logger.LogDebug("File {File} changed. Triggering file refresh ...",
e.Name);
_fileChanges.Writer.TryWrite(false);
}

/// <summary>
/// Handle creation of file
/// </summary>
/// <param name="sender"></param>
/// <param name="e"></param>
private void OnCreated(object? sender, FileSystemEventArgs e)
{
_logger.LogDebug("File {File} created. Triggering file refresh ...",
e.Name);
_fileChanges.Writer.TryWrite(false);
}

/// <summary>
/// Handle removal of file
/// </summary>
/// <param name="sender"></param>
/// <param name="e"></param>
private void OnRenamed(object? sender, FileSystemEventArgs e)
{
_logger.LogDebug("File {File} renamed. Triggering file refresh ...",
e.Name);
_fileChanges.Writer.TryWrite(false);
}

/// <summary>
/// Handle deletion of the file
/// </summary>
/// <param name="sender"></param>
/// <param name="e"></param>
private void OnDeleted(object? sender, FileSystemEventArgs e)
{
_logger.LogDebug("File {File} deleted. Clearing configuration ...",
e.Name);
_fileChanges.Writer.TryWrite(true);
_logger.LogDebug("File {File} {Action}. Triggering file refresh ...",
e.ChangeType, e.Name);
_fileChanges.Writer.TryWrite(e.ChangeType == WatcherChangeTypes.Deleted);
}

/// <summary>
Expand All @@ -921,10 +885,6 @@ private async Task ProcessFileChangesAsync()
try
{
_publishedNodesProvider.Changed += OnChanged;
_publishedNodesProvider.Created += OnCreated;
_publishedNodesProvider.Renamed += OnRenamed;
_publishedNodesProvider.Deleted += OnDeleted;
_publishedNodesProvider.EnableRaisingEvents = true;

var retryCount = 0;
await foreach (var clear in _fileChanges.Reader.ReadAllAsync())
Expand All @@ -943,7 +903,7 @@ private async Task ProcessFileChangesAsync()
if (currentFileHash != _lastKnownFileHash)
{
var jobs = Enumerable.Empty<WriterGroupModel>();
if (!clear && !string.IsNullOrEmpty(content))
if (!string.IsNullOrEmpty(content))
{
if (string.IsNullOrEmpty(_lastKnownFileHash))
{
Expand All @@ -961,9 +921,9 @@ private async Task ProcessFileChangesAsync()
var entries = _publishedNodesJobConverter.Read(content).ToList();
TransformFromLegacyNodeId(entries);
jobs = _publishedNodesJobConverter.ToWriterGroups(entries);
_logger.LogInformation("{Action} publisher configuration completed.",
clear ? "Resetting" : "Refreshing");
}
_logger.LogInformation("{Action} publisher configuration completed.",
clear ? "Resetting" : "Refreshing");
try
{
await _publisherHost.UpdateAsync(jobs).ConfigureAwait(false);
Expand Down Expand Up @@ -1044,13 +1004,7 @@ private async Task ProcessFileChangesAsync()
}
finally
{
_publishedNodesProvider.EnableRaisingEvents = false;

_publishedNodesProvider.Changed -= OnChanged;
_publishedNodesProvider.Created -= OnCreated;
_publishedNodesProvider.Renamed -= OnRenamed;
_publishedNodesProvider.Deleted -= OnDeleted;

_started = null;
}
}
Expand Down
Loading

0 comments on commit 6b5f13d

Please sign in to comment.