diff --git a/tutorials/Dashboards_Low_Code_Apps.md b/tutorials/Dashboards_Low_Code_Apps.md index cadc00a068..6705332eb4 100644 --- a/tutorials/Dashboards_Low_Code_Apps.md +++ b/tutorials/Dashboards_Low_Code_Apps.md @@ -54,3 +54,4 @@ keywords: low-code, low code, lowcode | [Building a GQI data source that fetches satellites](xref:Ad_hoc_Tutorials_Satellites) | Go through an example script that forwards satellites data to the GQI. | | [Creating a parameter table connected to an element feed](xref:Creating_a_parameter_table_connected_to_an_element_feed) | Create a parameter table connected to an element feed. | | [Building a GQI data source that retrieves data from a DMS](xref:Ad_hoc_Tutorials_GQIDMS) | Create an ad hoc data source to retrieve data from your DMS. | +| [Scaling an ad hoc data source](xref:Ad_hoc_Tutorials_Scalable) | An example of how to build a highly scalable data source capable of serving many concurrent users. | diff --git a/user-guide/Advanced_Modules/Dashboards_and_Low_Code_Apps/GQI/Extensions/Ad_hoc_data/Ad_hoc_Tutorials_Scaling.md b/user-guide/Advanced_Modules/Dashboards_and_Low_Code_Apps/GQI/Extensions/Ad_hoc_data/Ad_hoc_Tutorials_Scaling.md new file mode 100644 index 0000000000..1e57dc0ce3 --- /dev/null +++ b/user-guide/Advanced_Modules/Dashboards_and_Low_Code_Apps/GQI/Extensions/Ad_hoc_data/Ad_hoc_Tutorials_Scaling.md @@ -0,0 +1,233 @@ +--- +uid: Ad_hoc_Tutorials_Scalable +--- + +# Scaling an ad hoc data source + +Each time a query is executed, a new instance of the data source is created. This makes scalability crucial, especially when building the underlying data source is time-consuming or resource-intensive. Keep in mind that the code you're writing could be executed concurrently by many users. + +To optimize performance when real-time data isn't required, implement caching where appropriate, using a static cache. When querying data through the DMS interface, ensure that data is stored on a security group basis. + +Be aware that multithreading affects both the static cache object and the cache itself. Use consistent locking mechanisms during heavy operations to prevent multiple costly computations from running simultaneously. + +If needed, consider adding a circuit breaker to the ad hoc data source to reset the cache when necessary. + +> [!NOTE] +> The code example below is incomplete. You will need to design the cache object to fit your specific requirements. + +```csharp +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using Skyline.DataMiner.Analytics.GenericInterface; +using Skyline.DataMiner.Net.Messages; + +[GQIMetaData(Name = "Results")] +public class Results : IGQIDataSource, IGQIOnInit +{ + private static readonly GQIStringColumn _nameColumn = new GQIStringColumn("Name"); + private static readonly GQIStringColumn _kpiColumn = new GQIStringColumn("KPI"); + + private static readonly ConcurrentDictionary _groupToCache = new ConcurrentDictionary(StringComparer.OrdinalIgnoreCase); + private static readonly string _cacheBreakerPath = @"C:\Skyline DataMiner\Documents\Ad hoc cache\resultsCacheBreaker.txt"; + + private GQIDMS _dms; + private IGQILogger _logger; + + public OnInitOutputArgs OnInit(OnInitInputArgs args) + { + _dms = args.DMS; + _logger = args.Logger; + return default; + } + + public GQIColumn[] GetColumns() + { + return new GQIColumn[] + { + _nameColumn, + _kpiColumn, + }; + } + + public GQIPage GetNextPage(GetNextPageInputArgs args) + { + var results = GetResults(); + var rows = results.Select(v => + { + var cells = new GQICell[] + { + new GQICell() { Value = v.Identifier }, + new GQICell() { Value = v.Name }, + }; + return new GQIRow(v.Identifier, cells); + }).ToArray(); + + return new GQIPage(rows) { HasNextPage = false }; + } + + public IEnumerable GetResults() + { + var cache = GetCache(); + cache.EnsureInitialized(_dms, _logger); + cache.EnsureUpdated(_dms, _logger); + + return cache.Results; + } + + private Cache GetCache() + { + CheckCacheValidation(); + + var securityGroupKey = GetSecurityGroupKey(_dms, _logger); + if (!_groupToCache.TryGetValue(securityGroupKey, out var siteCache)) + { + lock (_groupToCache) + { + if (!_groupToCache.TryGetValue(securityGroupKey, out siteCache)) + { + siteCache = new Cache(securityGroupKey); + _groupToCache[securityGroupKey] = siteCache; + } + } + } + + return siteCache; + } + + private void CheckCacheValidation() + { + if (!File.Exists(_cacheBreakerPath)) + return; + + lock (_groupToCache) + { + try + { + _logger.Information("Going to remove cache."); + + if (!File.Exists(_cacheBreakerPath)) + return; + + _groupToCache.Clear(); + + File.Delete(_cacheBreakerPath); + + _logger.Information("Remove cache."); + } + catch (Exception ex) + { + _logger.Error(ex, "Could not delete cache."); + } + } + } + + private string GetSecurityGroupKey(GQIDMS dms, IGQILogger logger) + { + if (dms == null) + throw new ArgumentNullException(nameof(dms)); + if (logger == null) + throw new ArgumentNullException(nameof(logger)); + + var responses = dms.SendMessages(new DMSMessage[] { new GetUserFullNameMessage(), new GetInfoMessage(InfoType.SecurityInfo) }); + var userName = responses?.OfType().FirstOrDefault()?.User; + if (string.IsNullOrEmpty(userName)) + { + logger.Error("User not found."); + return null; + } + + var securityResponse = responses?.OfType().FirstOrDefault(); + var userGroups = securityResponse?.Users?.Where(u => u.Name == userName).FirstOrDefault()?.Groups?.OrderBy(x => x)?.ToArray() ?? new int[0]; + if (userGroups.Length == 0) + { + logger.Error("User is not part of any group."); + return null; + } + + return string.Join(";", userGroups); + } +} +``` + +```csharp +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using Skyline.DataMiner.Analytics.GenericInterface; + +public class Cache +{ + private const int TIMEOUT_SECONDS = 30; + private readonly string _securityKey; + private readonly object _lock = new object(); + private readonly ConcurrentDictionary _results = new ConcurrentDictionary(StringComparer.OrdinalIgnoreCase); + private bool _isInitialized; + + public Cache(string securityKey) + { + if (string.IsNullOrWhiteSpace(securityKey)) + throw new ArgumentException("SecurityKey can't be empty."); + + _securityKey = securityKey; + } + + public DateTime LastUpdate { get; private set; } + + public IEnumerable Results => _results.Values; + + internal void EnsureInitialized(GQIDMS dms, IGQILogger logger) + { + if (_isInitialized) + return; + + lock (_lock) + { + if (_isInitialized) + return; + + Initialize(dms, logger); + } + } + + internal void EnsureUpdated(GQIDMS dms, IGQILogger logger) + { + if (!_isInitialized) + return; + + if (LastUpdate > DateTime.UtcNow - TimeSpan.FromSeconds(TIMEOUT_SECONDS)) + return; + + lock (_lock) + { + if (LastUpdate > DateTime.UtcNow - TimeSpan.FromSeconds(TIMEOUT_SECONDS)) + return; + + UpdateData(dms, logger); + } + } + + private void UpdateData(GQIDMS dms, IGQILogger logger) + { + logger.Information($"{_securityKey} - Fetching update."); + + /* Fetch update */ + + logger.Information($"{_securityKey} - Fetching update done."); + LastUpdate = DateTime.UtcNow; + } + + private void Initialize(GQIDMS dms, IGQILogger logger) + { + logger.Information($"{_securityKey} - Initializing."); + + /* Initialize */ + + logger.Information($"{_securityKey} - Initializing done."); + + _isInitialized = true; + } +} +``` diff --git a/user-guide/Advanced_Modules/Dashboards_and_Low_Code_Apps/GQI/Extensions/GQI_Extensions_Best_Practices.md b/user-guide/Advanced_Modules/Dashboards_and_Low_Code_Apps/GQI/Extensions/GQI_Extensions_Best_Practices.md index f68ee3a9de..70698a89da 100644 --- a/user-guide/Advanced_Modules/Dashboards_and_Low_Code_Apps/GQI/Extensions/GQI_Extensions_Best_Practices.md +++ b/user-guide/Advanced_Modules/Dashboards_and_Low_Code_Apps/GQI/Extensions/GQI_Extensions_Best_Practices.md @@ -42,3 +42,9 @@ Types and methods in this namespace have no use in a GQI extension and, more imp ## Only use 64-bit assembly references GQI runs in a 64-bit process and cannot load any extensions that require 32-bit assemblies. You should therefore only use 64-bit assembly references. + +## Make your ad hoc data source scalable + +Each time a query is executed, a new instance of the data source is created. This makes scalability crucial, especially when building the underlying data source is time-consuming or resource-intensive. Keep in mind that the code you're writing could be executed concurrently by many users. + +An example of how to build a highly scalable data source capable of serving many concurrent users: [Scaling an ad hoc data source](xref:Ad_hoc_Tutorials_Scalable). diff --git a/user-guide/Advanced_Modules/toc.yml b/user-guide/Advanced_Modules/toc.yml index 48172c3d2b..4cd076b59c 100644 --- a/user-guide/Advanced_Modules/toc.yml +++ b/user-guide/Advanced_Modules/toc.yml @@ -585,6 +585,8 @@ items: topicUid: Ad_hoc_Tutorials_Satellites - name: Building a GQI data source that retrieves data from a DMS topicUid: Ad_hoc_Tutorials_GQIDMS + - name: Scaling an ad hoc data source + topicUid: Ad_hoc_Tutorials_Scalable - name: Configuring a custom operator topicUid: GQI_Custom_Operator items: