GQI - Added scaling best practice and example

tutorials/Dashboards_Low_Code_Apps.md

@@ -54,3 +54,4 @@ keywords: low-code, low code, lowcode
 | [Building a GQI data source that fetches satellites](xref:Ad_hoc_Tutorials_Satellites) | Go through an example script that forwards satellites data to the GQI. |
 | [Creating a parameter table connected to an element feed](xref:Creating_a_parameter_table_connected_to_an_element_feed) | Create a parameter table connected to an element feed. |
 | [Building a GQI data source that retrieves data from a DMS](xref:Ad_hoc_Tutorials_GQIDMS) | Create an ad hoc data source to retrieve data from your DMS. |
+| [Scaling an ad hoc data source](xref:Ad_hoc_Tutorials_Scalable) | An example of how to build a highly scalable data source capable of serving many concurrent users. |

user-guide/Advanced_Modules/Dashboards_and_Low_Code_Apps/GQI/Extensions/Ad_hoc_data/Ad_hoc_Tutorials_Scaling.md

@@ -0,0 +1,233 @@
+---
+uid: Ad_hoc_Tutorials_Scalable
+---
+
+# Scaling an ad hoc data source
+
+Each time a query is executed, a new instance of the data source is created. This makes scalability crucial, especially when building the underlying data source is time-consuming or resource-intensive. Keep in mind that the code you're writing could be executed concurrently by many users.
+
+To optimize performance when real-time data isn't required, implement caching where appropriate, using a static cache. When querying data through the DMS interface, ensure that data is stored on a security group basis.
+
+Be aware that multithreading affects both the static cache object and the cache itself. Use consistent locking mechanisms during heavy operations to prevent multiple costly computations from running simultaneously.
+
+If needed, consider adding a circuit breaker to the ad hoc data source to reset the cache when necessary.
+
+> [!NOTE]
+> The code example below is incomplete. You will need to design the cache object to fit your specific requirements.
+
+```csharp
+using System;
+using System.Collections.Concurrent;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using Skyline.DataMiner.Analytics.GenericInterface;
+using Skyline.DataMiner.Net.Messages;
+
+[GQIMetaData(Name = "Results")]
+public class Results : IGQIDataSource, IGQIOnInit
+{
+    private static readonly GQIStringColumn _nameColumn = new GQIStringColumn("Name");
+    private static readonly GQIStringColumn _kpiColumn = new GQIStringColumn("KPI");
+
+    private static readonly ConcurrentDictionary<string, Cache> _groupToCache = new ConcurrentDictionary<string, Cache>(StringComparer.OrdinalIgnoreCase);
+    private static readonly string _cacheBreakerPath = @"C:\Skyline DataMiner\Documents\Ad hoc cache\resultsCacheBreaker.txt";
+
+    private GQIDMS _dms;
+    private IGQILogger _logger;
+
+    public OnInitOutputArgs OnInit(OnInitInputArgs args)
+    {
+        _dms = args.DMS;
+        _logger = args.Logger;
+        return default;
+    }
+
+    public GQIColumn[] GetColumns()
+    {
+        return new GQIColumn[]
+        {
+                _nameColumn,
+                _kpiColumn,
+        };
+    }
+
+    public GQIPage GetNextPage(GetNextPageInputArgs args)
+    {
+        var results = GetResults();
+        var rows = results.Select(v =>
+        {
+            var cells = new GQICell[]
+            {
+                new GQICell() { Value = v.Identifier },
+                new GQICell() { Value = v.Name },
+            };
+            return new GQIRow(v.Identifier, cells);
+        }).ToArray();
+
+        return new GQIPage(rows) { HasNextPage = false };
+    }
+
+    public IEnumerable<Result> GetResults()
+    {
+        var cache = GetCache();
+        cache.EnsureInitialized(_dms, _logger);
+        cache.EnsureUpdated(_dms, _logger);
+
+        return cache.Results;
+    }
+
+    private Cache GetCache()
+    {
+        CheckCacheValidation();
+
+        var securityGroupKey = GetSecurityGroupKey(_dms, _logger);
+        if (!_groupToCache.TryGetValue(securityGroupKey, out var siteCache))
+        {
+            lock (_groupToCache)
+            {
+                if (!_groupToCache.TryGetValue(securityGroupKey, out siteCache))
+                {
+                    siteCache = new Cache(securityGroupKey);
+                    _groupToCache[securityGroupKey] = siteCache;
+                }
+            }
+        }
+
+        return siteCache;
+    }
+
+    private void CheckCacheValidation()
+    {
+        if (!File.Exists(_cacheBreakerPath))
+            return;
+
+        lock (_groupToCache)
+        {
+            try
+            {
+                _logger.Information("Going to remove cache.");
+
+                if (!File.Exists(_cacheBreakerPath))
+                    return;
+
+                _groupToCache.Clear();
+
+                File.Delete(_cacheBreakerPath);
+
+                _logger.Information("Remove cache.");
+            }
+            catch (Exception ex)
+            {
+                _logger.Error(ex, "Could not delete cache.");
+            }
+        }
+    }
+
+    private string GetSecurityGroupKey(GQIDMS dms, IGQILogger logger)
+    {
+        if (dms == null)
+            throw new ArgumentNullException(nameof(dms));
+        if (logger == null)
+            throw new ArgumentNullException(nameof(logger));
+
+        var responses = dms.SendMessages(new DMSMessage[] { new GetUserFullNameMessage(), new GetInfoMessage(InfoType.SecurityInfo) });
+        var userName = responses?.OfType<GetUserFullNameResponseMessage>().FirstOrDefault()?.User;
+        if (string.IsNullOrEmpty(userName))
+        {
+            logger.Error("User not found.");
+            return null;
+        }
+
+        var securityResponse = responses?.OfType<GetUserInfoResponseMessage>().FirstOrDefault();
+        var userGroups = securityResponse?.Users?.Where(u => u.Name == userName).FirstOrDefault()?.Groups?.OrderBy(x => x)?.ToArray() ?? new int[0];
+        if (userGroups.Length == 0)
+        {
+            logger.Error("User is not part of any group.");
+            return null;
+        }
+
+        return string.Join(";", userGroups);
+    }
+}
+```
+
+```csharp
+using System;
+using System.Collections.Concurrent;
+using System.Collections.Generic;
+using Skyline.DataMiner.Analytics.GenericInterface;
+
+public class Cache
+{
+    private const int TIMEOUT_SECONDS = 30;
+    private readonly string _securityKey;
+    private readonly object _lock = new object();
+    private readonly ConcurrentDictionary<string, Result> _results = new ConcurrentDictionary<string, Result>(StringComparer.OrdinalIgnoreCase);
+    private bool _isInitialized;
+
+    public Cache(string securityKey)
+    {
+        if (string.IsNullOrWhiteSpace(securityKey))
+            throw new ArgumentException("SecurityKey can't be empty.");
+
+        _securityKey = securityKey;
+    }
+
+    public DateTime LastUpdate { get; private set; }
+
+    public IEnumerable<Result> Results => _results.Values;
+
+    internal void EnsureInitialized(GQIDMS dms, IGQILogger logger)
+    {
+        if (_isInitialized)
+            return;
+
+        lock (_lock)
+        {
+            if (_isInitialized)
+                return;
+
+            Initialize(dms, logger);
+        }
+    }
+
+    internal void EnsureUpdated(GQIDMS dms, IGQILogger logger)
+    {
+        if (!_isInitialized)
+            return;
+
+        if (LastUpdate > DateTime.UtcNow - TimeSpan.FromSeconds(TIMEOUT_SECONDS))
+            return;
+
+        lock (_lock)
+        {
+            if (LastUpdate > DateTime.UtcNow - TimeSpan.FromSeconds(TIMEOUT_SECONDS))
+                return;
+
+            UpdateData(dms, logger);
+        }
+    }
+
+    private void UpdateData(GQIDMS dms, IGQILogger logger)
+    {
+        logger.Information($"{_securityKey} - Fetching update.");
+
+        /* Fetch update */
+
+        logger.Information($"{_securityKey} - Fetching update done.");
+        LastUpdate = DateTime.UtcNow;
+    }
+
+    private void Initialize(GQIDMS dms, IGQILogger logger)
+    {
+        logger.Information($"{_securityKey} - Initializing.");
+
+        /* Initialize */
+
+        logger.Information($"{_securityKey} - Initializing done.");
+
+        _isInitialized = true;
+    }
+}
+```

user-guide/Advanced_Modules/Dashboards_and_Low_Code_Apps/GQI/Extensions/GQI_Extensions_Best_Practices.md

@@ -42,3 +42,9 @@ Types and methods in this namespace have no use in a GQI extension and, more imp
 ## Only use 64-bit assembly references
 
 GQI runs in a 64-bit process and cannot load any extensions that require 32-bit assemblies. You should therefore only use 64-bit assembly references.
+
+## Make your ad hoc data source scalable
+
+Each time a query is executed, a new instance of the data source is created. This makes scalability crucial, especially when building the underlying data source is time-consuming or resource-intensive. Keep in mind that the code you're writing could be executed concurrently by many users.
+
+An example of how to build a highly scalable data source capable of serving many concurrent users: [Scaling an ad hoc data source](xref:Ad_hoc_Tutorials_Scalable).

user-guide/Advanced_Modules/toc.yml

@@ -585,6 +585,8 @@ items:
                         topicUid: Ad_hoc_Tutorials_Satellites
                       - name: Building a GQI data source that retrieves data from a DMS
                         topicUid: Ad_hoc_Tutorials_GQIDMS
+                      - name: Scaling an ad hoc data source
+                        topicUid: Ad_hoc_Tutorials_Scalable
               - name: Configuring a custom operator
                 topicUid: GQI_Custom_Operator
                 items: