Running LoRaWAN Network Server decoupled from IoT Edge (#1746)

Co-authored-by: Bastian Burger <[email protected]> Co-authored-by: Andrew Doing <[email protected]> Co-authored-by: mkcomer <[email protected]> Co-authored-by: Patrick Schuler <[email protected]>
Azure · Jul 4, 2022 · bbf3e4a · bbf3e4a
1 parent 81a6af1
commit bbf3e4a
Show file tree

Hide file tree

Showing 45 changed files with 2,181 additions and 618 deletions.
diff --git a/.github/workflows/e2e-ci.yaml b/.github/workflows/e2e-ci.yaml
@@ -14,7 +14,7 @@ on: # rebuild any PRs and main branch changes
         description: 'Include Load Tests in current run'
         default: 'false'
       TestsToRun:
-        default: '[SensorDecodingTest,OTAAJoinTest,ABPTest,OTAATest,MacTest,ClassCTest,C2DMessageTest,MultiGatewayTest,MultiConcentratorTest,CupsTest,LnsDiscoveryTest]'
+        default: '[SensorDecodingTest,OTAAJoinTest,ABPTest,OTAATest,MacTest,ClassCTest,C2DMessageTest,MultiGatewayTest,MultiConcentratorTest,CupsTest,LnsDiscoveryTest,CloudDeploymentTest]'
         description: 'tests to run'
       TxPower:
         description: 'TXPower value to use in E2E tests'
@@ -58,10 +58,10 @@ jobs:
           echo "::set-output name=E2ETestsToRun::${{ github.event.inputs.TestsToRun }}"
         elif [ ${{ github.event_name }} == 'pull_request' ]; then
           echo "Set up for pull request"
-          echo "::set-output name=E2ETestsToRun::[SensorDecodingTest,OTAAJoinTest,ABPTest,OTAATest,MacTest,ClassCTest,C2DMessageTest,MultiGatewayTest,MultiConcentratorTest,CupsTest,LnsDiscoveryTest]"
+          echo "::set-output name=E2ETestsToRun::[SensorDecodingTest,OTAAJoinTest,ABPTest,OTAATest,MacTest,ClassCTest,C2DMessageTest,MultiGatewayTest,MultiConcentratorTest,CupsTest,LnsDiscoveryTest,CloudDeploymentTest]"
         else
           echo "Set up for cron trigger"
-          echo "::set-output name=E2ETestsToRun::[SensorDecodingTest,OTAAJoinTest,ABPTest,OTAATest,MacTest,ClassCTest,C2DMessageTest,MultiGatewayTest,MultiConcentratorTest,CupsTest,LnsDiscoveryTest]"
+          echo "::set-output name=E2ETestsToRun::[SensorDecodingTest,OTAAJoinTest,ABPTest,OTAATest,MacTest,ClassCTest,C2DMessageTest,MultiGatewayTest,MultiConcentratorTest,CupsTest,LnsDiscoveryTest,CloudDeploymentTest]"
         fi
 
     - id: check-if-run
@@ -170,7 +170,8 @@ jobs:
       with:
         fetch-depth: '2'
 
-    - run: |
+    - id: image_tag_definition
+      run: |
         if [ ${{ github.ref	}} = "refs/heads/dev" ]; then
         echo "dev"
         IMAGE_TAG="$DEV_IMAGE_TAG"
@@ -189,6 +190,7 @@ jobs:
         echo "Using image tag $IMAGE_TAG"
         echo "::set-env name=NET_SRV_VERSION::$IMAGE_TAG"
         echo "::set-env name=LBS_VERSION::$IMAGE_TAG"
+        echo "::set-output name=imagetag::$IMAGE_TAG"
       env:
         ACTIONS_ALLOW_UNSECURE_COMMANDS: true
 
@@ -216,6 +218,9 @@ jobs:
       env:
         CONTAINER_REGISTRY_ADDRESS: ${{ env.CONTAINER_REGISTRY_ADDRESS }}
 
+    outputs:
+      imagetag: ${{ steps.image_tag_definition.outputs.imagetag }}
+
   # Generate root and server certificates and copy required files to RPi
   certificates_job :
     timeout-minutes: 5
@@ -287,6 +292,36 @@ jobs:
       clientfwdigest: ${{ steps.generate_step.outputs.clientfwdigest }}
       clientfwversion: ${{ steps.generate_step.outputs.clientfwversion }}
 
+  # Deploy Cloud based LoRaWAN Network Server
+  deploy_cloud_lns:
+    needs:
+    - env_var
+    - build_push_docker_images
+    - certificates_job
+    runs-on: ubuntu-latest
+    if: needs.env_var.outputs.RunE2ETestsOnly != 'true' && needs.env_var.outputs.StopFullCi != 'true'
+    name: Deploy Cloud based LNS
+    steps:
+      - name: "Deploy container instance"
+        id: "deploycloudlns"
+        shell: bash
+        run: |
+          az login --service-principal -u ${{ secrets.AZURE_SP_CLIENTID }} -p ${{ secrets.AZURE_SP_SECRET }} --tenant ${{ secrets.AZURE_TENANTID }}
+          az container create --resource-group ${{ secrets.AZURE_RG }} --ip-address Private --location westeurope --name cloudlns \
+                    --environment-variables LOG_TO_TCP_ADDRESS=${{ needs.certificates_job.outputs.itestupip }} LOG_TO_TCP_PORT=6100 LOG_TO_TCP=true LOG_LEVEL=1 IOTHUBHOSTNAME=${{secrets.IOTHUB_HOSTNAME}} ENABLE_GATEWAY=false CLOUD_DEPLOYMENT=true \
+                    --image ${{ env.CONTAINER_REGISTRY_ADDRESS }}/lorawannetworksrvmodule:${{needs.build_push_docker_images.outputs.imagetag}}-amd64 \
+                    --ports 5000 \
+                    --protocol TCP \
+                    --registry-username ${{ env.CONTAINER_REGISTRY_USERNAME }} \
+                    --registry-password ${{ env.CONTAINER_REGISTRY_PASSWORD }} \
+                    --restart-policy Never \
+                    --secure-environment-variables FACADE_AUTH_CODE=${{ secrets.FUNCTION_FACADE_AUTH_CODE }} FACADE_SERVER_URL=${{ secrets.FUNCTION_FACADE_SERVER_URL }} REDIS_CONNECTION_STRING=${{ secrets.REDIS_HOSTNAME }}.redis.cache.windows.net:6380,password=${{ secrets.REDIS_PASSWORD }},ssl=True,abortConnect=False \
+                    --subnet ${{ secrets.AZURE_SUBNET_NAME }} \
+                    --vnet ${{ secrets.AZURE_VNET_NAME }} --output none
+          echo "::set-output name=cloudlnsprivateip::$(az container show --name cloudlns --resource-group ${{ secrets.AZURE_RG }} --query ipAddress.ip -o tsv)"
+    outputs:
+      cloudlnsprivateip: ${{ steps.deploycloudlns.outputs.cloudlnsprivateip }}
+
   # Deploy IoT Edge solution to ARM gateway
   deploy_arm_gw_iot_edge:
     timeout-minutes: 20
@@ -433,7 +468,9 @@ jobs:
       INTEGRATIONTEST_LoadTestLnsEndpoints: ${{ secrets.LOAD_TEST_LNS_ENDPOINTS }}
       INTEGRATIONTEST_NumberOfLoadTestDevices: 10
       INTEGRATIONTEST_NumberOfLoadTestConcentrators: 4
-
+      INTEGRATIONTEST_FunctionAppCode: ${{ secrets.FUNCTION_FACADE_AUTH_CODE }}
+      INTEGRATIONTEST_FunctionAppBaseUrl: ${{ secrets.FUNCTION_FACADE_SERVER_URL }}
+      INTEGRATIONTEST_TcpLogPort: 6000
     steps:
     - uses: actions/checkout@v3
       name: Checkout current branch
@@ -460,7 +497,7 @@ jobs:
       shell: bash
       run: |
         dotnet test --logger trx --no-build --configuration ${{ env.BUILD_CONFIGURATION }} \
-        -r ${{ env.TESTS_RESULTS_FOLDER }}/LoadTest/ \
+        -r ${{ env.TESTS_RESULTS_FOLDER }}/LoadTest/ --filter "SimulatedLoadTests" \
         ${{ env.TESTS_FOLDER }}/Simulation/LoRaWan.Tests.Simulation.csproj
 
     # Upload test results as artifact
@@ -470,6 +507,77 @@ jobs:
         name: load-test-results
         path: ${{ env.TESTS_RESULTS_FOLDER }}/LoadTest/
 
+  cloud_test_job:
+    timeout-minutes: 150
+    name: Run cloud only deployment Tests
+    environment:
+      name: CI_AZURE_ENVIRONMENT
+      url: ${{ needs.env_var.outputs.CheckSuiteUrl }}
+    if: always() && needs.deploy_cloud_lns.result == 'success' && needs.deploy_facade_function.result == 'success' && needs.env_var.outputs.StopFullCi != 'true' && contains(needs.env_var.outputs.E2ETestsToRun, 'CloudDeploymentTest') && !(github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'CloudDeploymentTest'))
+    needs:
+      - deploy_facade_function
+      - deploy_cloud_lns
+      - env_var
+      - certificates_job
+    runs-on: [ self-hosted, x64 ]
+    env:
+      INTEGRATIONTEST_IoTHubEventHubConnectionString: ${{ secrets.IOTHUB_EVENT_HUB_CONNECTION_STRING }}
+      INTEGRATIONTEST_IoTHubEventHubConsumerGroup: ${{ secrets.IOTHUB_CI_CONSUMER_GROUP }}
+      INTEGRATIONTEST_IoTHubConnectionString: ${{ secrets.IOTHUB_OWNER_CONNECTION_STRING }}
+      INTEGRATIONTEST_LeafDeviceGatewayID: itestarm1
+      INTEGRATIONTEST_DevicePrefix: '12'
+      INTEGRATIONTEST_RunningInCI: true
+      INTEGRATIONTEST_LoadTestLnsEndpoints: "{ \\\"1\\\": \\\"ws://${{ needs.deploy_cloud_lns.outputs.cloudlnsprivateip }}:5000\\\" }"
+      INTEGRATIONTEST_NumberOfLoadTestDevices: 1
+      INTEGRATIONTEST_NumberOfLoadTestConcentrators: 2
+      INTEGRATIONTEST_FunctionAppCode: ${{ secrets.FUNCTION_FACADE_AUTH_CODE }}
+      INTEGRATIONTEST_FunctionAppBaseUrl: ${{ secrets.FUNCTION_FACADE_SERVER_URL }}
+      INTEGRATIONTEST_TcpLogPort: 6100
+
+    steps:
+    - uses: actions/checkout@v2
+      name: Checkout current branch
+
+    - name: Setup .NET
+      uses: actions/[email protected]
+      with:
+        dotnet-version: '6.0.x' # SDK Version to use.
+
+    - name: .NET SDK Information
+      run:
+        dotnet --info
+
+    - name: Configuration for simulated cloud tests
+      uses: cschleiden/replace-tokens@v1
+      with:
+        files: '${{ env.TESTS_FOLDER }}/Simulation/appsettings.json'
+
+    - name: Build simulated cloud tests
+      run: |
+        dotnet build --configuration ${{ env.BUILD_CONFIGURATION }} ${{ env.TESTS_FOLDER }}/Simulation/LoRaWan.Tests.Simulation.csproj
+
+    - name: Runs simulated cloud tests
+      shell: bash
+      run: |
+        dotnet test --logger trx --no-build --configuration ${{ env.BUILD_CONFIGURATION }} \
+        -r ${{ env.TESTS_RESULTS_FOLDER }}/LoadTest/ --filter "SimulatedCloudTests" \
+        ${{ env.TESTS_FOLDER }}/Simulation/LoRaWan.Tests.Simulation.csproj
+
+    - name: Add CloudDeploymentTest Test Label
+      uses: buildsville/add-remove-label@v1
+      if: github.event_name == 'pull_request' && success()
+      with:
+        token: ${{ github.token }}
+        label: 'CloudDeploymentTest'
+        type: add
+
+    # Upload simulated cloud results as artifact
+    - uses: actions/upload-artifact@v1
+      if: always()
+      with:
+        name: simulated-cloud-test-results
+        path: ${{ env.TESTS_RESULTS_FOLDER }}/LoadTest/
+
   # Runs E2E tests in dedicated agent, while having modules deployed into PI (arm32v7)
   e2e_tests_job:
     timeout-minutes: 150
@@ -647,3 +755,18 @@ jobs:
           AZURE_SP_CLIENTID: ${{ secrets.AZURE_SP_CLIENTID }}
           AZURE_SP_SECRET: ${{ secrets.AZURE_SP_SECRET }}
           AZURE_TENANTID: ${{ secrets.AZURE_TENANTID }}
+
+  delete_cloud_lns:
+    name: Delete cloud LNS
+    if: always()
+    runs-on: ubuntu-latest
+    needs:
+      - e2e_tests_job
+    steps:
+      - uses: actions/checkout@v2
+      - name: "Delete azure container instance for cloudlns"
+        shell: bash
+        run: |
+          az login --service-principal -u ${{ secrets.AZURE_SP_CLIENTID }} -p ${{ secrets.AZURE_SP_SECRET }} --tenant ${{ secrets.AZURE_TENANTID }}
+          az container logs -g ${{ secrets.AZURE_RG }} --name cloudlns
+          az container delete --yes --resource-group ${{ secrets.AZURE_RG }} --name cloudlns --output none
diff --git a/AssemblyInfo.cs b/AssemblyInfo.cs
@@ -11,4 +11,5 @@
 [assembly: InternalsVisibleTo("LoRaWan.Tests.Integration")]
 [assembly: InternalsVisibleTo("LoRaWan.Tests.E2E")]
 [assembly: InternalsVisibleTo("LoRaWan.Tests.Common")]
+[assembly: InternalsVisibleTo("LoRaWan.Tests.Simulation")]
 [assembly: InternalsVisibleTo("DynamicProxyGenAssembly2")]
diff --git a/LoRaEngine/LoraKeysManagerFacade/ RedisChannelPublisher.cs b/LoRaEngine/LoraKeysManagerFacade/ RedisChannelPublisher.cs
@@ -0,0 +1,29 @@
+// Copyright (c) Microsoft. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+namespace LoraKeysManagerFacade
+{
+    using StackExchange.Redis;
+    using System.Threading.Tasks;
+    using Microsoft.Extensions.Logging;
+    using LoRaTools;
+    using System.Text.Json;
+
+    public class RedisChannelPublisher : IChannelPublisher
+    {
+        private readonly ConnectionMultiplexer redis;
+        private readonly ILogger logger;
+
+        public RedisChannelPublisher(ConnectionMultiplexer redis, ILogger<RedisChannelPublisher> logger)
+        {
+            this.redis = redis;
+            this.logger = logger;
+        }
+
+        public async Task PublishAsync(string channel, LnsRemoteCall lnsRemoteCall)
+        {
+            this.logger.LogDebug("Publishing message to channel '{Channel}'.", channel);
+            _ = await this.redis.GetSubscriber().PublishAsync(channel, JsonSerializer.Serialize(lnsRemoteCall));
+        }
+    }
+}
diff --git a/LoRaEngine/LoraKeysManagerFacade/ClearLnsCache.cs b/LoRaEngine/LoraKeysManagerFacade/ClearLnsCache.cs
@@ -0,0 +1,96 @@
+// Copyright (c) Microsoft. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+#nullable enable
+
+namespace LoraKeysManagerFacade
+{
+    using System;
+    using System.Linq;
+    using System.Threading;
+    using System.Threading.Tasks;
+    using LoRaTools;
+    using Microsoft.AspNetCore.Http;
+    using Microsoft.AspNetCore.Mvc;
+    using Microsoft.Azure.Devices;
+    using Microsoft.Azure.WebJobs;
+    using Microsoft.Azure.WebJobs.Extensions.Http;
+    using Microsoft.Extensions.Logging;
+
+    public sealed class ClearLnsCache
+    {
+        private readonly IEdgeDeviceGetter edgeDeviceGetter;
+        private readonly IServiceClient serviceClient;
+        private readonly IChannelPublisher channelPublisher;
+        private readonly ILogger<ClearLnsCache> logger;
+
+        public ClearLnsCache(IEdgeDeviceGetter edgeDeviceGetter,
+                             IServiceClient serviceClient,
+                             IChannelPublisher channelPublisher,
+                             ILogger<ClearLnsCache> logger)
+        {
+            this.edgeDeviceGetter = edgeDeviceGetter;
+            this.serviceClient = serviceClient;
+            this.channelPublisher = channelPublisher;
+            this.logger = logger;
+        }
+
+        [FunctionName(nameof(ClearNetworkServerCache))]
+        public async Task<IActionResult> ClearNetworkServerCache([HttpTrigger(AuthorizationLevel.Function, "post")] HttpRequest req, CancellationToken cancellationToken)
+        {
+            if (req is null) throw new ArgumentNullException(nameof(req));
+
+            try
+            {
+                VersionValidator.Validate(req);
+            }
+            catch (IncompatibleVersionException ex)
+            {
+                return new BadRequestObjectResult(ex.Message);
+            }
+
+            await ClearLnsCacheInternalAsync(cancellationToken);
+
+            return new AcceptedResult();
+        }
+
+        internal async Task ClearLnsCacheInternalAsync(CancellationToken cancellationToken)
+        {
+            this.logger.LogInformation("Clearing device cache for all edge and Pub/Sub channel based Network Servers.");
+            // Edge device discovery for invoking direct methods
+            var edgeDevices = await this.edgeDeviceGetter.ListEdgeDevicesAsync(cancellationToken);
+            if (this.logger.IsEnabled(LogLevel.Debug))
+            {
+                this.logger.LogDebug("Invoking clear cache direct method for following devices: {deviceList}", string.Join(',', edgeDevices));
+            }
+            var tasks = edgeDevices.Select(e => InvokeClearViaDirectMethodAsync(e, cancellationToken)).ToArray();
+            // Publishing a single message for all cloud based LNSes
+            await PublishClearMessageAsync();
+            await Task.WhenAll(tasks);
+        }
+
+        internal async Task PublishClearMessageAsync()
+        {
+            await this.channelPublisher.PublishAsync(LoraKeysManagerFacadeConstants.ClearCacheMethodName, new LnsRemoteCall(RemoteCallKind.ClearCache, null));
+            this.logger.LogInformation("Cache clear message published on Pub/Sub channel");
+        }
+
+        internal async Task InvokeClearViaDirectMethodAsync(string lnsId, CancellationToken cancellationToken)
+        {
+            //Reason why the yield is needed is to avoid any potential "synchronous" code that might fail the publishing of a message on the pub/sub channel
+            await Task.Yield();
+            var res = await this.serviceClient.InvokeDeviceMethodAsync(lnsId,
+                                                                       LoraKeysManagerFacadeConstants.NetworkServerModuleId,
+                                                                       new CloudToDeviceMethod(LoraKeysManagerFacadeConstants.ClearCacheMethodName),
+                                                                       cancellationToken);
+            if (HttpUtilities.IsSuccessStatusCode(res.Status))
+            {
+                this.logger.LogInformation("Cache cleared for {gatewayID} via direct method", lnsId);
+            }
+            else
+            {
+                throw new InvalidOperationException($"Direct method call to {lnsId} failed with {res.Status}. Response: {res.GetPayloadAsJson()}");
+            }
+        }
+    }
+}