Merge pull request #37 from nehmebilal/namedpipes

Add health monitoring and graceful shutdown

.gitignore

@@ -30,6 +30,9 @@ bld/
 # Visual Studo 2015 cache/options directory
 .vs/
 
+# Rider ide directory
+.idea/
+
 # MSTest test Results
 [Tt]est[Rr]esult*/
 [Bb]uild[Ll]og.*

Docs/Deploy&Host_an_App_in_YAMS.md

@@ -250,5 +250,112 @@ That's it! The two versions should be now happily running side-by-side.
 # Removing or reverting a deployment
 To remove an app from YAMS, simply remove the corresponding entry from the `DeploymentConfig.json` file. YAMS will terminate the corresponding process and remove the app. You can also delete the associated files from the blob storage if you are never going to use this app again or keep it there to preserve history of deployments and to allow reverts. In fact, to revert a deployment in YAMS, simply edit the `DeploymentConfig.json` file and replace the current version of the app (the version to be reverted) with the old version (the version to revert to).
 
+# Advanced features
+
+Yams has support for health monitoring and graceful shutdown of apps as described [here](../Docs/Overview.md#health-monitoring-and-graceful-shutdown). Note that you can choose to enable one or multiple features and apps within the same cluster can use different features. 
+
+## Monitored initialization
+By default, Yams does not monitor the initialization of apps. In other words, when an app is deployed, the associated process is launched and then Yams assumes that the app is running and ready to receive requests. With the monitored initialization feature enabled, Yams would wait for the app to finish initialization before moving on to the next app (the app would notify Yams that it's done initializing through an IPC message).
+
+To enable *monitored initialization* for a given app, the corresponding flag must be added to the `AppConfig.json` file as shown below:
+```json
+{
+  "ExeName": "MyProcess.exe",
+  "ExeArgs": "Foo Bar",
+  "MonitorInitialization": true
+}
+```
+
+The app source code will also need to be updated so that the app can communicate with Yams (using IPC). Install the `Etg.Yams` NuGet package and modify the app source code so that Yams is notified when initialization is done, as shown in the code below:
+
+```csharp
+        public static void Main(string[] args)
+        {
+            MainAsync(args).Wait();
+        }
+
+        private static async Task MainAsync(string[] args)
+        {
+            var yamsClientConfig = new YamsClientConfigBuilder(args).Build();
+            var yamsClientFactory = new YamsClientFactory();
+            IYamsClient yamsClient = yamsClientFactory.CreateYamsClient(yamsClientConfig);
+
+            await Task.WhenAll(yamsClient.Connect(), Initialize());
+
+            await yamsClient.SendInitializationDoneMessage();
+
+	    // ...
+```
+
+## Heart beats
+With this feature enabled, the app is expected to send heart beat messages to Yams at steady intervals. If heart beats are not received in time, errors will be logged (more complex handling will be added in the future). To enable this feature, update the `AppConfig.json` and your app source code as shown below:
+
+```json
+{
+  "ExeName": "MyProcess.exe",
+  "ExeArgs": "Foo Bar",
+  "MonitorHealth":  true
+}
+```
+
+```csharp
+        public static void Main(string[] args)
+        {
+            MainAsync(args).Wait();
+        }
+
+        private static async Task MainAsync(string[] args)
+        {
+            var yamsClientConfig = new YamsClientConfigBuilder(args).Build();
+            var yamsClientFactory = new YamsClientFactory();
+            IYamsClient yamsClient = yamsClientFactory.CreateYamsClient(yamsClientConfig);
+
+            await Task.WhenAll(yamsClient.Connect(), Initialize());
+
+            while (true)
+            {
+                await Task.Delay(heartBeatPeriod);
+                await yamsClient.SendHeartBeat();
+            }
+```
+
+## Graceful Shutdown
+
+When graceful shutdown is enabled for a given app, Yams will deliver an event to the app and allow it a configurable amount of time to exit gracefully before closing/killing it. The graceful shutdown event will be delivered through the `YamsClient` as a normal C# event. To enable this feature, update the `AppConfig.json` and your app source code as shown below:
+
+```json
+{
+  "ExeName": "MyProcess.exe",
+  "ExeArgs": "Foo Bar",
+  "GracefulShutdown":  true
+}
+```
+
+```csharp
+        public static void Main(string[] args)
+        {
+            MainAsync(args).Wait();
+        }
+
+        private static async Task MainAsync(string[] args)
+        {
+            var yamsClientConfig = new YamsClientConfigBuilder(args).Build();
+            var yamsClientFactory = new YamsClientFactory();
+            IYamsClient yamsClient = yamsClientFactory.CreateYamsClient(yamsClientConfig);
+
+            await Task.WhenAll(yamsClient.Connect(), Initialize());
+
+            bool exitMessageReceived = false;
+            yamsClient.ExitMessageReceived += (sender, eventArgs) =>
+            {
+                exitMessageReceived = true;
+            };
+
+            while (!exitMessageReceived)
+            {
+                await DoWork();                
+            }	    
+```
+
 # Source code
 The source code associated with this tutorial can be found in the [Samples/WebApp](../Samples/WebApp) directory.

Docs/Overview.md

@@ -15,6 +15,7 @@ To deploy such a Microservices-based application to Azure using the Azure cloud
 * **Versioning** of microservices, quick **updates**, **reverts**, etc. 
 * Support for **Upgrade Domains** to minimize (and potentially eliminate) application downtime during updates, including first-class support for **Azure Upgrade Domains**.
 * Microservices can be developed in **any programming language** and deployed with YAMS (as long as your service can be started with an exe).
+* **Health monitoring** and **graceful shutdown** of microservices.
 
 YAMS has first-class support for deploying applications from Azure **blob storage**, but with its pluggable storage architecture, other providers such as SQL Server or file storage can be created and plugged in as well. 
 
@@ -101,6 +102,19 @@ Note that if an update fails, Yams will not try to revert back to the old versio
 
 To revert a deployment, simply edit the `DeploymentConfig.json` file and replace the current version of the app (the version to be reverted) with the old version (the version to revert to).
 
+## Health monitoring and graceful shutdown
+
+Apps deployed with Yams can optionally enable health monitoring and/or graceful shutdown. Yams uses inter-process-communication (currently [named pipes](https://msdn.microsoft.com/en-us/library/bb546085(v=vs.110).aspx)) to communicate with apps. 
+
+There are three features available:
+* *Monitored initialization*: Yams waits for the app to finish initialization before considering it ready to receive requests. If an app takes longer than expected to finish initialization (the timeout is configurable), it's considered unhealthy and is killed.
+* *Monitored heart beats*: With this feature enabled, Yams expects to receive heart beats from apps at steady intervals. If a heart beat is not received in time, an error is logged (more complex handling will be added in the future).
+* *Graceful shutdown*: A event is sent to the app to signal shutdown. If the app does not exit gracefully in time (the timeout is configurable), the app will be closed or killed.
+
+Note that each of these features can be enabled/disabled separately. In addition, apps running within the same cluster can choose to enable/disable different features.
+
+The [Yams Client API](../src/Etg.Yams.Core/Client/IYamsClient.cs) can be used by apps to communicate with Yams. See [Deploy and host applications in YAMS tutorial](../Docs/Deploy&Host_an_App_in_YAMS.md) to learn how you can enable these features (one ore more features can be enabled at a time).
+
 ## Sharing infrastructure
 One of the main goals of Yams is sharing infrastructure to reduce cost. In fact, some microservices consume little resources and can be deployed alongside other microservices. In addition, sharing infrastructure reduces the cost of over-provisioning resources. To illustrate this, consider an application composed of two microservices. Each microservice requires 2 VMs at normal operation load and 4 VMs at peak time. If each microservice is deployed separately, 8 VMs are needed in total (4 VMs per microservice). However, in practice, the peak time resources are over estimated and the peak time of one microservice does not necessarily overlap with the peak time of another microservice. If the same VMs are shared by both microservices and peak times are not likely to overlap, 6 VMs can be sufficient for both microservices (which saves us 2 VMs). In fact, this strategy works better for a large number of microservices where the probability of all microservices peaking at the same time decreases with the number of microservices and as a result, sharing infrastructure can result in large savings.
 

Etg.Yams.sln

@@ -37,6 +37,16 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "AzureUtilsTest", "test\Azur
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Etg.Yams.Core", "src\Etg.Yams.Core\Etg.Yams.Core.csproj", "{7145E485-34FA-4632-89B0-BD27C96AF69C}"
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Stubs", "test\Stubs\Stubs.csproj", "{68CD41F8-A6C3-4D43-93CA-92E898254CBD}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "MonitorInitProcess", "test\MonitorInitProcess\MonitorInitProcess.csproj", "{BB84CC4C-EB11-4A61-8ED4-791EADAA46E1}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "GracefullShutdownProcess", "test\GracefullShutdownProcess\GracefullShutdownProcess.csproj", "{706E37D7-B148-4734-9695-0D1AE6D4B3D5}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "HeartBeatProcess", "test\HeartBeatProcess\HeartBeatProcess.csproj", "{257044FB-F7A8-499D-8EF2-B5CAD76D617A}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "FullIpcProcess", "test\FullIpcProcess\FullIpcProcess.csproj", "{99F3A36C-7930-4670-A8B3-7137D891671D}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -103,6 +113,26 @@ Global
 		{7145E485-34FA-4632-89B0-BD27C96AF69C}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{7145E485-34FA-4632-89B0-BD27C96AF69C}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{7145E485-34FA-4632-89B0-BD27C96AF69C}.Release|Any CPU.Build.0 = Release|Any CPU
+		{68CD41F8-A6C3-4D43-93CA-92E898254CBD}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{68CD41F8-A6C3-4D43-93CA-92E898254CBD}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{68CD41F8-A6C3-4D43-93CA-92E898254CBD}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{68CD41F8-A6C3-4D43-93CA-92E898254CBD}.Release|Any CPU.Build.0 = Release|Any CPU
+		{BB84CC4C-EB11-4A61-8ED4-791EADAA46E1}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{BB84CC4C-EB11-4A61-8ED4-791EADAA46E1}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{BB84CC4C-EB11-4A61-8ED4-791EADAA46E1}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{BB84CC4C-EB11-4A61-8ED4-791EADAA46E1}.Release|Any CPU.Build.0 = Release|Any CPU
+		{706E37D7-B148-4734-9695-0D1AE6D4B3D5}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{706E37D7-B148-4734-9695-0D1AE6D4B3D5}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{706E37D7-B148-4734-9695-0D1AE6D4B3D5}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{706E37D7-B148-4734-9695-0D1AE6D4B3D5}.Release|Any CPU.Build.0 = Release|Any CPU
+		{257044FB-F7A8-499D-8EF2-B5CAD76D617A}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{257044FB-F7A8-499D-8EF2-B5CAD76D617A}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{257044FB-F7A8-499D-8EF2-B5CAD76D617A}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{257044FB-F7A8-499D-8EF2-B5CAD76D617A}.Release|Any CPU.Build.0 = Release|Any CPU
+		{99F3A36C-7930-4670-A8B3-7137D891671D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{99F3A36C-7930-4670-A8B3-7137D891671D}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{99F3A36C-7930-4670-A8B3-7137D891671D}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{99F3A36C-7930-4670-A8B3-7137D891671D}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -123,6 +153,11 @@ Global
 		{E66F281C-BD50-45B0-AAEF-FB87E4C9D0CC} = {2A52BDC8-4B16-43FE-9E9D-A7D0A72C17C8}
 		{2FE963BD-D826-4CC5-8A63-864CDA212233} = {2A52BDC8-4B16-43FE-9E9D-A7D0A72C17C8}
 		{7145E485-34FA-4632-89B0-BD27C96AF69C} = {5925E681-1BEA-456D-B9E0-CA175ABBFA9D}
+		{68CD41F8-A6C3-4D43-93CA-92E898254CBD} = {2A52BDC8-4B16-43FE-9E9D-A7D0A72C17C8}
+		{BB84CC4C-EB11-4A61-8ED4-791EADAA46E1} = {2A52BDC8-4B16-43FE-9E9D-A7D0A72C17C8}
+		{706E37D7-B148-4734-9695-0D1AE6D4B3D5} = {2A52BDC8-4B16-43FE-9E9D-A7D0A72C17C8}
+		{257044FB-F7A8-499D-8EF2-B5CAD76D617A} = {2A52BDC8-4B16-43FE-9E9D-A7D0A72C17C8}
+		{99F3A36C-7930-4670-A8B3-7137D891671D} = {2A52BDC8-4B16-43FE-9E9D-A7D0A72C17C8}
 	EndGlobalSection
 	GlobalSection(ExtensibilityGlobals) = postSolution
 		EnterpriseLibraryConfigurationToolBinariesPathV6 = packages\EnterpriseLibrary.TransientFaultHandling.6.0.1304.0\lib\portable-net45+win+wp8

README.md

@@ -10,6 +10,7 @@ YAMS
 * **Versioning** of microservices, quick **updates**, **reverts**, etc. 
 * Support for **Upgrade Domains** to minimize (and potentially eliminate) application downtime during updates, including first-class support for **Azure Upgrade Domains**.
 * Microservices can be developed in **any programming language** and deployed with YAMS (as long as your service can be started with an exe).
+* **Health monitoring** and **graceful shutdown** of microservices.
 
 YAMS has first-class support for deploying applications from Azure **blob storage**, but with its pluggable storage architecture, other providers such as SQL Server or file storage can be created and plugged in as well.
 

src/Etg.Yams.Core/Application/Application.cs

@@ -13,17 +13,17 @@ protected Application(AppIdentity identity, string path)
             Path = path;
         }
 
-        protected async Task<bool> StartProcess(IProcess process)
+        protected async Task<bool> StartProcess(IProcess process, string args)
         {
             try
             {
-                await process.Start();
+                await process.Start(args);
                 process.Exited += OnProcessExited;
                 return true;
             }
-            catch (Exception)
+            catch (Exception e)
             {
-                Trace.TraceInformation("Could not start the host process for application {0}", Identity);
+                Trace.TraceError($"Could not start the host process for application {Identity}, Inner Exception: {e}");
                 return false;    
             }
         }

src/Etg.Yams.Core/Application/ApplicationConfig.cs

@@ -2,14 +2,26 @@
 {
     public class ApplicationConfig
     {
-        public AppIdentity Identity { get; private set; }
-        public string ExeName { get; private set; }
-        public string ExeArgs { get; private set; }
+        public AppIdentity Identity { get; }
+        public string ExeName { get; }
+        public string ExeArgs { get; }
+        public bool MonitorInitialization { get; }
+        public bool MonitorHealth { get; }
+        public bool GracefulShutdown { get; }
 
-        public ApplicationConfig(AppIdentity identity, string exeName, string exeArgs)
+        public ApplicationConfig(
+            AppIdentity identity,
+            string exeName,
+            string exeArgs,
+            bool monitorInitialization = false,
+            bool monitorHealth = false,
+            bool gracefulShutdown = false)
         {
             Identity = identity;
             ExeArgs = exeArgs;
+            MonitorInitialization = monitorInitialization;
+            MonitorHealth = monitorHealth;
+            GracefulShutdown = gracefulShutdown;
             ExeName = exeName;
         }
     }

src/Etg.Yams.Core/Application/ApplicationConfigParser.cs

@@ -1,11 +1,9 @@
-using System;
-using System.Collections.Generic;
+using System.Collections.Generic;
 using System.IO;
 using System.Text.RegularExpressions;
 using System.Threading.Tasks;
 using Etg.Yams.Install;
 using Etg.Yams.Json;
-using Etg.Yams.Storage.Config;
 
 namespace Etg.Yams.Application
 {
@@ -20,6 +18,9 @@ private class ApplicationConfigData
 #pragma warning disable 649
             public string ExeName;
             public string ExeArgs;
+            public bool MonitorInitialization;
+            public bool MonitorHealth;
+            public bool GracefulShutdown;
 #pragma warning restore 649
         }
 
@@ -33,14 +34,16 @@ public async Task<ApplicationConfig> ParseFile(string path, AppInstallConfig app
         {
             using (StreamReader r = new StreamReader(path))
             {
-                return await Parse(await _jsonSerializer.DeserializeAsync<ApplicationConfigData>(await r.ReadToEndAsync()), appInstallConfig);
+                return await Parse(await _jsonSerializer.DeserializeAsync<ApplicationConfigData>(
+                    await r.ReadToEndAsync()), appInstallConfig);
             }
         }
 
         private async Task<ApplicationConfig> Parse(ApplicationConfigData appConfigData, AppInstallConfig appInstallConfig)
         {
             string args = await SubstituteSymbols(appConfigData.ExeArgs, appInstallConfig);
-            return new ApplicationConfig(appInstallConfig.AppIdentity, appConfigData.ExeName, args);
+            return new ApplicationConfig(appInstallConfig.AppIdentity, appConfigData.ExeName, args,
+                appConfigData.MonitorInitialization, appConfigData.MonitorHealth, appConfigData.GracefulShutdown);
         }
 
         private async Task<string> SubstituteSymbols(string str, AppInstallConfig appInstallConfig)
@@ -62,7 +65,7 @@ private async Task<string> SubstituteSymbols(string str, AppInstallConfig appIns
         private async Task<string> SubstitueSymbol(string str, string symbol, AppInstallConfig appInstallConfig)
         {
             string symbolValue = await _symbolResolver.ResolveSymbol(appInstallConfig, symbol);
-            return str.Replace(string.Format("${{{0}}}", symbol), symbolValue);
+            return str.Replace($"${{{symbol}}}", symbolValue);
         }
     }
 }