Support deploy context read from aws secrets

Summary: This is a significant change to the config format

Reviewers: jeeva

Subscribers: #helix_omniscient

Differential Revision: http://phab.helixta.com.au/D22317

adl/config.adl

@@ -23,10 +23,7 @@ struct ToolConfig {
     /// The storage location for release zip files
     BlobStoreConfig releases;
 
-    /// The storage location for deployment context files
-    BlobStoreConfig deployContext;
-
-    Vector<DeployContextFile> deployContextFiles;
+    Vector<DeployContext> deployContexts;
     DeployMode deployMode = "select";
 };
 
@@ -86,10 +83,22 @@ struct SslCertPaths {
   FilePath sslCertificateKey;
 };
 
-struct DeployContextFile
+struct DeployContext
+{
+    String name;
+    DeployContextSource source;
+};
+
+union DeployContextSource
 {
-    FilePath name;
-    String sourceName;
+    // Context from a local file
+    FilePath file;
+
+    /// Context from an S3 object
+    S3Path s3;
+
+    /// Context from AWS secrets manager secret
+    String awsSecretArn;
 };
 
 union Verbosity

adl/config.adl-hs

@@ -2,7 +2,8 @@ module config
 {
 import adlc.config.haskell.HaskellFieldPrefix;
 
-annotation DeployContextFile HaskellFieldPrefix "dcf_";
+annotation DeployContext HaskellFieldPrefix "dc_";
+annotation DeployContextSource HaskellFieldPrefix "dcs_";
 annotation EndPoint HaskellFieldPrefix "ep_";
 annotation EndPointType HaskellFieldPrefix "ep_";
 annotation ToolConfig HaskellFieldPrefix "tc_";

hx-deploy-tool.cabal

@@ -28,6 +28,7 @@ executable hx-deploy-tool
                      , ADL.Types
                      , Blobs
                      , Blobs.S3
+                     , Blobs.Secrets
                      , Commands
                      , Commands.LetsEncrypt
                      , Commands.ProxyMode
@@ -47,6 +48,7 @@ executable hx-deploy-tool
                      , amazonka-ecr >= 1.4 && < 1.7
                      , amazonka-route53 >= 1.4.5 && < 1.7
                      , amazonka-s3 >= 1.4 && < 1.7
+                     , amazonka-secretsmanager >= 1.4 && < 1.7
                      , base64-bytestring >= 1.0 && < 1.1
                      , binary >= 0.8.5.1 && < 0.9
                      , bytestring >= 0.10 && < 0.11

src/ADL/Config.hs

@@ -1,7 +1,8 @@
 {-# LANGUAGE OverloadedStrings #-}
 module ADL.Config(
     BlobStoreConfig(..),
-    DeployContextFile(..),
+    DeployContext(..),
+    DeployContextSource(..),
     DeployMode(..),
     EndPoint(..),
     EndPointType(..),
@@ -44,26 +45,47 @@ instance AdlValue BlobStoreConfig where
         <|> parseUnionValue "localdir" BlobStoreConfig_localdir
         <|> parseFail "expected a BlobStoreConfig"
 
-data DeployContextFile = DeployContextFile
-    { dcf_name :: ADL.Types.FilePath
-    , dcf_sourceName :: T.Text
+data DeployContext = DeployContext
+    { dc_name :: T.Text
+    , dc_source :: DeployContextSource
     }
     deriving (Prelude.Eq,Prelude.Ord,Prelude.Show)
 
-mkDeployContextFile :: ADL.Types.FilePath -> T.Text -> DeployContextFile
-mkDeployContextFile name sourceName = DeployContextFile name sourceName
+mkDeployContext :: T.Text -> DeployContextSource -> DeployContext
+mkDeployContext name source = DeployContext name source
 
-instance AdlValue DeployContextFile where
-    atype _ = "config.DeployContextFile"
+instance AdlValue DeployContext where
+    atype _ = "config.DeployContext"
 
     jsonGen = genObject
-        [ genField "name" dcf_name
-        , genField "sourceName" dcf_sourceName
+        [ genField "name" dc_name
+        , genField "source" dc_source
         ]
 
-    jsonParser = DeployContextFile
+    jsonParser = DeployContext
         <$> parseField "name"
-        <*> parseField "sourceName"
+        <*> parseField "source"
+
+data DeployContextSource
+    = Dcs_file ADL.Types.FilePath
+    | Dcs_s3 ADL.Types.S3Path
+    | Dcs_awsSecretArn T.Text
+    deriving (Prelude.Eq,Prelude.Ord,Prelude.Show)
+
+instance AdlValue DeployContextSource where
+    atype _ = "config.DeployContextSource"
+
+    jsonGen = genUnion (\jv -> case jv of
+        Dcs_file v -> genUnionValue "file" v
+        Dcs_s3 v -> genUnionValue "s3" v
+        Dcs_awsSecretArn v -> genUnionValue "awsSecretArn" v
+        )
+
+    jsonParser
+        =   parseUnionValue "file" Dcs_file
+        <|> parseUnionValue "s3" Dcs_s3
+        <|> parseUnionValue "awsSecretArn" Dcs_awsSecretArn
+        <|> parseFail "expected a DeployContextSource"
 
 data DeployMode
     = DeployMode_select
@@ -251,14 +273,13 @@ data ToolConfig = ToolConfig
     , tc_autoCertName :: T.Text
     , tc_autoCertContactEmail :: T.Text
     , tc_releases :: BlobStoreConfig
-    , tc_deployContext :: BlobStoreConfig
-    , tc_deployContextFiles :: [DeployContextFile]
+    , tc_deployContexts :: [DeployContext]
     , tc_deployMode :: DeployMode
     }
     deriving (Prelude.Eq,Prelude.Ord,Prelude.Show)
 
-mkToolConfig :: BlobStoreConfig -> BlobStoreConfig -> [DeployContextFile] -> ToolConfig
-mkToolConfig releases deployContext deployContextFiles = ToolConfig "/opt/releases" "/opt/etc/deployment" "/opt/var/log/hx-deploy-tool.log" "/opt" "/opt/var/www" "hxdeploytoolcert" "" releases deployContext deployContextFiles DeployMode_select
+mkToolConfig :: BlobStoreConfig -> [DeployContext] -> ToolConfig
+mkToolConfig releases deployContexts = ToolConfig "/opt/releases" "/opt/etc/deployment" "/opt/var/log/hx-deploy-tool.log" "/opt" "/opt/var/www" "hxdeploytoolcert" "" releases deployContexts DeployMode_select
 
 instance AdlValue ToolConfig where
     atype _ = "config.ToolConfig"
@@ -272,8 +293,7 @@ instance AdlValue ToolConfig where
         , genField "autoCertName" tc_autoCertName
         , genField "autoCertContactEmail" tc_autoCertContactEmail
         , genField "releases" tc_releases
-        , genField "deployContext" tc_deployContext
-        , genField "deployContextFiles" tc_deployContextFiles
+        , genField "deployContexts" tc_deployContexts
         , genField "deployMode" tc_deployMode
         ]
 
@@ -286,8 +306,7 @@ instance AdlValue ToolConfig where
         <*> parseFieldDef "autoCertName" "hxdeploytoolcert"
         <*> parseFieldDef "autoCertContactEmail" ""
         <*> parseField "releases"
-        <*> parseField "deployContext"
-        <*> parseField "deployContextFiles"
+        <*> parseField "deployContexts"
         <*> parseFieldDef "deployMode" DeployMode_select
 
 data Verbosity

src/Blobs.hs

@@ -27,11 +27,6 @@ releaseBlobStore = do
   tcfg <- getToolConfig
   createBlobStore (tc_releases tcfg)
 
-deployContextBlobStore :: IOR BlobStore
-deployContextBlobStore = do
-  tcfg <- getToolConfig
-  createBlobStore (tc_deployContext tcfg)
-
 createBlobStore :: BlobStoreConfig -> IOR BlobStore
 createBlobStore (BlobStoreConfig_s3 s3Path) = awsBlobStore s3Path
 createBlobStore (BlobStoreConfig_localdir dirpath)  = localBlobStore (T.unpack dirpath)

src/Blobs/S3.hs

@@ -5,7 +5,8 @@ module Blobs.S3(
   downloadFileFrom,
   listFiles,
   extendObjectKey,
-  mkAwsEnv
+  mkAwsEnv,
+  mkAwsEnvFn
   ) where
 
 import qualified Data.ByteString as BS
@@ -20,6 +21,7 @@ import Data.Maybe(catMaybes)
 import Data.List(sortOn)
 import Data.Traversable(for)
 import Control.Concurrent(threadDelay)
+import Control.Concurrent.MVar
 import Control.Exception.Lens
 import Control.Lens
 import Control.Monad.Trans.AWS
@@ -41,6 +43,22 @@ mkAwsEnv = do
     env0 <- newEnv Discover
     return (env0 & envLogger .~ (L.awsLogger logger))
 
+type AwsEnvFn = IOR Env
+
+-- Create an AWS environment when required, and reuse it subsequently.
+mkAwsEnvFn :: IOR AwsEnvFn
+mkAwsEnvFn = do
+  mv <- liftIO $ newEmptyMVar
+  return $ do
+    menv <- liftIO $ tryReadMVar mv
+    case menv of
+      (Just awsEnv) -> return awsEnv
+      Nothing -> do
+        awsEnv <- mkAwsEnv
+        liftIO $ putMVar mv awsEnv
+        return awsEnv
+
+
 splitPath :: T.Text -> (S3.BucketName,S3.ObjectKey)
 splitPath s3Path = case T.stripPrefix "s3://" s3Path of
   Nothing -> error "s3Path must start with s3://"

src/Blobs/Secrets.hs

@@ -0,0 +1,39 @@
+{-# LANGUAGE OverloadedStrings #-}
+module Blobs.Secrets(
+  secretExists,
+  downloadSecretFrom
+  ) where
+
+import qualified Data.ByteString as BS
+import qualified Data.Text as T
+import qualified Data.Text.IO as T
+import qualified Network.AWS.SecretsManager as SM
+import qualified Network.AWS.SecretsManager.DescribeSecret as SM
+import qualified Network.AWS.SecretsManager.GetSecretValue as SM
+
+import Control.Exception.Lens
+import Control.Lens
+import Control.Monad.IO.Class(liftIO)
+import Control.Monad.Trans.AWS
+import Control.Monad.Trans.Resource
+import Data.Conduit((.|), ($$+-), ConduitT, sealConduitT, runConduit)
+import Network.HTTP.Types.Status(notFound404)
+
+secretExists :: Env -> T.Text -> IO Bool
+secretExists awsEnv secretArn = do
+  handling _ServiceError onServiceError $ do
+    runResourceT . runAWST awsEnv $ do
+      send (SM.describeSecret secretArn)
+      return True
+  where
+    onServiceError :: ServiceError -> IO Bool
+    onServiceError se | view serviceStatus se == notFound404 = return False
+                      | otherwise                            = throwing _ServiceError se
+
+downloadSecretFrom :: Env -> T.Text -> FilePath -> IO ()
+downloadSecretFrom awsEnv secretArn toFilePath = do
+  runResourceT . runAWST awsEnv $ do
+    resp <- send (SM.getSecretValue secretArn)
+    case view SM.gsvrsSecretString resp of
+      Nothing -> return ()
+      (Just secret) -> liftIO $ T.writeFile toFilePath secret

src/Commands.hs

@@ -17,7 +17,7 @@ import qualified Text.Mustache.Types as TM
 import qualified Commands.ProxyMode as P
 import qualified Blobs.S3 as S3
 
-import ADL.Config(ToolConfig(..), DeployContextFile(..), DeployMode(..), ProxyModeConfig(..))
+import ADL.Config(ToolConfig(..), DeployMode(..), ProxyModeConfig(..))
 import ADL.Release(ReleaseConfig(..))
 import ADL.Core(adlFromJsonFile')
 import Blobs(releaseBlobStore, BlobStore(..))

src/Commands/ProxyMode.hs

@@ -25,7 +25,7 @@ import qualified Network.AWS.EC2.Metadata as EC2M
 import ADL.Config(EndPoint(..), EndPointType(..))
 import ADL.Core(adlFromJsonFile', adlToJsonFile)
 import ADL.Release(ReleaseConfig(..))
-import ADL.Config(ToolConfig(..), DeployContextFile(..), DeployMode(..), ProxyModeConfig(..), MachineLabel(..))
+import ADL.Config(ToolConfig(..), DeployMode(..), ProxyModeConfig(..), MachineLabel(..))
 import ADL.State(State(..), Deploy(..))
 import ADL.Types(EndPointLabel, DeployLabel)
 import Util(unpackRelease,fetchDeployContext, checkReleaseExists)

src/Commands/ProxyMode/LocalState.hs

@@ -16,7 +16,7 @@ import qualified Data.Set as S
 import ADL.Config(EndPoint(..), EndPointType(..))
 import ADL.Core(adlFromJsonFile', adlToJsonFile)
 import ADL.Release(ReleaseConfig(..))
-import ADL.Config(ToolConfig(..), DeployContextFile(..), DeployMode(..), ProxyModeConfig(..), SslCertMode(..),  SslCertPaths(..))
+import ADL.Config(ToolConfig(..), DeployMode(..), ProxyModeConfig(..), SslCertMode(..),  SslCertPaths(..))
 import ADL.State(State(..), Deploy(..))
 import ADL.Types(EndPointLabel, DeployLabel)
 import Commands.ProxyMode.Types

src/Commands/ProxyMode/Types.hs

@@ -3,7 +3,7 @@ module Commands.ProxyMode.Types where
 import qualified ADL.Core.StringMap as SM
 import qualified Data.Text as T
 
-import ADL.Config(EndPoint(..), ToolConfig(..), DeployContextFile(..), DeployMode(..), ProxyModeConfig(..))
+import ADL.Config(EndPoint(..), ToolConfig(..), DeployMode(..), ProxyModeConfig(..))
 import ADL.State(State(..), Deploy(..))
 import Types(IOR, REnv(..), getToolConfig, scopeInfo)