WIP: Try mounting shared home

It fails due to:
```
2021-04-23T16:36:59,942Z DEBUG Test worker [] [com.atlassian.performance.tools.ssh.SshjConnection] root$ sudo service nfs-kernel-server restart
2021-04-23T16:37:00,977Z DEBUG Test worker [] [com.atlassian.performance.tools.ssh.WaitingCommand.Companion]  * Stopping NFS kernel daemon
   ...done.
 * Unexporting directories for NFS kernel daemon...
   ...done.
 * Exporting directories for NFS kernel daemon...
   ...fail!

2021-04-23T16:37:00,979Z WARN  Test worker [] [com.atlassian.performance.tools.ssh.WaitingCommand.Companion] exportfs: /home/ubuntu/jira-shared-home does not support NFS export
```

Options:
* keep trying NFS in Docker (in WSL)
* try other FS sharing (FUSE? SAN?)
* use different FS sharing in Docker (volumes?) and different in AWS (so
SPI, but what is default? maybe no default or maybe bound to
`Infrastructure` SPI itself)

src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/Infrastructure.kt

@@ -1,11 +1,15 @@
 package com.atlassian.performance.tools.infrastructure.api
 
 import com.atlassian.performance.tools.infrastructure.api.jira.install.TcpHost
+import com.atlassian.performance.tools.ssh.api.Ssh
+
+interface Infrastructure : AutoCloseable { // TODO rename to ServerRoom
+
+    val subnet: String
 
-interface Infrastructure : AutoCloseable {
-
     /**
      * @return can be reached by the caller via [TcpHost.publicIp] and by the rest of the infra via [TcpHost.privateIp]
      */
-    fun serve(port: Int, name: String): TcpHost
+    fun serveTcp(name: String): TcpHost
+    fun serveSsh(name: String): Ssh
 }

src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/database/DockerMysqlServer.kt

@@ -28,7 +28,7 @@ class DockerMysqlServer private constructor(
         hooks: PreInstanceHooks,
         reports: Reports
     ) {
-        val server = infrastructure.serve(3306, "mysql")
+        val server = infrastructure.serveTcp("mysql")
         val client = server.ssh.newConnection().use { setup(it, server) }
         nodes.forEach { node ->
             node.postInstall.insert(DatabaseIpConfig(server.privateIp))

src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/JiraDataCenterPlan.kt

@@ -27,7 +27,7 @@ class JiraDataCenterPlan constructor(
         instanceHooks.call(nodePlans.map { it.hooks }, reports)
         val nodes = nodePlans.mapIndexed { nodeIndex, nodePlan ->
             val nodeNumber = nodeIndex + 1
-            val host = infrastructure.serve(8080, "jira-node-$nodeNumber")
+            val host = infrastructure.serveTcp("jira-node-$nodeNumber")
             nodePlan.materialize(host)
         }
         val balancer = balancerPlan.materialize(nodes)

src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/JiraServerPlan.kt

@@ -17,7 +17,7 @@ class JiraServerPlan private constructor(
     override fun materialize(): JiraInstance {
         val nodeHooks = listOf(plan).map { it.hooks }
         hooks.call(nodeHooks, reports)
-        val jiraNode = infrastructure.serve(8080, "jira-node")
+        val jiraNode = infrastructure.serveTcp("jira-node")
         val installed = plan.installation.install(jiraNode, reports)
         val started = plan.start.start(installed, reports)
         val instance = JiraServer(started)

src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/SharedHomeHook.kt

@@ -0,0 +1,62 @@
+package com.atlassian.performance.tools.infrastructure.api.jira.instance
+
+import com.atlassian.performance.tools.infrastructure.api.Infrastructure
+import com.atlassian.performance.tools.infrastructure.api.jira.JiraHomeSource
+import com.atlassian.performance.tools.infrastructure.api.jira.install.InstalledJira
+import com.atlassian.performance.tools.infrastructure.api.jira.install.hook.PostInstallHook
+import com.atlassian.performance.tools.infrastructure.api.jira.install.hook.PostInstallHooks
+import com.atlassian.performance.tools.infrastructure.api.jira.install.hook.PreInstallHooks
+import com.atlassian.performance.tools.infrastructure.api.jira.report.Reports
+import com.atlassian.performance.tools.infrastructure.api.os.RemotePath
+import com.atlassian.performance.tools.infrastructure.api.os.Ubuntu
+import com.atlassian.performance.tools.ssh.api.SshConnection
+
+internal class SharedHomeHook(
+    private val jiraHomeSource: JiraHomeSource,
+    private val infrastructure: Infrastructure
+) : PreInstanceHook {
+    private val localHome = "/home/ubuntu/jira-shared-home"
+
+    override fun call(nodes: List<PreInstallHooks>, hooks: PreInstanceHooks, reports: Reports) {
+        val server = infrastructure.serveSsh("shared-home")
+        server.newConnection().use { ssh ->
+            download(ssh)
+            export(ssh)
+        }
+        val sharedHome = RemotePath(server.host, localHome)
+        nodes.forEach { it.postInstall.insert(SharedHomeMount(sharedHome)) }
+    }
+
+    private fun download(ssh: SshConnection) {
+        ssh.execute("sudo mkdir -p $localHome")
+        val jiraHome = jiraHomeSource.download(ssh)
+        ssh.execute("sudo mv $jiraHome/{data,plugins,import,export} $localHome")
+        ssh.safeExecute("sudo mv $jiraHome/logos $localHome")
+    }
+
+    private fun export(ssh: SshConnection): SshConnection.SshResult {
+        Ubuntu().install(ssh, listOf("nfs-kernel-server"))
+        val options = "rw,sync,no_subtree_check,no_root_squash"
+        ssh.execute("sudo echo '$localHome ${infrastructure.subnet}($options)' | sudo tee -a /etc/exports")
+        return ssh.execute("sudo service nfs-kernel-server restart")
+    }
+
+    private class SharedHomeMount(
+        private val sharedHome: RemotePath
+    ) : PostInstallHook {
+
+        override fun call(ssh: SshConnection, jira: InstalledJira, hooks: PostInstallHooks, reports: Reports) {
+            Ubuntu().install(ssh, listOf("nfs-common"))
+            val mountSource = "${sharedHome.host.ipAddress}:${sharedHome.path}"
+            val mountTarget = "mounted-shared-home"
+            ssh.execute("mkdir -p $mountTarget")
+            ssh.execute("sudo mount -o soft,intr,rsize=8192,wsize=8192 $mountSource $mountTarget")
+            ssh.execute("sudo chown ubuntu:ubuntu $mountTarget")
+            val mountedPath = "`realpath $mountTarget`"
+            val jiraHome = jira.home.path
+            ssh.execute("echo ehcache.object.port = 40011 >> $jiraHome/cluster.properties")
+            ssh.execute("echo jira.node.id = ${jira.host.name} >> $jiraHome/cluster.properties")
+            ssh.execute("echo jira.shared.home = $mountedPath >> $jiraHome/cluster.properties")
+        }
+    }
+}

src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/report/Reports.kt

@@ -10,7 +10,7 @@ import java.nio.file.Paths
 import java.util.*
 import java.util.concurrent.ConcurrentLinkedQueue
 
-class Reports private constructor(
+class Reports private constructor( // TODO turn into SPI to allow AWS CLI transport (S3)
     private val hostReports: Queue<HostReport>
 ) {
     constructor() : this(ConcurrentLinkedQueue())

src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/loadbalancer/ApacheProxyPlan.kt

@@ -15,20 +15,19 @@ import java.net.URI
 import java.time.Duration
 
 class ApacheProxyPlan(
-    private val httpPort: Int,
     private val infrastructure: Infrastructure
 ) : LoadBalancerPlan {
 
     private val configPath = "/etc/apache2/sites-enabled/000-default.conf"
 
     override fun materialize(nodes: List<JiraNode>): LoadBalancer {
-        val proxyNode = infrastructure.serve(httpPort, "apache-proxy")
+        val proxyNode = infrastructure.serveTcp("apache-proxy")
         IdempotentAction("Installing and configuring apache load balancer") {
             proxyNode.ssh.newConnection().use { connection ->
                 tryToProvision(connection, nodes)
             }
         }.retry(2, ExponentialBackoff(Duration.ofSeconds(5)))
-        val balancerEndpoint = URI("http://${proxyNode.privateIp}:$httpPort/")
+        val balancerEndpoint = URI("http://${proxyNode.privateIp}:${proxyNode.port}/")
         nodes.forEach { it.plan.hooks.preStart.insert(InjectProxy(balancerEndpoint)) }
         return ApacheProxy(balancerEndpoint)
     }

src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/DockerInfrastructure.kt

@@ -26,6 +26,7 @@ internal class DockerInfrastructure : Infrastructure {
     private val allocatedResources: Deque<AutoCloseable> = ConcurrentLinkedDeque()
     private val docker: DockerClient
     private val network: DockerNetwork
+    override val subnet: String
 
     init {
         val dockerConfig = DefaultDockerClientConfig.createDefaultConfigBuilder().build()
@@ -37,17 +38,37 @@ internal class DockerInfrastructure : Infrastructure {
             .withName(randomUUID().toString())
             .execAsResource(docker)
         allocatedResources.add(network)
+        subnet = docker
+            .inspectNetworkCmd()
+            .withNetworkId(network.response.id)
+            .exec()
+            .ipam
+            .config
+            .first()
+            .subnet
+    }
+
+    fun serveTest(): Ssh {
+        return serveSsh("ssh")
     }
 
-    fun serve(): Ssh {
-        return serve("ssh")
+    override fun serveSsh(name: String): Ssh {
+        return serveTcp(888, name).ssh
     }
 
-    fun serve(name: String): Ssh {
-        return serve(888, name).ssh
+
+    override fun serveTcp(name: String): TcpHost {
+        return when {
+            name.startsWith("jira-node") -> serveTcp(8080, name) // TODO this is a contract on undocumented behavior
+            name.startsWith("mysql") -> serveTcp(3306, name)
+            else -> serveTcp(
+                888,
+                name
+            ) // TODO pre-provision all the hosts rather than on-demand - unlock batch provisioning (CFN Stack), picking EC2 types, SSD storage, TCP port ranges, subnets, etc.
+        }
     }
 
-    override fun serve(port: Int, name: String): TcpHost {
+    private fun serveTcp(port: Int, name: String): TcpHost {
         docker
             .pullImageCmd("rastasheep/ubuntu-sshd")
             .withTag("18.04")

src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/browser/ChromeIT.kt

@@ -12,7 +12,7 @@ class ChromeIT {
     @Test
     fun shouldInstallChromeBrowser() {
         DockerInfrastructure().use { infra ->
-            infra.serve(80, "ChromeIT").ssh.newConnection().use { connection ->
+            infra.serveSsh("ChromeIT").newConnection().use { connection ->
                 val wasInstalledBefore = isChromeInstalled(connection)
 
                 Chrome().install(connection)

src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/browser/chromium/Chromium69IT.kt

@@ -14,7 +14,7 @@ class Chromium69IT {
     @Test
     fun shouldInstallBrowser() {
         DockerInfrastructure().use { infra ->
-            infra.serve().newConnection().use { connection ->
+            infra.serveTest().newConnection().use { connection ->
                 val installedBefore = isChromiumInstalled(connection)
 
                 Chromium69().install(connection)

src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/browser/chromium/PageLoadTimeoutRecoveryTest.kt

@@ -20,7 +20,7 @@ internal class PageLoadTimeoutRecoveryTest {
             val fastResource = httpServer.register(FastResponse())
             val slowResource = httpServer.register(SlowResponse())
             DockerInfrastructure().use { infra ->
-                val ssh = infra.serve()
+                val ssh = infra.serveTest()
                 ssh.forwardRemotePort(httpServer.getPort(), httpServer.getPort()).use {
                     val localChromedriverPort = findFreePort()
                     ssh.forwardLocalPort(localChromedriverPort, remoteChromedriverPort).use {

src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/dataset/HttpDatasetPackageIT.kt

@@ -21,7 +21,7 @@ class HttpDatasetPackageIT {
         )
 
         val filesInDataset = DockerInfrastructure().use { infra ->
-            val ssh = infra.serve(80, "HttpDatasetPackageIT").ssh
+            val ssh = infra.serveSsh("HttpDatasetPackageIT")
             return@use RandomFilesGenerator(ssh).start().use {
                 ssh.newConnection().use { connection ->
                     val unpackedPath = dataset.download(connection)

src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/distribution/PublicJiraServiceDeskDistributionIT.kt

@@ -9,7 +9,7 @@ class PublicJiraServiceDeskDistributionIT {
     @Test
     fun shouldDownloadJiraServiceDesk() {
         DockerInfrastructure().use { infra ->
-            infra.serve().newConnection().use { connection ->
+            infra.serveTest().newConnection().use { connection ->
                 val serviceDeskDistribution: ProductDistribution = PublicJiraServiceDeskDistribution("4.0.1")
                 val targetFolder = "test"
                 connection.execute("mkdir $targetFolder")

src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/distribution/PublicJiraSoftwareDistributionsIT.kt

@@ -9,7 +9,7 @@ class PublicJiraSoftwareDistributionsIT {
     @Test
     fun shouldDownloadJiraSoftware() {
         DockerInfrastructure().use { infra ->
-            infra.serve().newConnection().use { connection ->
+            infra.serveTest().newConnection().use { connection ->
                 val jiraDistribution: ProductDistribution = PublicJiraSoftwareDistribution("7.2.0")
                 val targetFolder = "test"
                 connection.execute("mkdir $targetFolder")

src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/docker/DockerIT.kt

@@ -9,7 +9,7 @@ class DockerIT {
     @Test
     fun installWorks() {
         DockerInfrastructure().use { infra ->
-            infra.serve().newConnection().use { connection ->
+            infra.serveTest().newConnection().use { connection ->
                 //workaround for a bug in Docker download site for bionic
                 val packageFile = "containerd.io_1.2.2-3_amd64.deb"
                 Ubuntu().install(connection,listOf( "curl"))

src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/JiraDataCenterPlanIT.kt

@@ -20,7 +20,6 @@ import org.assertj.core.api.Assertions.assertThat
 import org.junit.After
 import org.junit.Before
 import org.junit.Test
-import java.lang.Exception
 import java.nio.file.Files
 
 class JiraDataCenterPlanIT {
@@ -40,22 +39,26 @@ class JiraDataCenterPlanIT {
     @Test
     fun shouldStartDataCenter() {
         // given
+        val jiraHomeSource = JiraHomePackage(Datasets.JiraSevenDataset.jiraHome)
         val nodePlans = listOf(1, 2).map {
+            val nodeHooks = PreInstallHooks.default()
+                .also { Datasets.JiraSevenDataset.hookMysql(it.postStart) }
             JiraNodePlan.Builder()
                 .installation(
                     ParallelInstallation(
-                        jiraHomeSource = JiraHomePackage(Datasets.JiraSevenDataset.jiraHome),
+                        jiraHomeSource = jiraHomeSource,
                         productDistribution = PublicJiraSoftwareDistribution("7.13.0"),
                         jdk = AdoptOpenJDK()
                     )
                 )
                 .start(JiraLaunchScript())
-                .hooks(PreInstallHooks.default().also { Datasets.JiraSevenDataset.hookMysql(it.postStart) })
+                .hooks(nodeHooks)
                 .build()
         }
         val instanceHooks = PreInstanceHooks.default()
             .also { Datasets.JiraSevenDataset.hookMysql(it, infrastructure) }
-        val balancerPlan = ApacheProxyPlan(80, infrastructure)
+            .also { it.insert(SharedHomeHook(jiraHomeSource, infrastructure)) }
+        val balancerPlan = ApacheProxyPlan(infrastructure)
         val dcPlan = JiraDataCenterPlan(nodePlans, instanceHooks, balancerPlan, infrastructure)
 
         // when
@@ -96,7 +99,7 @@ class JiraDataCenterPlanIT {
                 .hooks(PreInstallHooks.default().also { it.preStart.insert(FailingHook()) })
                 .build()
         }
-        val balancerPlan = ApacheProxyPlan(80, infrastructure)
+        val balancerPlan = ApacheProxyPlan(infrastructure)
         val dcPlan = JiraDataCenterPlan(nodePlans, PreInstanceHooks.default(), balancerPlan, infrastructure)
 
         try {

src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/jvm/AdoptOpenJdk11IT.kt

@@ -8,7 +8,7 @@ class AdoptOpenJdk11IT {
     @Test
     fun shouldSupportJstat() {
         DockerInfrastructure().use { infra ->
-            infra.serve().newConnection().use { connection ->
+            infra.serveTest().newConnection().use { connection ->
                 JstatSupport(AdoptOpenJDK11()).shouldSupportJstat(connection)
             }
         }

src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/jvm/AdoptOpenJdkIT.kt

@@ -8,7 +8,7 @@ class AdoptOpenJdkIT {
     @Test
     fun shouldSupportJstat() {
         DockerInfrastructure().use { infra ->
-            infra.serve().newConnection().use { connection ->
+            infra.serveTest().newConnection().use { connection ->
                 JstatSupport(AdoptOpenJDK()).shouldSupportJstat(connection)
             }
         }

src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/jvm/OpenJdk11IT.kt

@@ -8,7 +8,7 @@ class OpenJdk11IT {
     @Test
     fun shouldSupportJstat() {
         DockerInfrastructure().use { infra ->
-            infra.serve().newConnection().use { connection ->
+            infra.serveTest().newConnection().use { connection ->
                 JstatSupport(OpenJDK11()).shouldSupportJstat(connection)
             }
         }

src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/jvm/OpenJdkIT.kt

@@ -8,7 +8,7 @@ class OpenJdkIT {
     @Test
     fun shouldSupportJstat() {
         DockerInfrastructure().use { infra ->
-            infra.serve().newConnection().use { connection ->
+            infra.serveTest().newConnection().use { connection ->
                 JstatSupport(OpenJDK()).shouldSupportJstat(connection)
             }
         }

src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/jvm/OracleJdkIT.kt

@@ -8,7 +8,7 @@ class OracleJdkIT {
     @Test
     fun shouldSupportJstatAndThreadDumps() {
         DockerInfrastructure().use { infra ->
-            infra.serve().newConnection().use { connection ->
+            infra.serveTest().newConnection().use { connection ->
                 val jdk = OracleJDK()
                 JstatSupport(jdk).shouldSupportJstat(connection)
                 ThreadDumpTest().shouldGatherThreadDump(jdk, connection)