Bump a bunch of timeouts, because my Internet is awful

Show the problem with hooking onto the tail of the hook queue during hook iteration.
atlassian · Sep 6, 2019 · e18d03e · e18d03e
1 parent 7694417
commit e18d03e
Show file tree

Hide file tree

Showing 11 changed files with 167 additions and 55 deletions.
diff --git a/...tlassian/performance/tools/infrastructure/api/jira/flow/install/DefaultPostInstallHook.kt b/...tlassian/performance/tools/infrastructure/api/jira/flow/install/DefaultPostInstallHook.kt
@@ -2,7 +2,7 @@ package com.atlassian.performance.tools.infrastructure.api.jira.flow.install
 
 import com.atlassian.performance.tools.infrastructure.api.jira.JiraNodeConfig
 import com.atlassian.performance.tools.infrastructure.api.jira.flow.JiraNodeFlow
-import com.atlassian.performance.tools.infrastructure.api.jira.flow.server.UbuntuSysstat
+import com.atlassian.performance.tools.infrastructure.api.jira.flow.server.LateUbuntuSysstat
 import com.atlassian.performance.tools.infrastructure.jira.flow.install.ProfilerHook
 import com.atlassian.performance.tools.infrastructure.jira.flow.install.SplunkForwarderHook
 import com.atlassian.performance.tools.ssh.api.SshConnection
@@ -22,7 +22,7 @@ class DefaultPostInstallHook(
             JvmConfig(config),
             ProfilerHook(config.profiler),
             SplunkForwarderHook(config.splunkForwarder),
-            UbuntuSysstat()
+            LateUbuntuSysstat()
         ).forEach { it.run(ssh, jira, flow) }
     }
 }
diff --git a/...n/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/flow/install/JiraLogs.kt b/...n/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/flow/install/JiraLogs.kt
@@ -3,13 +3,26 @@ package com.atlassian.performance.tools.infrastructure.api.jira.flow.install
 import com.atlassian.performance.tools.infrastructure.api.jira.flow.JiraNodeFlow
 import com.atlassian.performance.tools.infrastructure.api.jira.flow.report.StaticReport
 import com.atlassian.performance.tools.ssh.api.SshConnection
+import java.nio.file.Path
+import java.nio.file.Paths
 
 class JiraLogs : InstalledJiraHook {
 
     override fun run(ssh: SshConnection, jira: InstalledJira, flow: JiraNodeFlow) {
         listOf(
-            StaticReport("${jira.home}/log/atlassian-jira.log"),
-            StaticReport("${jira.installation}/logs/catalina.out")
-        ).forEach { flow.reports.add(it) }
+            "${jira.home}/log/atlassian-jira.log",
+            "${jira.installation}/logs/catalina.out"
+        )
+            .onEach { ensureFile(Paths.get(it), ssh) }
+            .map { StaticReport(it) }
+            .forEach { flow.reports.add(it) }
+    }
+
+    private fun ensureFile(
+        path: Path,
+        ssh: SshConnection
+    ) {
+        ssh.execute("mkdir -p ${path.parent!!}")
+        ssh.execute("touch $path")
     }
 }
diff --git a/.../com/atlassian/performance/tools/infrastructure/api/jira/flow/server/LateUbuntuSysstat.kt b/.../com/atlassian/performance/tools/infrastructure/api/jira/flow/server/LateUbuntuSysstat.kt
@@ -0,0 +1,19 @@
+package com.atlassian.performance.tools.infrastructure.api.jira.flow.server
+
+import com.atlassian.performance.tools.infrastructure.api.jira.flow.JiraNodeFlow
+import com.atlassian.performance.tools.infrastructure.api.jira.flow.TcpServer
+import com.atlassian.performance.tools.infrastructure.ubuntu.UbuntuSysstat
+import com.atlassian.performance.tools.ssh.api.SshConnection
+
+class LateUbuntuSysstat : TcpServerHook {
+
+    override fun run(
+        ssh: SshConnection,
+        server: TcpServer,
+        flow: JiraNodeFlow
+    ) {
+        UbuntuSysstat()
+            .install(ssh)
+            .forEach { flow.hookPostStart(it) }
+    }
+}
diff --git a/...tlin/com/atlassian/performance/tools/infrastructure/api/jira/flow/server/UbuntuSysstat.kt b/...tlin/com/atlassian/performance/tools/infrastructure/api/jira/flow/server/UbuntuSysstat.kt
diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/os/Ubuntu.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/os/Ubuntu.kt
@@ -45,7 +45,7 @@ class Ubuntu {
         val lock = LOCKS.computeIfAbsent(ssh.getHost().ipAddress) { Object() }
         synchronized(lock) {
             ssh.execute("sudo rm -rf /var/lib/apt/lists/*")
-            ssh.execute("sudo apt-get update -qq", Duration.ofMinutes(1))
+            ssh.execute("sudo apt-get update -qq", Duration.ofMinutes(5))
             ssh.execute(
                 cmd = "sudo DEBIAN_FRONTEND=noninteractive apt-get install -qq $joinedPackages",
                 timeout = timeout,
@@ -65,4 +65,4 @@ class Ubuntu {
             Iostat()
         )
     }
-}
+}
diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/ubuntu/EarlyUbuntuSysstat.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/ubuntu/EarlyUbuntuSysstat.kt
@@ -0,0 +1,19 @@
+package com.atlassian.performance.tools.infrastructure.ubuntu
+
+import com.atlassian.performance.tools.infrastructure.api.jira.flow.JiraNodeFlow
+import com.atlassian.performance.tools.infrastructure.api.jira.flow.TcpServer
+import com.atlassian.performance.tools.infrastructure.api.jira.flow.server.TcpServerHook
+import com.atlassian.performance.tools.ssh.api.SshConnection
+
+class EarlyUbuntuSysstat : TcpServerHook {
+
+    override fun run(
+        ssh: SshConnection,
+        server: TcpServer,
+        flow: JiraNodeFlow
+    ) {
+        UbuntuSysstat()
+            .install(ssh)
+            .forEach { flow.hookPreInstall(it) }
+    }
+}
diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/ubuntu/InstalledOsMetric.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/ubuntu/InstalledOsMetric.kt
@@ -0,0 +1,18 @@
+package com.atlassian.performance.tools.infrastructure.ubuntu
+
+import com.atlassian.performance.tools.infrastructure.api.jira.flow.JiraNodeFlow
+import com.atlassian.performance.tools.infrastructure.api.jira.flow.TcpServer
+import com.atlassian.performance.tools.infrastructure.api.jira.flow.server.TcpServerHook
+import com.atlassian.performance.tools.infrastructure.api.os.OsMetric
+import com.atlassian.performance.tools.infrastructure.jira.flow.RemoteMonitoringProcessReport
+import com.atlassian.performance.tools.ssh.api.SshConnection
+
+internal class InstalledOsMetric(
+    private val metric: OsMetric
+) : TcpServerHook {
+
+    override fun run(ssh: SshConnection, server: TcpServer, flow: JiraNodeFlow) {
+        val process = metric.start(ssh)
+        flow.reports.add(RemoteMonitoringProcessReport(process))
+    }
+}
diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/ubuntu/UbuntuSysstat.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/ubuntu/UbuntuSysstat.kt
@@ -0,0 +1,17 @@
+package com.atlassian.performance.tools.infrastructure.ubuntu
+
+import com.atlassian.performance.tools.infrastructure.Iostat
+import com.atlassian.performance.tools.infrastructure.api.os.Ubuntu
+import com.atlassian.performance.tools.infrastructure.api.os.Vmstat
+import com.atlassian.performance.tools.ssh.api.SshConnection
+
+internal class UbuntuSysstat {
+
+    fun install(
+        ssh: SshConnection
+    ): List<InstalledOsMetric> {
+        val ubuntu = Ubuntu()
+        ubuntu.install(ssh, listOf("sysstat"))
+        return listOf(Vmstat(), Iostat()).map { InstalledOsMetric(it) }
+    }
+}
diff --git a/src/test/kotlin/com/atlassian/performance/tools/infrastructure/SshUbuntuExtensions.kt b/src/test/kotlin/com/atlassian/performance/tools/infrastructure/SshUbuntuExtensions.kt
@@ -16,8 +16,8 @@ internal fun SshUbuntu.toSsh(): Ssh {
         )
     })
     ssh.newConnection().use { connection ->
-        connection.execute("apt-get update -qq", Duration.ofMinutes(3))
-        connection.execute("apt-get install sudo curl screen gnupg2 -y -qq")
+        connection.execute("apt-get update -qq", Duration.ofMinutes(7))
+        connection.execute("apt-get install sudo curl screen gnupg2 -y -qq", Duration.ofMinutes(2))
     }
     return ssh
 }
diff --git a/...t/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/flow/JiraNodeFlowTest.kt b/...t/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/flow/JiraNodeFlowTest.kt
@@ -28,6 +28,23 @@ class JiraNodeFlowTest {
 
         assertThat(counter.count).isEqualTo(3)
     }
+
+    @Test
+    fun shouldHookToTheTailDuringListing() {
+        val counter = CountingHook()
+        val flow = JiraNodeFlow().apply {
+            hookPreInstall(counter)
+            hookPreInstall(counter)
+            hookPreInstall(HookingHook(counter))
+        }
+        val server = TcpServer("doesn't matter", 123, "fake-server")
+
+        flow.listPreInstallHooks().forEach {
+            it.run(FailingSshConnection(), server, flow)
+        }
+
+        assertThat(counter.count).isEqualTo(3)
+    }
 }
 
 private class CountingHook : TcpServerHook {

diff --git a/...n/com/atlassian/performance/tools/infrastructure/api/jira/flow/start/HookedJiraStartIT.kt b/...n/com/atlassian/performance/tools/infrastructure/api/jira/flow/start/HookedJiraStartIT.kt
@@ -8,15 +8,20 @@ import com.atlassian.performance.tools.infrastructure.api.jira.flow.TcpServer
 import com.atlassian.performance.tools.infrastructure.api.jira.flow.install.*
 import com.atlassian.performance.tools.infrastructure.api.jira.flow.server.StartedJira
 import com.atlassian.performance.tools.infrastructure.api.jira.flow.server.TcpServerHook
-import com.atlassian.performance.tools.infrastructure.api.jira.flow.server.UbuntuSysstat
-import com.atlassian.performance.tools.infrastructure.api.jvm.OracleJDK
+import com.atlassian.performance.tools.infrastructure.api.jvm.Jstat
+import com.atlassian.performance.tools.infrastructure.api.jvm.VersionedJavaDevelopmentKit
 import com.atlassian.performance.tools.infrastructure.toSsh
+import com.atlassian.performance.tools.infrastructure.ubuntu.EarlyUbuntuSysstat
+import com.atlassian.performance.tools.jvmtasks.api.Backoff
+import com.atlassian.performance.tools.jvmtasks.api.IdempotentAction
 import com.atlassian.performance.tools.ssh.api.SshConnection
 import com.atlassian.performance.tools.sshubuntu.api.SshUbuntuContainer
 import org.assertj.core.api.Assertions.assertThat
 import org.junit.Test
 import java.io.File
+import java.net.URI
 import java.nio.file.Files
+import java.time.Duration
 import java.util.function.Consumer
 
 class HookedJiraStartIT {
@@ -31,7 +36,7 @@ class HookedJiraStartIT {
         val jiraInstallation = HookedJiraInstallation(ParallelInstallation(
             jiraHomeSource = EmptyJiraHome(),
             productDistribution = PublicJiraSoftwareDistribution("7.13.0"),
-            jdk = OracleJDK()
+            jdk = S3HostedJdk()
         ))
         val jiraStart = HookedJiraStart(JiraLaunchScript())
         val privatePort = 8080
@@ -73,13 +78,12 @@ class HookedJiraStartIT {
     fun shouldDownloadPartialReportsInCaseOfFailure() {
         // given
         val flow = JiraNodeFlow()
-        val sysstat = UbuntuSysstat()
-        flow.hookPreInstall(sysstat)
-        flow.hookPreInstall(FailingHook())
+        flow.hookPreInstall(EarlyUbuntuSysstat())
+        flow.hookPostInstall(FailingHook())
         val jiraInstallation = HookedJiraInstallation(ParallelInstallation(
             jiraHomeSource = EmptyJiraHome(),
             productDistribution = PublicJiraSoftwareDistribution("7.13.0"),
-            jdk = OracleJDK()
+            jdk = S3HostedJdk()
         ))
         val privatePort = 8080
         val container = SshUbuntuContainer(Consumer {
@@ -132,11 +136,51 @@ class HookedJiraStartIT {
 }
 
 private class FailingHook : TcpServerHook, InstalledJiraHook {
-    override fun run(ssh: SshConnection, server: TcpServer, flow: JiraNodeFlow) {
-        throw Exception("Failed")
+    override fun run(ssh: SshConnection, server: TcpServer, flow: JiraNodeFlow) = throw Exception("Expected failure")
+    override fun run(ssh: SshConnection, jira: InstalledJira, flow: JiraNodeFlow) = throw Exception("Expected failure")
+}
+
+/**
+ * Harvested from https://stash.atlassian.com/projects/JIRASERVER/repos/jira-performance-tests/pull-requests/630
+ */
+class S3HostedJdk : VersionedJavaDevelopmentKit {
+    private val jdkVersion = "1.8.0"
+    private val jdkUpdate = 131
+    private val jdkArchive = "jdk${jdkVersion}_$jdkUpdate-linux-x64.tar.gz"
+    private val jdkUrl = URI.create("https://s3.amazonaws.com/packages_java/$jdkArchive")
+    private val jdkBin = "~/jdk${jdkVersion}_$jdkUpdate/jre/bin/"
+    private val bin = "~/jdk${jdkVersion}_$jdkUpdate/bin/"
+    override val jstatMonitoring = Jstat(bin)
+
+    override fun getMajorVersion() = 8
+
+    override fun install(connection: SshConnection) {
+        download(connection)
+        connection.execute("tar -xzf $jdkArchive")
+        connection.execute("echo '${use()}' >> ~/.bashrc")
     }
 
-    override fun run(ssh: SshConnection, jira: InstalledJira, flow: JiraNodeFlow) {
-        throw Exception("Failed")
+    private fun download(connection: SshConnection) {
+        IdempotentAction("download JDK") {
+            connection.execute(
+                cmd = "curl -s -L -O -k $jdkUrl",
+                timeout = Duration.ofMinutes(4)
+            )
+        }.retry(
+            maxAttempts = 3,
+            backoff = StaticBackoff(Duration.ofSeconds(4))
+        )
     }
+
+    override fun use(): String = "export PATH=$jdkBin:$bin:${'$'}PATH"
+
+    override fun command(options: String) = "${jdkBin}java $options"
+}
+
+
+class StaticBackoff(
+    private val backoff: Duration
+) : Backoff {
+
+    override fun backOff(attempt: Int): Duration = backoff
 }