From e18d03eb630594e942faf3ace55fdd4d7bbbfe41 Mon Sep 17 00:00:00 2001 From: Maciej Kwidzinski Date: Fri, 6 Sep 2019 17:05:43 +0200 Subject: [PATCH] Bump a bunch of timeouts, because my Internet is awful Show the problem with hooking onto the tail of the hook queue during hook iteration. --- .../flow/install/DefaultPostInstallHook.kt | 4 +- .../api/jira/flow/install/JiraLogs.kt | 19 +++++- .../api/jira/flow/server/LateUbuntuSysstat.kt | 19 ++++++ .../api/jira/flow/server/UbuntuSysstat.kt | 35 ---------- .../tools/infrastructure/api/os/Ubuntu.kt | 4 +- .../ubuntu/EarlyUbuntuSysstat.kt | 19 ++++++ .../ubuntu/InstalledOsMetric.kt | 18 +++++ .../infrastructure/ubuntu/UbuntuSysstat.kt | 17 +++++ .../infrastructure/SshUbuntuExtensions.kt | 4 +- .../api/jira/flow/JiraNodeFlowTest.kt | 17 +++++ .../api/jira/flow/start/HookedJiraStartIT.kt | 66 +++++++++++++++---- 11 files changed, 167 insertions(+), 55 deletions(-) create mode 100644 src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/flow/server/LateUbuntuSysstat.kt delete mode 100644 src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/flow/server/UbuntuSysstat.kt create mode 100644 src/main/kotlin/com/atlassian/performance/tools/infrastructure/ubuntu/EarlyUbuntuSysstat.kt create mode 100644 src/main/kotlin/com/atlassian/performance/tools/infrastructure/ubuntu/InstalledOsMetric.kt create mode 100644 src/main/kotlin/com/atlassian/performance/tools/infrastructure/ubuntu/UbuntuSysstat.kt diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/flow/install/DefaultPostInstallHook.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/flow/install/DefaultPostInstallHook.kt index b42a983a..50363f27 100644 --- a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/flow/install/DefaultPostInstallHook.kt +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/flow/install/DefaultPostInstallHook.kt @@ -2,7 +2,7 @@ package com.atlassian.performance.tools.infrastructure.api.jira.flow.install import com.atlassian.performance.tools.infrastructure.api.jira.JiraNodeConfig import com.atlassian.performance.tools.infrastructure.api.jira.flow.JiraNodeFlow -import com.atlassian.performance.tools.infrastructure.api.jira.flow.server.UbuntuSysstat +import com.atlassian.performance.tools.infrastructure.api.jira.flow.server.LateUbuntuSysstat import com.atlassian.performance.tools.infrastructure.jira.flow.install.ProfilerHook import com.atlassian.performance.tools.infrastructure.jira.flow.install.SplunkForwarderHook import com.atlassian.performance.tools.ssh.api.SshConnection @@ -22,7 +22,7 @@ class DefaultPostInstallHook( JvmConfig(config), ProfilerHook(config.profiler), SplunkForwarderHook(config.splunkForwarder), - UbuntuSysstat() + LateUbuntuSysstat() ).forEach { it.run(ssh, jira, flow) } } } diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/flow/install/JiraLogs.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/flow/install/JiraLogs.kt index 7c7013e3..8e2cab01 100644 --- a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/flow/install/JiraLogs.kt +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/flow/install/JiraLogs.kt @@ -3,13 +3,26 @@ package com.atlassian.performance.tools.infrastructure.api.jira.flow.install import com.atlassian.performance.tools.infrastructure.api.jira.flow.JiraNodeFlow import com.atlassian.performance.tools.infrastructure.api.jira.flow.report.StaticReport import com.atlassian.performance.tools.ssh.api.SshConnection +import java.nio.file.Path +import java.nio.file.Paths class JiraLogs : InstalledJiraHook { override fun run(ssh: SshConnection, jira: InstalledJira, flow: JiraNodeFlow) { listOf( - StaticReport("${jira.home}/log/atlassian-jira.log"), - StaticReport("${jira.installation}/logs/catalina.out") - ).forEach { flow.reports.add(it) } + "${jira.home}/log/atlassian-jira.log", + "${jira.installation}/logs/catalina.out" + ) + .onEach { ensureFile(Paths.get(it), ssh) } + .map { StaticReport(it) } + .forEach { flow.reports.add(it) } + } + + private fun ensureFile( + path: Path, + ssh: SshConnection + ) { + ssh.execute("mkdir -p ${path.parent!!}") + ssh.execute("touch $path") } } diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/flow/server/LateUbuntuSysstat.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/flow/server/LateUbuntuSysstat.kt new file mode 100644 index 00000000..0fe5945d --- /dev/null +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/flow/server/LateUbuntuSysstat.kt @@ -0,0 +1,19 @@ +package com.atlassian.performance.tools.infrastructure.api.jira.flow.server + +import com.atlassian.performance.tools.infrastructure.api.jira.flow.JiraNodeFlow +import com.atlassian.performance.tools.infrastructure.api.jira.flow.TcpServer +import com.atlassian.performance.tools.infrastructure.ubuntu.UbuntuSysstat +import com.atlassian.performance.tools.ssh.api.SshConnection + +class LateUbuntuSysstat : TcpServerHook { + + override fun run( + ssh: SshConnection, + server: TcpServer, + flow: JiraNodeFlow + ) { + UbuntuSysstat() + .install(ssh) + .forEach { flow.hookPostStart(it) } + } +} diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/flow/server/UbuntuSysstat.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/flow/server/UbuntuSysstat.kt deleted file mode 100644 index 5941136b..00000000 --- a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/flow/server/UbuntuSysstat.kt +++ /dev/null @@ -1,35 +0,0 @@ -package com.atlassian.performance.tools.infrastructure.api.jira.flow.server - -import com.atlassian.performance.tools.infrastructure.Iostat -import com.atlassian.performance.tools.infrastructure.api.jira.flow.JiraNodeFlow -import com.atlassian.performance.tools.infrastructure.api.jira.flow.TcpServer -import com.atlassian.performance.tools.infrastructure.api.os.OsMetric -import com.atlassian.performance.tools.infrastructure.api.os.Ubuntu -import com.atlassian.performance.tools.infrastructure.api.os.Vmstat -import com.atlassian.performance.tools.infrastructure.jira.flow.RemoteMonitoringProcessReport -import com.atlassian.performance.tools.ssh.api.SshConnection - -class UbuntuSysstat : TcpServerHook { - - override fun run( - ssh: SshConnection, - server: TcpServer, - flow: JiraNodeFlow - ) { - val ubuntu = Ubuntu() - ubuntu.install(ssh, listOf("sysstat")) - listOf(Vmstat(), Iostat()) - .map { InstalledOsMetric(it) } - .forEach { flow.hookPostStart(it) } - } -} - -private class InstalledOsMetric( - private val metric: OsMetric -) : TcpServerHook { - - override fun run(ssh: SshConnection, server: TcpServer, flow: JiraNodeFlow) { - val process = metric.start(ssh) - flow.reports.add(RemoteMonitoringProcessReport(process)) - } -} diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/os/Ubuntu.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/os/Ubuntu.kt index 4cbab549..d909b316 100644 --- a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/os/Ubuntu.kt +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/os/Ubuntu.kt @@ -45,7 +45,7 @@ class Ubuntu { val lock = LOCKS.computeIfAbsent(ssh.getHost().ipAddress) { Object() } synchronized(lock) { ssh.execute("sudo rm -rf /var/lib/apt/lists/*") - ssh.execute("sudo apt-get update -qq", Duration.ofMinutes(1)) + ssh.execute("sudo apt-get update -qq", Duration.ofMinutes(5)) ssh.execute( cmd = "sudo DEBIAN_FRONTEND=noninteractive apt-get install -qq $joinedPackages", timeout = timeout, @@ -65,4 +65,4 @@ class Ubuntu { Iostat() ) } -} \ No newline at end of file +} diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/ubuntu/EarlyUbuntuSysstat.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/ubuntu/EarlyUbuntuSysstat.kt new file mode 100644 index 00000000..3a49cafc --- /dev/null +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/ubuntu/EarlyUbuntuSysstat.kt @@ -0,0 +1,19 @@ +package com.atlassian.performance.tools.infrastructure.ubuntu + +import com.atlassian.performance.tools.infrastructure.api.jira.flow.JiraNodeFlow +import com.atlassian.performance.tools.infrastructure.api.jira.flow.TcpServer +import com.atlassian.performance.tools.infrastructure.api.jira.flow.server.TcpServerHook +import com.atlassian.performance.tools.ssh.api.SshConnection + +class EarlyUbuntuSysstat : TcpServerHook { + + override fun run( + ssh: SshConnection, + server: TcpServer, + flow: JiraNodeFlow + ) { + UbuntuSysstat() + .install(ssh) + .forEach { flow.hookPreInstall(it) } + } +} diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/ubuntu/InstalledOsMetric.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/ubuntu/InstalledOsMetric.kt new file mode 100644 index 00000000..d10ed0cb --- /dev/null +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/ubuntu/InstalledOsMetric.kt @@ -0,0 +1,18 @@ +package com.atlassian.performance.tools.infrastructure.ubuntu + +import com.atlassian.performance.tools.infrastructure.api.jira.flow.JiraNodeFlow +import com.atlassian.performance.tools.infrastructure.api.jira.flow.TcpServer +import com.atlassian.performance.tools.infrastructure.api.jira.flow.server.TcpServerHook +import com.atlassian.performance.tools.infrastructure.api.os.OsMetric +import com.atlassian.performance.tools.infrastructure.jira.flow.RemoteMonitoringProcessReport +import com.atlassian.performance.tools.ssh.api.SshConnection + +internal class InstalledOsMetric( + private val metric: OsMetric +) : TcpServerHook { + + override fun run(ssh: SshConnection, server: TcpServer, flow: JiraNodeFlow) { + val process = metric.start(ssh) + flow.reports.add(RemoteMonitoringProcessReport(process)) + } +} diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/ubuntu/UbuntuSysstat.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/ubuntu/UbuntuSysstat.kt new file mode 100644 index 00000000..8a76c645 --- /dev/null +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/ubuntu/UbuntuSysstat.kt @@ -0,0 +1,17 @@ +package com.atlassian.performance.tools.infrastructure.ubuntu + +import com.atlassian.performance.tools.infrastructure.Iostat +import com.atlassian.performance.tools.infrastructure.api.os.Ubuntu +import com.atlassian.performance.tools.infrastructure.api.os.Vmstat +import com.atlassian.performance.tools.ssh.api.SshConnection + +internal class UbuntuSysstat { + + fun install( + ssh: SshConnection + ): List { + val ubuntu = Ubuntu() + ubuntu.install(ssh, listOf("sysstat")) + return listOf(Vmstat(), Iostat()).map { InstalledOsMetric(it) } + } +} diff --git a/src/test/kotlin/com/atlassian/performance/tools/infrastructure/SshUbuntuExtensions.kt b/src/test/kotlin/com/atlassian/performance/tools/infrastructure/SshUbuntuExtensions.kt index 4eab4502..bf612007 100644 --- a/src/test/kotlin/com/atlassian/performance/tools/infrastructure/SshUbuntuExtensions.kt +++ b/src/test/kotlin/com/atlassian/performance/tools/infrastructure/SshUbuntuExtensions.kt @@ -16,8 +16,8 @@ internal fun SshUbuntu.toSsh(): Ssh { ) }) ssh.newConnection().use { connection -> - connection.execute("apt-get update -qq", Duration.ofMinutes(3)) - connection.execute("apt-get install sudo curl screen gnupg2 -y -qq") + connection.execute("apt-get update -qq", Duration.ofMinutes(7)) + connection.execute("apt-get install sudo curl screen gnupg2 -y -qq", Duration.ofMinutes(2)) } return ssh } diff --git a/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/flow/JiraNodeFlowTest.kt b/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/flow/JiraNodeFlowTest.kt index d92b0505..1f7125db 100644 --- a/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/flow/JiraNodeFlowTest.kt +++ b/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/flow/JiraNodeFlowTest.kt @@ -28,6 +28,23 @@ class JiraNodeFlowTest { assertThat(counter.count).isEqualTo(3) } + + @Test + fun shouldHookToTheTailDuringListing() { + val counter = CountingHook() + val flow = JiraNodeFlow().apply { + hookPreInstall(counter) + hookPreInstall(counter) + hookPreInstall(HookingHook(counter)) + } + val server = TcpServer("doesn't matter", 123, "fake-server") + + flow.listPreInstallHooks().forEach { + it.run(FailingSshConnection(), server, flow) + } + + assertThat(counter.count).isEqualTo(3) + } } private class CountingHook : TcpServerHook { diff --git a/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/flow/start/HookedJiraStartIT.kt b/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/flow/start/HookedJiraStartIT.kt index 3eaa6e88..5caf7b4c 100644 --- a/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/flow/start/HookedJiraStartIT.kt +++ b/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/flow/start/HookedJiraStartIT.kt @@ -8,15 +8,20 @@ import com.atlassian.performance.tools.infrastructure.api.jira.flow.TcpServer import com.atlassian.performance.tools.infrastructure.api.jira.flow.install.* import com.atlassian.performance.tools.infrastructure.api.jira.flow.server.StartedJira import com.atlassian.performance.tools.infrastructure.api.jira.flow.server.TcpServerHook -import com.atlassian.performance.tools.infrastructure.api.jira.flow.server.UbuntuSysstat -import com.atlassian.performance.tools.infrastructure.api.jvm.OracleJDK +import com.atlassian.performance.tools.infrastructure.api.jvm.Jstat +import com.atlassian.performance.tools.infrastructure.api.jvm.VersionedJavaDevelopmentKit import com.atlassian.performance.tools.infrastructure.toSsh +import com.atlassian.performance.tools.infrastructure.ubuntu.EarlyUbuntuSysstat +import com.atlassian.performance.tools.jvmtasks.api.Backoff +import com.atlassian.performance.tools.jvmtasks.api.IdempotentAction import com.atlassian.performance.tools.ssh.api.SshConnection import com.atlassian.performance.tools.sshubuntu.api.SshUbuntuContainer import org.assertj.core.api.Assertions.assertThat import org.junit.Test import java.io.File +import java.net.URI import java.nio.file.Files +import java.time.Duration import java.util.function.Consumer class HookedJiraStartIT { @@ -31,7 +36,7 @@ class HookedJiraStartIT { val jiraInstallation = HookedJiraInstallation(ParallelInstallation( jiraHomeSource = EmptyJiraHome(), productDistribution = PublicJiraSoftwareDistribution("7.13.0"), - jdk = OracleJDK() + jdk = S3HostedJdk() )) val jiraStart = HookedJiraStart(JiraLaunchScript()) val privatePort = 8080 @@ -73,13 +78,12 @@ class HookedJiraStartIT { fun shouldDownloadPartialReportsInCaseOfFailure() { // given val flow = JiraNodeFlow() - val sysstat = UbuntuSysstat() - flow.hookPreInstall(sysstat) - flow.hookPreInstall(FailingHook()) + flow.hookPreInstall(EarlyUbuntuSysstat()) + flow.hookPostInstall(FailingHook()) val jiraInstallation = HookedJiraInstallation(ParallelInstallation( jiraHomeSource = EmptyJiraHome(), productDistribution = PublicJiraSoftwareDistribution("7.13.0"), - jdk = OracleJDK() + jdk = S3HostedJdk() )) val privatePort = 8080 val container = SshUbuntuContainer(Consumer { @@ -132,11 +136,51 @@ class HookedJiraStartIT { } private class FailingHook : TcpServerHook, InstalledJiraHook { - override fun run(ssh: SshConnection, server: TcpServer, flow: JiraNodeFlow) { - throw Exception("Failed") + override fun run(ssh: SshConnection, server: TcpServer, flow: JiraNodeFlow) = throw Exception("Expected failure") + override fun run(ssh: SshConnection, jira: InstalledJira, flow: JiraNodeFlow) = throw Exception("Expected failure") +} + +/** + * Harvested from https://stash.atlassian.com/projects/JIRASERVER/repos/jira-performance-tests/pull-requests/630 + */ +class S3HostedJdk : VersionedJavaDevelopmentKit { + private val jdkVersion = "1.8.0" + private val jdkUpdate = 131 + private val jdkArchive = "jdk${jdkVersion}_$jdkUpdate-linux-x64.tar.gz" + private val jdkUrl = URI.create("https://s3.amazonaws.com/packages_java/$jdkArchive") + private val jdkBin = "~/jdk${jdkVersion}_$jdkUpdate/jre/bin/" + private val bin = "~/jdk${jdkVersion}_$jdkUpdate/bin/" + override val jstatMonitoring = Jstat(bin) + + override fun getMajorVersion() = 8 + + override fun install(connection: SshConnection) { + download(connection) + connection.execute("tar -xzf $jdkArchive") + connection.execute("echo '${use()}' >> ~/.bashrc") } - override fun run(ssh: SshConnection, jira: InstalledJira, flow: JiraNodeFlow) { - throw Exception("Failed") + private fun download(connection: SshConnection) { + IdempotentAction("download JDK") { + connection.execute( + cmd = "curl -s -L -O -k $jdkUrl", + timeout = Duration.ofMinutes(4) + ) + }.retry( + maxAttempts = 3, + backoff = StaticBackoff(Duration.ofSeconds(4)) + ) } + + override fun use(): String = "export PATH=$jdkBin:$bin:${'$'}PATH" + + override fun command(options: String) = "${jdkBin}java $options" +} + + +class StaticBackoff( + private val backoff: Duration +) : Backoff { + + override fun backOff(attempt: Int): Duration = backoff }