diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/AsyncProfilerHook.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/AsyncProfilerHook.kt new file mode 100644 index 00000000..5f60deaf --- /dev/null +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/AsyncProfilerHook.kt @@ -0,0 +1,57 @@ +package com.atlassian.performance.tools.infrastructure.api.jira.install.hook + +import com.atlassian.performance.tools.infrastructure.api.jira.install.TcpServer +import com.atlassian.performance.tools.infrastructure.api.jira.report.Report +import com.atlassian.performance.tools.infrastructure.api.jira.start.StartedJira +import com.atlassian.performance.tools.infrastructure.api.jira.start.hook.PostStartHook +import com.atlassian.performance.tools.infrastructure.api.jira.start.hook.PostStartHooks +import com.atlassian.performance.tools.ssh.api.SshConnection +import java.net.URI + +class AsyncProfilerHook : PreInstallHook { + + override fun call( + ssh: SshConnection, + server: TcpServer, + hooks: PreInstallHooks + ) { + val directory = "async-profiler" + val downloads = URI("https://github.com/jvm-profiling-tools/async-profiler/releases/download/") + val distribution = downloads.resolve("v1.4/async-profiler-1.4-linux-x64.tar.gz") + ssh.execute("wget -q $distribution") + ssh.execute("mkdir $directory") + ssh.execute("tar -xzf async-profiler-1.4-linux-x64.tar.gz -C $directory") + ssh.execute("sudo sh -c 'echo 1 > /proc/sys/kernel/perf_event_paranoid'") + ssh.execute("sudo sh -c 'echo 0 > /proc/sys/kernel/kptr_restrict'") + val profilerPath = "./$directory/profiler.sh" + val profiler = InstalledAsyncProfiler(profilerPath) + hooks.postStart.insert(profiler) + } +} + +private class InstalledAsyncProfiler( + private val profilerPath: String +) : PostStartHook { + + override fun call( + ssh: SshConnection, + jira: StartedJira, + hooks: PostStartHooks + ) { + ssh.execute("$profilerPath -b 20000000 start ${jira.pid}") + val profiler = StartedAsyncProfiler(jira.pid, profilerPath) + hooks.reports.add(profiler) + } +} + +private class StartedAsyncProfiler( + private val pid: Int, + private val profilerPath: String +) : Report { + + override fun locate(ssh: SshConnection): List { + val flameGraphFile = "flamegraph.svg" + ssh.execute("$profilerPath stop $pid -o svg > $flameGraphFile") + return listOf(flameGraphFile) + } +} diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/DataCenterHook.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/DataCenterHook.kt new file mode 100644 index 00000000..4b4b549f --- /dev/null +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/DataCenterHook.kt @@ -0,0 +1,24 @@ +package com.atlassian.performance.tools.infrastructure.api.jira.install.hook + +import com.atlassian.performance.tools.infrastructure.api.jira.SharedHome +import com.atlassian.performance.tools.infrastructure.api.jira.install.InstalledJira +import com.atlassian.performance.tools.ssh.api.SshConnection + +class DataCenterHook( + private val nodeId: String, + private val sharedHome: SharedHome +) : PostInstallHook { + + override fun call( + ssh: SshConnection, + jira: InstalledJira, + hooks: PostInstallHooks + ) { + val localSharedHome = sharedHome.localSharedHome + sharedHome.mount(ssh) + val jiraHome = jira.home.path // TODO what's the difference between localSharedHome and jiraHome? should both be hookable? + ssh.execute("echo ehcache.object.port = 40011 >> $jiraHome/cluster.properties") + ssh.execute("echo jira.node.id = $nodeId >> $jiraHome/cluster.properties") + ssh.execute("echo jira.shared.home = `realpath $localSharedHome` >> $jiraHome/cluster.properties") + } +} diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/DisabledAutoBackup.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/DisabledAutoBackup.kt new file mode 100644 index 00000000..8cc6e973 --- /dev/null +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/DisabledAutoBackup.kt @@ -0,0 +1,15 @@ +package com.atlassian.performance.tools.infrastructure.api.jira.install.hook + +import com.atlassian.performance.tools.infrastructure.api.jira.install.InstalledJira +import com.atlassian.performance.tools.ssh.api.SshConnection + +class DisabledAutoBackup : PostInstallHook { + + override fun call( + ssh: SshConnection, + jira: InstalledJira, + hooks: PostInstallHooks + ) { + ssh.execute("echo jira.autoexport=false > ${jira.home.path}/jira-config.properties") + } +} diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/HookedJiraInstallation.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/HookedJiraInstallation.kt new file mode 100644 index 00000000..e8fc3ae0 --- /dev/null +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/HookedJiraInstallation.kt @@ -0,0 +1,24 @@ +package com.atlassian.performance.tools.infrastructure.api.jira.install.hook + +import com.atlassian.performance.tools.infrastructure.api.jira.install.InstalledJira +import com.atlassian.performance.tools.infrastructure.api.jira.install.JiraInstallation +import com.atlassian.performance.tools.infrastructure.api.jira.install.TcpServer + +class HookedJiraInstallation( + private val installation: JiraInstallation, + private val hooks: PreInstallHooks +) : JiraInstallation { + + override fun install( + server: TcpServer + ): InstalledJira { + server.ssh.newConnection().use { ssh -> + hooks.call(ssh, server) + } + val installed = installation.install(server) + server.ssh.newConnection().use { ssh -> + hooks.postInstall.call(ssh, installed) + } + return installed + } +} diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/JiraHomeProperty.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/JiraHomeProperty.kt new file mode 100644 index 00000000..9d8d1ed4 --- /dev/null +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/JiraHomeProperty.kt @@ -0,0 +1,16 @@ +package com.atlassian.performance.tools.infrastructure.api.jira.install.hook + +import com.atlassian.performance.tools.infrastructure.api.jira.install.InstalledJira +import com.atlassian.performance.tools.ssh.api.SshConnection + +class JiraHomeProperty : PostInstallHook { + + override fun call( + ssh: SshConnection, + jira: InstalledJira, + hooks: PostInstallHooks + ) { + val properties = "${jira.installation.path}/atlassian-jira/WEB-INF/classes/jira-application.properties" + ssh.execute("echo jira.home=`realpath ${jira.home.path}` > $properties") + } +} diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/JiraLogs.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/JiraLogs.kt new file mode 100644 index 00000000..ef6f803c --- /dev/null +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/JiraLogs.kt @@ -0,0 +1,28 @@ +package com.atlassian.performance.tools.infrastructure.api.jira.install.hook + +import com.atlassian.performance.tools.infrastructure.api.jira.install.InstalledJira +import com.atlassian.performance.tools.infrastructure.api.jira.report.StaticReport +import com.atlassian.performance.tools.ssh.api.SshConnection +import java.nio.file.Path +import java.nio.file.Paths + +class JiraLogs : PostInstallHook { + + override fun call(ssh: SshConnection, jira: InstalledJira, hooks: PostInstallHooks) { + listOf( + "${jira.home.path}/log/atlassian-jira.log", + "${jira.installation.path}/logs/catalina.out" + ) + .onEach { ensureFile(Paths.get(it), ssh) } + .map { StaticReport(it) } + .forEach { hooks.reports.add(it) } + } + + private fun ensureFile( + path: Path, + ssh: SshConnection + ) { + ssh.execute("mkdir -p ${path.parent!!}") + ssh.execute("touch $path") + } +} diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/JvmConfig.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/JvmConfig.kt new file mode 100644 index 00000000..d7f68580 --- /dev/null +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/JvmConfig.kt @@ -0,0 +1,29 @@ +package com.atlassian.performance.tools.infrastructure.api.jira.install.hook + +import com.atlassian.performance.tools.infrastructure.api.jira.JiraGcLog +import com.atlassian.performance.tools.infrastructure.api.jira.JiraNodeConfig +import com.atlassian.performance.tools.infrastructure.api.jira.SetenvSh +import com.atlassian.performance.tools.infrastructure.api.jira.install.InstalledJira +import com.atlassian.performance.tools.infrastructure.api.jira.report.FileListing +import com.atlassian.performance.tools.ssh.api.SshConnection + +class JvmConfig( + private val config: JiraNodeConfig +) : PostInstallHook { + + override fun call( + ssh: SshConnection, + jira: InstalledJira, + hooks: PostInstallHooks + ) { + val gcLog = JiraGcLog(jira.installation.path) + SetenvSh(jira.installation.path).setup( + connection = ssh, + config = config, + gcLog = gcLog, + jiraIp = jira.server.ip + ) + val report = FileListing(gcLog.path("*")) + hooks.reports.add(report) + } +} diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/LateUbuntuSysstat.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/LateUbuntuSysstat.kt new file mode 100644 index 00000000..2273c81c --- /dev/null +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/LateUbuntuSysstat.kt @@ -0,0 +1,35 @@ +package com.atlassian.performance.tools.infrastructure.api.jira.install.hook + +import com.atlassian.performance.tools.infrastructure.Iostat +import com.atlassian.performance.tools.infrastructure.api.jira.install.InstalledJira +import com.atlassian.performance.tools.infrastructure.api.jira.start.StartedJira +import com.atlassian.performance.tools.infrastructure.api.jira.start.hook.PostStartHook +import com.atlassian.performance.tools.infrastructure.api.jira.start.hook.PostStartHooks +import com.atlassian.performance.tools.infrastructure.api.os.OsMetric +import com.atlassian.performance.tools.infrastructure.api.os.Ubuntu +import com.atlassian.performance.tools.infrastructure.api.os.Vmstat +import com.atlassian.performance.tools.infrastructure.jira.report.RemoteMonitoringProcessReport +import com.atlassian.performance.tools.ssh.api.SshConnection + +class LateUbuntuSysstat : PostInstallHook { + override fun call( + ssh: SshConnection, + jira: InstalledJira, + hooks: PostInstallHooks + ) { + val ubuntu = Ubuntu() + ubuntu.install(ssh, listOf("sysstat")) + listOf(Vmstat(), Iostat()) + .map { PostStartOsMetric(it) } + .forEach { hooks.postStart.insert(it) } + } +} + +private class PostStartOsMetric( + private val metric: OsMetric +) : PostStartHook { + override fun call(ssh: SshConnection, jira: StartedJira, hooks: PostStartHooks) { + val process = metric.start(ssh) + hooks.reports.add(RemoteMonitoringProcessReport(process)) + } +} diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/PostInstallHook.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/PostInstallHook.kt new file mode 100644 index 00000000..0fe19b0a --- /dev/null +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/PostInstallHook.kt @@ -0,0 +1,21 @@ +package com.atlassian.performance.tools.infrastructure.api.jira.install.hook + +import com.atlassian.performance.tools.infrastructure.api.jira.install.InstalledJira +import com.atlassian.performance.tools.ssh.api.SshConnection + +/** + * Intercepts a call after Jira is installed. + */ +interface PostInstallHook { + + /** + * @param [ssh] connects to the [jira] + * @param [jira] points to the installed Jira + * @param [hooks] inserts future hooks and reports + */ + fun call( + ssh: SshConnection, + jira: InstalledJira, + hooks: PostInstallHooks + ) +} diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/PostInstallHooks.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/PostInstallHooks.kt new file mode 100644 index 00000000..0b285cf5 --- /dev/null +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/PostInstallHooks.kt @@ -0,0 +1,54 @@ +package com.atlassian.performance.tools.infrastructure.api.jira.install.hook + +import com.atlassian.performance.tools.infrastructure.api.jira.JiraNodeConfig +import com.atlassian.performance.tools.infrastructure.api.jira.install.InstalledJira +import com.atlassian.performance.tools.infrastructure.api.jira.start.hook.PreStartHooks +import com.atlassian.performance.tools.infrastructure.jira.install.hook.ProfilerHook +import com.atlassian.performance.tools.infrastructure.jira.install.hook.SplunkForwarderHook +import com.atlassian.performance.tools.ssh.api.SshConnection +import java.util.* +import java.util.concurrent.ConcurrentLinkedQueue + +class PostInstallHooks private constructor( + val preStart: PreStartHooks +) { + + private val hooks: Queue = ConcurrentLinkedQueue() + val postStart = preStart.postStart + val reports = postStart.reports + + fun insert( + hook: PostInstallHook + ) { + hooks.add(hook) + } + + internal fun call( + ssh: SshConnection, + jira: InstalledJira + ) { + while (true) { + hooks + .poll() + ?.call(ssh, jira, this) + ?: break + } + } + + companion object Factory { + fun default(): PostInstallHooks = PostInstallHooks(PreStartHooks.default()).apply { + val config = JiraNodeConfig.Builder().build() + listOf( + JiraHomeProperty(), + DisabledAutoBackup(), + JvmConfig(config), + ProfilerHook(config.profiler), + SplunkForwarderHook(config.splunkForwarder), + JiraLogs(), + LateUbuntuSysstat() + ).forEach { insert(it) } + } + + fun empty(): PostInstallHooks = PostInstallHooks(PreStartHooks.empty()) + } +} diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/PreInstallHook.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/PreInstallHook.kt new file mode 100644 index 00000000..1e58b1af --- /dev/null +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/PreInstallHook.kt @@ -0,0 +1,21 @@ +package com.atlassian.performance.tools.infrastructure.api.jira.install.hook + +import com.atlassian.performance.tools.infrastructure.api.jira.install.TcpServer +import com.atlassian.performance.tools.ssh.api.SshConnection + +/** + * Intercepts a call before Jira is installed. + */ +interface PreInstallHook { + + /** + * @param [ssh] connects to the [server] + * @param [server] will install Jira + * @param [hooks] inserts future hooks and reports + */ + fun call( + ssh: SshConnection, + server: TcpServer, + hooks: PreInstallHooks + ) +} diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/PreInstallHooks.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/PreInstallHooks.kt new file mode 100644 index 00000000..5506de0d --- /dev/null +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/PreInstallHooks.kt @@ -0,0 +1,40 @@ +package com.atlassian.performance.tools.infrastructure.api.jira.install.hook + +import com.atlassian.performance.tools.infrastructure.api.jira.install.TcpServer +import com.atlassian.performance.tools.ssh.api.SshConnection +import java.util.* +import java.util.concurrent.ConcurrentLinkedQueue + +class PreInstallHooks private constructor( + val postInstall: PostInstallHooks +) { + + private val hooks: Queue = ConcurrentLinkedQueue() + val preStart = postInstall.preStart + val postStart = preStart.postStart + val reports = postStart.reports + + fun insert( + hook: PreInstallHook + ) { + hooks.add(hook) + } + + internal fun call( + ssh: SshConnection, + server: TcpServer + ) { + while (true) { + hooks + .poll() + ?.call(ssh, server, this) + ?: break + } + } + + companion object Factory { + fun default(): PreInstallHooks = PreInstallHooks(PostInstallHooks.default()) + + fun empty(): PreInstallHooks = PreInstallHooks(PostInstallHooks.empty()) + } +} diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/SystemLog.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/SystemLog.kt new file mode 100644 index 00000000..861b0ec8 --- /dev/null +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/SystemLog.kt @@ -0,0 +1,11 @@ +package com.atlassian.performance.tools.infrastructure.api.jira.install.hook + +import com.atlassian.performance.tools.infrastructure.api.jira.report.Report +import com.atlassian.performance.tools.ssh.api.SshConnection + +class SystemLog : Report { + + override fun locate(ssh: SshConnection): List { + return listOf("/var/log/syslog") + } +} diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/report/FileListing.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/report/FileListing.kt new file mode 100644 index 00000000..75979fca --- /dev/null +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/report/FileListing.kt @@ -0,0 +1,16 @@ +package com.atlassian.performance.tools.infrastructure.api.jira.report + +import com.atlassian.performance.tools.ssh.api.SshConnection + +class FileListing( + private val pattern: String +) : Report { + + override fun locate( + ssh: SshConnection + ): List = ssh + .execute("ls $pattern") + .output + .lines() + .filter { it.isNotBlank() } +} diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/report/Report.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/report/Report.kt new file mode 100644 index 00000000..65059517 --- /dev/null +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/report/Report.kt @@ -0,0 +1,15 @@ +package com.atlassian.performance.tools.infrastructure.api.jira.report + +import com.atlassian.performance.tools.ssh.api.SshConnection + +/** + * Reports back about remote events. E.g. points to interesting logs, dumps, charts. + */ +interface Report { + + /** + * @param [ssh] connects to the server, which holds interesting data + * @return interesting file paths, which could be downloaded + */ + fun locate(ssh: SshConnection): List +} diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/report/Reports.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/report/Reports.kt new file mode 100644 index 00000000..bfc8b1cd --- /dev/null +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/report/Reports.kt @@ -0,0 +1,18 @@ +package com.atlassian.performance.tools.infrastructure.api.jira.report + +import java.util.* +import java.util.concurrent.ConcurrentLinkedQueue + +class Reports { + private val reports: Queue = ConcurrentLinkedQueue() + + fun add( + report: Report + ) { + reports.add(report) + } + + fun list(): Iterable { + return reports.toList() + } +} diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/report/StaticReport.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/report/StaticReport.kt new file mode 100644 index 00000000..da90e4d1 --- /dev/null +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/report/StaticReport.kt @@ -0,0 +1,10 @@ +package com.atlassian.performance.tools.infrastructure.api.jira.report + +import com.atlassian.performance.tools.ssh.api.SshConnection + +class StaticReport( + private val remotePath: String +) : Report { + + override fun locate(ssh: SshConnection): List = listOf(remotePath) +} diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/start/hook/AccessLogs.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/start/hook/AccessLogs.kt new file mode 100644 index 00000000..6bc2f30e --- /dev/null +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/start/hook/AccessLogs.kt @@ -0,0 +1,12 @@ +package com.atlassian.performance.tools.infrastructure.api.jira.start.hook + +import com.atlassian.performance.tools.infrastructure.api.jira.report.FileListing +import com.atlassian.performance.tools.infrastructure.api.jira.start.StartedJira +import com.atlassian.performance.tools.ssh.api.SshConnection + +class AccessLogs : PostStartHook { + + override fun call(ssh: SshConnection, jira: StartedJira, hooks: PostStartHooks) { + hooks.reports.add(FileListing("${jira.installed.installation.path}/logs/*access*")) + } +} diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/start/hook/HookedJiraStart.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/start/hook/HookedJiraStart.kt new file mode 100644 index 00000000..70afbfff --- /dev/null +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/start/hook/HookedJiraStart.kt @@ -0,0 +1,24 @@ +package com.atlassian.performance.tools.infrastructure.api.jira.start.hook + +import com.atlassian.performance.tools.infrastructure.api.jira.install.InstalledJira +import com.atlassian.performance.tools.infrastructure.api.jira.start.JiraStart +import com.atlassian.performance.tools.infrastructure.api.jira.start.StartedJira + +class HookedJiraStart( + private val start: JiraStart, + private val hooks: PreStartHooks +) : JiraStart { + + override fun start( + installed: InstalledJira + ): StartedJira { + installed.server.ssh.newConnection().use { ssh -> + hooks.call(ssh, installed) + } + val started = start.start(installed) + installed.server.ssh.newConnection().use { ssh -> + hooks.postStart.call(ssh, started) + } + return started + } +} diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/start/hook/JstatHook.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/start/hook/JstatHook.kt new file mode 100644 index 00000000..e4485af4 --- /dev/null +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/start/hook/JstatHook.kt @@ -0,0 +1,17 @@ +package com.atlassian.performance.tools.infrastructure.api.jira.start.hook + +import com.atlassian.performance.tools.infrastructure.api.jira.start.StartedJira +import com.atlassian.performance.tools.infrastructure.jira.report.RemoteMonitoringProcessReport +import com.atlassian.performance.tools.ssh.api.SshConnection + +class JstatHook : PostStartHook { + + override fun call( + ssh: SshConnection, + jira: StartedJira, + hooks: PostStartHooks + ) { + val process = jira.installed.jdk.jstatMonitoring.start(ssh, jira.pid) + hooks.reports.add(RemoteMonitoringProcessReport(process)) + } +} diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/start/hook/PostStartHook.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/start/hook/PostStartHook.kt new file mode 100644 index 00000000..c3c47123 --- /dev/null +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/start/hook/PostStartHook.kt @@ -0,0 +1,21 @@ +package com.atlassian.performance.tools.infrastructure.api.jira.start.hook + +import com.atlassian.performance.tools.infrastructure.api.jira.start.StartedJira +import com.atlassian.performance.tools.ssh.api.SshConnection + +/** + * Intercepts a call after Jira is started. + */ +interface PostStartHook { + + /** + * @param [ssh] connects to the [jira] + * @param [jira] points to the started Jira + * @param [hooks] inserts future reports + */ + fun call( + ssh: SshConnection, + jira: StartedJira, + hooks: PostStartHooks + ) +} diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/start/hook/PostStartHooks.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/start/hook/PostStartHooks.kt new file mode 100644 index 00000000..21b0b46a --- /dev/null +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/start/hook/PostStartHooks.kt @@ -0,0 +1,39 @@ +package com.atlassian.performance.tools.infrastructure.api.jira.start.hook + +import com.atlassian.performance.tools.infrastructure.api.jira.report.Reports +import com.atlassian.performance.tools.infrastructure.api.jira.start.StartedJira +import com.atlassian.performance.tools.ssh.api.SshConnection +import java.util.* +import java.util.concurrent.ConcurrentLinkedQueue + +class PostStartHooks private constructor() { + + private val hooks: Queue = ConcurrentLinkedQueue() + val reports = Reports() + + fun insert( + hook: PostStartHook + ) { + hooks.add(hook) + } + + internal fun call( + ssh: SshConnection, + jira: StartedJira + ) { + while (true) { + hooks + .poll() + ?.call(ssh, jira, this) + ?: break + } + } + + companion object Factory { + fun default(): PostStartHooks = empty().apply { + insert(JstatHook()) + } + + fun empty(): PostStartHooks = PostStartHooks() + } +} diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/start/hook/PreStartHook.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/start/hook/PreStartHook.kt new file mode 100644 index 00000000..d0fa8326 --- /dev/null +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/start/hook/PreStartHook.kt @@ -0,0 +1,21 @@ +package com.atlassian.performance.tools.infrastructure.api.jira.start.hook + +import com.atlassian.performance.tools.infrastructure.api.jira.install.InstalledJira +import com.atlassian.performance.tools.ssh.api.SshConnection + +/** + * Intercepts a call before Jira is started. + */ +interface PreStartHook { + + /** + * @param [ssh] connects to the [jira] + * @param [jira] points to the installed Jira + * @param [hooks] inserts future hooks and reports + */ + fun call( + ssh: SshConnection, + jira: InstalledJira, + hooks: PreStartHooks + ) +} diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/start/hook/PreStartHooks.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/start/hook/PreStartHooks.kt new file mode 100644 index 00000000..6ce16b42 --- /dev/null +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/start/hook/PreStartHooks.kt @@ -0,0 +1,38 @@ +package com.atlassian.performance.tools.infrastructure.api.jira.start.hook + +import com.atlassian.performance.tools.infrastructure.api.jira.install.InstalledJira +import com.atlassian.performance.tools.ssh.api.SshConnection +import java.util.* +import java.util.concurrent.ConcurrentLinkedQueue + +class PreStartHooks private constructor( + val postStart: PostStartHooks +) { + + private val hooks: Queue = ConcurrentLinkedQueue() + val reports = postStart.reports + + fun insert( + hook: PreStartHook + ) { + hooks.add(hook) + } + + internal fun call( + ssh: SshConnection, + jira: InstalledJira + ) { + while (true) { + hooks + .poll() + ?.call(ssh, jira, this) + ?: break + } + } + + companion object Factory { + fun default() = PreStartHooks(PostStartHooks.default()) + + fun empty() = PreStartHooks(PostStartHooks.empty()) + } +} diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/start/hook/RestUpgrade.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/start/hook/RestUpgrade.kt new file mode 100644 index 00000000..84aa39f4 --- /dev/null +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/start/hook/RestUpgrade.kt @@ -0,0 +1,74 @@ +package com.atlassian.performance.tools.infrastructure.api.jira.start.hook + +import com.atlassian.performance.tools.infrastructure.api.jira.JiraLaunchTimeouts +import com.atlassian.performance.tools.infrastructure.api.jira.report.StaticReport +import com.atlassian.performance.tools.infrastructure.api.jira.start.StartedJira +import com.atlassian.performance.tools.infrastructure.api.jvm.ThreadDump +import com.atlassian.performance.tools.ssh.api.SshConnection +import java.net.URI +import java.time.Duration +import java.time.Instant + +class RestUpgrade( + private val timeouts: JiraLaunchTimeouts, + private val adminUsername: String, + private val adminPassword: String +) : PostStartHook { + + override fun call(ssh: SshConnection, jira: StartedJira, hooks: PostStartHooks) { + val threadDump = ThreadDump(jira.pid, jira.installed.jdk) + val privatePort = jira.installed.server.privatePort + val upgradesEndpoint = URI("http://$adminUsername:$adminPassword@localhost:$privatePort/rest/api/2/upgrade") + hooks.reports.add(StaticReport("thread-dumps")) + waitForStatusToChange( + statusQuo = "000", + timeout = timeouts.offlineTimeout, + ssh = ssh, + uri = upgradesEndpoint, + threadDump = threadDump + ) + waitForStatusToChange( + statusQuo = "503", + timeout = timeouts.initTimeout, + ssh = ssh, + uri = upgradesEndpoint, + threadDump = threadDump + ) + ssh.execute( + cmd = "curl --silent --retry 6 -X POST $upgradesEndpoint", + timeout = Duration.ofSeconds(15) + ) + waitForStatusToChange( + statusQuo = "303", + timeout = timeouts.upgradeTimeout, + ssh = ssh, + uri = upgradesEndpoint, + threadDump = threadDump + ) + } + + private fun waitForStatusToChange( + statusQuo: String, + uri: URI, + timeout: Duration, + ssh: SshConnection, + threadDump: ThreadDump + ) { + val backoff = Duration.ofSeconds(10) + val deadline = Instant.now() + timeout + while (true) { + val currentStatus = ssh.safeExecute( + cmd = "curl --silent --write-out '%{http_code}' --output /dev/null -X GET $uri", + timeout = timeouts.unresponsivenessTimeout + ).output + if (currentStatus != statusQuo) { + break + } + if (deadline < Instant.now()) { + throw Exception("$uri failed to get out of $statusQuo status within $timeout") + } + threadDump.gather(ssh, "thread-dumps") + Thread.sleep(backoff.toMillis()) + } + } +} diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/jira/install/hook/ProfilerHook.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/jira/install/hook/ProfilerHook.kt new file mode 100644 index 00000000..9ce1d904 --- /dev/null +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/jira/install/hook/ProfilerHook.kt @@ -0,0 +1,40 @@ +package com.atlassian.performance.tools.infrastructure.jira.install.hook + +import com.atlassian.performance.tools.infrastructure.api.jira.install.InstalledJira +import com.atlassian.performance.tools.infrastructure.api.jira.install.hook.PostInstallHook +import com.atlassian.performance.tools.infrastructure.api.jira.install.hook.PostInstallHooks +import com.atlassian.performance.tools.infrastructure.api.jira.start.StartedJira +import com.atlassian.performance.tools.infrastructure.api.jira.start.hook.PostStartHook +import com.atlassian.performance.tools.infrastructure.api.jira.start.hook.PostStartHooks +import com.atlassian.performance.tools.infrastructure.api.profiler.Profiler +import com.atlassian.performance.tools.infrastructure.jira.report.RemoteMonitoringProcessReport +import com.atlassian.performance.tools.ssh.api.SshConnection + +/** + * Bridges the [Profiler] SPI with the [PostInstallHook] SPI. + * In general any [Profiler] can be rewritten as an [PostInstallHook] or any other hookPreStart without this bridge. + */ +class ProfilerHook( + private val profiler: Profiler +) : PostInstallHook { + override fun call(ssh: SshConnection, jira: InstalledJira, hooks: PostInstallHooks) { + profiler.install(ssh) + hooks.preStart.postStart.insert(InstalledProfiler(profiler)) + } +} + +private class InstalledProfiler( + private val profiler: Profiler +) : PostStartHook { + + override fun call( + ssh: SshConnection, + jira: StartedJira, + hooks: PostStartHooks + ) { + val process = profiler.start(ssh, jira.pid) + if (process != null) { + hooks.reports.add(RemoteMonitoringProcessReport(process)) + } + } +} diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/jira/install/hook/SplunkForwarderHook.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/jira/install/hook/SplunkForwarderHook.kt new file mode 100644 index 00000000..59c05bd2 --- /dev/null +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/jira/install/hook/SplunkForwarderHook.kt @@ -0,0 +1,21 @@ +package com.atlassian.performance.tools.infrastructure.jira.install.hook + +import com.atlassian.performance.tools.infrastructure.api.jira.install.hook.PostInstallHooks +import com.atlassian.performance.tools.infrastructure.api.jira.install.InstalledJira +import com.atlassian.performance.tools.infrastructure.api.jira.install.hook.PostInstallHook +import com.atlassian.performance.tools.infrastructure.api.splunk.SplunkForwarder +import com.atlassian.performance.tools.ssh.api.SshConnection + +internal class SplunkForwarderHook( + private val splunk: SplunkForwarder +) : PostInstallHook { + + override fun call( + ssh: SshConnection, + jira: InstalledJira, + hooks: PostInstallHooks + ) { + splunk.jsonifyLog4j(ssh, "${jira.installation.path}/atlassian-jira/WEB-INF/classes/log4j.properties") + splunk.run(ssh, jira.server.name, "/home/ubuntu/jirahome/log") + } +} diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/jira/report/RemoteMonitoringProcessReport.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/jira/report/RemoteMonitoringProcessReport.kt new file mode 100644 index 00000000..6d904047 --- /dev/null +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/jira/report/RemoteMonitoringProcessReport.kt @@ -0,0 +1,14 @@ +package com.atlassian.performance.tools.infrastructure.jira.report + +import com.atlassian.performance.tools.infrastructure.api.jira.report.Report +import com.atlassian.performance.tools.infrastructure.api.process.RemoteMonitoringProcess +import com.atlassian.performance.tools.ssh.api.SshConnection + +internal class RemoteMonitoringProcessReport( + private val process: RemoteMonitoringProcess +) : Report { + override fun locate(ssh: SshConnection): List { + process.stop(ssh) + return listOf(process.getResultPath()) + } +} diff --git a/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/JiraLaunchScriptIT.kt b/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/HookedJiraStartIT.kt similarity index 59% rename from src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/JiraLaunchScriptIT.kt rename to src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/HookedJiraStartIT.kt index ca914726..f1fb9814 100644 --- a/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/JiraLaunchScriptIT.kt +++ b/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/HookedJiraStartIT.kt @@ -2,7 +2,10 @@ package com.atlassian.performance.tools.infrastructure.api.jira.install import com.atlassian.performance.tools.infrastructure.api.distribution.PublicJiraSoftwareDistribution import com.atlassian.performance.tools.infrastructure.api.jira.EmptyJiraHome +import com.atlassian.performance.tools.infrastructure.api.jira.install.hook.HookedJiraInstallation +import com.atlassian.performance.tools.infrastructure.api.jira.install.hook.PreInstallHooks import com.atlassian.performance.tools.infrastructure.api.jira.start.JiraLaunchScript +import com.atlassian.performance.tools.infrastructure.api.jira.start.hook.HookedJiraStart import com.atlassian.performance.tools.infrastructure.api.jvm.AdoptOpenJDK import com.atlassian.performance.tools.infrastructure.toSsh import com.atlassian.performance.tools.sshubuntu.api.SshUbuntuContainer @@ -11,22 +14,29 @@ import org.junit.Test import java.nio.file.Files import java.util.function.Consumer -class JiraLaunchScriptIT { +class HookedJiraStartIT { @Test - fun shouldInstallJira() { + fun shouldStartJiraWithHooks() { // given - val installation = ParallelInstallation( - jiraHomeSource = EmptyJiraHome(), - productDistribution = PublicJiraSoftwareDistribution("7.13.0"), - jdk = AdoptOpenJDK() + val hooks = PreInstallHooks.default() + val installation = HookedJiraInstallation( + ParallelInstallation( + jiraHomeSource = EmptyJiraHome(), + productDistribution = PublicJiraSoftwareDistribution("7.13.0"), + jdk = AdoptOpenJDK() + ), + hooks ) - val start = JiraLaunchScript() + val start = HookedJiraStart(JiraLaunchScript(), hooks.preStart) testOnServer { server -> // when val installed = installation.install(server) val started = start.start(installed) + val reports = server.ssh.newConnection().use { ssh -> + hooks.reports.list().flatMap { it.locate(ssh) } + } // then val serverXml = installed @@ -35,6 +45,13 @@ class JiraLaunchScriptIT { .download(Files.createTempFile("downloaded-config", ".xml")) assertThat(serverXml.readText()).contains("