From 814f8860513e2a4add3e99b12bac48363d8b5922 Mon Sep 17 00:00:00 2001 From: Maciej Kwidzinski Date: Fri, 7 May 2021 16:49:27 +0200 Subject: [PATCH 01/52] Revert "JPERF-273: Hide untested new API" This reverts commit 71d7136625e9a1425f293786819f893b9bedf7fc. --- CHANGELOG.md | 6 ++++++ .../{ => api}/jira/install/InstalledJira.kt | 4 +++- .../{ => api}/jira/install/JiraInstallation.kt | 5 ++++- .../{ => api}/jira/install/ParallelInstallation.kt | 2 +- .../{ => api}/jira/install/SequentialInstallation.kt | 2 +- .../{ => api}/jira/install/TcpServer.kt | 2 +- .../{ => api}/jira/start/JiraLaunchScript.kt | 4 ++-- .../tools/infrastructure/api/jira/start/JiraStart.kt | 12 ++++++++++++ .../infrastructure/api/jira/start/StartedJira.kt | 8 ++++++++ .../tools/infrastructure/jira/start/JiraStart.kt | 12 ------------ .../tools/infrastructure/jira/start/StartedJira.kt | 8 -------- .../{ => api}/jira/install/JiraLaunchScriptIT.kt | 4 ++-- 12 files changed, 40 insertions(+), 29 deletions(-) rename src/main/kotlin/com/atlassian/performance/tools/infrastructure/{ => api}/jira/install/InstalledJira.kt (87%) rename src/main/kotlin/com/atlassian/performance/tools/infrastructure/{ => api}/jira/install/JiraInstallation.kt (71%) rename src/main/kotlin/com/atlassian/performance/tools/infrastructure/{ => api}/jira/install/ParallelInstallation.kt (95%) rename src/main/kotlin/com/atlassian/performance/tools/infrastructure/{ => api}/jira/install/SequentialInstallation.kt (93%) rename src/main/kotlin/com/atlassian/performance/tools/infrastructure/{ => api}/jira/install/TcpServer.kt (84%) rename src/main/kotlin/com/atlassian/performance/tools/infrastructure/{ => api}/jira/start/JiraLaunchScript.kt (81%) create mode 100644 src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/start/JiraStart.kt create mode 100644 src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/start/StartedJira.kt delete mode 100644 src/main/kotlin/com/atlassian/performance/tools/infrastructure/jira/start/JiraStart.kt delete mode 100644 src/main/kotlin/com/atlassian/performance/tools/infrastructure/jira/start/StartedJira.kt rename src/test/kotlin/com/atlassian/performance/tools/infrastructure/{ => api}/jira/install/JiraLaunchScriptIT.kt (92%) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1cb48b3b..b6ee5a15 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -32,10 +32,16 @@ Dropping a requirement of a major version of a dependency is a new contract. ### Added - Point to remote files on SSH hosts via `RemotePath`. +Progress on [JPERF-273]: +- Allow multiple ways of installing Jira via `JiraInstallation` or starting it via `JiraStart`. +- Represent the information required to use an already installed Jira via `InstalledJira` or `JiraStart` if started. +- Represent a brand-new Jira instance via `EmptyJiraHome`. + ### Fixed - Increase network-level retries for Jira/browser downloads. Decrease flakiness of such downloads on Ubuntu on WSL2. - Download ChromeDriver version that matches installed Chrome version. Fix [JPERF-732]. +[JPERF-273]: https://ecosystem.atlassian.net/browse/JPERF-273 [JPERF-732]: https://ecosystem.atlassian.net/browse/JPERF-732 ## [4.17.5] - 2020-12-15 diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/jira/install/InstalledJira.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/InstalledJira.kt similarity index 87% rename from src/main/kotlin/com/atlassian/performance/tools/infrastructure/jira/install/InstalledJira.kt rename to src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/InstalledJira.kt index f989223e..0668ea98 100644 --- a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/jira/install/InstalledJira.kt +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/InstalledJira.kt @@ -1,10 +1,12 @@ -package com.atlassian.performance.tools.infrastructure.jira.install +package com.atlassian.performance.tools.infrastructure.api.jira.install import com.atlassian.performance.tools.infrastructure.api.jvm.JavaDevelopmentKit import com.atlassian.performance.tools.infrastructure.api.os.RemotePath /** * Points to an already installed Jira. + * + * @since 4.18.0 */ class InstalledJira( /** diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/jira/install/JiraInstallation.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/JiraInstallation.kt similarity index 71% rename from src/main/kotlin/com/atlassian/performance/tools/infrastructure/jira/install/JiraInstallation.kt rename to src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/JiraInstallation.kt index ecf1383b..5f390f47 100644 --- a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/jira/install/JiraInstallation.kt +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/JiraInstallation.kt @@ -1,7 +1,10 @@ -package com.atlassian.performance.tools.infrastructure.jira.install +package com.atlassian.performance.tools.infrastructure.api.jira.install import net.jcip.annotations.ThreadSafe +/** + * @since 4.18.0 + */ @ThreadSafe interface JiraInstallation { diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/jira/install/ParallelInstallation.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/ParallelInstallation.kt similarity index 95% rename from src/main/kotlin/com/atlassian/performance/tools/infrastructure/jira/install/ParallelInstallation.kt rename to src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/ParallelInstallation.kt index 8dcd537e..8f228003 100644 --- a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/jira/install/ParallelInstallation.kt +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/ParallelInstallation.kt @@ -1,4 +1,4 @@ -package com.atlassian.performance.tools.infrastructure.jira.install +package com.atlassian.performance.tools.infrastructure.api.jira.install import com.atlassian.performance.tools.concurrency.api.submitWithLogContext import com.atlassian.performance.tools.infrastructure.api.distribution.ProductDistribution diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/jira/install/SequentialInstallation.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/SequentialInstallation.kt similarity index 93% rename from src/main/kotlin/com/atlassian/performance/tools/infrastructure/jira/install/SequentialInstallation.kt rename to src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/SequentialInstallation.kt index 0d9ca677..7615a397 100644 --- a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/jira/install/SequentialInstallation.kt +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/SequentialInstallation.kt @@ -1,4 +1,4 @@ -package com.atlassian.performance.tools.infrastructure.jira.install +package com.atlassian.performance.tools.infrastructure.api.jira.install import com.atlassian.performance.tools.infrastructure.api.distribution.ProductDistribution import com.atlassian.performance.tools.infrastructure.api.jira.JiraHomeSource diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/jira/install/TcpServer.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/TcpServer.kt similarity index 84% rename from src/main/kotlin/com/atlassian/performance/tools/infrastructure/jira/install/TcpServer.kt rename to src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/TcpServer.kt index 13cf3dd3..1b7f1eeb 100644 --- a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/jira/install/TcpServer.kt +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/TcpServer.kt @@ -1,4 +1,4 @@ -package com.atlassian.performance.tools.infrastructure.jira.install +package com.atlassian.performance.tools.infrastructure.api.jira.install import com.atlassian.performance.tools.ssh.api.Ssh diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/jira/start/JiraLaunchScript.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/start/JiraLaunchScript.kt similarity index 81% rename from src/main/kotlin/com/atlassian/performance/tools/infrastructure/jira/start/JiraLaunchScript.kt rename to src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/start/JiraLaunchScript.kt index 14086413..55e8ec6d 100644 --- a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/jira/start/JiraLaunchScript.kt +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/start/JiraLaunchScript.kt @@ -1,6 +1,6 @@ -package com.atlassian.performance.tools.infrastructure.jira.start +package com.atlassian.performance.tools.infrastructure.api.jira.start -import com.atlassian.performance.tools.infrastructure.jira.install.InstalledJira +import com.atlassian.performance.tools.infrastructure.api.jira.install.InstalledJira import com.atlassian.performance.tools.ssh.api.Ssh import java.time.Duration diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/start/JiraStart.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/start/JiraStart.kt new file mode 100644 index 00000000..9f2d9cd8 --- /dev/null +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/start/JiraStart.kt @@ -0,0 +1,12 @@ +package com.atlassian.performance.tools.infrastructure.api.jira.start + +import com.atlassian.performance.tools.infrastructure.api.jira.install.InstalledJira +import net.jcip.annotations.ThreadSafe + +@ThreadSafe +interface JiraStart { + + fun start( + installed: InstalledJira + ): StartedJira +} diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/start/StartedJira.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/start/StartedJira.kt new file mode 100644 index 00000000..da6f4955 --- /dev/null +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/start/StartedJira.kt @@ -0,0 +1,8 @@ +package com.atlassian.performance.tools.infrastructure.api.jira.start + +import com.atlassian.performance.tools.infrastructure.api.jira.install.InstalledJira + +class StartedJira( + val installed: InstalledJira, + val pid: Int +) diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/jira/start/JiraStart.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/jira/start/JiraStart.kt deleted file mode 100644 index 9cb166bb..00000000 --- a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/jira/start/JiraStart.kt +++ /dev/null @@ -1,12 +0,0 @@ -package com.atlassian.performance.tools.infrastructure.jira.start - -import com.atlassian.performance.tools.infrastructure.jira.install.InstalledJira -import net.jcip.annotations.ThreadSafe - -@ThreadSafe -interface JiraStart { - - fun start( - installed: InstalledJira - ): StartedJira -} diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/jira/start/StartedJira.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/jira/start/StartedJira.kt deleted file mode 100644 index a7f29f61..00000000 --- a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/jira/start/StartedJira.kt +++ /dev/null @@ -1,8 +0,0 @@ -package com.atlassian.performance.tools.infrastructure.jira.start - -import com.atlassian.performance.tools.infrastructure.jira.install.InstalledJira - -class StartedJira( - val installed: InstalledJira, - val pid: Int -) diff --git a/src/test/kotlin/com/atlassian/performance/tools/infrastructure/jira/install/JiraLaunchScriptIT.kt b/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/JiraLaunchScriptIT.kt similarity index 92% rename from src/test/kotlin/com/atlassian/performance/tools/infrastructure/jira/install/JiraLaunchScriptIT.kt rename to src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/JiraLaunchScriptIT.kt index 2d8d0889..ca914726 100644 --- a/src/test/kotlin/com/atlassian/performance/tools/infrastructure/jira/install/JiraLaunchScriptIT.kt +++ b/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/JiraLaunchScriptIT.kt @@ -1,8 +1,8 @@ -package com.atlassian.performance.tools.infrastructure.jira.install +package com.atlassian.performance.tools.infrastructure.api.jira.install import com.atlassian.performance.tools.infrastructure.api.distribution.PublicJiraSoftwareDistribution import com.atlassian.performance.tools.infrastructure.api.jira.EmptyJiraHome -import com.atlassian.performance.tools.infrastructure.jira.start.JiraLaunchScript +import com.atlassian.performance.tools.infrastructure.api.jira.start.JiraLaunchScript import com.atlassian.performance.tools.infrastructure.api.jvm.AdoptOpenJDK import com.atlassian.performance.tools.infrastructure.toSsh import com.atlassian.performance.tools.sshubuntu.api.SshUbuntuContainer From c19447f0ca13da218f817c94e80d0897c8133524 Mon Sep 17 00:00:00 2001 From: Maciej Kwidzinski Date: Fri, 12 Feb 2021 16:00:01 +0100 Subject: [PATCH 02/52] JPERF-273: Hook into Jira provisioning --- CHANGELOG.md | 6 +- .../jira/install/hook/AsyncProfilerHook.kt | 57 ++++++++++++++ .../api/jira/install/hook/DataCenterHook.kt | 24 ++++++ .../jira/install/hook/DisabledAutoBackup.kt | 15 ++++ .../install/hook/HookedJiraInstallation.kt | 24 ++++++ .../api/jira/install/hook/JiraHomeProperty.kt | 16 ++++ .../api/jira/install/hook/JiraLogs.kt | 28 +++++++ .../api/jira/install/hook/JvmConfig.kt | 29 ++++++++ .../jira/install/hook/LateUbuntuSysstat.kt | 35 +++++++++ .../api/jira/install/hook/PostInstallHook.kt | 21 ++++++ .../api/jira/install/hook/PostInstallHooks.kt | 54 ++++++++++++++ .../api/jira/install/hook/PreInstallHook.kt | 21 ++++++ .../api/jira/install/hook/PreInstallHooks.kt | 40 ++++++++++ .../api/jira/install/hook/SystemLog.kt | 11 +++ .../api/jira/report/FileListing.kt | 16 ++++ .../infrastructure/api/jira/report/Report.kt | 15 ++++ .../infrastructure/api/jira/report/Reports.kt | 18 +++++ .../api/jira/report/StaticReport.kt | 10 +++ .../api/jira/start/hook/AccessLogs.kt | 12 +++ .../api/jira/start/hook/HookedJiraStart.kt | 24 ++++++ .../api/jira/start/hook/JstatHook.kt | 17 +++++ .../api/jira/start/hook/PostStartHook.kt | 21 ++++++ .../api/jira/start/hook/PostStartHooks.kt | 39 ++++++++++ .../api/jira/start/hook/PreStartHook.kt | 21 ++++++ .../api/jira/start/hook/PreStartHooks.kt | 38 ++++++++++ .../api/jira/start/hook/RestUpgrade.kt | 74 +++++++++++++++++++ .../jira/install/hook/ProfilerHook.kt | 41 ++++++++++ .../jira/install/hook/SplunkForwarderHook.kt | 21 ++++++ .../report/RemoteMonitoringProcessReport.kt | 14 ++++ ...LaunchScriptIT.kt => HookedJiraStartIT.kt} | 31 ++++++-- .../jira/install/hook/PreInstallHooksTest.kt | 61 +++++++++++++++ 31 files changed, 846 insertions(+), 8 deletions(-) create mode 100644 src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/AsyncProfilerHook.kt create mode 100644 src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/DataCenterHook.kt create mode 100644 src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/DisabledAutoBackup.kt create mode 100644 src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/HookedJiraInstallation.kt create mode 100644 src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/JiraHomeProperty.kt create mode 100644 src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/JiraLogs.kt create mode 100644 src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/JvmConfig.kt create mode 100644 src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/LateUbuntuSysstat.kt create mode 100644 src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/PostInstallHook.kt create mode 100644 src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/PostInstallHooks.kt create mode 100644 src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/PreInstallHook.kt create mode 100644 src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/PreInstallHooks.kt create mode 100644 src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/SystemLog.kt create mode 100644 src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/report/FileListing.kt create mode 100644 src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/report/Report.kt create mode 100644 src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/report/Reports.kt create mode 100644 src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/report/StaticReport.kt create mode 100644 src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/start/hook/AccessLogs.kt create mode 100644 src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/start/hook/HookedJiraStart.kt create mode 100644 src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/start/hook/JstatHook.kt create mode 100644 src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/start/hook/PostStartHook.kt create mode 100644 src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/start/hook/PostStartHooks.kt create mode 100644 src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/start/hook/PreStartHook.kt create mode 100644 src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/start/hook/PreStartHooks.kt create mode 100644 src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/start/hook/RestUpgrade.kt create mode 100644 src/main/kotlin/com/atlassian/performance/tools/infrastructure/jira/install/hook/ProfilerHook.kt create mode 100644 src/main/kotlin/com/atlassian/performance/tools/infrastructure/jira/install/hook/SplunkForwarderHook.kt create mode 100644 src/main/kotlin/com/atlassian/performance/tools/infrastructure/jira/report/RemoteMonitoringProcessReport.kt rename src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/{JiraLaunchScriptIT.kt => HookedJiraStartIT.kt} (59%) create mode 100644 src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/PreInstallHooksTest.kt diff --git a/CHANGELOG.md b/CHANGELOG.md index b6ee5a15..bff00388 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -32,10 +32,14 @@ Dropping a requirement of a major version of a dependency is a new contract. ### Added - Point to remote files on SSH hosts via `RemotePath`. -Progress on [JPERF-273]: +Fix [JPERF-273]: - Allow multiple ways of installing Jira via `JiraInstallation` or starting it via `JiraStart`. - Represent the information required to use an already installed Jira via `InstalledJira` or `JiraStart` if started. - Represent a brand-new Jira instance via `EmptyJiraHome`. +- Hook into Jira installation via `PreInstallHooks` and `PostInstallHooks`. +- Hook into Jira start via `PreStartHooks` and `PostStartHooks`. +- Let hooks insert new hooks. +- Locate and download any logs, charts, profiles and other reports via `Report` (rather than hardcoding the paths). ### Fixed - Increase network-level retries for Jira/browser downloads. Decrease flakiness of such downloads on Ubuntu on WSL2. diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/AsyncProfilerHook.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/AsyncProfilerHook.kt new file mode 100644 index 00000000..5f60deaf --- /dev/null +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/AsyncProfilerHook.kt @@ -0,0 +1,57 @@ +package com.atlassian.performance.tools.infrastructure.api.jira.install.hook + +import com.atlassian.performance.tools.infrastructure.api.jira.install.TcpServer +import com.atlassian.performance.tools.infrastructure.api.jira.report.Report +import com.atlassian.performance.tools.infrastructure.api.jira.start.StartedJira +import com.atlassian.performance.tools.infrastructure.api.jira.start.hook.PostStartHook +import com.atlassian.performance.tools.infrastructure.api.jira.start.hook.PostStartHooks +import com.atlassian.performance.tools.ssh.api.SshConnection +import java.net.URI + +class AsyncProfilerHook : PreInstallHook { + + override fun call( + ssh: SshConnection, + server: TcpServer, + hooks: PreInstallHooks + ) { + val directory = "async-profiler" + val downloads = URI("https://github.com/jvm-profiling-tools/async-profiler/releases/download/") + val distribution = downloads.resolve("v1.4/async-profiler-1.4-linux-x64.tar.gz") + ssh.execute("wget -q $distribution") + ssh.execute("mkdir $directory") + ssh.execute("tar -xzf async-profiler-1.4-linux-x64.tar.gz -C $directory") + ssh.execute("sudo sh -c 'echo 1 > /proc/sys/kernel/perf_event_paranoid'") + ssh.execute("sudo sh -c 'echo 0 > /proc/sys/kernel/kptr_restrict'") + val profilerPath = "./$directory/profiler.sh" + val profiler = InstalledAsyncProfiler(profilerPath) + hooks.postStart.insert(profiler) + } +} + +private class InstalledAsyncProfiler( + private val profilerPath: String +) : PostStartHook { + + override fun call( + ssh: SshConnection, + jira: StartedJira, + hooks: PostStartHooks + ) { + ssh.execute("$profilerPath -b 20000000 start ${jira.pid}") + val profiler = StartedAsyncProfiler(jira.pid, profilerPath) + hooks.reports.add(profiler) + } +} + +private class StartedAsyncProfiler( + private val pid: Int, + private val profilerPath: String +) : Report { + + override fun locate(ssh: SshConnection): List { + val flameGraphFile = "flamegraph.svg" + ssh.execute("$profilerPath stop $pid -o svg > $flameGraphFile") + return listOf(flameGraphFile) + } +} diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/DataCenterHook.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/DataCenterHook.kt new file mode 100644 index 00000000..4b4b549f --- /dev/null +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/DataCenterHook.kt @@ -0,0 +1,24 @@ +package com.atlassian.performance.tools.infrastructure.api.jira.install.hook + +import com.atlassian.performance.tools.infrastructure.api.jira.SharedHome +import com.atlassian.performance.tools.infrastructure.api.jira.install.InstalledJira +import com.atlassian.performance.tools.ssh.api.SshConnection + +class DataCenterHook( + private val nodeId: String, + private val sharedHome: SharedHome +) : PostInstallHook { + + override fun call( + ssh: SshConnection, + jira: InstalledJira, + hooks: PostInstallHooks + ) { + val localSharedHome = sharedHome.localSharedHome + sharedHome.mount(ssh) + val jiraHome = jira.home.path // TODO what's the difference between localSharedHome and jiraHome? should both be hookable? + ssh.execute("echo ehcache.object.port = 40011 >> $jiraHome/cluster.properties") + ssh.execute("echo jira.node.id = $nodeId >> $jiraHome/cluster.properties") + ssh.execute("echo jira.shared.home = `realpath $localSharedHome` >> $jiraHome/cluster.properties") + } +} diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/DisabledAutoBackup.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/DisabledAutoBackup.kt new file mode 100644 index 00000000..8cc6e973 --- /dev/null +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/DisabledAutoBackup.kt @@ -0,0 +1,15 @@ +package com.atlassian.performance.tools.infrastructure.api.jira.install.hook + +import com.atlassian.performance.tools.infrastructure.api.jira.install.InstalledJira +import com.atlassian.performance.tools.ssh.api.SshConnection + +class DisabledAutoBackup : PostInstallHook { + + override fun call( + ssh: SshConnection, + jira: InstalledJira, + hooks: PostInstallHooks + ) { + ssh.execute("echo jira.autoexport=false > ${jira.home.path}/jira-config.properties") + } +} diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/HookedJiraInstallation.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/HookedJiraInstallation.kt new file mode 100644 index 00000000..e8fc3ae0 --- /dev/null +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/HookedJiraInstallation.kt @@ -0,0 +1,24 @@ +package com.atlassian.performance.tools.infrastructure.api.jira.install.hook + +import com.atlassian.performance.tools.infrastructure.api.jira.install.InstalledJira +import com.atlassian.performance.tools.infrastructure.api.jira.install.JiraInstallation +import com.atlassian.performance.tools.infrastructure.api.jira.install.TcpServer + +class HookedJiraInstallation( + private val installation: JiraInstallation, + private val hooks: PreInstallHooks +) : JiraInstallation { + + override fun install( + server: TcpServer + ): InstalledJira { + server.ssh.newConnection().use { ssh -> + hooks.call(ssh, server) + } + val installed = installation.install(server) + server.ssh.newConnection().use { ssh -> + hooks.postInstall.call(ssh, installed) + } + return installed + } +} diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/JiraHomeProperty.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/JiraHomeProperty.kt new file mode 100644 index 00000000..9d8d1ed4 --- /dev/null +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/JiraHomeProperty.kt @@ -0,0 +1,16 @@ +package com.atlassian.performance.tools.infrastructure.api.jira.install.hook + +import com.atlassian.performance.tools.infrastructure.api.jira.install.InstalledJira +import com.atlassian.performance.tools.ssh.api.SshConnection + +class JiraHomeProperty : PostInstallHook { + + override fun call( + ssh: SshConnection, + jira: InstalledJira, + hooks: PostInstallHooks + ) { + val properties = "${jira.installation.path}/atlassian-jira/WEB-INF/classes/jira-application.properties" + ssh.execute("echo jira.home=`realpath ${jira.home.path}` > $properties") + } +} diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/JiraLogs.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/JiraLogs.kt new file mode 100644 index 00000000..ef6f803c --- /dev/null +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/JiraLogs.kt @@ -0,0 +1,28 @@ +package com.atlassian.performance.tools.infrastructure.api.jira.install.hook + +import com.atlassian.performance.tools.infrastructure.api.jira.install.InstalledJira +import com.atlassian.performance.tools.infrastructure.api.jira.report.StaticReport +import com.atlassian.performance.tools.ssh.api.SshConnection +import java.nio.file.Path +import java.nio.file.Paths + +class JiraLogs : PostInstallHook { + + override fun call(ssh: SshConnection, jira: InstalledJira, hooks: PostInstallHooks) { + listOf( + "${jira.home.path}/log/atlassian-jira.log", + "${jira.installation.path}/logs/catalina.out" + ) + .onEach { ensureFile(Paths.get(it), ssh) } + .map { StaticReport(it) } + .forEach { hooks.reports.add(it) } + } + + private fun ensureFile( + path: Path, + ssh: SshConnection + ) { + ssh.execute("mkdir -p ${path.parent!!}") + ssh.execute("touch $path") + } +} diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/JvmConfig.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/JvmConfig.kt new file mode 100644 index 00000000..d7f68580 --- /dev/null +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/JvmConfig.kt @@ -0,0 +1,29 @@ +package com.atlassian.performance.tools.infrastructure.api.jira.install.hook + +import com.atlassian.performance.tools.infrastructure.api.jira.JiraGcLog +import com.atlassian.performance.tools.infrastructure.api.jira.JiraNodeConfig +import com.atlassian.performance.tools.infrastructure.api.jira.SetenvSh +import com.atlassian.performance.tools.infrastructure.api.jira.install.InstalledJira +import com.atlassian.performance.tools.infrastructure.api.jira.report.FileListing +import com.atlassian.performance.tools.ssh.api.SshConnection + +class JvmConfig( + private val config: JiraNodeConfig +) : PostInstallHook { + + override fun call( + ssh: SshConnection, + jira: InstalledJira, + hooks: PostInstallHooks + ) { + val gcLog = JiraGcLog(jira.installation.path) + SetenvSh(jira.installation.path).setup( + connection = ssh, + config = config, + gcLog = gcLog, + jiraIp = jira.server.ip + ) + val report = FileListing(gcLog.path("*")) + hooks.reports.add(report) + } +} diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/LateUbuntuSysstat.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/LateUbuntuSysstat.kt new file mode 100644 index 00000000..2273c81c --- /dev/null +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/LateUbuntuSysstat.kt @@ -0,0 +1,35 @@ +package com.atlassian.performance.tools.infrastructure.api.jira.install.hook + +import com.atlassian.performance.tools.infrastructure.Iostat +import com.atlassian.performance.tools.infrastructure.api.jira.install.InstalledJira +import com.atlassian.performance.tools.infrastructure.api.jira.start.StartedJira +import com.atlassian.performance.tools.infrastructure.api.jira.start.hook.PostStartHook +import com.atlassian.performance.tools.infrastructure.api.jira.start.hook.PostStartHooks +import com.atlassian.performance.tools.infrastructure.api.os.OsMetric +import com.atlassian.performance.tools.infrastructure.api.os.Ubuntu +import com.atlassian.performance.tools.infrastructure.api.os.Vmstat +import com.atlassian.performance.tools.infrastructure.jira.report.RemoteMonitoringProcessReport +import com.atlassian.performance.tools.ssh.api.SshConnection + +class LateUbuntuSysstat : PostInstallHook { + override fun call( + ssh: SshConnection, + jira: InstalledJira, + hooks: PostInstallHooks + ) { + val ubuntu = Ubuntu() + ubuntu.install(ssh, listOf("sysstat")) + listOf(Vmstat(), Iostat()) + .map { PostStartOsMetric(it) } + .forEach { hooks.postStart.insert(it) } + } +} + +private class PostStartOsMetric( + private val metric: OsMetric +) : PostStartHook { + override fun call(ssh: SshConnection, jira: StartedJira, hooks: PostStartHooks) { + val process = metric.start(ssh) + hooks.reports.add(RemoteMonitoringProcessReport(process)) + } +} diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/PostInstallHook.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/PostInstallHook.kt new file mode 100644 index 00000000..0fe19b0a --- /dev/null +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/PostInstallHook.kt @@ -0,0 +1,21 @@ +package com.atlassian.performance.tools.infrastructure.api.jira.install.hook + +import com.atlassian.performance.tools.infrastructure.api.jira.install.InstalledJira +import com.atlassian.performance.tools.ssh.api.SshConnection + +/** + * Intercepts a call after Jira is installed. + */ +interface PostInstallHook { + + /** + * @param [ssh] connects to the [jira] + * @param [jira] points to the installed Jira + * @param [hooks] inserts future hooks and reports + */ + fun call( + ssh: SshConnection, + jira: InstalledJira, + hooks: PostInstallHooks + ) +} diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/PostInstallHooks.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/PostInstallHooks.kt new file mode 100644 index 00000000..0b285cf5 --- /dev/null +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/PostInstallHooks.kt @@ -0,0 +1,54 @@ +package com.atlassian.performance.tools.infrastructure.api.jira.install.hook + +import com.atlassian.performance.tools.infrastructure.api.jira.JiraNodeConfig +import com.atlassian.performance.tools.infrastructure.api.jira.install.InstalledJira +import com.atlassian.performance.tools.infrastructure.api.jira.start.hook.PreStartHooks +import com.atlassian.performance.tools.infrastructure.jira.install.hook.ProfilerHook +import com.atlassian.performance.tools.infrastructure.jira.install.hook.SplunkForwarderHook +import com.atlassian.performance.tools.ssh.api.SshConnection +import java.util.* +import java.util.concurrent.ConcurrentLinkedQueue + +class PostInstallHooks private constructor( + val preStart: PreStartHooks +) { + + private val hooks: Queue = ConcurrentLinkedQueue() + val postStart = preStart.postStart + val reports = postStart.reports + + fun insert( + hook: PostInstallHook + ) { + hooks.add(hook) + } + + internal fun call( + ssh: SshConnection, + jira: InstalledJira + ) { + while (true) { + hooks + .poll() + ?.call(ssh, jira, this) + ?: break + } + } + + companion object Factory { + fun default(): PostInstallHooks = PostInstallHooks(PreStartHooks.default()).apply { + val config = JiraNodeConfig.Builder().build() + listOf( + JiraHomeProperty(), + DisabledAutoBackup(), + JvmConfig(config), + ProfilerHook(config.profiler), + SplunkForwarderHook(config.splunkForwarder), + JiraLogs(), + LateUbuntuSysstat() + ).forEach { insert(it) } + } + + fun empty(): PostInstallHooks = PostInstallHooks(PreStartHooks.empty()) + } +} diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/PreInstallHook.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/PreInstallHook.kt new file mode 100644 index 00000000..1e58b1af --- /dev/null +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/PreInstallHook.kt @@ -0,0 +1,21 @@ +package com.atlassian.performance.tools.infrastructure.api.jira.install.hook + +import com.atlassian.performance.tools.infrastructure.api.jira.install.TcpServer +import com.atlassian.performance.tools.ssh.api.SshConnection + +/** + * Intercepts a call before Jira is installed. + */ +interface PreInstallHook { + + /** + * @param [ssh] connects to the [server] + * @param [server] will install Jira + * @param [hooks] inserts future hooks and reports + */ + fun call( + ssh: SshConnection, + server: TcpServer, + hooks: PreInstallHooks + ) +} diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/PreInstallHooks.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/PreInstallHooks.kt new file mode 100644 index 00000000..5506de0d --- /dev/null +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/PreInstallHooks.kt @@ -0,0 +1,40 @@ +package com.atlassian.performance.tools.infrastructure.api.jira.install.hook + +import com.atlassian.performance.tools.infrastructure.api.jira.install.TcpServer +import com.atlassian.performance.tools.ssh.api.SshConnection +import java.util.* +import java.util.concurrent.ConcurrentLinkedQueue + +class PreInstallHooks private constructor( + val postInstall: PostInstallHooks +) { + + private val hooks: Queue = ConcurrentLinkedQueue() + val preStart = postInstall.preStart + val postStart = preStart.postStart + val reports = postStart.reports + + fun insert( + hook: PreInstallHook + ) { + hooks.add(hook) + } + + internal fun call( + ssh: SshConnection, + server: TcpServer + ) { + while (true) { + hooks + .poll() + ?.call(ssh, server, this) + ?: break + } + } + + companion object Factory { + fun default(): PreInstallHooks = PreInstallHooks(PostInstallHooks.default()) + + fun empty(): PreInstallHooks = PreInstallHooks(PostInstallHooks.empty()) + } +} diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/SystemLog.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/SystemLog.kt new file mode 100644 index 00000000..861b0ec8 --- /dev/null +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/SystemLog.kt @@ -0,0 +1,11 @@ +package com.atlassian.performance.tools.infrastructure.api.jira.install.hook + +import com.atlassian.performance.tools.infrastructure.api.jira.report.Report +import com.atlassian.performance.tools.ssh.api.SshConnection + +class SystemLog : Report { + + override fun locate(ssh: SshConnection): List { + return listOf("/var/log/syslog") + } +} diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/report/FileListing.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/report/FileListing.kt new file mode 100644 index 00000000..75979fca --- /dev/null +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/report/FileListing.kt @@ -0,0 +1,16 @@ +package com.atlassian.performance.tools.infrastructure.api.jira.report + +import com.atlassian.performance.tools.ssh.api.SshConnection + +class FileListing( + private val pattern: String +) : Report { + + override fun locate( + ssh: SshConnection + ): List = ssh + .execute("ls $pattern") + .output + .lines() + .filter { it.isNotBlank() } +} diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/report/Report.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/report/Report.kt new file mode 100644 index 00000000..65059517 --- /dev/null +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/report/Report.kt @@ -0,0 +1,15 @@ +package com.atlassian.performance.tools.infrastructure.api.jira.report + +import com.atlassian.performance.tools.ssh.api.SshConnection + +/** + * Reports back about remote events. E.g. points to interesting logs, dumps, charts. + */ +interface Report { + + /** + * @param [ssh] connects to the server, which holds interesting data + * @return interesting file paths, which could be downloaded + */ + fun locate(ssh: SshConnection): List +} diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/report/Reports.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/report/Reports.kt new file mode 100644 index 00000000..bfc8b1cd --- /dev/null +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/report/Reports.kt @@ -0,0 +1,18 @@ +package com.atlassian.performance.tools.infrastructure.api.jira.report + +import java.util.* +import java.util.concurrent.ConcurrentLinkedQueue + +class Reports { + private val reports: Queue = ConcurrentLinkedQueue() + + fun add( + report: Report + ) { + reports.add(report) + } + + fun list(): Iterable { + return reports.toList() + } +} diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/report/StaticReport.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/report/StaticReport.kt new file mode 100644 index 00000000..da90e4d1 --- /dev/null +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/report/StaticReport.kt @@ -0,0 +1,10 @@ +package com.atlassian.performance.tools.infrastructure.api.jira.report + +import com.atlassian.performance.tools.ssh.api.SshConnection + +class StaticReport( + private val remotePath: String +) : Report { + + override fun locate(ssh: SshConnection): List = listOf(remotePath) +} diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/start/hook/AccessLogs.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/start/hook/AccessLogs.kt new file mode 100644 index 00000000..6bc2f30e --- /dev/null +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/start/hook/AccessLogs.kt @@ -0,0 +1,12 @@ +package com.atlassian.performance.tools.infrastructure.api.jira.start.hook + +import com.atlassian.performance.tools.infrastructure.api.jira.report.FileListing +import com.atlassian.performance.tools.infrastructure.api.jira.start.StartedJira +import com.atlassian.performance.tools.ssh.api.SshConnection + +class AccessLogs : PostStartHook { + + override fun call(ssh: SshConnection, jira: StartedJira, hooks: PostStartHooks) { + hooks.reports.add(FileListing("${jira.installed.installation.path}/logs/*access*")) + } +} diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/start/hook/HookedJiraStart.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/start/hook/HookedJiraStart.kt new file mode 100644 index 00000000..70afbfff --- /dev/null +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/start/hook/HookedJiraStart.kt @@ -0,0 +1,24 @@ +package com.atlassian.performance.tools.infrastructure.api.jira.start.hook + +import com.atlassian.performance.tools.infrastructure.api.jira.install.InstalledJira +import com.atlassian.performance.tools.infrastructure.api.jira.start.JiraStart +import com.atlassian.performance.tools.infrastructure.api.jira.start.StartedJira + +class HookedJiraStart( + private val start: JiraStart, + private val hooks: PreStartHooks +) : JiraStart { + + override fun start( + installed: InstalledJira + ): StartedJira { + installed.server.ssh.newConnection().use { ssh -> + hooks.call(ssh, installed) + } + val started = start.start(installed) + installed.server.ssh.newConnection().use { ssh -> + hooks.postStart.call(ssh, started) + } + return started + } +} diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/start/hook/JstatHook.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/start/hook/JstatHook.kt new file mode 100644 index 00000000..e4485af4 --- /dev/null +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/start/hook/JstatHook.kt @@ -0,0 +1,17 @@ +package com.atlassian.performance.tools.infrastructure.api.jira.start.hook + +import com.atlassian.performance.tools.infrastructure.api.jira.start.StartedJira +import com.atlassian.performance.tools.infrastructure.jira.report.RemoteMonitoringProcessReport +import com.atlassian.performance.tools.ssh.api.SshConnection + +class JstatHook : PostStartHook { + + override fun call( + ssh: SshConnection, + jira: StartedJira, + hooks: PostStartHooks + ) { + val process = jira.installed.jdk.jstatMonitoring.start(ssh, jira.pid) + hooks.reports.add(RemoteMonitoringProcessReport(process)) + } +} diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/start/hook/PostStartHook.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/start/hook/PostStartHook.kt new file mode 100644 index 00000000..c3c47123 --- /dev/null +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/start/hook/PostStartHook.kt @@ -0,0 +1,21 @@ +package com.atlassian.performance.tools.infrastructure.api.jira.start.hook + +import com.atlassian.performance.tools.infrastructure.api.jira.start.StartedJira +import com.atlassian.performance.tools.ssh.api.SshConnection + +/** + * Intercepts a call after Jira is started. + */ +interface PostStartHook { + + /** + * @param [ssh] connects to the [jira] + * @param [jira] points to the started Jira + * @param [hooks] inserts future reports + */ + fun call( + ssh: SshConnection, + jira: StartedJira, + hooks: PostStartHooks + ) +} diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/start/hook/PostStartHooks.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/start/hook/PostStartHooks.kt new file mode 100644 index 00000000..21b0b46a --- /dev/null +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/start/hook/PostStartHooks.kt @@ -0,0 +1,39 @@ +package com.atlassian.performance.tools.infrastructure.api.jira.start.hook + +import com.atlassian.performance.tools.infrastructure.api.jira.report.Reports +import com.atlassian.performance.tools.infrastructure.api.jira.start.StartedJira +import com.atlassian.performance.tools.ssh.api.SshConnection +import java.util.* +import java.util.concurrent.ConcurrentLinkedQueue + +class PostStartHooks private constructor() { + + private val hooks: Queue = ConcurrentLinkedQueue() + val reports = Reports() + + fun insert( + hook: PostStartHook + ) { + hooks.add(hook) + } + + internal fun call( + ssh: SshConnection, + jira: StartedJira + ) { + while (true) { + hooks + .poll() + ?.call(ssh, jira, this) + ?: break + } + } + + companion object Factory { + fun default(): PostStartHooks = empty().apply { + insert(JstatHook()) + } + + fun empty(): PostStartHooks = PostStartHooks() + } +} diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/start/hook/PreStartHook.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/start/hook/PreStartHook.kt new file mode 100644 index 00000000..d0fa8326 --- /dev/null +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/start/hook/PreStartHook.kt @@ -0,0 +1,21 @@ +package com.atlassian.performance.tools.infrastructure.api.jira.start.hook + +import com.atlassian.performance.tools.infrastructure.api.jira.install.InstalledJira +import com.atlassian.performance.tools.ssh.api.SshConnection + +/** + * Intercepts a call before Jira is started. + */ +interface PreStartHook { + + /** + * @param [ssh] connects to the [jira] + * @param [jira] points to the installed Jira + * @param [hooks] inserts future hooks and reports + */ + fun call( + ssh: SshConnection, + jira: InstalledJira, + hooks: PreStartHooks + ) +} diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/start/hook/PreStartHooks.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/start/hook/PreStartHooks.kt new file mode 100644 index 00000000..6ce16b42 --- /dev/null +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/start/hook/PreStartHooks.kt @@ -0,0 +1,38 @@ +package com.atlassian.performance.tools.infrastructure.api.jira.start.hook + +import com.atlassian.performance.tools.infrastructure.api.jira.install.InstalledJira +import com.atlassian.performance.tools.ssh.api.SshConnection +import java.util.* +import java.util.concurrent.ConcurrentLinkedQueue + +class PreStartHooks private constructor( + val postStart: PostStartHooks +) { + + private val hooks: Queue = ConcurrentLinkedQueue() + val reports = postStart.reports + + fun insert( + hook: PreStartHook + ) { + hooks.add(hook) + } + + internal fun call( + ssh: SshConnection, + jira: InstalledJira + ) { + while (true) { + hooks + .poll() + ?.call(ssh, jira, this) + ?: break + } + } + + companion object Factory { + fun default() = PreStartHooks(PostStartHooks.default()) + + fun empty() = PreStartHooks(PostStartHooks.empty()) + } +} diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/start/hook/RestUpgrade.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/start/hook/RestUpgrade.kt new file mode 100644 index 00000000..84aa39f4 --- /dev/null +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/start/hook/RestUpgrade.kt @@ -0,0 +1,74 @@ +package com.atlassian.performance.tools.infrastructure.api.jira.start.hook + +import com.atlassian.performance.tools.infrastructure.api.jira.JiraLaunchTimeouts +import com.atlassian.performance.tools.infrastructure.api.jira.report.StaticReport +import com.atlassian.performance.tools.infrastructure.api.jira.start.StartedJira +import com.atlassian.performance.tools.infrastructure.api.jvm.ThreadDump +import com.atlassian.performance.tools.ssh.api.SshConnection +import java.net.URI +import java.time.Duration +import java.time.Instant + +class RestUpgrade( + private val timeouts: JiraLaunchTimeouts, + private val adminUsername: String, + private val adminPassword: String +) : PostStartHook { + + override fun call(ssh: SshConnection, jira: StartedJira, hooks: PostStartHooks) { + val threadDump = ThreadDump(jira.pid, jira.installed.jdk) + val privatePort = jira.installed.server.privatePort + val upgradesEndpoint = URI("http://$adminUsername:$adminPassword@localhost:$privatePort/rest/api/2/upgrade") + hooks.reports.add(StaticReport("thread-dumps")) + waitForStatusToChange( + statusQuo = "000", + timeout = timeouts.offlineTimeout, + ssh = ssh, + uri = upgradesEndpoint, + threadDump = threadDump + ) + waitForStatusToChange( + statusQuo = "503", + timeout = timeouts.initTimeout, + ssh = ssh, + uri = upgradesEndpoint, + threadDump = threadDump + ) + ssh.execute( + cmd = "curl --silent --retry 6 -X POST $upgradesEndpoint", + timeout = Duration.ofSeconds(15) + ) + waitForStatusToChange( + statusQuo = "303", + timeout = timeouts.upgradeTimeout, + ssh = ssh, + uri = upgradesEndpoint, + threadDump = threadDump + ) + } + + private fun waitForStatusToChange( + statusQuo: String, + uri: URI, + timeout: Duration, + ssh: SshConnection, + threadDump: ThreadDump + ) { + val backoff = Duration.ofSeconds(10) + val deadline = Instant.now() + timeout + while (true) { + val currentStatus = ssh.safeExecute( + cmd = "curl --silent --write-out '%{http_code}' --output /dev/null -X GET $uri", + timeout = timeouts.unresponsivenessTimeout + ).output + if (currentStatus != statusQuo) { + break + } + if (deadline < Instant.now()) { + throw Exception("$uri failed to get out of $statusQuo status within $timeout") + } + threadDump.gather(ssh, "thread-dumps") + Thread.sleep(backoff.toMillis()) + } + } +} diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/jira/install/hook/ProfilerHook.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/jira/install/hook/ProfilerHook.kt new file mode 100644 index 00000000..5b26eaa1 --- /dev/null +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/jira/install/hook/ProfilerHook.kt @@ -0,0 +1,41 @@ +package com.atlassian.performance.tools.infrastructure.jira.install.hook + +import com.atlassian.performance.tools.infrastructure.api.jira.install.InstalledJira +import com.atlassian.performance.tools.infrastructure.api.jira.install.hook.PostInstallHook +import com.atlassian.performance.tools.infrastructure.api.jira.install.hook.PostInstallHooks +import com.atlassian.performance.tools.infrastructure.api.jira.start.StartedJira +import com.atlassian.performance.tools.infrastructure.api.jira.start.hook.PostStartHook +import com.atlassian.performance.tools.infrastructure.api.jira.start.hook.PostStartHooks +import com.atlassian.performance.tools.infrastructure.api.jira.start.hook.PreStartHook +import com.atlassian.performance.tools.infrastructure.api.profiler.Profiler +import com.atlassian.performance.tools.infrastructure.jira.report.RemoteMonitoringProcessReport +import com.atlassian.performance.tools.ssh.api.SshConnection + +/** + * Bridges the [Profiler] SPI with the [PostInstallHook] SPI. + * In general any [Profiler] can be rewritten as a [PreStartHook] or [PostStartHook] without this bridge. + */ +class ProfilerHook( + private val profiler: Profiler +) : PostInstallHook { + override fun call(ssh: SshConnection, jira: InstalledJira, hooks: PostInstallHooks) { + profiler.install(ssh) + hooks.preStart.postStart.insert(InstalledProfiler(profiler)) + } +} + +private class InstalledProfiler( + private val profiler: Profiler +) : PostStartHook { + + override fun call( + ssh: SshConnection, + jira: StartedJira, + hooks: PostStartHooks + ) { + val process = profiler.start(ssh, jira.pid) + if (process != null) { + hooks.reports.add(RemoteMonitoringProcessReport(process)) + } + } +} diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/jira/install/hook/SplunkForwarderHook.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/jira/install/hook/SplunkForwarderHook.kt new file mode 100644 index 00000000..59c05bd2 --- /dev/null +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/jira/install/hook/SplunkForwarderHook.kt @@ -0,0 +1,21 @@ +package com.atlassian.performance.tools.infrastructure.jira.install.hook + +import com.atlassian.performance.tools.infrastructure.api.jira.install.hook.PostInstallHooks +import com.atlassian.performance.tools.infrastructure.api.jira.install.InstalledJira +import com.atlassian.performance.tools.infrastructure.api.jira.install.hook.PostInstallHook +import com.atlassian.performance.tools.infrastructure.api.splunk.SplunkForwarder +import com.atlassian.performance.tools.ssh.api.SshConnection + +internal class SplunkForwarderHook( + private val splunk: SplunkForwarder +) : PostInstallHook { + + override fun call( + ssh: SshConnection, + jira: InstalledJira, + hooks: PostInstallHooks + ) { + splunk.jsonifyLog4j(ssh, "${jira.installation.path}/atlassian-jira/WEB-INF/classes/log4j.properties") + splunk.run(ssh, jira.server.name, "/home/ubuntu/jirahome/log") + } +} diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/jira/report/RemoteMonitoringProcessReport.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/jira/report/RemoteMonitoringProcessReport.kt new file mode 100644 index 00000000..6d904047 --- /dev/null +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/jira/report/RemoteMonitoringProcessReport.kt @@ -0,0 +1,14 @@ +package com.atlassian.performance.tools.infrastructure.jira.report + +import com.atlassian.performance.tools.infrastructure.api.jira.report.Report +import com.atlassian.performance.tools.infrastructure.api.process.RemoteMonitoringProcess +import com.atlassian.performance.tools.ssh.api.SshConnection + +internal class RemoteMonitoringProcessReport( + private val process: RemoteMonitoringProcess +) : Report { + override fun locate(ssh: SshConnection): List { + process.stop(ssh) + return listOf(process.getResultPath()) + } +} diff --git a/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/JiraLaunchScriptIT.kt b/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/HookedJiraStartIT.kt similarity index 59% rename from src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/JiraLaunchScriptIT.kt rename to src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/HookedJiraStartIT.kt index ca914726..f1fb9814 100644 --- a/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/JiraLaunchScriptIT.kt +++ b/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/HookedJiraStartIT.kt @@ -2,7 +2,10 @@ package com.atlassian.performance.tools.infrastructure.api.jira.install import com.atlassian.performance.tools.infrastructure.api.distribution.PublicJiraSoftwareDistribution import com.atlassian.performance.tools.infrastructure.api.jira.EmptyJiraHome +import com.atlassian.performance.tools.infrastructure.api.jira.install.hook.HookedJiraInstallation +import com.atlassian.performance.tools.infrastructure.api.jira.install.hook.PreInstallHooks import com.atlassian.performance.tools.infrastructure.api.jira.start.JiraLaunchScript +import com.atlassian.performance.tools.infrastructure.api.jira.start.hook.HookedJiraStart import com.atlassian.performance.tools.infrastructure.api.jvm.AdoptOpenJDK import com.atlassian.performance.tools.infrastructure.toSsh import com.atlassian.performance.tools.sshubuntu.api.SshUbuntuContainer @@ -11,22 +14,29 @@ import org.junit.Test import java.nio.file.Files import java.util.function.Consumer -class JiraLaunchScriptIT { +class HookedJiraStartIT { @Test - fun shouldInstallJira() { + fun shouldStartJiraWithHooks() { // given - val installation = ParallelInstallation( - jiraHomeSource = EmptyJiraHome(), - productDistribution = PublicJiraSoftwareDistribution("7.13.0"), - jdk = AdoptOpenJDK() + val hooks = PreInstallHooks.default() + val installation = HookedJiraInstallation( + ParallelInstallation( + jiraHomeSource = EmptyJiraHome(), + productDistribution = PublicJiraSoftwareDistribution("7.13.0"), + jdk = AdoptOpenJDK() + ), + hooks ) - val start = JiraLaunchScript() + val start = HookedJiraStart(JiraLaunchScript(), hooks.preStart) testOnServer { server -> // when val installed = installation.install(server) val started = start.start(installed) + val reports = server.ssh.newConnection().use { ssh -> + hooks.reports.list().flatMap { it.locate(ssh) } + } // then val serverXml = installed @@ -35,6 +45,13 @@ class JiraLaunchScriptIT { .download(Files.createTempFile("downloaded-config", ".xml")) assertThat(serverXml.readText()).contains(" Date: Fri, 18 Oct 2019 10:40:03 +0200 Subject: [PATCH 03/52] JPERF-273 Postgres 9.6.15 --- .../api/database/PostgresDatabase.kt | 51 +++++++++++++++++++ 1 file changed, 51 insertions(+) create mode 100644 src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/database/PostgresDatabase.kt diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/database/PostgresDatabase.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/database/PostgresDatabase.kt new file mode 100644 index 00000000..602889df --- /dev/null +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/database/PostgresDatabase.kt @@ -0,0 +1,51 @@ +package com.atlassian.performance.tools.infrastructure.api.database + +import com.atlassian.performance.tools.infrastructure.DockerImage +import com.atlassian.performance.tools.infrastructure.api.dataset.DatasetPackage +import com.atlassian.performance.tools.ssh.api.SshConnection +import org.apache.logging.log4j.LogManager +import org.apache.logging.log4j.Logger +import java.net.URI +import java.time.Duration + +class PostgresDatabase( + private val source: DatasetPackage, + private val maxConnections: Int +) : Database { + private val logger: Logger = LogManager.getLogger(this::class.java) + + private val image: DockerImage = DockerImage( + name = "postgres:9.6.15", + pullTimeout = Duration.ofMinutes(5) + ) + + constructor( + source: DatasetPackage + ) : this( + source = source, + maxConnections = 200 + ) + + override fun setup(ssh: SshConnection): String { + val data = source.download(ssh) + val containerName = image.run( + ssh = ssh, +// TODO Dataset for Postgres +// parameters = "-p 5432:5432 -v `realpath $data`:/", + parameters = "-p 5432:5432", + arguments = "-c 'listen_addresses='*'' -c 'max_connections=$maxConnections'" + ) + Thread.sleep(Duration.ofSeconds(15).toMillis()) + logger.debug("Postgres - creating jira user and database") + ssh.execute("sudo docker exec -u postgres $containerName psql --command \"CREATE USER jira WITH NOSUPERUSER INHERIT NOCREATEROLE NOCREATEDB LOGIN PASSWORD 'jira';\"") + ssh.execute("sudo docker exec -u postgres $containerName createdb -E UNICODE -l C -T template0 -O jira jira") + return data + } + + override fun start(jira: URI, ssh: SshConnection) { + // TODO Check logs for the following entry + // LOG: database system is ready to accept connections + Thread.sleep(Duration.ofSeconds(15).toMillis()) + } + +} \ No newline at end of file From 22cd86e051e335cb7b1711ab0bdb293c7f2ed701 Mon Sep 17 00:00:00 2001 From: Maciej Kwidzinski Date: Thu, 17 Oct 2019 14:50:19 +0200 Subject: [PATCH 04/52] Offer `Docker` and `DockerImage` for customization and injection --- CHANGELOG.md | 3 + .../tools/infrastructure/DockerImage.kt | 31 ----------- .../api/database/MySqlDatabase.kt | 10 ++-- .../api/database/PostgresDatabase.kt | 9 +-- .../infrastructure/{ => api/docker}/Docker.kt | 22 ++++++-- .../infrastructure/api/docker/DockerImage.kt | 55 +++++++++++++++++++ .../api/splunk/AtlassianSplunkForwarder.kt | 8 ++- .../api/splunk/UniversalSplunkForwarder.kt | 6 +- .../{ => api/docker}/DockerIT.kt | 7 ++- 9 files changed, 96 insertions(+), 55 deletions(-) delete mode 100644 src/main/kotlin/com/atlassian/performance/tools/infrastructure/DockerImage.kt rename src/main/kotlin/com/atlassian/performance/tools/infrastructure/{ => api/docker}/Docker.kt (64%) create mode 100644 src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/docker/DockerImage.kt rename src/test/kotlin/com/atlassian/performance/tools/infrastructure/{ => api/docker}/DockerIT.kt (81%) diff --git a/CHANGELOG.md b/CHANGELOG.md index bff00388..ec7b81ca 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -140,6 +140,9 @@ This is an erroneous release. Don't use new APIs from this version, switch to `4 ## [4.14.3] - 2019-11-27 [4.14.3]: https://github.com/atlassian/infrastructure/compare/release-4.14.2...release-4.14.3 +### Added +- Offer `Docker` and `DockerImage` for customization and injection. Unblock Docker to work with other Ubuntu versions. + ### Fixed - Quote VU CLI args. Fix [JPERF-569]. - Clean up after `apt-get install` flakes. Help fix [JPERF-219]. diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/DockerImage.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/DockerImage.kt deleted file mode 100644 index 3a77ecf3..00000000 --- a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/DockerImage.kt +++ /dev/null @@ -1,31 +0,0 @@ -package com.atlassian.performance.tools.infrastructure - -import com.atlassian.performance.tools.ssh.api.SshConnection -import org.apache.logging.log4j.Level -import java.time.Duration -import java.util.* - -internal class DockerImage( - private val name: String, - private val pullTimeout: Duration = Duration.ofMinutes(1) -) { - - private val docker = Docker() - - fun run( - ssh: SshConnection, - parameters: String = "", - arguments: String = "" - ): String { - docker.install(ssh) - val containerName = "jpt-" + UUID.randomUUID() - ssh.execute( - cmd = "sudo docker pull $name", - timeout = pullTimeout, - stdout = Level.TRACE, - stderr = Level.WARN - ) - ssh.execute("sudo docker run -d $parameters --name $containerName $name $arguments") - return containerName - } -} \ No newline at end of file diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/database/MySqlDatabase.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/database/MySqlDatabase.kt index 7746d94e..7b4d61fb 100644 --- a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/database/MySqlDatabase.kt +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/database/MySqlDatabase.kt @@ -1,7 +1,7 @@ package com.atlassian.performance.tools.infrastructure.api.database -import com.atlassian.performance.tools.infrastructure.DockerImage import com.atlassian.performance.tools.infrastructure.api.dataset.DatasetPackage +import com.atlassian.performance.tools.infrastructure.api.docker.DockerImage import com.atlassian.performance.tools.infrastructure.api.os.Ubuntu import com.atlassian.performance.tools.ssh.api.SshConnection import org.apache.logging.log4j.LogManager @@ -19,10 +19,10 @@ class MySqlDatabase( ) : Database { private val logger: Logger = LogManager.getLogger(this::class.java) - private val image: DockerImage = DockerImage( - name = "mysql:5.7.32", - pullTimeout = Duration.ofMinutes(5) - ) + private val image: DockerImage = DockerImage.Builder("mysql:5.7.32") + .pullTimeout(Duration.ofMinutes(5)) + .build() + private val ubuntu = Ubuntu() /** diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/database/PostgresDatabase.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/database/PostgresDatabase.kt index 602889df..c79970f0 100644 --- a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/database/PostgresDatabase.kt +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/database/PostgresDatabase.kt @@ -1,7 +1,7 @@ package com.atlassian.performance.tools.infrastructure.api.database -import com.atlassian.performance.tools.infrastructure.DockerImage import com.atlassian.performance.tools.infrastructure.api.dataset.DatasetPackage +import com.atlassian.performance.tools.infrastructure.api.docker.DockerImage import com.atlassian.performance.tools.ssh.api.SshConnection import org.apache.logging.log4j.LogManager import org.apache.logging.log4j.Logger @@ -14,10 +14,7 @@ class PostgresDatabase( ) : Database { private val logger: Logger = LogManager.getLogger(this::class.java) - private val image: DockerImage = DockerImage( - name = "postgres:9.6.15", - pullTimeout = Duration.ofMinutes(5) - ) + private val image: DockerImage = DockerImage.Builder("postgres:9.6.15").build() constructor( source: DatasetPackage @@ -48,4 +45,4 @@ class PostgresDatabase( Thread.sleep(Duration.ofSeconds(15).toMillis()) } -} \ No newline at end of file +} diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/Docker.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/docker/Docker.kt similarity index 64% rename from src/main/kotlin/com/atlassian/performance/tools/infrastructure/Docker.kt rename to src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/docker/Docker.kt index f4ebf8fc..53adea3d 100644 --- a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/Docker.kt +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/docker/Docker.kt @@ -1,10 +1,13 @@ -package com.atlassian.performance.tools.infrastructure +package com.atlassian.performance.tools.infrastructure.api.docker import com.atlassian.performance.tools.infrastructure.api.os.Ubuntu import com.atlassian.performance.tools.ssh.api.SshConnection import java.time.Duration -internal class Docker { +class Docker private constructor( + private val dependencyPackagesTimeout: Duration, + private val mainPackageTimeout: Duration +) { private val ubuntu = Ubuntu() @@ -21,7 +24,7 @@ internal class Docker { "ca-certificates", "curl" ), - timeout = Duration.ofMinutes(2) + timeout = dependencyPackagesTimeout ) ubuntu.addKey(ssh, "7EA0A9C3F273FCD8") @@ -32,8 +35,19 @@ internal class Docker { ubuntu.install( ssh = ssh, packages = listOf("docker-ce=$version"), - timeout = Duration.ofMinutes(5) + timeout = mainPackageTimeout ) ssh.execute("sudo service docker status || sudo service docker start") } + + class Builder { + + private var dependencyPackagesTimeout: Duration = Duration.ofMinutes(2) + private var mainPackageTimeout: Duration = Duration.ofMinutes(5) + + fun build(): Docker = Docker( + dependencyPackagesTimeout, + mainPackageTimeout + ) + } } diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/docker/DockerImage.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/docker/DockerImage.kt new file mode 100644 index 00000000..86c204af --- /dev/null +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/docker/DockerImage.kt @@ -0,0 +1,55 @@ +package com.atlassian.performance.tools.infrastructure.api.docker + +import com.atlassian.performance.tools.ssh.api.SshConnection +import org.apache.logging.log4j.Level +import java.time.Duration +import java.util.* + +class DockerImage private constructor( + private val docker: Docker, + private val name: String, + private val pullTimeout: Duration +) { + + fun run( + ssh: SshConnection + ) = run( + ssh = ssh, + parameters = "", + arguments = "" + ) + + fun run( + ssh: SshConnection, + parameters: String, + arguments: String + ): String { + docker.install(ssh) + val containerName = "jpt-" + UUID.randomUUID() + ssh.execute( + cmd = "sudo docker pull $name", + timeout = pullTimeout, + stdout = Level.TRACE, + stderr = Level.WARN + ) + ssh.execute("sudo docker run -d $parameters --name $containerName $name $arguments") + return containerName + } + + class Builder( + private val name: String + ) { + + private var docker = Docker.Builder().build() + private var pullTimeout: Duration = Duration.ofMinutes(1) + + fun docker(docker: Docker) = apply { this.docker = docker } + fun pullTimeout(pullTimeout: Duration) = apply { this.pullTimeout = pullTimeout } + + fun build(): DockerImage = DockerImage( + docker, + name, + pullTimeout + ) + } +} diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/splunk/AtlassianSplunkForwarder.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/splunk/AtlassianSplunkForwarder.kt index 60b3425a..ef17273a 100644 --- a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/splunk/AtlassianSplunkForwarder.kt +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/splunk/AtlassianSplunkForwarder.kt @@ -1,7 +1,7 @@ package com.atlassian.performance.tools.infrastructure.api.splunk -import com.atlassian.performance.tools.infrastructure.DockerImage import com.atlassian.performance.tools.infrastructure.api.Sed +import com.atlassian.performance.tools.infrastructure.api.docker.DockerImage import com.atlassian.performance.tools.ssh.api.SshConnection import java.time.Duration @@ -12,7 +12,9 @@ class AtlassianSplunkForwarder( ) : SplunkForwarder { override fun run(sshConnection: SshConnection, name: String, logsPath: String) { - val logstashImage = DockerImage("docker.elastic.co/logstash/logstash-oss:6.2.4", Duration.ofMinutes(5)) + val logstashImage = DockerImage.Builder("docker.elastic.co/logstash/logstash-oss:6.2.4") + .pullTimeout(Duration.ofMinutes(5)) + .build() val logstashConfFilePath = "~/logstash.conf" sshConnection.execute("""cat > $logstashConfFilePath <<'EOF' @@ -92,4 +94,4 @@ internal class LogStashConfigBuilder(private val additionalEventFields: Map Date: Thu, 17 Oct 2019 14:51:03 +0200 Subject: [PATCH 05/52] Add `DockerMysqlServer` --- CHANGELOG.md | 1 + .../api/database/DockerMysqlServer.kt | 63 +++++++++++++++++++ 2 files changed, 64 insertions(+) create mode 100644 src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/database/DockerMysqlServer.kt diff --git a/CHANGELOG.md b/CHANGELOG.md index ec7b81ca..a3806ff1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -142,6 +142,7 @@ This is an erroneous release. Don't use new APIs from this version, switch to `4 ### Added - Offer `Docker` and `DockerImage` for customization and injection. Unblock Docker to work with other Ubuntu versions. +- Add `DockerMysqlServer`. ### Fixed - Quote VU CLI args. Fix [JPERF-569]. diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/database/DockerMysqlServer.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/database/DockerMysqlServer.kt new file mode 100644 index 00000000..57451ad4 --- /dev/null +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/database/DockerMysqlServer.kt @@ -0,0 +1,63 @@ +package com.atlassian.performance.tools.infrastructure.api.database + +import com.atlassian.performance.tools.infrastructure.api.dataset.DatasetPackage +import com.atlassian.performance.tools.infrastructure.api.docker.DockerImage +import com.atlassian.performance.tools.infrastructure.api.os.Ubuntu +import com.atlassian.performance.tools.ssh.api.Ssh +import com.atlassian.performance.tools.ssh.api.SshConnection +import com.atlassian.performance.tools.ssh.api.SshHost +import org.apache.logging.log4j.LogManager +import org.apache.logging.log4j.Logger +import java.time.Duration +import java.time.Instant + +class DockerMysqlServer private constructor( + private val source: DatasetPackage, + private val dockerImage: DockerImage, + private val maxConnections: Int +) { + private val logger: Logger = LogManager.getLogger(this::class.java) + + fun setup(sshHost: SshHost) { + Ssh(host = sshHost, connectivityPatience = 4) + .newConnection() + .use { setup(it) } + } + + private fun setup(ssh: SshConnection) { + val mysqlData = source.download(ssh) + dockerImage.run( + ssh = ssh, + parameters = "-p 3306:3306 -v `realpath $mysqlData`:/var/lib/mysql", + arguments = "--skip-grant-tables --max_connections=$maxConnections" + ) + Ubuntu().install(ssh, listOf("mysql-client")) + val deadline = Instant.now() + Duration.ofMinutes(15) + while (ssh.safeExecute("mysql -h 127.0.0.1 -u root -e 'select 1;'").isSuccessful().not()) { + if (Instant.now() > deadline) { + throw Exception("MySQL didn't start in time") + } + logger.debug("Waiting for MySQL...") + Thread.sleep(Duration.ofSeconds(10).toMillis()) + } + } + + class Builder( + private val source: DatasetPackage + ) { + + private var dockerImage = DockerImage.Builder("mysql:5.6.42") + .pullTimeout(Duration.ofMinutes(5)) + .build() + private var maxConnections: Int = 151 + + fun dockerImage(dockerImage: DockerImage) = apply { this.dockerImage = dockerImage } + fun maxConnections(maxConnections: Int) = apply { this.maxConnections = maxConnections } + + fun build(): DockerMysqlServer = DockerMysqlServer( + source, + dockerImage, + maxConnections + ) + } +} From 1e33416da8056d3dcb8a728f7a51dbd8de2bc4f6 Mon Sep 17 00:00:00 2001 From: Maciej Kwidzinski Date: Fri, 18 Oct 2019 15:34:47 +0200 Subject: [PATCH 06/52] Introduce pre- and post- instance hooks --- .../api/jira/instance/PostInstanceHook.kt | 15 +++++++++++ .../api/jira/instance/PostInstanceHooks.kt | 26 +++++++++++++++++++ .../api/jira/instance/PreInstanceHook.kt | 11 ++++++++ .../api/jira/instance/PreInstanceHooks.kt | 26 +++++++++++++++++++ 4 files changed, 78 insertions(+) create mode 100644 src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/PostInstanceHook.kt create mode 100644 src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/PostInstanceHooks.kt create mode 100644 src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/PreInstanceHook.kt create mode 100644 src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/PreInstanceHooks.kt diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/PostInstanceHook.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/PostInstanceHook.kt new file mode 100644 index 00000000..48dcb0f8 --- /dev/null +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/PostInstanceHook.kt @@ -0,0 +1,15 @@ +package com.atlassian.performance.tools.infrastructure.api.jira.instance + +import java.net.URI + +interface PostInstanceHook { + + /** + * @param [instance] a standalone Jira Server node or a Jira Data Center cluster + * @param [hooks] inserts future hooks and reports + */ + fun call( + instance: URI, + hooks: PostInstanceHooks + ) +} diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/PostInstanceHooks.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/PostInstanceHooks.kt new file mode 100644 index 00000000..f519425e --- /dev/null +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/PostInstanceHooks.kt @@ -0,0 +1,26 @@ +package com.atlassian.performance.tools.infrastructure.api.jira.instance + +import com.atlassian.performance.tools.infrastructure.api.jira.install.hook.PreInstallHooks +import java.net.URI +import java.util.* +import java.util.concurrent.ConcurrentLinkedQueue + +class PostInstanceHooks( + val nodes: List +) { + + private val hooks: Queue = ConcurrentLinkedQueue() + + fun insert(hook: PostInstanceHook) { + hooks.add(hook) + } + + fun call(instance: URI) { + while (true) { + hooks + .poll() + ?.call(instance, this) + ?: break + } + } +} diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/PreInstanceHook.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/PreInstanceHook.kt new file mode 100644 index 00000000..31c4c777 --- /dev/null +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/PreInstanceHook.kt @@ -0,0 +1,11 @@ +package com.atlassian.performance.tools.infrastructure.api.jira.instance + +interface PreInstanceHook { + + /** + * @param [hooks] inserts future hooks and reports + */ + fun call( + hooks: PreInstanceHooks + ) +} diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/PreInstanceHooks.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/PreInstanceHooks.kt new file mode 100644 index 00000000..bd898228 --- /dev/null +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/PreInstanceHooks.kt @@ -0,0 +1,26 @@ +package com.atlassian.performance.tools.infrastructure.api.jira.instance + +import com.atlassian.performance.tools.infrastructure.api.jira.install.hook.PreInstallHooks +import java.util.* +import java.util.concurrent.ConcurrentLinkedQueue + +class PreInstanceHooks( + val nodes: List +) { + + private val hooks: Queue = ConcurrentLinkedQueue() + val postInstance = PostInstanceHooks(nodes) + + fun hook(hook: PreInstanceHook) { + hooks.add(hook) + } + + fun call() { + while (true) { + hooks + .poll() + ?.call(this) + ?: break + } + } +} From 5df283a68f30a42d47734bded6ebb7c9baac1da7 Mon Sep 17 00:00:00 2001 From: Maciej Kwidzinski Date: Mon, 21 Oct 2019 15:33:00 +0200 Subject: [PATCH 07/52] Turn `DockerMysqlServer` into a hook Add `DockerPostgresServer` as a hook too. --- .../api/database/DatabaseIpConfig.kt | 25 +++++++ .../api/database/DockerMysqlServer.kt | 45 +++++++++++-- .../api/database/DockerPostgresServer.kt | 67 +++++++++++++++++++ .../api/database/MysqlConnector.kt | 29 ++++++++ .../api/database/PostgresDatabase.kt | 48 ------------- 5 files changed, 159 insertions(+), 55 deletions(-) create mode 100644 src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/database/DatabaseIpConfig.kt create mode 100644 src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/database/DockerPostgresServer.kt create mode 100644 src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/database/MysqlConnector.kt delete mode 100644 src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/database/PostgresDatabase.kt diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/database/DatabaseIpConfig.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/database/DatabaseIpConfig.kt new file mode 100644 index 00000000..7fb79d0a --- /dev/null +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/database/DatabaseIpConfig.kt @@ -0,0 +1,25 @@ +package com.atlassian.performance.tools.infrastructure.api.database + +import com.atlassian.performance.tools.infrastructure.api.Sed +import com.atlassian.performance.tools.infrastructure.api.jira.install.InstalledJira +import com.atlassian.performance.tools.infrastructure.api.jira.install.hook.PostInstallHook +import com.atlassian.performance.tools.infrastructure.api.jira.install.hook.PostInstallHooks +import com.atlassian.performance.tools.ssh.api.SshConnection + +class DatabaseIpConfig( + private val databaseIp: String +) : PostInstallHook { + + override fun call( + ssh: SshConnection, + jira: InstalledJira, + hooks: PostInstallHooks + ) { + Sed().replace( + connection = ssh, + expression = "(.*(@(//)?|//))" + "([^:/]+)" + "(.*)", + output = """\1$databaseIp\5""", + file = "${jira.home}/dbconfig.xml" + ) + } +} \ No newline at end of file diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/database/DockerMysqlServer.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/database/DockerMysqlServer.kt index 57451ad4..ada1beb6 100644 --- a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/database/DockerMysqlServer.kt +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/database/DockerMysqlServer.kt @@ -2,26 +2,39 @@ package com.atlassian.performance.tools.infrastructure.api.database import com.atlassian.performance.tools.infrastructure.api.dataset.DatasetPackage import com.atlassian.performance.tools.infrastructure.api.docker.DockerImage +import com.atlassian.performance.tools.infrastructure.api.jira.install.TcpServer +import com.atlassian.performance.tools.infrastructure.api.jira.instance.PostInstanceHook +import com.atlassian.performance.tools.infrastructure.api.jira.instance.PostInstanceHooks +import com.atlassian.performance.tools.infrastructure.api.jira.instance.PreInstanceHook +import com.atlassian.performance.tools.infrastructure.api.jira.instance.PreInstanceHooks import com.atlassian.performance.tools.infrastructure.api.os.Ubuntu +import com.atlassian.performance.tools.infrastructure.database.SshMysqlClient import com.atlassian.performance.tools.ssh.api.Ssh import com.atlassian.performance.tools.ssh.api.SshConnection -import com.atlassian.performance.tools.ssh.api.SshHost import org.apache.logging.log4j.LogManager import org.apache.logging.log4j.Logger +import java.net.URI import java.time.Duration import java.time.Instant +import java.util.function.Supplier class DockerMysqlServer private constructor( + private val serverSupplier: Supplier, private val source: DatasetPackage, private val dockerImage: DockerImage, private val maxConnections: Int -) { +) : PreInstanceHook { + private val logger: Logger = LogManager.getLogger(this::class.java) - fun setup(sshHost: SshHost) { - Ssh(host = sshHost, connectivityPatience = 4) - .newConnection() - .use { setup(it) } + override fun call(hooks: PreInstanceHooks) { + val server = serverSupplier.get() + server.ssh.newConnection().use { setup(it) } + hooks.nodes.forEach { node -> + node.postInstall.insert(DatabaseIpConfig(server.ip)) + node.postInstall.insert(MysqlConnector()) + } + hooks.postInstance.insert(FixJiraUriViaMysql(server.ssh)) } private fun setup(ssh: SshConnection) { @@ -43,7 +56,8 @@ class DockerMysqlServer private constructor( } class Builder( - private val source: DatasetPackage + private var serverSupplier: Supplier, + private var source: DatasetPackage ) { private var dockerImage = DockerImage.Builder("mysql:5.6.42") @@ -51,13 +65,30 @@ class DockerMysqlServer private constructor( .build() private var maxConnections: Int = 151 + fun serverSupplier(serverSupplier: Supplier) = apply { this.serverSupplier = serverSupplier } + fun source(source: DatasetPackage) = apply { this.source = source } fun dockerImage(dockerImage: DockerImage) = apply { this.dockerImage = dockerImage } fun maxConnections(maxConnections: Int) = apply { this.maxConnections = maxConnections } fun build(): DockerMysqlServer = DockerMysqlServer( + serverSupplier, source, dockerImage, maxConnections ) } + + private class FixJiraUriViaMysql( + private val mysql: Ssh + ) : PostInstanceHook { + + override fun call(instance: URI, hooks: PostInstanceHooks) { + mysql.newConnection().use { ssh -> + val db = "jiradb" + val update = "UPDATE $db.propertystring SET propertyvalue = '$instance'" + val where = "WHERE id IN (select id from $db.propertyentry where property_key like '%baseurl%')" + SshMysqlClient().runSql(ssh, "$update $where;") + } + } + } } diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/database/DockerPostgresServer.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/database/DockerPostgresServer.kt new file mode 100644 index 00000000..432e66b4 --- /dev/null +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/database/DockerPostgresServer.kt @@ -0,0 +1,67 @@ +package com.atlassian.performance.tools.infrastructure.api.database + +import com.atlassian.performance.tools.infrastructure.api.docker.DockerImage +import com.atlassian.performance.tools.infrastructure.api.jira.install.TcpServer +import com.atlassian.performance.tools.infrastructure.api.jira.instance.PreInstanceHook +import com.atlassian.performance.tools.infrastructure.api.jira.instance.PreInstanceHooks +import com.atlassian.performance.tools.ssh.api.SshConnection +import org.apache.logging.log4j.LogManager +import org.apache.logging.log4j.Logger +import java.time.Duration +import java.util.function.Supplier + +class DockerPostgresServer private constructor( + private val serverSupplier: Supplier, + private val dockerImage: DockerImage, + private val maxConnections: Int +) : PreInstanceHook { + + private val logger: Logger = LogManager.getLogger(this::class.java) + + override fun call(hooks: PreInstanceHooks) { + val server = serverSupplier.get() + server.ssh.newConnection().use { setup(it) } + hooks.nodes.forEach { node -> + node.postInstall.insert(DatabaseIpConfig(server.ip)) + } + } + + private fun setup(ssh: SshConnection) { + val container = dockerImage.run( + ssh = ssh, + // TODO Dataset for Postgres + // parameters = "-p 5432:5432 -v `realpath $data`:/", + parameters = "-p 5432:5432", + arguments = "-c 'listen_addresses='*'' -c 'max_connections=$maxConnections'" + ) + Thread.sleep(Duration.ofSeconds(15).toMillis()) + logger.debug("Postgres - creating jira user and database") + ssh.execute("sudo docker exec -u postgres $container psql --command \"CREATE USER jira WITH NOSUPERUSER INHERIT NOCREATEROLE NOCREATEDB LOGIN PASSWORD 'jira';\"") + ssh.execute("sudo docker exec -u postgres $container createdb -E UNICODE -l C -T template0 -O jira jira") + /** + * TODO Check logs for the following entry + * `LOG: database system is ready to accept connections` + */ + Thread.sleep(Duration.ofSeconds(15).toMillis()) + } + + class Builder( + private var serverSupplier: Supplier + ) { + + private var dockerImage = DockerImage.Builder("postgres:9.6.15") + .pullTimeout(Duration.ofMinutes(5)) + .build() + private var maxConnections: Int = 200 + + fun serverSupplier(serverSupplier: Supplier) = apply { this.serverSupplier = serverSupplier } + fun dockerImage(dockerImage: DockerImage) = apply { this.dockerImage = dockerImage } + fun maxConnections(maxConnections: Int) = apply { this.maxConnections = maxConnections } + + fun build(): DockerPostgresServer = DockerPostgresServer( + serverSupplier, + dockerImage, + maxConnections + ) + } +} diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/database/MysqlConnector.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/database/MysqlConnector.kt new file mode 100644 index 00000000..edc0c8d1 --- /dev/null +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/database/MysqlConnector.kt @@ -0,0 +1,29 @@ +package com.atlassian.performance.tools.infrastructure.api.database + +import com.atlassian.performance.tools.infrastructure.api.jira.install.InstalledJira +import com.atlassian.performance.tools.infrastructure.api.jira.install.hook.PostInstallHook +import com.atlassian.performance.tools.infrastructure.api.jira.install.hook.PostInstallHooks +import com.atlassian.performance.tools.jvmtasks.api.IdempotentAction +import com.atlassian.performance.tools.jvmtasks.api.StaticBackoff +import com.atlassian.performance.tools.ssh.api.SshConnection +import java.time.Duration + +class MysqlConnector : PostInstallHook { + + override fun call( + ssh: SshConnection, + jira: InstalledJira, + hooks: PostInstallHooks + ) { + val connector = "https://dev.mysql.com/get/Downloads/Connector-J/mysql-connector-java-5.1.40.tar.gz" + IdempotentAction( + description = "Download MySQL connector", + action = { ssh.execute("wget -q $connector") } + ).retry( + maxAttempts = 3, + backoff = StaticBackoff(Duration.ofSeconds(5)) + ) + ssh.execute("tar -xzf mysql-connector-java-5.1.40.tar.gz") + ssh.execute("cp mysql-connector-java-5.1.40/mysql-connector-java-5.1.40-bin.jar ${jira.installation}/lib") + } +} \ No newline at end of file diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/database/PostgresDatabase.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/database/PostgresDatabase.kt deleted file mode 100644 index c79970f0..00000000 --- a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/database/PostgresDatabase.kt +++ /dev/null @@ -1,48 +0,0 @@ -package com.atlassian.performance.tools.infrastructure.api.database - -import com.atlassian.performance.tools.infrastructure.api.dataset.DatasetPackage -import com.atlassian.performance.tools.infrastructure.api.docker.DockerImage -import com.atlassian.performance.tools.ssh.api.SshConnection -import org.apache.logging.log4j.LogManager -import org.apache.logging.log4j.Logger -import java.net.URI -import java.time.Duration - -class PostgresDatabase( - private val source: DatasetPackage, - private val maxConnections: Int -) : Database { - private val logger: Logger = LogManager.getLogger(this::class.java) - - private val image: DockerImage = DockerImage.Builder("postgres:9.6.15").build() - - constructor( - source: DatasetPackage - ) : this( - source = source, - maxConnections = 200 - ) - - override fun setup(ssh: SshConnection): String { - val data = source.download(ssh) - val containerName = image.run( - ssh = ssh, -// TODO Dataset for Postgres -// parameters = "-p 5432:5432 -v `realpath $data`:/", - parameters = "-p 5432:5432", - arguments = "-c 'listen_addresses='*'' -c 'max_connections=$maxConnections'" - ) - Thread.sleep(Duration.ofSeconds(15).toMillis()) - logger.debug("Postgres - creating jira user and database") - ssh.execute("sudo docker exec -u postgres $containerName psql --command \"CREATE USER jira WITH NOSUPERUSER INHERIT NOCREATEROLE NOCREATEDB LOGIN PASSWORD 'jira';\"") - ssh.execute("sudo docker exec -u postgres $containerName createdb -E UNICODE -l C -T template0 -O jira jira") - return data - } - - override fun start(jira: URI, ssh: SshConnection) { - // TODO Check logs for the following entry - // LOG: database system is ready to accept connections - Thread.sleep(Duration.ofSeconds(15).toMillis()) - } - -} From e03f9b3200d85c31ed4c45d391e42dd94bbc5406 Mon Sep 17 00:00:00 2001 From: Maciej Kwidzinski Date: Mon, 21 Oct 2019 17:10:59 +0200 Subject: [PATCH 08/52] Respect the `TcpServer.publicPort` in `DockerMysqlServer` --- .../tools/infrastructure/api/database/DockerMysqlServer.kt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/database/DockerMysqlServer.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/database/DockerMysqlServer.kt index ada1beb6..e1d588a8 100644 --- a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/database/DockerMysqlServer.kt +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/database/DockerMysqlServer.kt @@ -29,7 +29,7 @@ class DockerMysqlServer private constructor( override fun call(hooks: PreInstanceHooks) { val server = serverSupplier.get() - server.ssh.newConnection().use { setup(it) } + server.ssh.newConnection().use { setup(it, server.publicPort) } hooks.nodes.forEach { node -> node.postInstall.insert(DatabaseIpConfig(server.ip)) node.postInstall.insert(MysqlConnector()) @@ -37,11 +37,11 @@ class DockerMysqlServer private constructor( hooks.postInstance.insert(FixJiraUriViaMysql(server.ssh)) } - private fun setup(ssh: SshConnection) { + private fun setup(ssh: SshConnection, publicPort: Int) { val mysqlData = source.download(ssh) dockerImage.run( ssh = ssh, - parameters = "-p 3306:3306 -v `realpath $mysqlData`:/var/lib/mysql", + parameters = "-p $publicPort:3306 -v `realpath $mysqlData`:/var/lib/mysql", arguments = "--skip-grant-tables --max_connections=$maxConnections" ) Ubuntu().install(ssh, listOf("mysql-client")) From 9f52a7c142f4c780e0bc25c558718a4624d4451f Mon Sep 17 00:00:00 2001 From: Maciej Kwidzinski Date: Mon, 21 Oct 2019 18:08:13 +0200 Subject: [PATCH 09/52] Fix `RestUpgrade` `Report` --- .../tools/infrastructure/api/jira/report/StaticReport.kt | 6 ++++++ .../tools/infrastructure/api/jira/start/hook/RestUpgrade.kt | 4 ++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/report/StaticReport.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/report/StaticReport.kt index da90e4d1..16edfafa 100644 --- a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/report/StaticReport.kt +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/report/StaticReport.kt @@ -2,6 +2,12 @@ package com.atlassian.performance.tools.infrastructure.api.jira.report import com.atlassian.performance.tools.ssh.api.SshConnection +/** + * Points to a remote SSH **file**. For directories use a [FileListing] instead. + * + * @param [remotePath] Points to a file on a remote system. + * Relative to the SSH shell default directory (predominantly the user home directory). + */ class StaticReport( private val remotePath: String ) : Report { diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/start/hook/RestUpgrade.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/start/hook/RestUpgrade.kt index 84aa39f4..d89e8443 100644 --- a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/start/hook/RestUpgrade.kt +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/start/hook/RestUpgrade.kt @@ -1,7 +1,7 @@ package com.atlassian.performance.tools.infrastructure.api.jira.start.hook import com.atlassian.performance.tools.infrastructure.api.jira.JiraLaunchTimeouts -import com.atlassian.performance.tools.infrastructure.api.jira.report.StaticReport +import com.atlassian.performance.tools.infrastructure.api.jira.report.FileListing import com.atlassian.performance.tools.infrastructure.api.jira.start.StartedJira import com.atlassian.performance.tools.infrastructure.api.jvm.ThreadDump import com.atlassian.performance.tools.ssh.api.SshConnection @@ -19,7 +19,7 @@ class RestUpgrade( val threadDump = ThreadDump(jira.pid, jira.installed.jdk) val privatePort = jira.installed.server.privatePort val upgradesEndpoint = URI("http://$adminUsername:$adminPassword@localhost:$privatePort/rest/api/2/upgrade") - hooks.reports.add(StaticReport("thread-dumps")) + hooks.reports.add(FileListing("thread-dumps/*")) waitForStatusToChange( statusQuo = "000", timeout = timeouts.offlineTimeout, From 5172d172bb210e3587deb2df6202f49db95212b9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maciej=20Kwidzi=C5=84ski?= Date: Fri, 12 Mar 2021 11:47:16 +0100 Subject: [PATCH 10/52] WIP test Data Center hooks --- .../jira/provision/JiraNodeProvisioning.kt | 52 +++++++++++++++++++ .../api/jira/install/HookedJiraStartIT.kt | 50 ++++++++++++++++-- 2 files changed, 98 insertions(+), 4 deletions(-) create mode 100644 src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/provision/JiraNodeProvisioning.kt diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/provision/JiraNodeProvisioning.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/provision/JiraNodeProvisioning.kt new file mode 100644 index 00000000..fa177670 --- /dev/null +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/provision/JiraNodeProvisioning.kt @@ -0,0 +1,52 @@ +package com.atlassian.performance.tools.infrastructure.api.jira.provision + +import com.atlassian.performance.tools.infrastructure.api.distribution.PublicJiraSoftwareDistribution +import com.atlassian.performance.tools.infrastructure.api.jira.EmptyJiraHome +import com.atlassian.performance.tools.infrastructure.api.jira.JiraHomeSource +import com.atlassian.performance.tools.infrastructure.api.jira.hook.JiraNodeHooks +import com.atlassian.performance.tools.infrastructure.api.jira.hook.install.HookedJiraInstallation +import com.atlassian.performance.tools.infrastructure.api.jira.hook.install.JiraInstallation +import com.atlassian.performance.tools.infrastructure.api.jira.hook.install.ParallelInstallation +import com.atlassian.performance.tools.infrastructure.api.jira.hook.start.HookedJiraStart +import com.atlassian.performance.tools.infrastructure.api.jira.hook.start.JiraLaunchScript +import com.atlassian.performance.tools.infrastructure.api.jira.hook.start.JiraStart +import com.atlassian.performance.tools.infrastructure.api.jira.install.JiraInstallation +import com.atlassian.performance.tools.infrastructure.api.jira.install.ParallelInstallation +import com.atlassian.performance.tools.infrastructure.api.jira.install.hook.HookedJiraInstallation +import com.atlassian.performance.tools.infrastructure.api.jira.install.hook.PreInstallHooks +import com.atlassian.performance.tools.infrastructure.api.jira.start.JiraLaunchScript +import com.atlassian.performance.tools.infrastructure.api.jira.start.JiraStart +import com.atlassian.performance.tools.infrastructure.api.jira.start.hook.HookedJiraStart +import com.atlassian.performance.tools.infrastructure.api.jvm.OracleJDK +import net.jcip.annotations.NotThreadSafe + +class JiraNodeProvisioning private constructor( + val hooks: PreInstallHooks, + val installation: JiraInstallation, + val start: JiraStart +) { + + @NotThreadSafe + class Builder { + private var hooks: PreInstallHooks = PreInstallHooks.default() + private var installation: JiraInstallation = HookedJiraInstallation( + ParallelInstallation( + EmptyJiraHome(), + PublicJiraSoftwareDistribution("7.13.0"), + OracleJDK() + ), + hooks + ) + private var start: JiraStart = HookedJiraStart(JiraLaunchScript()) + + fun hooks(hooks: PreInstallHooks) = apply { this.hooks = hooks } + fun installation(installation: JiraInstallation) = apply { this.installation = installation } + fun start(start: JiraStart) = apply { this.start = start } + + fun build() = JiraNodeProvisioning( + hooks = hooks, + installation = installation, + start = start + ) + } +} diff --git a/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/HookedJiraStartIT.kt b/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/HookedJiraStartIT.kt index f1fb9814..96252ebe 100644 --- a/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/HookedJiraStartIT.kt +++ b/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/HookedJiraStartIT.kt @@ -30,7 +30,7 @@ class HookedJiraStartIT { ) val start = HookedJiraStart(JiraLaunchScript(), hooks.preStart) - testOnServer { server -> + testOnServer("jira", 8080) { server -> // when val installed = installation.install(server) val started = start.start(installed) @@ -55,8 +55,50 @@ class HookedJiraStartIT { } } - private fun testOnServer(test: (TcpServer) -> T) { - val privatePort = 8080 + @Test + fun shouldStartDataCenter() { + // given + val hooks = PreInstallHooks.default() + val installation = HookedJiraInstallation( + ParallelInstallation( + jiraHomeSource = EmptyJiraHome(), + productDistribution = PublicJiraSoftwareDistribution("7.13.0"), + jdk = AdoptOpenJDK() + ), + hooks + ) + val start = HookedJiraStart(JiraLaunchScript(), hooks.preStart) + + testOnServer("jira1", 8080) { jira1 -> + testOnServer("jira2", 8080) { jira2 -> + testOnServer("mysql", 3306) { mysql -> + // when + val installed = installation.install(server) + val started = start.start(installed) + val reports = server.ssh.newConnection().use { ssh -> + hooks.reports.list().flatMap { it.locate(ssh) } + } + + // then + val serverXml = installed + .installation + .resolve("conf/server.xml") + .download(Files.createTempFile("downloaded-config", ".xml")) + assertThat(serverXml.readText()).contains(" testOnServer(name: String, privatePort: Int, test: (TcpServer) -> T) { val container = SshUbuntuContainer(Consumer { it.addExposedPort(privatePort) }) @@ -65,7 +107,7 @@ class HookedJiraStartIT { "localhost", sshUbuntu.container.getMappedPort(privatePort), privatePort, - "my-jira", + name, sshUbuntu.toSsh() ) test(server) From 0befe8b032a71bfdf9e9812f65059625559d5213 Mon Sep 17 00:00:00 2001 From: Maciej Kwidzinski Date: Fri, 12 Mar 2021 17:55:15 +0100 Subject: [PATCH 11/52] WIP include Jira instances Include overall Data Center plan with hook timings. Plans should be reusable with different ways of getting a `TcpHost`: * new via Docker * new via AWS * existing machines This commit injects an `Infrastructure` SPI to reuse a plan with different infra, but to prove the concept, it has to fit the current CloudFormation stack in aws-infrastructure / 2-nodes-dc*.yaml The API smells a little, because a big one-shot CFN stack is split into multiple fine-grained methods. Alternative API could be: `fun plan(host: TcpHost) : JiraNodePlan` Other matters to resolve: * loadbalancer (plan?) API * getting results out of DC instance and node hooks --- .../infrastructure/api/Infrastructure.kt | 11 ++ .../api/database/DockerMysqlServer.kt | 20 +-- .../api/database/DockerPostgresServer.kt | 12 +- .../api/jira/install/InstalledJira.kt | 2 +- .../api/jira/install/JiraInstallation.kt | 6 +- .../api/jira/install/ParallelInstallation.kt | 6 +- .../jira/install/SequentialInstallation.kt | 6 +- .../jira/install/{TcpServer.kt => TcpHost.kt} | 2 +- .../jira/install/hook/AsyncProfilerHook.kt | 8 +- .../install/hook/HookedJiraInstallation.kt | 12 +- .../api/jira/install/hook/JvmConfig.kt | 2 +- .../api/jira/install/hook/PreInstallHook.kt | 12 +- .../api/jira/install/hook/PreInstallHooks.kt | 8 +- .../api/jira/instance/JiraDataCenterPlan.kt | 69 ++++++++ .../api/jira/instance/JiraInstance.kt | 7 + .../api/jira/instance/JiraServerPlan.kt | 64 +++++++ .../api/jira/instance/PreInstanceHooks.kt | 2 +- .../infrastructure/api/jira/node/JiraNode.kt | 8 + .../JiraNodePlan.kt} | 21 +-- .../api/jira/start/hook/HookedJiraStart.kt | 4 +- .../api/jira/start/hook/RestUpgrade.kt | 2 +- .../jira/install/hook/SplunkForwarderHook.kt | 2 +- .../api/DockerInfrastructure.kt | 45 +++++ .../api/jira/install/HookedJiraStartIT.kt | 167 ++++++++++-------- .../jira/install/hook/PreInstallHooksTest.kt | 8 +- 25 files changed, 366 insertions(+), 140 deletions(-) create mode 100644 src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/Infrastructure.kt rename src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/{TcpServer.kt => TcpHost.kt} (96%) create mode 100644 src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/JiraDataCenterPlan.kt create mode 100644 src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/JiraInstance.kt create mode 100644 src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/JiraServerPlan.kt create mode 100644 src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/node/JiraNode.kt rename src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/{provision/JiraNodeProvisioning.kt => node/JiraNodePlan.kt} (69%) create mode 100644 src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/DockerInfrastructure.kt diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/Infrastructure.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/Infrastructure.kt new file mode 100644 index 00000000..219d74e4 --- /dev/null +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/Infrastructure.kt @@ -0,0 +1,11 @@ +package com.atlassian.performance.tools.infrastructure.api + +import com.atlassian.performance.tools.infrastructure.api.jira.install.TcpHost +import com.atlassian.performance.tools.infrastructure.api.jira.node.JiraNode +import com.atlassian.performance.tools.infrastructure.api.jira.node.JiraNodePlan + +interface Infrastructure { + fun serve(jiraNodePlans: List): List + fun serve(port: Int, name: String): TcpHost + fun releaseResources() +} \ No newline at end of file diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/database/DockerMysqlServer.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/database/DockerMysqlServer.kt index e1d588a8..29bcce60 100644 --- a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/database/DockerMysqlServer.kt +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/database/DockerMysqlServer.kt @@ -1,8 +1,9 @@ package com.atlassian.performance.tools.infrastructure.api.database +import com.atlassian.performance.tools.infrastructure.api.Infrastructure import com.atlassian.performance.tools.infrastructure.api.dataset.DatasetPackage import com.atlassian.performance.tools.infrastructure.api.docker.DockerImage -import com.atlassian.performance.tools.infrastructure.api.jira.install.TcpServer +import com.atlassian.performance.tools.infrastructure.api.jira.install.TcpHost import com.atlassian.performance.tools.infrastructure.api.jira.instance.PostInstanceHook import com.atlassian.performance.tools.infrastructure.api.jira.instance.PostInstanceHooks import com.atlassian.performance.tools.infrastructure.api.jira.instance.PreInstanceHook @@ -16,10 +17,9 @@ import org.apache.logging.log4j.Logger import java.net.URI import java.time.Duration import java.time.Instant -import java.util.function.Supplier class DockerMysqlServer private constructor( - private val serverSupplier: Supplier, + private val infrastructure: Infrastructure, private val source: DatasetPackage, private val dockerImage: DockerImage, private val maxConnections: Int @@ -28,8 +28,8 @@ class DockerMysqlServer private constructor( private val logger: Logger = LogManager.getLogger(this::class.java) override fun call(hooks: PreInstanceHooks) { - val server = serverSupplier.get() - server.ssh.newConnection().use { setup(it, server.publicPort) } + val server = infrastructure.serve(3306, "mysql") + server.ssh.newConnection().use { setup(it, server) } hooks.nodes.forEach { node -> node.postInstall.insert(DatabaseIpConfig(server.ip)) node.postInstall.insert(MysqlConnector()) @@ -37,11 +37,11 @@ class DockerMysqlServer private constructor( hooks.postInstance.insert(FixJiraUriViaMysql(server.ssh)) } - private fun setup(ssh: SshConnection, publicPort: Int) { + private fun setup(ssh: SshConnection, host: TcpHost) { val mysqlData = source.download(ssh) dockerImage.run( ssh = ssh, - parameters = "-p $publicPort:3306 -v `realpath $mysqlData`:/var/lib/mysql", + parameters = "-p ${host.publicPort}:${host.privatePort} -v `realpath $mysqlData`:/var/lib/mysql", arguments = "--skip-grant-tables --max_connections=$maxConnections" ) Ubuntu().install(ssh, listOf("mysql-client")) @@ -56,7 +56,7 @@ class DockerMysqlServer private constructor( } class Builder( - private var serverSupplier: Supplier, + private var infrastructure: Infrastructure, private var source: DatasetPackage ) { @@ -65,13 +65,13 @@ class DockerMysqlServer private constructor( .build() private var maxConnections: Int = 151 - fun serverSupplier(serverSupplier: Supplier) = apply { this.serverSupplier = serverSupplier } + fun infrastructure(infrastructure: Infrastructure) = apply { this.infrastructure = infrastructure } fun source(source: DatasetPackage) = apply { this.source = source } fun dockerImage(dockerImage: DockerImage) = apply { this.dockerImage = dockerImage } fun maxConnections(maxConnections: Int) = apply { this.maxConnections = maxConnections } fun build(): DockerMysqlServer = DockerMysqlServer( - serverSupplier, + infrastructure, source, dockerImage, maxConnections diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/database/DockerPostgresServer.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/database/DockerPostgresServer.kt index 432e66b4..e62cef4e 100644 --- a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/database/DockerPostgresServer.kt +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/database/DockerPostgresServer.kt @@ -1,7 +1,7 @@ package com.atlassian.performance.tools.infrastructure.api.database import com.atlassian.performance.tools.infrastructure.api.docker.DockerImage -import com.atlassian.performance.tools.infrastructure.api.jira.install.TcpServer +import com.atlassian.performance.tools.infrastructure.api.jira.install.TcpHost import com.atlassian.performance.tools.infrastructure.api.jira.instance.PreInstanceHook import com.atlassian.performance.tools.infrastructure.api.jira.instance.PreInstanceHooks import com.atlassian.performance.tools.ssh.api.SshConnection @@ -11,7 +11,7 @@ import java.time.Duration import java.util.function.Supplier class DockerPostgresServer private constructor( - private val serverSupplier: Supplier, + private val hostSupplier: Supplier, private val dockerImage: DockerImage, private val maxConnections: Int ) : PreInstanceHook { @@ -19,7 +19,7 @@ class DockerPostgresServer private constructor( private val logger: Logger = LogManager.getLogger(this::class.java) override fun call(hooks: PreInstanceHooks) { - val server = serverSupplier.get() + val server = hostSupplier.get() server.ssh.newConnection().use { setup(it) } hooks.nodes.forEach { node -> node.postInstall.insert(DatabaseIpConfig(server.ip)) @@ -46,7 +46,7 @@ class DockerPostgresServer private constructor( } class Builder( - private var serverSupplier: Supplier + private var hostSupplier: Supplier ) { private var dockerImage = DockerImage.Builder("postgres:9.6.15") @@ -54,12 +54,12 @@ class DockerPostgresServer private constructor( .build() private var maxConnections: Int = 200 - fun serverSupplier(serverSupplier: Supplier) = apply { this.serverSupplier = serverSupplier } + fun serverSupplier(hostSupplier: Supplier) = apply { this.hostSupplier = hostSupplier } fun dockerImage(dockerImage: DockerImage) = apply { this.dockerImage = dockerImage } fun maxConnections(maxConnections: Int) = apply { this.maxConnections = maxConnections } fun build(): DockerPostgresServer = DockerPostgresServer( - serverSupplier, + hostSupplier, dockerImage, maxConnections ) diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/InstalledJira.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/InstalledJira.kt index 0668ea98..1845ac86 100644 --- a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/InstalledJira.kt +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/InstalledJira.kt @@ -24,5 +24,5 @@ class InstalledJira( /** * Hosts Jira. Specifies sockets used by Jira to handle requests. */ - val server: TcpServer + val host: TcpHost ) diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/JiraInstallation.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/JiraInstallation.kt index 5f390f47..05bd3d96 100644 --- a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/JiraInstallation.kt +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/JiraInstallation.kt @@ -9,11 +9,11 @@ import net.jcip.annotations.ThreadSafe interface JiraInstallation { /** - * Installs Jira on [server]. + * Installs Jira on [host]. * - * @param [server] will host the Jira + * @param [host] will host the Jira */ fun install( - server: TcpServer + host: TcpHost ): InstalledJira } diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/ParallelInstallation.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/ParallelInstallation.kt index 8f228003..03af91c0 100644 --- a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/ParallelInstallation.kt +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/ParallelInstallation.kt @@ -15,9 +15,9 @@ class ParallelInstallation( ) : JiraInstallation { override fun install( - server: TcpServer + host: TcpHost ): InstalledJira { - server.ssh.newConnection().use { ssh -> + host.ssh.newConnection().use { ssh -> val pool = Executors.newCachedThreadPool { runnable -> Thread(runnable, "jira-installation-${runnable.hashCode()}") } @@ -30,7 +30,7 @@ class ParallelInstallation( val java = pool.submitWithLogContext("java") { jdk.also { it.install(ssh) } } - val jira = InstalledJira(home.get(), product.get(), java.get(), server) + val jira = InstalledJira(home.get(), product.get(), java.get(), host) pool.shutdownNow() return jira } diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/SequentialInstallation.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/SequentialInstallation.kt index 7615a397..382c82e8 100644 --- a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/SequentialInstallation.kt +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/SequentialInstallation.kt @@ -13,13 +13,13 @@ class SequentialInstallation( ) : JiraInstallation { override fun install( - server: TcpServer + host: TcpHost ): InstalledJira { - server.ssh.newConnection().use { ssh -> + host.ssh.newConnection().use { ssh -> val installation = productDistribution.installRemotely(ssh, ".") val home = jiraHomeSource.downloadRemotely(ssh) jdk.install(ssh) - return InstalledJira(home, installation, jdk, server) + return InstalledJira(home, installation, jdk, host) } } } diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/TcpServer.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/TcpHost.kt similarity index 96% rename from src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/TcpServer.kt rename to src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/TcpHost.kt index 1b7f1eeb..fc2aa038 100644 --- a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/TcpServer.kt +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/TcpHost.kt @@ -5,7 +5,7 @@ import com.atlassian.performance.tools.ssh.api.Ssh /** * Has open TCP sockets. */ -class TcpServer( +class TcpHost( val ip: String, val publicPort: Int, val privatePort: Int, diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/AsyncProfilerHook.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/AsyncProfilerHook.kt index 5f60deaf..95688f98 100644 --- a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/AsyncProfilerHook.kt +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/AsyncProfilerHook.kt @@ -1,6 +1,6 @@ package com.atlassian.performance.tools.infrastructure.api.jira.install.hook -import com.atlassian.performance.tools.infrastructure.api.jira.install.TcpServer +import com.atlassian.performance.tools.infrastructure.api.jira.install.TcpHost import com.atlassian.performance.tools.infrastructure.api.jira.report.Report import com.atlassian.performance.tools.infrastructure.api.jira.start.StartedJira import com.atlassian.performance.tools.infrastructure.api.jira.start.hook.PostStartHook @@ -11,9 +11,9 @@ import java.net.URI class AsyncProfilerHook : PreInstallHook { override fun call( - ssh: SshConnection, - server: TcpServer, - hooks: PreInstallHooks + ssh: SshConnection, + host: TcpHost, + hooks: PreInstallHooks ) { val directory = "async-profiler" val downloads = URI("https://github.com/jvm-profiling-tools/async-profiler/releases/download/") diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/HookedJiraInstallation.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/HookedJiraInstallation.kt index e8fc3ae0..a7633b23 100644 --- a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/HookedJiraInstallation.kt +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/HookedJiraInstallation.kt @@ -2,7 +2,7 @@ package com.atlassian.performance.tools.infrastructure.api.jira.install.hook import com.atlassian.performance.tools.infrastructure.api.jira.install.InstalledJira import com.atlassian.performance.tools.infrastructure.api.jira.install.JiraInstallation -import com.atlassian.performance.tools.infrastructure.api.jira.install.TcpServer +import com.atlassian.performance.tools.infrastructure.api.jira.install.TcpHost class HookedJiraInstallation( private val installation: JiraInstallation, @@ -10,13 +10,13 @@ class HookedJiraInstallation( ) : JiraInstallation { override fun install( - server: TcpServer + host: TcpHost ): InstalledJira { - server.ssh.newConnection().use { ssh -> - hooks.call(ssh, server) + host.ssh.newConnection().use { ssh -> + hooks.call(ssh, host) } - val installed = installation.install(server) - server.ssh.newConnection().use { ssh -> + val installed = installation.install(host) + host.ssh.newConnection().use { ssh -> hooks.postInstall.call(ssh, installed) } return installed diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/JvmConfig.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/JvmConfig.kt index d7f68580..7c72be25 100644 --- a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/JvmConfig.kt +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/JvmConfig.kt @@ -21,7 +21,7 @@ class JvmConfig( connection = ssh, config = config, gcLog = gcLog, - jiraIp = jira.server.ip + jiraIp = jira.host.ip ) val report = FileListing(gcLog.path("*")) hooks.reports.add(report) diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/PreInstallHook.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/PreInstallHook.kt index 1e58b1af..e51cfd64 100644 --- a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/PreInstallHook.kt +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/PreInstallHook.kt @@ -1,6 +1,6 @@ package com.atlassian.performance.tools.infrastructure.api.jira.install.hook -import com.atlassian.performance.tools.infrastructure.api.jira.install.TcpServer +import com.atlassian.performance.tools.infrastructure.api.jira.install.TcpHost import com.atlassian.performance.tools.ssh.api.SshConnection /** @@ -9,13 +9,13 @@ import com.atlassian.performance.tools.ssh.api.SshConnection interface PreInstallHook { /** - * @param [ssh] connects to the [server] - * @param [server] will install Jira + * @param [ssh] connects to the [host] + * @param [host] will install Jira * @param [hooks] inserts future hooks and reports */ fun call( - ssh: SshConnection, - server: TcpServer, - hooks: PreInstallHooks + ssh: SshConnection, + host: TcpHost, + hooks: PreInstallHooks ) } diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/PreInstallHooks.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/PreInstallHooks.kt index 5506de0d..84842cff 100644 --- a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/PreInstallHooks.kt +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/PreInstallHooks.kt @@ -1,6 +1,6 @@ package com.atlassian.performance.tools.infrastructure.api.jira.install.hook -import com.atlassian.performance.tools.infrastructure.api.jira.install.TcpServer +import com.atlassian.performance.tools.infrastructure.api.jira.install.TcpHost import com.atlassian.performance.tools.ssh.api.SshConnection import java.util.* import java.util.concurrent.ConcurrentLinkedQueue @@ -21,13 +21,13 @@ class PreInstallHooks private constructor( } internal fun call( - ssh: SshConnection, - server: TcpServer + ssh: SshConnection, + host: TcpHost ) { while (true) { hooks .poll() - ?.call(ssh, server, this) + ?.call(ssh, host, this) ?: break } } diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/JiraDataCenterPlan.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/JiraDataCenterPlan.kt new file mode 100644 index 00000000..97de16e6 --- /dev/null +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/JiraDataCenterPlan.kt @@ -0,0 +1,69 @@ +package com.atlassian.performance.tools.infrastructure.api.jira.instance + +import com.atlassian.performance.tools.infrastructure.api.Infrastructure +import com.atlassian.performance.tools.infrastructure.api.jira.node.JiraNodePlan +import com.atlassian.performance.tools.infrastructure.api.jira.start.StartedJira +import com.atlassian.performance.tools.infrastructure.api.loadbalancer.LoadBalancer +import java.net.URI +import java.time.Duration +import java.util.function.Supplier + +class JiraDataCenterPlan constructor( + val nodePlans: List, + val instanceHooks: PreInstanceHooks, + val loadBalancerSupplier: Supplier, + val infrastructure: Infrastructure +) : Supplier { + + private val loadBalancingPatience = Duration.ofMinutes(5) + + override fun get(): JiraDataCenter { + instanceHooks.call() + val started = infrastructure.serve(nodePlans).map { jiraNode -> + jiraNode + .plan + .installation + .install(jiraNode.host) + .let { jiraNode.plan.start.start(it) } + } + val loadBalancer = loadBalancerSupplier.get() + instanceHooks.postInstance.call(loadBalancer.uri) + loadBalancer.waitUntilHealthy(loadBalancingPatience) + return JiraDataCenter(started, loadBalancer) + } + + class JiraDataCenter( + val nodes: List, + val loadBalancer: LoadBalancer + ) : JiraInstance { + override val address: URI + get() = loadBalancer.uri + } +// +// class Builder( +// private var jiraNode: TcpServer +// ) { +// private var hooks: PreInstallHooks = PreInstallHooks.default() +// private var installation: JiraInstallation = HookedJiraInstallation( +// ParallelInstallation( +// EmptyJiraHome(), +// PublicJiraSoftwareDistribution("7.13.0"), +// OracleJDK() +// ), +// hooks +// ) +// private var start: JiraStart = HookedJiraStart(JiraLaunchScript(), hooks.preStart) +// +// fun jiraNode(jiraNode: TcpServer) = apply { this.jiraNode = jiraNode } +// fun hooks(hooks: PreInstallHooks) = apply { this.hooks = hooks } // TODO this doesn't affect the start or installation +// fun installation(installation: JiraInstallation) = apply { this.installation = installation } +// fun start(start: JiraStart) = apply { this.start = start } +// +// fun build() = JiraDataCenterPlan( +// jiraNode = jiraNode, +// hooks = hooks, +// installation = installation, +// start = start +// ) +// } +} \ No newline at end of file diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/JiraInstance.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/JiraInstance.kt new file mode 100644 index 00000000..8d2f4e3c --- /dev/null +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/JiraInstance.kt @@ -0,0 +1,7 @@ +package com.atlassian.performance.tools.infrastructure.api.jira.instance + +import java.net.URI + +interface JiraInstance { + val address: URI +} \ No newline at end of file diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/JiraServerPlan.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/JiraServerPlan.kt new file mode 100644 index 00000000..7d1ad934 --- /dev/null +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/JiraServerPlan.kt @@ -0,0 +1,64 @@ +package com.atlassian.performance.tools.infrastructure.api.jira.instance + +import com.atlassian.performance.tools.infrastructure.api.Infrastructure +import com.atlassian.performance.tools.infrastructure.api.distribution.PublicJiraSoftwareDistribution +import com.atlassian.performance.tools.infrastructure.api.jira.EmptyJiraHome +import com.atlassian.performance.tools.infrastructure.api.jira.install.JiraInstallation +import com.atlassian.performance.tools.infrastructure.api.jira.install.ParallelInstallation +import com.atlassian.performance.tools.infrastructure.api.jira.install.hook.HookedJiraInstallation +import com.atlassian.performance.tools.infrastructure.api.jira.install.hook.PreInstallHooks +import com.atlassian.performance.tools.infrastructure.api.jira.start.JiraLaunchScript +import com.atlassian.performance.tools.infrastructure.api.jira.start.JiraStart +import com.atlassian.performance.tools.infrastructure.api.jira.start.StartedJira +import com.atlassian.performance.tools.infrastructure.api.jira.start.hook.HookedJiraStart +import com.atlassian.performance.tools.infrastructure.api.jvm.OracleJDK +import java.net.URI +import java.util.function.Supplier + +class JiraServerPlan private constructor( + private val infrastructure: Infrastructure, + val hooks: PreInstallHooks, + val installation: JiraInstallation, + val start: JiraStart +) : Supplier { + + override fun get(): JiraServer { + val jiraNode = infrastructure.serve(8080, "jira-node") + val installed = installation.install(jiraNode) + val started = start.start(installed) + return JiraServer(started) + } + + class JiraServer( + val node: StartedJira + ) : JiraInstance { + override val address: URI = node.installed.host.run { URI("http://$ip:$publicPort/") } + } + + class Builder( + private var infrastructure: Infrastructure + ) { + private var hooks: PreInstallHooks = PreInstallHooks.default() + private var installation: JiraInstallation = HookedJiraInstallation( + ParallelInstallation( + EmptyJiraHome(), + PublicJiraSoftwareDistribution("7.13.0"), + OracleJDK() + ), + hooks + ) + private var start: JiraStart = HookedJiraStart(JiraLaunchScript(), hooks.preStart) + + fun infrastructure(infrastructure: Infrastructure) = apply { this.infrastructure = infrastructure } + fun hooks(hooks: PreInstallHooks) = apply { this.hooks = hooks } // TODO this doesn't affect the start or installation + fun installation(installation: JiraInstallation) = apply { this.installation = installation } + fun start(start: JiraStart) = apply { this.start = start } + + fun build() = JiraServerPlan( + infrastructure = infrastructure, + hooks = hooks, + installation = installation, + start = start + ) + } +} \ No newline at end of file diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/PreInstanceHooks.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/PreInstanceHooks.kt index bd898228..238db71a 100644 --- a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/PreInstanceHooks.kt +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/PreInstanceHooks.kt @@ -11,7 +11,7 @@ class PreInstanceHooks( private val hooks: Queue = ConcurrentLinkedQueue() val postInstance = PostInstanceHooks(nodes) - fun hook(hook: PreInstanceHook) { + fun insert(hook: PreInstanceHook) { hooks.add(hook) } diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/node/JiraNode.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/node/JiraNode.kt new file mode 100644 index 00000000..3e0edb01 --- /dev/null +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/node/JiraNode.kt @@ -0,0 +1,8 @@ +package com.atlassian.performance.tools.infrastructure.api.jira.node + +import com.atlassian.performance.tools.infrastructure.api.jira.install.TcpHost + +class JiraNode constructor( + val host: TcpHost, + val plan: JiraNodePlan +) \ No newline at end of file diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/provision/JiraNodeProvisioning.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/node/JiraNodePlan.kt similarity index 69% rename from src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/provision/JiraNodeProvisioning.kt rename to src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/node/JiraNodePlan.kt index fa177670..f0b0e746 100644 --- a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/provision/JiraNodeProvisioning.kt +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/node/JiraNodePlan.kt @@ -1,17 +1,10 @@ -package com.atlassian.performance.tools.infrastructure.api.jira.provision +package com.atlassian.performance.tools.infrastructure.api.jira.node import com.atlassian.performance.tools.infrastructure.api.distribution.PublicJiraSoftwareDistribution import com.atlassian.performance.tools.infrastructure.api.jira.EmptyJiraHome -import com.atlassian.performance.tools.infrastructure.api.jira.JiraHomeSource -import com.atlassian.performance.tools.infrastructure.api.jira.hook.JiraNodeHooks -import com.atlassian.performance.tools.infrastructure.api.jira.hook.install.HookedJiraInstallation -import com.atlassian.performance.tools.infrastructure.api.jira.hook.install.JiraInstallation -import com.atlassian.performance.tools.infrastructure.api.jira.hook.install.ParallelInstallation -import com.atlassian.performance.tools.infrastructure.api.jira.hook.start.HookedJiraStart -import com.atlassian.performance.tools.infrastructure.api.jira.hook.start.JiraLaunchScript -import com.atlassian.performance.tools.infrastructure.api.jira.hook.start.JiraStart import com.atlassian.performance.tools.infrastructure.api.jira.install.JiraInstallation import com.atlassian.performance.tools.infrastructure.api.jira.install.ParallelInstallation +import com.atlassian.performance.tools.infrastructure.api.jira.install.TcpHost import com.atlassian.performance.tools.infrastructure.api.jira.install.hook.HookedJiraInstallation import com.atlassian.performance.tools.infrastructure.api.jira.install.hook.PreInstallHooks import com.atlassian.performance.tools.infrastructure.api.jira.start.JiraLaunchScript @@ -20,12 +13,14 @@ import com.atlassian.performance.tools.infrastructure.api.jira.start.hook.Hooked import com.atlassian.performance.tools.infrastructure.api.jvm.OracleJDK import net.jcip.annotations.NotThreadSafe -class JiraNodeProvisioning private constructor( +class JiraNodePlan private constructor( val hooks: PreInstallHooks, val installation: JiraInstallation, val start: JiraStart ) { + fun materialize(host: TcpHost) = JiraNode(host, this) + @NotThreadSafe class Builder { private var hooks: PreInstallHooks = PreInstallHooks.default() @@ -37,13 +32,13 @@ class JiraNodeProvisioning private constructor( ), hooks ) - private var start: JiraStart = HookedJiraStart(JiraLaunchScript()) + private var start: JiraStart = HookedJiraStart(JiraLaunchScript(), hooks.preStart) - fun hooks(hooks: PreInstallHooks) = apply { this.hooks = hooks } + fun hooks(hooks: PreInstallHooks) = apply { this.hooks = hooks } // TODO this doesn't affect the start or installation fun installation(installation: JiraInstallation) = apply { this.installation = installation } fun start(start: JiraStart) = apply { this.start = start } - fun build() = JiraNodeProvisioning( + fun build() = JiraNodePlan( hooks = hooks, installation = installation, start = start diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/start/hook/HookedJiraStart.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/start/hook/HookedJiraStart.kt index 70afbfff..911c2b43 100644 --- a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/start/hook/HookedJiraStart.kt +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/start/hook/HookedJiraStart.kt @@ -12,11 +12,11 @@ class HookedJiraStart( override fun start( installed: InstalledJira ): StartedJira { - installed.server.ssh.newConnection().use { ssh -> + installed.host.ssh.newConnection().use { ssh -> hooks.call(ssh, installed) } val started = start.start(installed) - installed.server.ssh.newConnection().use { ssh -> + installed.host.ssh.newConnection().use { ssh -> hooks.postStart.call(ssh, started) } return started diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/start/hook/RestUpgrade.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/start/hook/RestUpgrade.kt index d89e8443..f1239d3a 100644 --- a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/start/hook/RestUpgrade.kt +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/start/hook/RestUpgrade.kt @@ -17,7 +17,7 @@ class RestUpgrade( override fun call(ssh: SshConnection, jira: StartedJira, hooks: PostStartHooks) { val threadDump = ThreadDump(jira.pid, jira.installed.jdk) - val privatePort = jira.installed.server.privatePort + val privatePort = jira.installed.host.privatePort val upgradesEndpoint = URI("http://$adminUsername:$adminPassword@localhost:$privatePort/rest/api/2/upgrade") hooks.reports.add(FileListing("thread-dumps/*")) waitForStatusToChange( diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/jira/install/hook/SplunkForwarderHook.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/jira/install/hook/SplunkForwarderHook.kt index 59c05bd2..78282a69 100644 --- a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/jira/install/hook/SplunkForwarderHook.kt +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/jira/install/hook/SplunkForwarderHook.kt @@ -16,6 +16,6 @@ internal class SplunkForwarderHook( hooks: PostInstallHooks ) { splunk.jsonifyLog4j(ssh, "${jira.installation.path}/atlassian-jira/WEB-INF/classes/log4j.properties") - splunk.run(ssh, jira.server.name, "/home/ubuntu/jirahome/log") + splunk.run(ssh, jira.host.name, "/home/ubuntu/jirahome/log") } } diff --git a/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/DockerInfrastructure.kt b/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/DockerInfrastructure.kt new file mode 100644 index 00000000..019ead0e --- /dev/null +++ b/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/DockerInfrastructure.kt @@ -0,0 +1,45 @@ +package com.atlassian.performance.tools.infrastructure.api + +import com.atlassian.performance.tools.infrastructure.api.jira.install.TcpHost +import com.atlassian.performance.tools.infrastructure.api.jira.node.JiraNode +import com.atlassian.performance.tools.infrastructure.api.jira.node.JiraNodePlan +import com.atlassian.performance.tools.infrastructure.toSsh +import com.atlassian.performance.tools.sshubuntu.api.SshUbuntuContainer +import java.util.* +import java.util.concurrent.ConcurrentLinkedQueue +import java.util.function.Consumer + +internal class DockerInfrastructure : Infrastructure { + + private val allocatedResources: Queue = ConcurrentLinkedQueue() + + override fun serve(jiraNodePlans: List): List { + return jiraNodePlans.mapIndexed { nodeIndex, plan -> + plan.materialize(serve(8080, "jira-node-$nodeIndex")) + } + } + + override fun serve(port: Int, name: String): TcpHost { + val container = SshUbuntuContainer(Consumer { + it.addExposedPort(port) + }) + val sshUbuntu = container.start() + allocatedResources.offer(sshUbuntu) + return TcpHost( + "localhost", + sshUbuntu.container.getMappedPort(port), + port, + name, + sshUbuntu.toSsh() + ) + } + + override fun releaseResources() { + while (true) { + allocatedResources + .poll() + ?.use {} + ?: break + } + } +} \ No newline at end of file diff --git a/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/HookedJiraStartIT.kt b/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/HookedJiraStartIT.kt index 96252ebe..874a0956 100644 --- a/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/HookedJiraStartIT.kt +++ b/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/HookedJiraStartIT.kt @@ -1,21 +1,44 @@ package com.atlassian.performance.tools.infrastructure.api.jira.install +import com.atlassian.performance.tools.infrastructure.api.Infrastructure +import com.atlassian.performance.tools.infrastructure.api.database.DockerMysqlServer +import com.atlassian.performance.tools.infrastructure.api.dataset.HttpDatasetPackage import com.atlassian.performance.tools.infrastructure.api.distribution.PublicJiraSoftwareDistribution +import com.atlassian.performance.tools.infrastructure.api.DockerInfrastructure import com.atlassian.performance.tools.infrastructure.api.jira.EmptyJiraHome import com.atlassian.performance.tools.infrastructure.api.jira.install.hook.HookedJiraInstallation import com.atlassian.performance.tools.infrastructure.api.jira.install.hook.PreInstallHooks +import com.atlassian.performance.tools.infrastructure.api.jira.instance.JiraDataCenterPlan +import com.atlassian.performance.tools.infrastructure.api.jira.instance.JiraServerPlan +import com.atlassian.performance.tools.infrastructure.api.jira.instance.PreInstanceHooks +import com.atlassian.performance.tools.infrastructure.api.jira.node.JiraNodePlan import com.atlassian.performance.tools.infrastructure.api.jira.start.JiraLaunchScript import com.atlassian.performance.tools.infrastructure.api.jira.start.hook.HookedJiraStart import com.atlassian.performance.tools.infrastructure.api.jvm.AdoptOpenJDK -import com.atlassian.performance.tools.infrastructure.toSsh -import com.atlassian.performance.tools.sshubuntu.api.SshUbuntuContainer import org.assertj.core.api.Assertions.assertThat +import org.junit.After +import org.junit.Before import org.junit.Test +import java.net.URI import java.nio.file.Files -import java.util.function.Consumer +import java.time.Duration +import java.util.* +import java.util.function.Supplier class HookedJiraStartIT { + private lateinit var infrastructure: Infrastructure + + @Before + fun setUp() { + infrastructure = DockerInfrastructure() + } + + @After + fun tearDown() { + infrastructure.releaseResources() + } + @Test fun shouldStartJiraWithHooks() { // given @@ -30,21 +53,83 @@ class HookedJiraStartIT { ) val start = HookedJiraStart(JiraLaunchScript(), hooks.preStart) - testOnServer("jira", 8080) { server -> - // when - val installed = installation.install(server) - val started = start.start(installed) - val reports = server.ssh.newConnection().use { ssh -> - hooks.reports.list().flatMap { it.locate(ssh) } + // when + val jiraServer = JiraServerPlan.Builder(infrastructure) + .installation(installation) + .start(start) + .hooks(hooks) + .build() + .get() + + val host = jiraServer.node.installed.host + val reports = host.ssh.newConnection().use { ssh -> + hooks.reports.list().flatMap { it.locate(ssh) } + } + + // then + val serverXml = jiraServer + .node + .installed + .installation + .resolve("conf/server.xml") + .download(Files.createTempFile("downloaded-config", ".xml")) + assertThat(serverXml.readText()).contains(" + HttpDatasetPackage( + uri = uri.resolve("database.tar.bz2"), + downloadTimeout = Duration.ofMinutes(6) + ) } + instanceHooks.insert(DockerMysqlServer.Builder(infrastructure, smallJiraSeven).build()) + val dcPlan = JiraDataCenterPlan(nodePlans, instanceHooks, Supplier { TODO() }, infrastructure) - // then + // when + val dataCenter = dcPlan.get() + val reports = dataCenter.nodes.ssh.newConnection().use { ssh -> + nodeHooks.reports.list().flatMap { it.locate(ssh) } + } + + // then + dataCenter.nodes.forEach { node -> + val installed = node.installed val serverXml = installed .installation .resolve("conf/server.xml") .download(Files.createTempFile("downloaded-config", ".xml")) - assertThat(serverXml.readText()).contains(" - testOnServer("jira2", 8080) { jira2 -> - testOnServer("mysql", 3306) { mysql -> - // when - val installed = installation.install(server) - val started = start.start(installed) - val reports = server.ssh.newConnection().use { ssh -> - hooks.reports.list().flatMap { it.locate(ssh) } - } - - // then - val serverXml = installed - .installation - .resolve("conf/server.xml") - .download(Files.createTempFile("downloaded-config", ".xml")) - assertThat(serverXml.readText()).contains(" testOnServer(name: String, privatePort: Int, test: (TcpServer) -> T) { - val container = SshUbuntuContainer(Consumer { - it.addExposedPort(privatePort) - }) - container.start().use { sshUbuntu -> - val server = TcpServer( - "localhost", - sshUbuntu.container.getMappedPort(privatePort), - privatePort, - name, - sshUbuntu.toSsh() - ) - test(server) - } - } } diff --git a/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/PreInstallHooksTest.kt b/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/PreInstallHooksTest.kt index 9e5b8bef..eb6dd82a 100644 --- a/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/PreInstallHooksTest.kt +++ b/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/PreInstallHooksTest.kt @@ -1,6 +1,6 @@ package com.atlassian.performance.tools.infrastructure.api.jira.install.hook -import com.atlassian.performance.tools.infrastructure.api.jira.install.TcpServer +import com.atlassian.performance.tools.infrastructure.api.jira.install.TcpHost import com.atlassian.performance.tools.infrastructure.mock.UnimplementedSshConnection import com.atlassian.performance.tools.ssh.api.Ssh import com.atlassian.performance.tools.ssh.api.SshConnection @@ -12,7 +12,7 @@ import java.nio.file.Paths class PreInstallHooksTest { private val dummySsh = Ssh(SshHost("localhost", "dummyUser", Paths.get("dummyKey"))) - private val dummyServer = TcpServer("doesn't matter", 123, "fake-server", dummySsh) + private val dummyServer = TcpHost("doesn't matter", 123, "fake-server", dummySsh) @Test fun shouldInsertDuringListing() { @@ -47,7 +47,7 @@ private class CountingHook : PreInstallHook { var count = 0 - override fun call(ssh: SshConnection, server: TcpServer, hooks: PreInstallHooks) { + override fun call(ssh: SshConnection, host: TcpHost, hooks: PreInstallHooks) { count++ } } @@ -55,7 +55,7 @@ private class CountingHook : PreInstallHook { private class InsertingHook( private val hook: PreInstallHook ) : PreInstallHook { - override fun call(ssh: SshConnection, server: TcpServer, hooks: PreInstallHooks) { + override fun call(ssh: SshConnection, host: TcpHost, hooks: PreInstallHooks) { hooks.insert(hook) } } From c240e7e9df4032eb9416731873881c8d4cd0b697 Mon Sep 17 00:00:00 2001 From: Maciej Kwidzinski Date: Fri, 26 Mar 2021 17:27:55 +0100 Subject: [PATCH 12/52] JPERF-273: Download reports even if a plan fails Download reports per host. --- .../api/database/DatabaseIpConfig.kt | 6 +- .../api/database/DockerMysqlServer.kt | 24 +-- .../api/database/DockerPostgresServer.kt | 6 +- .../api/database/MysqlConnector.kt | 4 +- .../tools/infrastructure/api/docker/Docker.kt | 11 +- .../api/jira/install/JiraInstallation.kt | 5 +- .../api/jira/install/ParallelInstallation.kt | 4 +- .../jira/install/SequentialInstallation.kt | 4 +- .../api/jira/install/TcpHost.kt | 15 +- .../jira/install/hook/AsyncProfilerHook.kt | 13 +- .../api/jira/install/hook/DataCenterHook.kt | 4 +- .../jira/install/hook/DisabledAutoBackup.kt | 4 +- .../api/jira/install/hook/JiraHomeProperty.kt | 4 +- .../api/jira/install/hook/JiraLogs.kt | 5 +- .../api/jira/install/hook/JvmConfig.kt | 6 +- .../jira/install/hook/LateUbuntuSysstat.kt | 8 +- .../api/jira/install/hook/PostInstallHook.kt | 7 +- .../api/jira/install/hook/PostInstallHooks.kt | 7 +- .../api/jira/install/hook/PreInstallHook.kt | 11 +- .../api/jira/install/hook/PreInstallHooks.kt | 13 +- .../api/jira/install/hook/SystemLog.kt | 10 +- .../api/jira/instance/JiraDataCenterPlan.kt | 90 ++++++----- .../api/jira/instance/JiraInstance.kt | 2 + .../api/jira/instance/JiraInstancePlan.kt | 8 + .../api/jira/instance/JiraServerPlan.kt | 65 +++----- .../api/jira/instance/PostInstanceHook.kt | 10 +- .../api/jira/instance/PostInstanceHooks.kt | 17 ++- .../api/jira/instance/PreInstanceHook.kt | 11 +- .../api/jira/instance/PreInstanceHooks.kt | 15 +- .../api/jira/node/JiraNodePlan.kt | 31 ++-- .../api/jira/report/FileListing.kt | 10 +- .../infrastructure/api/jira/report/Reports.kt | 44 +++++- .../api/jira/start/JiraLaunchScript.kt | 4 +- .../api/jira/start/JiraStart.kt | 10 +- .../api/jira/start/hook/AccessLogs.kt | 9 +- .../api/jira/start/hook/JstatHook.kt | 6 +- .../api/jira/start/hook/PostStartHook.kt | 7 +- .../api/jira/start/hook/PostStartHooks.kt | 6 +- .../api/jira/start/hook/PreStartHook.kt | 7 +- .../api/jira/start/hook/PreStartHooks.kt | 11 +- .../api/jira/start/hook/RestUpgrade.kt | 5 +- .../install/hook/HookedJiraInstallation.kt | 13 +- .../jira/install/hook/ProfilerHook.kt | 8 +- .../jira/install/hook/SplunkForwarderHook.kt | 6 +- .../jira/start/hook/HookedJiraStart.kt | 13 +- .../tools/infrastructure/Datasets.kt | 18 +++ .../api/DockerInfrastructure.kt | 3 +- .../api/jira/install/HookedJiraStartIT.kt | 143 ------------------ .../jira/install/hook/PreInstallHooksTest.kt | 11 +- .../api/jira/instance/JiraDataCenterPlanIT.kt | 83 ++++++++++ .../api/jira/instance/JiraServerPlanIT.kt | 80 ++++++++++ 51 files changed, 540 insertions(+), 377 deletions(-) create mode 100644 src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/JiraInstancePlan.kt rename src/main/kotlin/com/atlassian/performance/tools/infrastructure/{api => }/jira/install/hook/HookedJiraInstallation.kt (56%) rename src/main/kotlin/com/atlassian/performance/tools/infrastructure/{api => }/jira/start/hook/HookedJiraStart.kt (55%) create mode 100644 src/test/kotlin/com/atlassian/performance/tools/infrastructure/Datasets.kt delete mode 100644 src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/HookedJiraStartIT.kt create mode 100644 src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/JiraDataCenterPlanIT.kt create mode 100644 src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/JiraServerPlanIT.kt diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/database/DatabaseIpConfig.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/database/DatabaseIpConfig.kt index 7fb79d0a..da00606b 100644 --- a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/database/DatabaseIpConfig.kt +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/database/DatabaseIpConfig.kt @@ -4,6 +4,7 @@ import com.atlassian.performance.tools.infrastructure.api.Sed import com.atlassian.performance.tools.infrastructure.api.jira.install.InstalledJira import com.atlassian.performance.tools.infrastructure.api.jira.install.hook.PostInstallHook import com.atlassian.performance.tools.infrastructure.api.jira.install.hook.PostInstallHooks +import com.atlassian.performance.tools.infrastructure.api.jira.report.Reports import com.atlassian.performance.tools.ssh.api.SshConnection class DatabaseIpConfig( @@ -13,13 +14,14 @@ class DatabaseIpConfig( override fun call( ssh: SshConnection, jira: InstalledJira, - hooks: PostInstallHooks + hooks: PostInstallHooks, + reports: Reports ) { Sed().replace( connection = ssh, expression = "(.*(@(//)?|//))" + "([^:/]+)" + "(.*)", output = """\1$databaseIp\5""", - file = "${jira.home}/dbconfig.xml" + file = "${jira.home.path}/dbconfig.xml" ) } } \ No newline at end of file diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/database/DockerMysqlServer.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/database/DockerMysqlServer.kt index 29bcce60..3f3416ac 100644 --- a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/database/DockerMysqlServer.kt +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/database/DockerMysqlServer.kt @@ -4,17 +4,15 @@ import com.atlassian.performance.tools.infrastructure.api.Infrastructure import com.atlassian.performance.tools.infrastructure.api.dataset.DatasetPackage import com.atlassian.performance.tools.infrastructure.api.docker.DockerImage import com.atlassian.performance.tools.infrastructure.api.jira.install.TcpHost -import com.atlassian.performance.tools.infrastructure.api.jira.instance.PostInstanceHook -import com.atlassian.performance.tools.infrastructure.api.jira.instance.PostInstanceHooks -import com.atlassian.performance.tools.infrastructure.api.jira.instance.PreInstanceHook -import com.atlassian.performance.tools.infrastructure.api.jira.instance.PreInstanceHooks +import com.atlassian.performance.tools.infrastructure.api.jira.install.hook.PreInstallHooks +import com.atlassian.performance.tools.infrastructure.api.jira.instance.* +import com.atlassian.performance.tools.infrastructure.api.jira.report.Reports import com.atlassian.performance.tools.infrastructure.api.os.Ubuntu import com.atlassian.performance.tools.infrastructure.database.SshMysqlClient import com.atlassian.performance.tools.ssh.api.Ssh import com.atlassian.performance.tools.ssh.api.SshConnection import org.apache.logging.log4j.LogManager import org.apache.logging.log4j.Logger -import java.net.URI import java.time.Duration import java.time.Instant @@ -27,10 +25,14 @@ class DockerMysqlServer private constructor( private val logger: Logger = LogManager.getLogger(this::class.java) - override fun call(hooks: PreInstanceHooks) { + override fun call( + nodes: List, + hooks: PreInstanceHooks, + reports: Reports + ) { val server = infrastructure.serve(3306, "mysql") server.ssh.newConnection().use { setup(it, server) } - hooks.nodes.forEach { node -> + nodes.forEach { node -> node.postInstall.insert(DatabaseIpConfig(server.ip)) node.postInstall.insert(MysqlConnector()) } @@ -41,12 +43,12 @@ class DockerMysqlServer private constructor( val mysqlData = source.download(ssh) dockerImage.run( ssh = ssh, - parameters = "-p ${host.publicPort}:${host.privatePort} -v `realpath $mysqlData`:/var/lib/mysql", + parameters = "-p ${host.privatePort}:${host.publicPort} -v `realpath $mysqlData`:/var/lib/mysql", arguments = "--skip-grant-tables --max_connections=$maxConnections" ) Ubuntu().install(ssh, listOf("mysql-client")) val deadline = Instant.now() + Duration.ofMinutes(15) - while (ssh.safeExecute("mysql -h 127.0.0.1 -u root -e 'select 1;'").isSuccessful().not()) { + while (ssh.safeExecute("mysql -h 127.0.0.1 -P ${host.privatePort} -u root -e 'select 1;'").isSuccessful().not()) { if (Instant.now() > deadline) { throw Exception("MySQL didn't start in time") } @@ -82,10 +84,10 @@ class DockerMysqlServer private constructor( private val mysql: Ssh ) : PostInstanceHook { - override fun call(instance: URI, hooks: PostInstanceHooks) { + override fun call(instance: JiraInstance, hooks: PostInstanceHooks, reports: Reports) { mysql.newConnection().use { ssh -> val db = "jiradb" - val update = "UPDATE $db.propertystring SET propertyvalue = '$instance'" + val update = "UPDATE $db.propertystring SET propertyvalue = '${instance.address}'" val where = "WHERE id IN (select id from $db.propertyentry where property_key like '%baseurl%')" SshMysqlClient().runSql(ssh, "$update $where;") } diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/database/DockerPostgresServer.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/database/DockerPostgresServer.kt index e62cef4e..269f9a0b 100644 --- a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/database/DockerPostgresServer.kt +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/database/DockerPostgresServer.kt @@ -2,8 +2,10 @@ package com.atlassian.performance.tools.infrastructure.api.database import com.atlassian.performance.tools.infrastructure.api.docker.DockerImage import com.atlassian.performance.tools.infrastructure.api.jira.install.TcpHost +import com.atlassian.performance.tools.infrastructure.api.jira.install.hook.PreInstallHooks import com.atlassian.performance.tools.infrastructure.api.jira.instance.PreInstanceHook import com.atlassian.performance.tools.infrastructure.api.jira.instance.PreInstanceHooks +import com.atlassian.performance.tools.infrastructure.api.jira.report.Reports import com.atlassian.performance.tools.ssh.api.SshConnection import org.apache.logging.log4j.LogManager import org.apache.logging.log4j.Logger @@ -18,10 +20,10 @@ class DockerPostgresServer private constructor( private val logger: Logger = LogManager.getLogger(this::class.java) - override fun call(hooks: PreInstanceHooks) { + override fun call(nodes: List, hooks: PreInstanceHooks, reports: Reports) { val server = hostSupplier.get() server.ssh.newConnection().use { setup(it) } - hooks.nodes.forEach { node -> + nodes.forEach { node -> node.postInstall.insert(DatabaseIpConfig(server.ip)) } } diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/database/MysqlConnector.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/database/MysqlConnector.kt index edc0c8d1..4aa5cd5b 100644 --- a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/database/MysqlConnector.kt +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/database/MysqlConnector.kt @@ -3,6 +3,7 @@ package com.atlassian.performance.tools.infrastructure.api.database import com.atlassian.performance.tools.infrastructure.api.jira.install.InstalledJira import com.atlassian.performance.tools.infrastructure.api.jira.install.hook.PostInstallHook import com.atlassian.performance.tools.infrastructure.api.jira.install.hook.PostInstallHooks +import com.atlassian.performance.tools.infrastructure.api.jira.report.Reports import com.atlassian.performance.tools.jvmtasks.api.IdempotentAction import com.atlassian.performance.tools.jvmtasks.api.StaticBackoff import com.atlassian.performance.tools.ssh.api.SshConnection @@ -13,7 +14,8 @@ class MysqlConnector : PostInstallHook { override fun call( ssh: SshConnection, jira: InstalledJira, - hooks: PostInstallHooks + hooks: PostInstallHooks, + reports: Reports ) { val connector = "https://dev.mysql.com/get/Downloads/Connector-J/mysql-connector-java-5.1.40.tar.gz" IdempotentAction( diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/docker/Docker.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/docker/Docker.kt index 53adea3d..dc79ca6d 100644 --- a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/docker/Docker.kt +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/docker/Docker.kt @@ -1,6 +1,8 @@ package com.atlassian.performance.tools.infrastructure.api.docker import com.atlassian.performance.tools.infrastructure.api.os.Ubuntu +import com.atlassian.performance.tools.jvmtasks.api.IdempotentAction +import com.atlassian.performance.tools.jvmtasks.api.StaticBackoff import com.atlassian.performance.tools.ssh.api.SshConnection import java.time.Duration @@ -29,7 +31,11 @@ class Docker private constructor( ubuntu.addKey(ssh, "7EA0A9C3F273FCD8") val release = ubuntu.getDistributionCodename(ssh) - ubuntu.addRepository(ssh, "deb [arch=amd64] https://download.docker.com/linux/ubuntu $release stable", "docker"); + ubuntu.addRepository( + ssh, + "deb [arch=amd64] https://download.docker.com/linux/ubuntu $release stable", + "docker" + ); val version = "5:19.03.13~3-0~ubuntu-$release" ubuntu.install( @@ -38,6 +44,9 @@ class Docker private constructor( timeout = mainPackageTimeout ) ssh.execute("sudo service docker status || sudo service docker start") + IdempotentAction("poll docker") { + ssh.execute("sudo docker ps") + }.retry(2, StaticBackoff(Duration.ofSeconds(1))) } class Builder { diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/JiraInstallation.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/JiraInstallation.kt index 05bd3d96..71c0f488 100644 --- a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/JiraInstallation.kt +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/JiraInstallation.kt @@ -1,5 +1,6 @@ package com.atlassian.performance.tools.infrastructure.api.jira.install +import com.atlassian.performance.tools.infrastructure.api.jira.report.Reports import net.jcip.annotations.ThreadSafe /** @@ -12,8 +13,10 @@ interface JiraInstallation { * Installs Jira on [host]. * * @param [host] will host the Jira + * @param [reports] accumulates reports */ fun install( - host: TcpHost + host: TcpHost, + reports: Reports ): InstalledJira } diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/ParallelInstallation.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/ParallelInstallation.kt index 03af91c0..452f1bea 100644 --- a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/ParallelInstallation.kt +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/ParallelInstallation.kt @@ -3,6 +3,7 @@ package com.atlassian.performance.tools.infrastructure.api.jira.install import com.atlassian.performance.tools.concurrency.api.submitWithLogContext import com.atlassian.performance.tools.infrastructure.api.distribution.ProductDistribution import com.atlassian.performance.tools.infrastructure.api.jira.JiraHomeSource +import com.atlassian.performance.tools.infrastructure.api.jira.report.Reports import com.atlassian.performance.tools.infrastructure.api.jvm.JavaDevelopmentKit import com.atlassian.performance.tools.infrastructure.downloadRemotely import com.atlassian.performance.tools.infrastructure.installRemotely @@ -15,7 +16,8 @@ class ParallelInstallation( ) : JiraInstallation { override fun install( - host: TcpHost + host: TcpHost, + reports: Reports ): InstalledJira { host.ssh.newConnection().use { ssh -> val pool = Executors.newCachedThreadPool { runnable -> diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/SequentialInstallation.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/SequentialInstallation.kt index 382c82e8..c5d124d3 100644 --- a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/SequentialInstallation.kt +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/SequentialInstallation.kt @@ -2,6 +2,7 @@ package com.atlassian.performance.tools.infrastructure.api.jira.install import com.atlassian.performance.tools.infrastructure.api.distribution.ProductDistribution import com.atlassian.performance.tools.infrastructure.api.jira.JiraHomeSource +import com.atlassian.performance.tools.infrastructure.api.jira.report.Reports import com.atlassian.performance.tools.infrastructure.api.jvm.JavaDevelopmentKit import com.atlassian.performance.tools.infrastructure.downloadRemotely import com.atlassian.performance.tools.infrastructure.installRemotely @@ -13,7 +14,8 @@ class SequentialInstallation( ) : JiraInstallation { override fun install( - host: TcpHost + host: TcpHost, + reports: Reports ): InstalledJira { host.ssh.newConnection().use { ssh -> val installation = productDistribution.installRemotely(ssh, ".") diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/TcpHost.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/TcpHost.kt index fc2aa038..83b3df2f 100644 --- a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/TcpHost.kt +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/TcpHost.kt @@ -11,17 +11,4 @@ class TcpHost( val privatePort: Int, val name: String, val ssh: Ssh -) { - constructor( - ip: String, - port: Int, - name: String, - ssh: Ssh - ) : this( - ip, - port, - port, - name, - ssh - ) -} +) diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/AsyncProfilerHook.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/AsyncProfilerHook.kt index 95688f98..186453e7 100644 --- a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/AsyncProfilerHook.kt +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/AsyncProfilerHook.kt @@ -2,6 +2,7 @@ package com.atlassian.performance.tools.infrastructure.api.jira.install.hook import com.atlassian.performance.tools.infrastructure.api.jira.install.TcpHost import com.atlassian.performance.tools.infrastructure.api.jira.report.Report +import com.atlassian.performance.tools.infrastructure.api.jira.report.Reports import com.atlassian.performance.tools.infrastructure.api.jira.start.StartedJira import com.atlassian.performance.tools.infrastructure.api.jira.start.hook.PostStartHook import com.atlassian.performance.tools.infrastructure.api.jira.start.hook.PostStartHooks @@ -11,9 +12,10 @@ import java.net.URI class AsyncProfilerHook : PreInstallHook { override fun call( - ssh: SshConnection, - host: TcpHost, - hooks: PreInstallHooks + ssh: SshConnection, + host: TcpHost, + hooks: PreInstallHooks, + reports: Reports ) { val directory = "async-profiler" val downloads = URI("https://github.com/jvm-profiling-tools/async-profiler/releases/download/") @@ -36,11 +38,12 @@ private class InstalledAsyncProfiler( override fun call( ssh: SshConnection, jira: StartedJira, - hooks: PostStartHooks + hooks: PostStartHooks, + reports: Reports ) { ssh.execute("$profilerPath -b 20000000 start ${jira.pid}") val profiler = StartedAsyncProfiler(jira.pid, profilerPath) - hooks.reports.add(profiler) + reports.add(profiler, jira.installed.host) } } diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/DataCenterHook.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/DataCenterHook.kt index 4b4b549f..5ec19a6d 100644 --- a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/DataCenterHook.kt +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/DataCenterHook.kt @@ -2,6 +2,7 @@ package com.atlassian.performance.tools.infrastructure.api.jira.install.hook import com.atlassian.performance.tools.infrastructure.api.jira.SharedHome import com.atlassian.performance.tools.infrastructure.api.jira.install.InstalledJira +import com.atlassian.performance.tools.infrastructure.api.jira.report.Reports import com.atlassian.performance.tools.ssh.api.SshConnection class DataCenterHook( @@ -12,7 +13,8 @@ class DataCenterHook( override fun call( ssh: SshConnection, jira: InstalledJira, - hooks: PostInstallHooks + hooks: PostInstallHooks, + reports: Reports ) { val localSharedHome = sharedHome.localSharedHome sharedHome.mount(ssh) diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/DisabledAutoBackup.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/DisabledAutoBackup.kt index 8cc6e973..164da451 100644 --- a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/DisabledAutoBackup.kt +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/DisabledAutoBackup.kt @@ -1,6 +1,7 @@ package com.atlassian.performance.tools.infrastructure.api.jira.install.hook import com.atlassian.performance.tools.infrastructure.api.jira.install.InstalledJira +import com.atlassian.performance.tools.infrastructure.api.jira.report.Reports import com.atlassian.performance.tools.ssh.api.SshConnection class DisabledAutoBackup : PostInstallHook { @@ -8,7 +9,8 @@ class DisabledAutoBackup : PostInstallHook { override fun call( ssh: SshConnection, jira: InstalledJira, - hooks: PostInstallHooks + hooks: PostInstallHooks, + reports: Reports ) { ssh.execute("echo jira.autoexport=false > ${jira.home.path}/jira-config.properties") } diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/JiraHomeProperty.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/JiraHomeProperty.kt index 9d8d1ed4..b09991b5 100644 --- a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/JiraHomeProperty.kt +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/JiraHomeProperty.kt @@ -1,6 +1,7 @@ package com.atlassian.performance.tools.infrastructure.api.jira.install.hook import com.atlassian.performance.tools.infrastructure.api.jira.install.InstalledJira +import com.atlassian.performance.tools.infrastructure.api.jira.report.Reports import com.atlassian.performance.tools.ssh.api.SshConnection class JiraHomeProperty : PostInstallHook { @@ -8,7 +9,8 @@ class JiraHomeProperty : PostInstallHook { override fun call( ssh: SshConnection, jira: InstalledJira, - hooks: PostInstallHooks + hooks: PostInstallHooks, + reports: Reports ) { val properties = "${jira.installation.path}/atlassian-jira/WEB-INF/classes/jira-application.properties" ssh.execute("echo jira.home=`realpath ${jira.home.path}` > $properties") diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/JiraLogs.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/JiraLogs.kt index ef6f803c..683f5389 100644 --- a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/JiraLogs.kt +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/JiraLogs.kt @@ -1,6 +1,7 @@ package com.atlassian.performance.tools.infrastructure.api.jira.install.hook import com.atlassian.performance.tools.infrastructure.api.jira.install.InstalledJira +import com.atlassian.performance.tools.infrastructure.api.jira.report.Reports import com.atlassian.performance.tools.infrastructure.api.jira.report.StaticReport import com.atlassian.performance.tools.ssh.api.SshConnection import java.nio.file.Path @@ -8,14 +9,14 @@ import java.nio.file.Paths class JiraLogs : PostInstallHook { - override fun call(ssh: SshConnection, jira: InstalledJira, hooks: PostInstallHooks) { + override fun call(ssh: SshConnection, jira: InstalledJira, hooks: PostInstallHooks, reports: Reports) { listOf( "${jira.home.path}/log/atlassian-jira.log", "${jira.installation.path}/logs/catalina.out" ) .onEach { ensureFile(Paths.get(it), ssh) } .map { StaticReport(it) } - .forEach { hooks.reports.add(it) } + .forEach { reports.add(it, jira.host) } } private fun ensureFile( diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/JvmConfig.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/JvmConfig.kt index 7c72be25..9fe8d5a7 100644 --- a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/JvmConfig.kt +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/JvmConfig.kt @@ -5,6 +5,7 @@ import com.atlassian.performance.tools.infrastructure.api.jira.JiraNodeConfig import com.atlassian.performance.tools.infrastructure.api.jira.SetenvSh import com.atlassian.performance.tools.infrastructure.api.jira.install.InstalledJira import com.atlassian.performance.tools.infrastructure.api.jira.report.FileListing +import com.atlassian.performance.tools.infrastructure.api.jira.report.Reports import com.atlassian.performance.tools.ssh.api.SshConnection class JvmConfig( @@ -14,7 +15,8 @@ class JvmConfig( override fun call( ssh: SshConnection, jira: InstalledJira, - hooks: PostInstallHooks + hooks: PostInstallHooks, + reports: Reports ) { val gcLog = JiraGcLog(jira.installation.path) SetenvSh(jira.installation.path).setup( @@ -24,6 +26,6 @@ class JvmConfig( jiraIp = jira.host.ip ) val report = FileListing(gcLog.path("*")) - hooks.reports.add(report) + reports.add(report, jira.host) } } diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/LateUbuntuSysstat.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/LateUbuntuSysstat.kt index 2273c81c..26f327e2 100644 --- a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/LateUbuntuSysstat.kt +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/LateUbuntuSysstat.kt @@ -2,6 +2,7 @@ package com.atlassian.performance.tools.infrastructure.api.jira.install.hook import com.atlassian.performance.tools.infrastructure.Iostat import com.atlassian.performance.tools.infrastructure.api.jira.install.InstalledJira +import com.atlassian.performance.tools.infrastructure.api.jira.report.Reports import com.atlassian.performance.tools.infrastructure.api.jira.start.StartedJira import com.atlassian.performance.tools.infrastructure.api.jira.start.hook.PostStartHook import com.atlassian.performance.tools.infrastructure.api.jira.start.hook.PostStartHooks @@ -15,7 +16,8 @@ class LateUbuntuSysstat : PostInstallHook { override fun call( ssh: SshConnection, jira: InstalledJira, - hooks: PostInstallHooks + hooks: PostInstallHooks, + reports: Reports ) { val ubuntu = Ubuntu() ubuntu.install(ssh, listOf("sysstat")) @@ -28,8 +30,8 @@ class LateUbuntuSysstat : PostInstallHook { private class PostStartOsMetric( private val metric: OsMetric ) : PostStartHook { - override fun call(ssh: SshConnection, jira: StartedJira, hooks: PostStartHooks) { + override fun call(ssh: SshConnection, jira: StartedJira, hooks: PostStartHooks, reports: Reports) { val process = metric.start(ssh) - hooks.reports.add(RemoteMonitoringProcessReport(process)) + reports.add(RemoteMonitoringProcessReport(process), jira.installed.host) } } diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/PostInstallHook.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/PostInstallHook.kt index 0fe19b0a..c95cd0cb 100644 --- a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/PostInstallHook.kt +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/PostInstallHook.kt @@ -1,6 +1,7 @@ package com.atlassian.performance.tools.infrastructure.api.jira.install.hook import com.atlassian.performance.tools.infrastructure.api.jira.install.InstalledJira +import com.atlassian.performance.tools.infrastructure.api.jira.report.Reports import com.atlassian.performance.tools.ssh.api.SshConnection /** @@ -11,11 +12,13 @@ interface PostInstallHook { /** * @param [ssh] connects to the [jira] * @param [jira] points to the installed Jira - * @param [hooks] inserts future hooks and reports + * @param [hooks] inserts future hooks + * @param [reports] accumulates reports */ fun call( ssh: SshConnection, jira: InstalledJira, - hooks: PostInstallHooks + hooks: PostInstallHooks, + reports: Reports ) } diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/PostInstallHooks.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/PostInstallHooks.kt index 0b285cf5..37088763 100644 --- a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/PostInstallHooks.kt +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/PostInstallHooks.kt @@ -2,6 +2,7 @@ package com.atlassian.performance.tools.infrastructure.api.jira.install.hook import com.atlassian.performance.tools.infrastructure.api.jira.JiraNodeConfig import com.atlassian.performance.tools.infrastructure.api.jira.install.InstalledJira +import com.atlassian.performance.tools.infrastructure.api.jira.report.Reports import com.atlassian.performance.tools.infrastructure.api.jira.start.hook.PreStartHooks import com.atlassian.performance.tools.infrastructure.jira.install.hook.ProfilerHook import com.atlassian.performance.tools.infrastructure.jira.install.hook.SplunkForwarderHook @@ -15,7 +16,6 @@ class PostInstallHooks private constructor( private val hooks: Queue = ConcurrentLinkedQueue() val postStart = preStart.postStart - val reports = postStart.reports fun insert( hook: PostInstallHook @@ -25,12 +25,13 @@ class PostInstallHooks private constructor( internal fun call( ssh: SshConnection, - jira: InstalledJira + jira: InstalledJira, + reports: Reports ) { while (true) { hooks .poll() - ?.call(ssh, jira, this) + ?.call(ssh, jira, this, reports) ?: break } } diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/PreInstallHook.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/PreInstallHook.kt index e51cfd64..b9be4a39 100644 --- a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/PreInstallHook.kt +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/PreInstallHook.kt @@ -1,6 +1,7 @@ package com.atlassian.performance.tools.infrastructure.api.jira.install.hook import com.atlassian.performance.tools.infrastructure.api.jira.install.TcpHost +import com.atlassian.performance.tools.infrastructure.api.jira.report.Reports import com.atlassian.performance.tools.ssh.api.SshConnection /** @@ -11,11 +12,13 @@ interface PreInstallHook { /** * @param [ssh] connects to the [host] * @param [host] will install Jira - * @param [hooks] inserts future hooks and reports + * @param [hooks] inserts future hooks + * @param [reports] accumulates reports */ fun call( - ssh: SshConnection, - host: TcpHost, - hooks: PreInstallHooks + ssh: SshConnection, + host: TcpHost, + hooks: PreInstallHooks, + reports: Reports ) } diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/PreInstallHooks.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/PreInstallHooks.kt index 84842cff..58e256fa 100644 --- a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/PreInstallHooks.kt +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/PreInstallHooks.kt @@ -1,6 +1,7 @@ package com.atlassian.performance.tools.infrastructure.api.jira.install.hook import com.atlassian.performance.tools.infrastructure.api.jira.install.TcpHost +import com.atlassian.performance.tools.infrastructure.api.jira.report.Reports import com.atlassian.performance.tools.ssh.api.SshConnection import java.util.* import java.util.concurrent.ConcurrentLinkedQueue @@ -12,7 +13,6 @@ class PreInstallHooks private constructor( private val hooks: Queue = ConcurrentLinkedQueue() val preStart = postInstall.preStart val postStart = preStart.postStart - val reports = postStart.reports fun insert( hook: PreInstallHook @@ -21,19 +21,22 @@ class PreInstallHooks private constructor( } internal fun call( - ssh: SshConnection, - host: TcpHost + ssh: SshConnection, + host: TcpHost, + reports: Reports ) { while (true) { hooks .poll() - ?.call(ssh, host, this) + ?.call(ssh, host, this, reports) ?: break } } companion object Factory { - fun default(): PreInstallHooks = PreInstallHooks(PostInstallHooks.default()) + fun default(): PreInstallHooks = PreInstallHooks(PostInstallHooks.default()).apply { + insert(SystemLog()) + } fun empty(): PreInstallHooks = PreInstallHooks(PostInstallHooks.empty()) } diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/SystemLog.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/SystemLog.kt index 861b0ec8..b3a50fd4 100644 --- a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/SystemLog.kt +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/SystemLog.kt @@ -1,11 +1,13 @@ package com.atlassian.performance.tools.infrastructure.api.jira.install.hook -import com.atlassian.performance.tools.infrastructure.api.jira.report.Report +import com.atlassian.performance.tools.infrastructure.api.jira.install.TcpHost +import com.atlassian.performance.tools.infrastructure.api.jira.report.Reports +import com.atlassian.performance.tools.infrastructure.api.jira.report.StaticReport import com.atlassian.performance.tools.ssh.api.SshConnection -class SystemLog : Report { +class SystemLog : PreInstallHook { - override fun locate(ssh: SshConnection): List { - return listOf("/var/log/syslog") + override fun call(ssh: SshConnection, host: TcpHost, hooks: PreInstallHooks, reports: Reports) { + reports.add(StaticReport("/var/log/syslog"), host) } } diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/JiraDataCenterPlan.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/JiraDataCenterPlan.kt index 97de16e6..ac5be1ca 100644 --- a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/JiraDataCenterPlan.kt +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/JiraDataCenterPlan.kt @@ -1,69 +1,79 @@ package com.atlassian.performance.tools.infrastructure.api.jira.instance import com.atlassian.performance.tools.infrastructure.api.Infrastructure +import com.atlassian.performance.tools.infrastructure.api.jira.install.InstalledJira +import com.atlassian.performance.tools.infrastructure.api.jira.node.JiraNode import com.atlassian.performance.tools.infrastructure.api.jira.node.JiraNodePlan +import com.atlassian.performance.tools.infrastructure.api.jira.report.Reports import com.atlassian.performance.tools.infrastructure.api.jira.start.StartedJira import com.atlassian.performance.tools.infrastructure.api.loadbalancer.LoadBalancer import java.net.URI import java.time.Duration import java.util.function.Supplier +import kotlin.streams.asStream +import kotlin.streams.toList class JiraDataCenterPlan constructor( val nodePlans: List, val instanceHooks: PreInstanceHooks, val loadBalancerSupplier: Supplier, val infrastructure: Infrastructure -) : Supplier { - +) : JiraInstancePlan { + + private val reports: Reports = Reports() private val loadBalancingPatience = Duration.ofMinutes(5) - override fun get(): JiraDataCenter { - instanceHooks.call() - val started = infrastructure.serve(nodePlans).map { jiraNode -> + override fun materialize(): JiraInstance { + instanceHooks.call(nodePlans.map { it.hooks }, reports) + val nodes = infrastructure.serve(nodePlans) + val installed = installInParallel(nodes) + val started = installed.map { it.start(reports) } + val loadBalancer = loadBalancerSupplier.get() + val instance = JiraDataCenter(started, loadBalancer) + instanceHooks.postInstance.call(instance, reports) + loadBalancer.waitUntilHealthy(loadBalancingPatience) + return instance + } + + override fun report(): Reports = reports.copy() + + private fun installInParallel(nodes: List) = nodes + .asSequence() + .asStream() + .parallel() + .map { jiraNode -> jiraNode .plan .installation - .install(jiraNode.host) - .let { jiraNode.plan.start.start(it) } + .install(jiraNode.host, reports) + .let { installedJira -> InstalledJiraNode(installedJira, jiraNode.plan) } + } + .toList() + + private class InstalledJiraNode( + private val installedJira: InstalledJira, + private val nodePlan: JiraNodePlan + ) { + fun start(reports: Reports): StartedJira { + return nodePlan.start.start(installedJira, reports) } - val loadBalancer = loadBalancerSupplier.get() - instanceHooks.postInstance.call(loadBalancer.uri) - loadBalancer.waitUntilHealthy(loadBalancingPatience) - return JiraDataCenter(started, loadBalancer) } - class JiraDataCenter( - val nodes: List, - val loadBalancer: LoadBalancer + private class JiraDataCenter( + override val nodes: List, + private val loadBalancer: LoadBalancer ) : JiraInstance { override val address: URI get() = loadBalancer.uri } + + class Builder( + private var infrastructure: Infrastructure + ) { + private var nodePlans: List = listOf(1, 2).map { JiraNodePlan.Builder().build() } +// TODO private var instanceHooks: PreInstanceHooks = +// PreInstanceHooks(listOf(1..2).map { PreInstallHooks.default() }) // TODO two lists in sync? gotta be a better way // -// class Builder( -// private var jiraNode: TcpServer -// ) { -// private var hooks: PreInstallHooks = PreInstallHooks.default() -// private var installation: JiraInstallation = HookedJiraInstallation( -// ParallelInstallation( -// EmptyJiraHome(), -// PublicJiraSoftwareDistribution("7.13.0"), -// OracleJDK() -// ), -// hooks -// ) -// private var start: JiraStart = HookedJiraStart(JiraLaunchScript(), hooks.preStart) -// -// fun jiraNode(jiraNode: TcpServer) = apply { this.jiraNode = jiraNode } -// fun hooks(hooks: PreInstallHooks) = apply { this.hooks = hooks } // TODO this doesn't affect the start or installation -// fun installation(installation: JiraInstallation) = apply { this.installation = installation } -// fun start(start: JiraStart) = apply { this.start = start } -// -// fun build() = JiraDataCenterPlan( -// jiraNode = jiraNode, -// hooks = hooks, -// installation = installation, -// start = start -// ) -// } +// fun build(): Supplier = JiraDataCenterPlan(nodePlans, infrastructure) + } } \ No newline at end of file diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/JiraInstance.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/JiraInstance.kt index 8d2f4e3c..173b48ac 100644 --- a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/JiraInstance.kt +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/JiraInstance.kt @@ -1,7 +1,9 @@ package com.atlassian.performance.tools.infrastructure.api.jira.instance +import com.atlassian.performance.tools.infrastructure.api.jira.start.StartedJira import java.net.URI interface JiraInstance { val address: URI + val nodes: List } \ No newline at end of file diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/JiraInstancePlan.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/JiraInstancePlan.kt new file mode 100644 index 00000000..2b80ac64 --- /dev/null +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/JiraInstancePlan.kt @@ -0,0 +1,8 @@ +package com.atlassian.performance.tools.infrastructure.api.jira.instance + +import com.atlassian.performance.tools.infrastructure.api.jira.report.Reports + +interface JiraInstancePlan { + fun materialize(): JiraInstance + fun report(): Reports +} \ No newline at end of file diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/JiraServerPlan.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/JiraServerPlan.kt index 7d1ad934..1640a6ba 100644 --- a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/JiraServerPlan.kt +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/JiraServerPlan.kt @@ -1,64 +1,49 @@ package com.atlassian.performance.tools.infrastructure.api.jira.instance import com.atlassian.performance.tools.infrastructure.api.Infrastructure -import com.atlassian.performance.tools.infrastructure.api.distribution.PublicJiraSoftwareDistribution -import com.atlassian.performance.tools.infrastructure.api.jira.EmptyJiraHome -import com.atlassian.performance.tools.infrastructure.api.jira.install.JiraInstallation -import com.atlassian.performance.tools.infrastructure.api.jira.install.ParallelInstallation -import com.atlassian.performance.tools.infrastructure.api.jira.install.hook.HookedJiraInstallation -import com.atlassian.performance.tools.infrastructure.api.jira.install.hook.PreInstallHooks -import com.atlassian.performance.tools.infrastructure.api.jira.start.JiraLaunchScript -import com.atlassian.performance.tools.infrastructure.api.jira.start.JiraStart +import com.atlassian.performance.tools.infrastructure.api.jira.node.JiraNodePlan +import com.atlassian.performance.tools.infrastructure.api.jira.report.Reports import com.atlassian.performance.tools.infrastructure.api.jira.start.StartedJira -import com.atlassian.performance.tools.infrastructure.api.jira.start.hook.HookedJiraStart -import com.atlassian.performance.tools.infrastructure.api.jvm.OracleJDK import java.net.URI -import java.util.function.Supplier class JiraServerPlan private constructor( + private val plan: JiraNodePlan, private val infrastructure: Infrastructure, - val hooks: PreInstallHooks, - val installation: JiraInstallation, - val start: JiraStart -) : Supplier { + private val hooks: PreInstanceHooks +) : JiraInstancePlan { - override fun get(): JiraServer { + private val reports: Reports = Reports() + + override fun materialize(): JiraInstance { + val nodeHooks = listOf(plan).map { it.hooks } + hooks.call(nodeHooks, reports) val jiraNode = infrastructure.serve(8080, "jira-node") - val installed = installation.install(jiraNode) - val started = start.start(installed) - return JiraServer(started) + val installed = plan.installation.install(jiraNode, reports) + val started = plan.start.start(installed, reports) + val instance = JiraServer(started) + hooks.postInstance.call(instance, reports) + return instance } - class JiraServer( - val node: StartedJira + override fun report(): Reports = reports.copy() + + private class JiraServer( + node: StartedJira ) : JiraInstance { override val address: URI = node.installed.host.run { URI("http://$ip:$publicPort/") } + override val nodes: List = listOf(node) } class Builder( private var infrastructure: Infrastructure ) { - private var hooks: PreInstallHooks = PreInstallHooks.default() - private var installation: JiraInstallation = HookedJiraInstallation( - ParallelInstallation( - EmptyJiraHome(), - PublicJiraSoftwareDistribution("7.13.0"), - OracleJDK() - ), - hooks - ) - private var start: JiraStart = HookedJiraStart(JiraLaunchScript(), hooks.preStart) + private var plan: JiraNodePlan = JiraNodePlan.Builder().build() + private var hooks: PreInstanceHooks = PreInstanceHooks.default() + fun plan(plan: JiraNodePlan) = apply { this.plan = plan } fun infrastructure(infrastructure: Infrastructure) = apply { this.infrastructure = infrastructure } - fun hooks(hooks: PreInstallHooks) = apply { this.hooks = hooks } // TODO this doesn't affect the start or installation - fun installation(installation: JiraInstallation) = apply { this.installation = installation } - fun start(start: JiraStart) = apply { this.start = start } + fun hooks(hooks: PreInstanceHooks) = apply { this.hooks = hooks } - fun build() = JiraServerPlan( - infrastructure = infrastructure, - hooks = hooks, - installation = installation, - start = start - ) + fun build(): JiraInstancePlan = JiraServerPlan(plan, infrastructure, hooks) } } \ No newline at end of file diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/PostInstanceHook.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/PostInstanceHook.kt index 48dcb0f8..42e3442c 100644 --- a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/PostInstanceHook.kt +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/PostInstanceHook.kt @@ -1,15 +1,17 @@ package com.atlassian.performance.tools.infrastructure.api.jira.instance -import java.net.URI +import com.atlassian.performance.tools.infrastructure.api.jira.report.Reports interface PostInstanceHook { /** * @param [instance] a standalone Jira Server node or a Jira Data Center cluster - * @param [hooks] inserts future hooks and reports + * @param [hooks] inserts future hooks + * @param [reports] accumulates reports */ fun call( - instance: URI, - hooks: PostInstanceHooks + instance: JiraInstance, + hooks: PostInstanceHooks, + reports: Reports ) } diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/PostInstanceHooks.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/PostInstanceHooks.kt index f519425e..b56147ae 100644 --- a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/PostInstanceHooks.kt +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/PostInstanceHooks.kt @@ -1,13 +1,10 @@ package com.atlassian.performance.tools.infrastructure.api.jira.instance -import com.atlassian.performance.tools.infrastructure.api.jira.install.hook.PreInstallHooks -import java.net.URI +import com.atlassian.performance.tools.infrastructure.api.jira.report.Reports import java.util.* import java.util.concurrent.ConcurrentLinkedQueue -class PostInstanceHooks( - val nodes: List -) { +class PostInstanceHooks private constructor() { private val hooks: Queue = ConcurrentLinkedQueue() @@ -15,12 +12,18 @@ class PostInstanceHooks( hooks.add(hook) } - fun call(instance: URI) { + fun call(instance: JiraInstance, reports: Reports) { while (true) { hooks .poll() - ?.call(instance, this) + ?.call(instance, this, reports) ?: break } } + + + companion object Factory { + fun default(): PostInstanceHooks = empty() + fun empty(): PostInstanceHooks = PostInstanceHooks() + } } diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/PreInstanceHook.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/PreInstanceHook.kt index 31c4c777..be674f40 100644 --- a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/PreInstanceHook.kt +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/PreInstanceHook.kt @@ -1,11 +1,18 @@ package com.atlassian.performance.tools.infrastructure.api.jira.instance +import com.atlassian.performance.tools.infrastructure.api.jira.install.hook.PreInstallHooks +import com.atlassian.performance.tools.infrastructure.api.jira.report.Reports + interface PreInstanceHook { /** - * @param [hooks] inserts future hooks and reports + * @param [nodes] inserts node hooks + * @param [hooks] inserts future hooks + * @param [reports] accumulates reports */ fun call( - hooks: PreInstanceHooks + nodes: List, + hooks: PreInstanceHooks, + reports: Reports ) } diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/PreInstanceHooks.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/PreInstanceHooks.kt index 238db71a..5643ca27 100644 --- a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/PreInstanceHooks.kt +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/PreInstanceHooks.kt @@ -1,26 +1,31 @@ package com.atlassian.performance.tools.infrastructure.api.jira.instance import com.atlassian.performance.tools.infrastructure.api.jira.install.hook.PreInstallHooks +import com.atlassian.performance.tools.infrastructure.api.jira.report.Reports import java.util.* import java.util.concurrent.ConcurrentLinkedQueue -class PreInstanceHooks( - val nodes: List +class PreInstanceHooks private constructor( + val postInstance: PostInstanceHooks ) { private val hooks: Queue = ConcurrentLinkedQueue() - val postInstance = PostInstanceHooks(nodes) fun insert(hook: PreInstanceHook) { hooks.add(hook) } - fun call() { + internal fun call(nodes: List, reports: Reports) { while (true) { hooks .poll() - ?.call(this) + ?.call(nodes, this, reports) ?: break } } + + companion object Factory { + fun default(): PreInstanceHooks = PreInstanceHooks(PostInstanceHooks.default()) + fun empty(): PreInstanceHooks = PreInstanceHooks(PostInstanceHooks.empty()) + } } diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/node/JiraNodePlan.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/node/JiraNodePlan.kt index f0b0e746..d4664efe 100644 --- a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/node/JiraNodePlan.kt +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/node/JiraNodePlan.kt @@ -5,43 +5,40 @@ import com.atlassian.performance.tools.infrastructure.api.jira.EmptyJiraHome import com.atlassian.performance.tools.infrastructure.api.jira.install.JiraInstallation import com.atlassian.performance.tools.infrastructure.api.jira.install.ParallelInstallation import com.atlassian.performance.tools.infrastructure.api.jira.install.TcpHost -import com.atlassian.performance.tools.infrastructure.api.jira.install.hook.HookedJiraInstallation import com.atlassian.performance.tools.infrastructure.api.jira.install.hook.PreInstallHooks import com.atlassian.performance.tools.infrastructure.api.jira.start.JiraLaunchScript import com.atlassian.performance.tools.infrastructure.api.jira.start.JiraStart -import com.atlassian.performance.tools.infrastructure.api.jira.start.hook.HookedJiraStart import com.atlassian.performance.tools.infrastructure.api.jvm.OracleJDK +import com.atlassian.performance.tools.infrastructure.jira.install.hook.HookedJiraInstallation +import com.atlassian.performance.tools.infrastructure.jira.start.hook.HookedJiraStart import net.jcip.annotations.NotThreadSafe class JiraNodePlan private constructor( - val hooks: PreInstallHooks, val installation: JiraInstallation, - val start: JiraStart + val start: JiraStart, + internal val hooks: PreInstallHooks ) { fun materialize(host: TcpHost) = JiraNode(host, this) @NotThreadSafe class Builder { - private var hooks: PreInstallHooks = PreInstallHooks.default() - private var installation: JiraInstallation = HookedJiraInstallation( - ParallelInstallation( - EmptyJiraHome(), - PublicJiraSoftwareDistribution("7.13.0"), - OracleJDK() - ), - hooks + private var installation: JiraInstallation = ParallelInstallation( + EmptyJiraHome(), + PublicJiraSoftwareDistribution("7.13.0"), + OracleJDK() ) - private var start: JiraStart = HookedJiraStart(JiraLaunchScript(), hooks.preStart) + private var start: JiraStart = JiraLaunchScript() + private var hooks: PreInstallHooks = PreInstallHooks.default() - fun hooks(hooks: PreInstallHooks) = apply { this.hooks = hooks } // TODO this doesn't affect the start or installation fun installation(installation: JiraInstallation) = apply { this.installation = installation } fun start(start: JiraStart) = apply { this.start = start } + fun hooks(hooks: PreInstallHooks) = apply { this.hooks = hooks } fun build() = JiraNodePlan( - hooks = hooks, - installation = installation, - start = start + HookedJiraInstallation(installation, hooks), + HookedJiraStart(start, hooks.preStart), + hooks ) } } diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/report/FileListing.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/report/FileListing.kt index 75979fca..b23e5965 100644 --- a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/report/FileListing.kt +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/report/FileListing.kt @@ -9,8 +9,10 @@ class FileListing( override fun locate( ssh: SshConnection ): List = ssh - .execute("ls $pattern") - .output - .lines() - .filter { it.isNotBlank() } + .safeExecute("ls $pattern") + .takeIf { it.isSuccessful() } + ?.output + ?.lines() + ?.filter { it.isNotBlank() } + ?: emptyList() } diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/report/Reports.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/report/Reports.kt index bfc8b1cd..59a66270 100644 --- a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/report/Reports.kt +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/report/Reports.kt @@ -1,18 +1,50 @@ package com.atlassian.performance.tools.infrastructure.api.jira.report +import com.atlassian.performance.tools.infrastructure.api.jira.install.TcpHost +import com.atlassian.performance.tools.infrastructure.api.os.RemotePath +import com.atlassian.performance.tools.io.api.ensureDirectory +import com.atlassian.performance.tools.io.api.resolveSafely +import java.io.File +import java.nio.file.Path import java.util.* import java.util.concurrent.ConcurrentLinkedQueue -class Reports { - private val reports: Queue = ConcurrentLinkedQueue() +class Reports private constructor( + private val hostReports: Queue +) { + constructor() : this(ConcurrentLinkedQueue()) fun add( - report: Report + report: Report, + host: TcpHost ) { - reports.add(report) + hostReports.add(HostReport(host, report)) } - fun list(): Iterable { - return reports.toList() + fun downloadTo( + localDirectory: Path + ): File { + localDirectory.ensureDirectory() + hostReports.groupBy { it.host }.map { (host, reports) -> + host.ssh.newConnection().use { ssh -> + reports + .flatMap { report -> + report.report.locate(ssh).map { path -> RemotePath(host.ssh.host, path) } + } + .forEach { remotePath -> + remotePath.download(localDirectory.resolveSafely(host.name)) + } + } + } + return localDirectory.toFile() } + + fun copy(): Reports { + return Reports(ConcurrentLinkedQueue(hostReports)) + } + + private class HostReport( + val host: TcpHost, + val report: Report + ) } diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/start/JiraLaunchScript.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/start/JiraLaunchScript.kt index 55e8ec6d..3ce78b6c 100644 --- a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/start/JiraLaunchScript.kt +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/start/JiraLaunchScript.kt @@ -1,13 +1,15 @@ package com.atlassian.performance.tools.infrastructure.api.jira.start import com.atlassian.performance.tools.infrastructure.api.jira.install.InstalledJira +import com.atlassian.performance.tools.infrastructure.api.jira.report.Reports import com.atlassian.performance.tools.ssh.api.Ssh import java.time.Duration class JiraLaunchScript : JiraStart { override fun start( - installed: InstalledJira + installed: InstalledJira, + reports: Reports ): StartedJira { val installation = installed.installation Ssh(installation.host).newConnection().use { ssh -> diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/start/JiraStart.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/start/JiraStart.kt index 9f2d9cd8..706e01e2 100644 --- a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/start/JiraStart.kt +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/start/JiraStart.kt @@ -1,12 +1,20 @@ package com.atlassian.performance.tools.infrastructure.api.jira.start import com.atlassian.performance.tools.infrastructure.api.jira.install.InstalledJira +import com.atlassian.performance.tools.infrastructure.api.jira.report.Reports import net.jcip.annotations.ThreadSafe @ThreadSafe interface JiraStart { + /** + * Starts the [installed] Jira. + * + * @param [installed] will start the Jira + * @param [reports] accumulates reports + */ fun start( - installed: InstalledJira + installed: InstalledJira, + reports: Reports ): StartedJira } diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/start/hook/AccessLogs.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/start/hook/AccessLogs.kt index 6bc2f30e..f6f35c5d 100644 --- a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/start/hook/AccessLogs.kt +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/start/hook/AccessLogs.kt @@ -1,12 +1,13 @@ package com.atlassian.performance.tools.infrastructure.api.jira.start.hook +import com.atlassian.performance.tools.infrastructure.api.jira.install.InstalledJira import com.atlassian.performance.tools.infrastructure.api.jira.report.FileListing -import com.atlassian.performance.tools.infrastructure.api.jira.start.StartedJira +import com.atlassian.performance.tools.infrastructure.api.jira.report.Reports import com.atlassian.performance.tools.ssh.api.SshConnection -class AccessLogs : PostStartHook { +class AccessLogs : PreStartHook { - override fun call(ssh: SshConnection, jira: StartedJira, hooks: PostStartHooks) { - hooks.reports.add(FileListing("${jira.installed.installation.path}/logs/*access*")) + override fun call(ssh: SshConnection, jira: InstalledJira, hooks: PreStartHooks, reports: Reports) { + reports.add(FileListing("${jira.installation.path}/logs/*access*"), jira.host) } } diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/start/hook/JstatHook.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/start/hook/JstatHook.kt index e4485af4..547e51af 100644 --- a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/start/hook/JstatHook.kt +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/start/hook/JstatHook.kt @@ -1,5 +1,6 @@ package com.atlassian.performance.tools.infrastructure.api.jira.start.hook +import com.atlassian.performance.tools.infrastructure.api.jira.report.Reports import com.atlassian.performance.tools.infrastructure.api.jira.start.StartedJira import com.atlassian.performance.tools.infrastructure.jira.report.RemoteMonitoringProcessReport import com.atlassian.performance.tools.ssh.api.SshConnection @@ -9,9 +10,10 @@ class JstatHook : PostStartHook { override fun call( ssh: SshConnection, jira: StartedJira, - hooks: PostStartHooks + hooks: PostStartHooks, + reports: Reports ) { val process = jira.installed.jdk.jstatMonitoring.start(ssh, jira.pid) - hooks.reports.add(RemoteMonitoringProcessReport(process)) + reports.add(RemoteMonitoringProcessReport(process), jira.installed.host) } } diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/start/hook/PostStartHook.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/start/hook/PostStartHook.kt index c3c47123..ee867e14 100644 --- a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/start/hook/PostStartHook.kt +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/start/hook/PostStartHook.kt @@ -1,5 +1,6 @@ package com.atlassian.performance.tools.infrastructure.api.jira.start.hook +import com.atlassian.performance.tools.infrastructure.api.jira.report.Reports import com.atlassian.performance.tools.infrastructure.api.jira.start.StartedJira import com.atlassian.performance.tools.ssh.api.SshConnection @@ -11,11 +12,13 @@ interface PostStartHook { /** * @param [ssh] connects to the [jira] * @param [jira] points to the started Jira - * @param [hooks] inserts future reports + * @param [hooks] inserts future hooks + * @param [reports] accumulates reports */ fun call( ssh: SshConnection, jira: StartedJira, - hooks: PostStartHooks + hooks: PostStartHooks, + reports: Reports ) } diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/start/hook/PostStartHooks.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/start/hook/PostStartHooks.kt index 21b0b46a..77671948 100644 --- a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/start/hook/PostStartHooks.kt +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/start/hook/PostStartHooks.kt @@ -9,7 +9,6 @@ import java.util.concurrent.ConcurrentLinkedQueue class PostStartHooks private constructor() { private val hooks: Queue = ConcurrentLinkedQueue() - val reports = Reports() fun insert( hook: PostStartHook @@ -19,12 +18,13 @@ class PostStartHooks private constructor() { internal fun call( ssh: SshConnection, - jira: StartedJira + jira: StartedJira, + reports: Reports ) { while (true) { hooks .poll() - ?.call(ssh, jira, this) + ?.call(ssh, jira, this, reports) ?: break } } diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/start/hook/PreStartHook.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/start/hook/PreStartHook.kt index d0fa8326..0123305e 100644 --- a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/start/hook/PreStartHook.kt +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/start/hook/PreStartHook.kt @@ -1,6 +1,7 @@ package com.atlassian.performance.tools.infrastructure.api.jira.start.hook import com.atlassian.performance.tools.infrastructure.api.jira.install.InstalledJira +import com.atlassian.performance.tools.infrastructure.api.jira.report.Reports import com.atlassian.performance.tools.ssh.api.SshConnection /** @@ -11,11 +12,13 @@ interface PreStartHook { /** * @param [ssh] connects to the [jira] * @param [jira] points to the installed Jira - * @param [hooks] inserts future hooks and reports + * @param [hooks] inserts future hooks + * @param [reports] accumulates reports */ fun call( ssh: SshConnection, jira: InstalledJira, - hooks: PreStartHooks + hooks: PreStartHooks, + reports: Reports ) } diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/start/hook/PreStartHooks.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/start/hook/PreStartHooks.kt index 6ce16b42..bc3082f4 100644 --- a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/start/hook/PreStartHooks.kt +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/start/hook/PreStartHooks.kt @@ -1,6 +1,7 @@ package com.atlassian.performance.tools.infrastructure.api.jira.start.hook import com.atlassian.performance.tools.infrastructure.api.jira.install.InstalledJira +import com.atlassian.performance.tools.infrastructure.api.jira.report.Reports import com.atlassian.performance.tools.ssh.api.SshConnection import java.util.* import java.util.concurrent.ConcurrentLinkedQueue @@ -10,7 +11,6 @@ class PreStartHooks private constructor( ) { private val hooks: Queue = ConcurrentLinkedQueue() - val reports = postStart.reports fun insert( hook: PreStartHook @@ -20,18 +20,21 @@ class PreStartHooks private constructor( internal fun call( ssh: SshConnection, - jira: InstalledJira + jira: InstalledJira, + reports: Reports ) { while (true) { hooks .poll() - ?.call(ssh, jira, this) + ?.call(ssh, jira, this, reports) ?: break } } companion object Factory { - fun default() = PreStartHooks(PostStartHooks.default()) + fun default() = PreStartHooks(PostStartHooks.default()).apply { + insert(AccessLogs()) + } fun empty() = PreStartHooks(PostStartHooks.empty()) } diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/start/hook/RestUpgrade.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/start/hook/RestUpgrade.kt index f1239d3a..1ea0c995 100644 --- a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/start/hook/RestUpgrade.kt +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/start/hook/RestUpgrade.kt @@ -2,6 +2,7 @@ package com.atlassian.performance.tools.infrastructure.api.jira.start.hook import com.atlassian.performance.tools.infrastructure.api.jira.JiraLaunchTimeouts import com.atlassian.performance.tools.infrastructure.api.jira.report.FileListing +import com.atlassian.performance.tools.infrastructure.api.jira.report.Reports import com.atlassian.performance.tools.infrastructure.api.jira.start.StartedJira import com.atlassian.performance.tools.infrastructure.api.jvm.ThreadDump import com.atlassian.performance.tools.ssh.api.SshConnection @@ -15,11 +16,11 @@ class RestUpgrade( private val adminPassword: String ) : PostStartHook { - override fun call(ssh: SshConnection, jira: StartedJira, hooks: PostStartHooks) { + override fun call(ssh: SshConnection, jira: StartedJira, hooks: PostStartHooks, reports: Reports) { val threadDump = ThreadDump(jira.pid, jira.installed.jdk) val privatePort = jira.installed.host.privatePort val upgradesEndpoint = URI("http://$adminUsername:$adminPassword@localhost:$privatePort/rest/api/2/upgrade") - hooks.reports.add(FileListing("thread-dumps/*")) + reports.add(FileListing("thread-dumps/*"), jira.installed.host) waitForStatusToChange( statusQuo = "000", timeout = timeouts.offlineTimeout, diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/HookedJiraInstallation.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/jira/install/hook/HookedJiraInstallation.kt similarity index 56% rename from src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/HookedJiraInstallation.kt rename to src/main/kotlin/com/atlassian/performance/tools/infrastructure/jira/install/hook/HookedJiraInstallation.kt index a7633b23..153fafe8 100644 --- a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/HookedJiraInstallation.kt +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/jira/install/hook/HookedJiraInstallation.kt @@ -1,8 +1,10 @@ -package com.atlassian.performance.tools.infrastructure.api.jira.install.hook +package com.atlassian.performance.tools.infrastructure.jira.install.hook import com.atlassian.performance.tools.infrastructure.api.jira.install.InstalledJira import com.atlassian.performance.tools.infrastructure.api.jira.install.JiraInstallation import com.atlassian.performance.tools.infrastructure.api.jira.install.TcpHost +import com.atlassian.performance.tools.infrastructure.api.jira.install.hook.PreInstallHooks +import com.atlassian.performance.tools.infrastructure.api.jira.report.Reports class HookedJiraInstallation( private val installation: JiraInstallation, @@ -10,14 +12,15 @@ class HookedJiraInstallation( ) : JiraInstallation { override fun install( - host: TcpHost + host: TcpHost, + reports: Reports ): InstalledJira { host.ssh.newConnection().use { ssh -> - hooks.call(ssh, host) + hooks.call(ssh, host, reports) } - val installed = installation.install(host) + val installed = installation.install(host, reports) host.ssh.newConnection().use { ssh -> - hooks.postInstall.call(ssh, installed) + hooks.postInstall.call(ssh, installed, reports) } return installed } diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/jira/install/hook/ProfilerHook.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/jira/install/hook/ProfilerHook.kt index 5b26eaa1..294ab3e5 100644 --- a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/jira/install/hook/ProfilerHook.kt +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/jira/install/hook/ProfilerHook.kt @@ -3,6 +3,7 @@ package com.atlassian.performance.tools.infrastructure.jira.install.hook import com.atlassian.performance.tools.infrastructure.api.jira.install.InstalledJira import com.atlassian.performance.tools.infrastructure.api.jira.install.hook.PostInstallHook import com.atlassian.performance.tools.infrastructure.api.jira.install.hook.PostInstallHooks +import com.atlassian.performance.tools.infrastructure.api.jira.report.Reports import com.atlassian.performance.tools.infrastructure.api.jira.start.StartedJira import com.atlassian.performance.tools.infrastructure.api.jira.start.hook.PostStartHook import com.atlassian.performance.tools.infrastructure.api.jira.start.hook.PostStartHooks @@ -18,7 +19,7 @@ import com.atlassian.performance.tools.ssh.api.SshConnection class ProfilerHook( private val profiler: Profiler ) : PostInstallHook { - override fun call(ssh: SshConnection, jira: InstalledJira, hooks: PostInstallHooks) { + override fun call(ssh: SshConnection, jira: InstalledJira, hooks: PostInstallHooks, reports: Reports) { profiler.install(ssh) hooks.preStart.postStart.insert(InstalledProfiler(profiler)) } @@ -31,11 +32,12 @@ private class InstalledProfiler( override fun call( ssh: SshConnection, jira: StartedJira, - hooks: PostStartHooks + hooks: PostStartHooks, + reports: Reports ) { val process = profiler.start(ssh, jira.pid) if (process != null) { - hooks.reports.add(RemoteMonitoringProcessReport(process)) + reports.add(RemoteMonitoringProcessReport(process), jira.installed.host) } } } diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/jira/install/hook/SplunkForwarderHook.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/jira/install/hook/SplunkForwarderHook.kt index 78282a69..1e27f25b 100644 --- a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/jira/install/hook/SplunkForwarderHook.kt +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/jira/install/hook/SplunkForwarderHook.kt @@ -1,8 +1,9 @@ package com.atlassian.performance.tools.infrastructure.jira.install.hook -import com.atlassian.performance.tools.infrastructure.api.jira.install.hook.PostInstallHooks import com.atlassian.performance.tools.infrastructure.api.jira.install.InstalledJira import com.atlassian.performance.tools.infrastructure.api.jira.install.hook.PostInstallHook +import com.atlassian.performance.tools.infrastructure.api.jira.install.hook.PostInstallHooks +import com.atlassian.performance.tools.infrastructure.api.jira.report.Reports import com.atlassian.performance.tools.infrastructure.api.splunk.SplunkForwarder import com.atlassian.performance.tools.ssh.api.SshConnection @@ -13,7 +14,8 @@ internal class SplunkForwarderHook( override fun call( ssh: SshConnection, jira: InstalledJira, - hooks: PostInstallHooks + hooks: PostInstallHooks, + reports: Reports ) { splunk.jsonifyLog4j(ssh, "${jira.installation.path}/atlassian-jira/WEB-INF/classes/log4j.properties") splunk.run(ssh, jira.host.name, "/home/ubuntu/jirahome/log") diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/start/hook/HookedJiraStart.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/jira/start/hook/HookedJiraStart.kt similarity index 55% rename from src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/start/hook/HookedJiraStart.kt rename to src/main/kotlin/com/atlassian/performance/tools/infrastructure/jira/start/hook/HookedJiraStart.kt index 911c2b43..e23395f9 100644 --- a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/start/hook/HookedJiraStart.kt +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/jira/start/hook/HookedJiraStart.kt @@ -1,8 +1,10 @@ -package com.atlassian.performance.tools.infrastructure.api.jira.start.hook +package com.atlassian.performance.tools.infrastructure.jira.start.hook import com.atlassian.performance.tools.infrastructure.api.jira.install.InstalledJira +import com.atlassian.performance.tools.infrastructure.api.jira.report.Reports import com.atlassian.performance.tools.infrastructure.api.jira.start.JiraStart import com.atlassian.performance.tools.infrastructure.api.jira.start.StartedJira +import com.atlassian.performance.tools.infrastructure.api.jira.start.hook.PreStartHooks class HookedJiraStart( private val start: JiraStart, @@ -10,14 +12,15 @@ class HookedJiraStart( ) : JiraStart { override fun start( - installed: InstalledJira + installed: InstalledJira, + reports: Reports ): StartedJira { installed.host.ssh.newConnection().use { ssh -> - hooks.call(ssh, installed) + hooks.call(ssh, installed, reports) } - val started = start.start(installed) + val started = start.start(installed, reports) installed.host.ssh.newConnection().use { ssh -> - hooks.postStart.call(ssh, started) + hooks.postStart.call(ssh, started, reports) } return started } diff --git a/src/test/kotlin/com/atlassian/performance/tools/infrastructure/Datasets.kt b/src/test/kotlin/com/atlassian/performance/tools/infrastructure/Datasets.kt new file mode 100644 index 00000000..f16f3d5a --- /dev/null +++ b/src/test/kotlin/com/atlassian/performance/tools/infrastructure/Datasets.kt @@ -0,0 +1,18 @@ +package com.atlassian.performance.tools.infrastructure + +import com.atlassian.performance.tools.infrastructure.api.dataset.HttpDatasetPackage +import java.net.URI +import java.time.Duration + +class Datasets { + + fun smallJiraSeven() = URI("https://s3-eu-west-1.amazonaws.com/") + .resolve("jpt-custom-datasets-storage-a008820-datasetbucket-1sjxdtrv5hdhj/") + .resolve("dataset-f8dba866-9d1b-492e-b76c-f4a78ac3958c/") + .let { uri -> + HttpDatasetPackage( + uri = uri.resolve("database.tar.bz2"), + downloadTimeout = Duration.ofMinutes(6) + ) + } +} \ No newline at end of file diff --git a/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/DockerInfrastructure.kt b/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/DockerInfrastructure.kt index 019ead0e..c44225f7 100644 --- a/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/DockerInfrastructure.kt +++ b/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/DockerInfrastructure.kt @@ -22,13 +22,14 @@ internal class DockerInfrastructure : Infrastructure { override fun serve(port: Int, name: String): TcpHost { val container = SshUbuntuContainer(Consumer { it.addExposedPort(port) + it.setPrivilegedMode(true) }) val sshUbuntu = container.start() allocatedResources.offer(sshUbuntu) return TcpHost( "localhost", - sshUbuntu.container.getMappedPort(port), port, + sshUbuntu.container.getMappedPort(port), name, sshUbuntu.toSsh() ) diff --git a/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/HookedJiraStartIT.kt b/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/HookedJiraStartIT.kt deleted file mode 100644 index 874a0956..00000000 --- a/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/HookedJiraStartIT.kt +++ /dev/null @@ -1,143 +0,0 @@ -package com.atlassian.performance.tools.infrastructure.api.jira.install - -import com.atlassian.performance.tools.infrastructure.api.Infrastructure -import com.atlassian.performance.tools.infrastructure.api.database.DockerMysqlServer -import com.atlassian.performance.tools.infrastructure.api.dataset.HttpDatasetPackage -import com.atlassian.performance.tools.infrastructure.api.distribution.PublicJiraSoftwareDistribution -import com.atlassian.performance.tools.infrastructure.api.DockerInfrastructure -import com.atlassian.performance.tools.infrastructure.api.jira.EmptyJiraHome -import com.atlassian.performance.tools.infrastructure.api.jira.install.hook.HookedJiraInstallation -import com.atlassian.performance.tools.infrastructure.api.jira.install.hook.PreInstallHooks -import com.atlassian.performance.tools.infrastructure.api.jira.instance.JiraDataCenterPlan -import com.atlassian.performance.tools.infrastructure.api.jira.instance.JiraServerPlan -import com.atlassian.performance.tools.infrastructure.api.jira.instance.PreInstanceHooks -import com.atlassian.performance.tools.infrastructure.api.jira.node.JiraNodePlan -import com.atlassian.performance.tools.infrastructure.api.jira.start.JiraLaunchScript -import com.atlassian.performance.tools.infrastructure.api.jira.start.hook.HookedJiraStart -import com.atlassian.performance.tools.infrastructure.api.jvm.AdoptOpenJDK -import org.assertj.core.api.Assertions.assertThat -import org.junit.After -import org.junit.Before -import org.junit.Test -import java.net.URI -import java.nio.file.Files -import java.time.Duration -import java.util.* -import java.util.function.Supplier - -class HookedJiraStartIT { - - private lateinit var infrastructure: Infrastructure - - @Before - fun setUp() { - infrastructure = DockerInfrastructure() - } - - @After - fun tearDown() { - infrastructure.releaseResources() - } - - @Test - fun shouldStartJiraWithHooks() { - // given - val hooks = PreInstallHooks.default() - val installation = HookedJiraInstallation( - ParallelInstallation( - jiraHomeSource = EmptyJiraHome(), - productDistribution = PublicJiraSoftwareDistribution("7.13.0"), - jdk = AdoptOpenJDK() - ), - hooks - ) - val start = HookedJiraStart(JiraLaunchScript(), hooks.preStart) - - // when - val jiraServer = JiraServerPlan.Builder(infrastructure) - .installation(installation) - .start(start) - .hooks(hooks) - .build() - .get() - - val host = jiraServer.node.installed.host - val reports = host.ssh.newConnection().use { ssh -> - hooks.reports.list().flatMap { it.locate(ssh) } - } - - // then - val serverXml = jiraServer - .node - .installed - .installation - .resolve("conf/server.xml") - .download(Files.createTempFile("downloaded-config", ".xml")) - assertThat(serverXml.readText()).contains(" - HttpDatasetPackage( - uri = uri.resolve("database.tar.bz2"), - downloadTimeout = Duration.ofMinutes(6) - ) - } - instanceHooks.insert(DockerMysqlServer.Builder(infrastructure, smallJiraSeven).build()) - val dcPlan = JiraDataCenterPlan(nodePlans, instanceHooks, Supplier { TODO() }, infrastructure) - - // when - val dataCenter = dcPlan.get() - val reports = dataCenter.nodes.ssh.newConnection().use { ssh -> - nodeHooks.reports.list().flatMap { it.locate(ssh) } - } - - // then - dataCenter.nodes.forEach { node -> - val installed = node.installed - val serverXml = installed - .installation - .resolve("conf/server.xml") - .download(Files.createTempFile("downloaded-config", ".xml")) - assertThat(serverXml.readText()).contains(" + val installed = node.installed + val serverXml = installed + .installation + .resolve("conf/server.xml") + .download(Files.createTempFile("downloaded-config", ".xml")) + assertThat(serverXml.readText()).contains(" Date: Fri, 2 Apr 2021 14:35:18 +0200 Subject: [PATCH 13/52] Fix hooked MySQL start --- .../api/database/DockerMysqlServer.kt | 36 ++++++++--------- .../api/database/MysqlConnector.kt | 2 +- .../api/jira/install/TcpHost.kt | 3 +- .../api/jira/instance/JiraServerPlan.kt | 2 +- .../api/jira/start/hook/RestUpgrade.kt | 4 +- .../infrastructure/database/SshMysqlClient.kt | 12 ++++-- .../tools/infrastructure/Datasets.kt | 39 +++++++++++++++---- .../api/DockerInfrastructure.kt | 2 +- .../jira/install/hook/PreInstallHooksTest.kt | 2 +- .../api/jira/instance/JiraDataCenterPlanIT.kt | 8 ++-- .../api/jira/instance/JiraServerPlanIT.kt | 8 ++-- 11 files changed, 71 insertions(+), 47 deletions(-) diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/database/DockerMysqlServer.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/database/DockerMysqlServer.kt index 3f3416ac..ace4306a 100644 --- a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/database/DockerMysqlServer.kt +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/database/DockerMysqlServer.kt @@ -9,12 +9,12 @@ import com.atlassian.performance.tools.infrastructure.api.jira.instance.* import com.atlassian.performance.tools.infrastructure.api.jira.report.Reports import com.atlassian.performance.tools.infrastructure.api.os.Ubuntu import com.atlassian.performance.tools.infrastructure.database.SshMysqlClient +import com.atlassian.performance.tools.infrastructure.database.SshSqlClient +import com.atlassian.performance.tools.jvmtasks.api.IdempotentAction +import com.atlassian.performance.tools.jvmtasks.api.StaticBackoff import com.atlassian.performance.tools.ssh.api.Ssh import com.atlassian.performance.tools.ssh.api.SshConnection -import org.apache.logging.log4j.LogManager -import org.apache.logging.log4j.Logger import java.time.Duration -import java.time.Instant class DockerMysqlServer private constructor( private val infrastructure: Infrastructure, @@ -23,38 +23,33 @@ class DockerMysqlServer private constructor( private val maxConnections: Int ) : PreInstanceHook { - private val logger: Logger = LogManager.getLogger(this::class.java) - override fun call( nodes: List, hooks: PreInstanceHooks, reports: Reports ) { val server = infrastructure.serve(3306, "mysql") - server.ssh.newConnection().use { setup(it, server) } + val client = server.ssh.newConnection().use { setup(it, server) } nodes.forEach { node -> node.postInstall.insert(DatabaseIpConfig(server.ip)) node.postInstall.insert(MysqlConnector()) } - hooks.postInstance.insert(FixJiraUriViaMysql(server.ssh)) + hooks.postInstance.insert(FixJiraUriViaMysql(server.ssh, client)) } - private fun setup(ssh: SshConnection, host: TcpHost) { + private fun setup(ssh: SshConnection, host: TcpHost): SshSqlClient { val mysqlData = source.download(ssh) + val port = host.port dockerImage.run( ssh = ssh, - parameters = "-p ${host.privatePort}:${host.publicPort} -v `realpath $mysqlData`:/var/lib/mysql", + parameters = "-p $port:$port -v `realpath $mysqlData`:/var/lib/mysql", arguments = "--skip-grant-tables --max_connections=$maxConnections" ) Ubuntu().install(ssh, listOf("mysql-client")) - val deadline = Instant.now() + Duration.ofMinutes(15) - while (ssh.safeExecute("mysql -h 127.0.0.1 -P ${host.privatePort} -u root -e 'select 1;'").isSuccessful().not()) { - if (Instant.now() > deadline) { - throw Exception("MySQL didn't start in time") - } - logger.debug("Waiting for MySQL...") - Thread.sleep(Duration.ofSeconds(10).toMillis()) - } + val client = SshMysqlClient("127.0.0.1", port, "root") + IdempotentAction("wait for MySQL start") { client.runSql(ssh, "select 1;") } + .retry(90, StaticBackoff(Duration.ofSeconds(10))) + return client } class Builder( @@ -81,15 +76,16 @@ class DockerMysqlServer private constructor( } private class FixJiraUriViaMysql( - private val mysql: Ssh + private val mysqlNode: Ssh, + private val client: SshSqlClient ) : PostInstanceHook { override fun call(instance: JiraInstance, hooks: PostInstanceHooks, reports: Reports) { - mysql.newConnection().use { ssh -> + mysqlNode.newConnection().use { ssh -> val db = "jiradb" val update = "UPDATE $db.propertystring SET propertyvalue = '${instance.address}'" val where = "WHERE id IN (select id from $db.propertyentry where property_key like '%baseurl%')" - SshMysqlClient().runSql(ssh, "$update $where;") + client.runSql(ssh, "$update $where;") } } } diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/database/MysqlConnector.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/database/MysqlConnector.kt index 4aa5cd5b..b4a13348 100644 --- a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/database/MysqlConnector.kt +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/database/MysqlConnector.kt @@ -26,6 +26,6 @@ class MysqlConnector : PostInstallHook { backoff = StaticBackoff(Duration.ofSeconds(5)) ) ssh.execute("tar -xzf mysql-connector-java-5.1.40.tar.gz") - ssh.execute("cp mysql-connector-java-5.1.40/mysql-connector-java-5.1.40-bin.jar ${jira.installation}/lib") + ssh.execute("cp mysql-connector-java-5.1.40/mysql-connector-java-5.1.40-bin.jar ${jira.installation.path}/lib") } } \ No newline at end of file diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/TcpHost.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/TcpHost.kt index 83b3df2f..8fef4548 100644 --- a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/TcpHost.kt +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/TcpHost.kt @@ -7,8 +7,7 @@ import com.atlassian.performance.tools.ssh.api.Ssh */ class TcpHost( val ip: String, - val publicPort: Int, - val privatePort: Int, + val port: Int, val name: String, val ssh: Ssh ) diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/JiraServerPlan.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/JiraServerPlan.kt index 1640a6ba..250c4f55 100644 --- a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/JiraServerPlan.kt +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/JiraServerPlan.kt @@ -30,7 +30,7 @@ class JiraServerPlan private constructor( private class JiraServer( node: StartedJira ) : JiraInstance { - override val address: URI = node.installed.host.run { URI("http://$ip:$publicPort/") } + override val address: URI = node.installed.host.run { URI("http://$ip:$port/") } override val nodes: List = listOf(node) } diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/start/hook/RestUpgrade.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/start/hook/RestUpgrade.kt index 1ea0c995..6cff1183 100644 --- a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/start/hook/RestUpgrade.kt +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/start/hook/RestUpgrade.kt @@ -18,8 +18,8 @@ class RestUpgrade( override fun call(ssh: SshConnection, jira: StartedJira, hooks: PostStartHooks, reports: Reports) { val threadDump = ThreadDump(jira.pid, jira.installed.jdk) - val privatePort = jira.installed.host.privatePort - val upgradesEndpoint = URI("http://$adminUsername:$adminPassword@localhost:$privatePort/rest/api/2/upgrade") + val port = jira.installed.host.port + val upgradesEndpoint = URI("http://$adminUsername:$adminPassword@localhost:$port/rest/api/2/upgrade") reports.add(FileListing("thread-dumps/*"), jira.installed.host) waitForStatusToChange( statusQuo = "000", diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/database/SshMysqlClient.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/database/SshMysqlClient.kt index 59bba7bb..c3050244 100644 --- a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/database/SshMysqlClient.kt +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/database/SshMysqlClient.kt @@ -3,14 +3,20 @@ package com.atlassian.performance.tools.infrastructure.database import com.atlassian.performance.tools.ssh.api.SshConnection import java.io.File -internal class SshMysqlClient : SshSqlClient { +internal class SshMysqlClient( + private val host: String, + private val port: Int, + private val user: String +) : SshSqlClient { + + internal constructor() : this("127.0.0.1", 3306, "root") override fun runSql( ssh: SshConnection, sql: String ): SshConnection.SshResult { val quotedSql = sql.quote('"') - return ssh.execute("mysql -h 127.0.0.1 -u root -e $quotedSql") + return ssh.execute("mysql -h $host -P $port -u $user -e $quotedSql") } override fun runSql( @@ -19,7 +25,7 @@ internal class SshMysqlClient : SshSqlClient { ): SshConnection.SshResult { val remoteSqlFile = sql.name ssh.upload(sql, remoteSqlFile) - val result = ssh.execute("mysql -h 127.0.0.1 -u root < $remoteSqlFile") + val result = ssh.execute("mysql -h $host -P $port -u $user < $remoteSqlFile") ssh.execute("rm $remoteSqlFile") return result } diff --git a/src/test/kotlin/com/atlassian/performance/tools/infrastructure/Datasets.kt b/src/test/kotlin/com/atlassian/performance/tools/infrastructure/Datasets.kt index f16f3d5a..3f3601ed 100644 --- a/src/test/kotlin/com/atlassian/performance/tools/infrastructure/Datasets.kt +++ b/src/test/kotlin/com/atlassian/performance/tools/infrastructure/Datasets.kt @@ -1,18 +1,41 @@ package com.atlassian.performance.tools.infrastructure +import com.atlassian.performance.tools.infrastructure.api.Infrastructure +import com.atlassian.performance.tools.infrastructure.api.database.DockerMysqlServer import com.atlassian.performance.tools.infrastructure.api.dataset.HttpDatasetPackage +import com.atlassian.performance.tools.infrastructure.api.jira.JiraLaunchTimeouts +import com.atlassian.performance.tools.infrastructure.api.jira.instance.PreInstanceHooks +import com.atlassian.performance.tools.infrastructure.api.jira.start.hook.PostStartHooks +import com.atlassian.performance.tools.infrastructure.api.jira.start.hook.RestUpgrade import java.net.URI import java.time.Duration class Datasets { - fun smallJiraSeven() = URI("https://s3-eu-west-1.amazonaws.com/") - .resolve("jpt-custom-datasets-storage-a008820-datasetbucket-1sjxdtrv5hdhj/") - .resolve("dataset-f8dba866-9d1b-492e-b76c-f4a78ac3958c/") - .let { uri -> - HttpDatasetPackage( - uri = uri.resolve("database.tar.bz2"), - downloadTimeout = Duration.ofMinutes(6) - ) + object JiraSevenDataset { + private val s3Bucket = URI("https://s3-eu-west-1.amazonaws.com/") + .resolve("jpt-custom-datasets-storage-a008820-datasetbucket-1sjxdtrv5hdhj/") + .resolve("dataset-f8dba866-9d1b-492e-b76c-f4a78ac3958c/") + + private val mysql = HttpDatasetPackage( + uri = s3Bucket.resolve("database.tar.bz2"), + downloadTimeout = Duration.ofMinutes(6) + ) + + val jiraHome = HttpDatasetPackage( + uri = s3Bucket.resolve("jirahome.tar.bz2"), + downloadTimeout = Duration.ofMinutes(6) + ) + + fun hookMysql( + preInstanceHooks: PreInstanceHooks, + postStartHooks: PostStartHooks, + infrastructure: Infrastructure + ) { + val mysqlServer = DockerMysqlServer.Builder(infrastructure, mysql).build() + preInstanceHooks.insert(mysqlServer) + val dataUpgrade = RestUpgrade(JiraLaunchTimeouts.Builder().build(), "admin", "admin") + postStartHooks.insert(dataUpgrade) } + } } \ No newline at end of file diff --git a/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/DockerInfrastructure.kt b/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/DockerInfrastructure.kt index c44225f7..0e7bde15 100644 --- a/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/DockerInfrastructure.kt +++ b/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/DockerInfrastructure.kt @@ -22,6 +22,7 @@ internal class DockerInfrastructure : Infrastructure { override fun serve(port: Int, name: String): TcpHost { val container = SshUbuntuContainer(Consumer { it.addExposedPort(port) + it.setPortBindings(listOf("$port:$port")) it.setPrivilegedMode(true) }) val sshUbuntu = container.start() @@ -29,7 +30,6 @@ internal class DockerInfrastructure : Infrastructure { return TcpHost( "localhost", port, - sshUbuntu.container.getMappedPort(port), name, sshUbuntu.toSsh() ) diff --git a/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/PreInstallHooksTest.kt b/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/PreInstallHooksTest.kt index aba5e0ce..aeb25f99 100644 --- a/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/PreInstallHooksTest.kt +++ b/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/PreInstallHooksTest.kt @@ -13,7 +13,7 @@ import java.nio.file.Paths class PreInstallHooksTest { private val dummySsh = Ssh(SshHost("localhost", "dummyUser", Paths.get("dummyKey"))) - private val dummyServer = TcpHost("doesn't matter", 123, 123, "fake-server", dummySsh) + private val dummyServer = TcpHost("doesn't matter", 123, "fake-server", dummySsh) @Test fun shouldInsertDuringListing() { diff --git a/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/JiraDataCenterPlanIT.kt b/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/JiraDataCenterPlanIT.kt index bc04ccfa..99698e7d 100644 --- a/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/JiraDataCenterPlanIT.kt +++ b/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/JiraDataCenterPlanIT.kt @@ -5,7 +5,7 @@ import com.atlassian.performance.tools.infrastructure.api.DockerInfrastructure import com.atlassian.performance.tools.infrastructure.api.Infrastructure import com.atlassian.performance.tools.infrastructure.api.database.DockerMysqlServer import com.atlassian.performance.tools.infrastructure.api.distribution.PublicJiraSoftwareDistribution -import com.atlassian.performance.tools.infrastructure.api.jira.EmptyJiraHome +import com.atlassian.performance.tools.infrastructure.api.jira.JiraHomePackage import com.atlassian.performance.tools.infrastructure.api.jira.install.ParallelInstallation import com.atlassian.performance.tools.infrastructure.api.jira.install.hook.PreInstallHooks import com.atlassian.performance.tools.infrastructure.api.jira.node.JiraNodePlan @@ -39,7 +39,7 @@ class JiraDataCenterPlanIT { JiraNodePlan.Builder() .installation( ParallelInstallation( - jiraHomeSource = EmptyJiraHome(), + jiraHomeSource = JiraHomePackage(Datasets.JiraSevenDataset.jiraHome), productDistribution = PublicJiraSoftwareDistribution("7.13.0"), jdk = AdoptOpenJDK() ) @@ -50,7 +50,7 @@ class JiraDataCenterPlanIT { } val instanceHooks = PreInstanceHooks.default().also { // TODO this plus `EmptyJiraHome()` = failing `DatabaseIpConfig` - couple them together or stop expecting a preexisting `dbconfig.xml`? but then what about missing lucene indexes? - it.insert(DockerMysqlServer.Builder(infrastructure, Datasets().smallJiraSeven()).build()) + it.insert(DockerMysqlServer.Builder(infrastructure, Datasets.JiraSevenDataset.mysql).build()) } val dcPlan = JiraDataCenterPlan(nodePlans, instanceHooks, Supplier { TODO() }, infrastructure) @@ -65,7 +65,7 @@ class JiraDataCenterPlanIT { .installation .resolve("conf/server.xml") .download(Files.createTempFile("downloaded-config", ".xml")) - assertThat(serverXml.readText()).contains(" Date: Fri, 2 Apr 2021 14:35:33 +0200 Subject: [PATCH 14/52] Fix downloaded reports overwriting each other --- .../infrastructure/api/jira/install/hook/SystemLog.kt | 4 ++-- .../tools/infrastructure/api/jira/report/Reports.kt | 11 ++++++++++- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/SystemLog.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/SystemLog.kt index b3a50fd4..235e41d2 100644 --- a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/SystemLog.kt +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/SystemLog.kt @@ -1,13 +1,13 @@ package com.atlassian.performance.tools.infrastructure.api.jira.install.hook import com.atlassian.performance.tools.infrastructure.api.jira.install.TcpHost +import com.atlassian.performance.tools.infrastructure.api.jira.report.FileListing import com.atlassian.performance.tools.infrastructure.api.jira.report.Reports -import com.atlassian.performance.tools.infrastructure.api.jira.report.StaticReport import com.atlassian.performance.tools.ssh.api.SshConnection class SystemLog : PreInstallHook { override fun call(ssh: SshConnection, host: TcpHost, hooks: PreInstallHooks, reports: Reports) { - reports.add(StaticReport("/var/log/syslog"), host) + reports.add(FileListing("/var/log/syslog"), host) } } diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/report/Reports.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/report/Reports.kt index 59a66270..1a579e41 100644 --- a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/report/Reports.kt +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/report/Reports.kt @@ -6,6 +6,7 @@ import com.atlassian.performance.tools.io.api.ensureDirectory import com.atlassian.performance.tools.io.api.resolveSafely import java.io.File import java.nio.file.Path +import java.nio.file.Paths import java.util.* import java.util.concurrent.ConcurrentLinkedQueue @@ -27,18 +28,26 @@ class Reports private constructor( localDirectory.ensureDirectory() hostReports.groupBy { it.host }.map { (host, reports) -> host.ssh.newConnection().use { ssh -> + val remoteBase = RemotePath(ssh.getHost(), ssh.execute("pwd").output.trim()) reports .flatMap { report -> report.report.locate(ssh).map { path -> RemotePath(host.ssh.host, path) } } .forEach { remotePath -> - remotePath.download(localDirectory.resolveSafely(host.name)) + localDirectory + .resolveSafely(host.name) + .resolve(remoteBase.toLocalRelativePath()) + .resolve(remotePath.toLocalRelativePath()) + .normalize() + .let { remotePath.download(it) } } } } return localDirectory.toFile() } + private fun RemotePath.toLocalRelativePath(): Path = Paths.get(path.trimStart('/')) + fun copy(): Reports { return Reports(ConcurrentLinkedQueue(hostReports)) } From ae486003e05801a856268a492383a1de9599e225 Mon Sep 17 00:00:00 2001 From: Maciej Kwidzinski Date: Fri, 2 Apr 2021 15:25:19 +0200 Subject: [PATCH 15/52] fixup: Fix hooked MySQL start --- .../performance/tools/infrastructure/Datasets.kt | 14 ++++++++------ .../api/jira/instance/JiraDataCenterPlanIT.kt | 8 +++----- .../api/jira/instance/JiraServerPlanIT.kt | 4 ++-- 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/src/test/kotlin/com/atlassian/performance/tools/infrastructure/Datasets.kt b/src/test/kotlin/com/atlassian/performance/tools/infrastructure/Datasets.kt index 3f3601ed..5c475ff2 100644 --- a/src/test/kotlin/com/atlassian/performance/tools/infrastructure/Datasets.kt +++ b/src/test/kotlin/com/atlassian/performance/tools/infrastructure/Datasets.kt @@ -27,14 +27,16 @@ class Datasets { downloadTimeout = Duration.ofMinutes(6) ) - fun hookMysql( - preInstanceHooks: PreInstanceHooks, - postStartHooks: PostStartHooks, - infrastructure: Infrastructure - ) { + fun hookMysql(preInstanceHooks: PreInstanceHooks, infrastructure: Infrastructure) { val mysqlServer = DockerMysqlServer.Builder(infrastructure, mysql).build() preInstanceHooks.insert(mysqlServer) - val dataUpgrade = RestUpgrade(JiraLaunchTimeouts.Builder().build(), "admin", "admin") + } + + fun hookMysql(postStartHooks: PostStartHooks) { + val timeouts = JiraLaunchTimeouts.Builder() + .initTimeout(Duration.ofSeconds(30)) + .build() + val dataUpgrade = RestUpgrade(timeouts, "admin", "admin") postStartHooks.insert(dataUpgrade) } } diff --git a/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/JiraDataCenterPlanIT.kt b/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/JiraDataCenterPlanIT.kt index 99698e7d..21fc7f8c 100644 --- a/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/JiraDataCenterPlanIT.kt +++ b/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/JiraDataCenterPlanIT.kt @@ -3,7 +3,6 @@ package com.atlassian.performance.tools.infrastructure.api.jira.instance import com.atlassian.performance.tools.infrastructure.Datasets import com.atlassian.performance.tools.infrastructure.api.DockerInfrastructure import com.atlassian.performance.tools.infrastructure.api.Infrastructure -import com.atlassian.performance.tools.infrastructure.api.database.DockerMysqlServer import com.atlassian.performance.tools.infrastructure.api.distribution.PublicJiraSoftwareDistribution import com.atlassian.performance.tools.infrastructure.api.jira.JiraHomePackage import com.atlassian.performance.tools.infrastructure.api.jira.install.ParallelInstallation @@ -45,13 +44,12 @@ class JiraDataCenterPlanIT { ) ) .start(JiraLaunchScript()) - .hooks(PreInstallHooks.default()) + .hooks(PreInstallHooks.default().also { Datasets.JiraSevenDataset.hookMysql(it.postStart) }) .build() } - val instanceHooks = PreInstanceHooks.default().also { + val instanceHooks = PreInstanceHooks.default() // TODO this plus `EmptyJiraHome()` = failing `DatabaseIpConfig` - couple them together or stop expecting a preexisting `dbconfig.xml`? but then what about missing lucene indexes? - it.insert(DockerMysqlServer.Builder(infrastructure, Datasets.JiraSevenDataset.mysql).build()) - } + .also { Datasets.JiraSevenDataset.hookMysql(it, infrastructure) } val dcPlan = JiraDataCenterPlan(nodePlans, instanceHooks, Supplier { TODO() }, infrastructure) try { diff --git a/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/JiraServerPlanIT.kt b/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/JiraServerPlanIT.kt index fb17f464..584528d1 100644 --- a/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/JiraServerPlanIT.kt +++ b/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/JiraServerPlanIT.kt @@ -3,7 +3,6 @@ package com.atlassian.performance.tools.infrastructure.api.jira.instance import com.atlassian.performance.tools.infrastructure.Datasets import com.atlassian.performance.tools.infrastructure.api.DockerInfrastructure import com.atlassian.performance.tools.infrastructure.api.Infrastructure -import com.atlassian.performance.tools.infrastructure.api.database.DockerMysqlServer import com.atlassian.performance.tools.infrastructure.api.distribution.PublicJiraSoftwareDistribution import com.atlassian.performance.tools.infrastructure.api.jira.JiraHomePackage import com.atlassian.performance.tools.infrastructure.api.jira.install.ParallelInstallation @@ -35,6 +34,7 @@ class JiraServerPlanIT { fun shouldStartJiraWithHooks() { // given val hooks = PreInstallHooks.default() + .also { Datasets.JiraSevenDataset.hookMysql(it.postStart) } val nodePlan = JiraNodePlan.Builder() .hooks(hooks) .installation( @@ -48,7 +48,7 @@ class JiraServerPlanIT { .hooks(hooks) .build() val instanceHooks = PreInstanceHooks.default() - .also { it.insert(DockerMysqlServer.Builder(infrastructure, Datasets.JiraSevenDataset.mysql).build()) } + .also { Datasets.JiraSevenDataset.hookMysql(it, infrastructure) } val jiraServerPlan = JiraServerPlan.Builder(infrastructure) .plan(nodePlan) .hooks(instanceHooks) From c89b0403d634b119e4907a33ab7066cb3aef59f9 Mon Sep 17 00:00:00 2001 From: Maciej Kwidzinski Date: Fri, 2 Apr 2021 15:26:35 +0200 Subject: [PATCH 16/52] Make `Infrastructure` `AutoCloseable` --- .../performance/tools/infrastructure/api/Infrastructure.kt | 3 +-- .../tools/infrastructure/api/DockerInfrastructure.kt | 4 ++-- .../infrastructure/api/jira/instance/JiraDataCenterPlanIT.kt | 2 +- .../infrastructure/api/jira/instance/JiraServerPlanIT.kt | 2 +- 4 files changed, 5 insertions(+), 6 deletions(-) diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/Infrastructure.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/Infrastructure.kt index 219d74e4..de596b36 100644 --- a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/Infrastructure.kt +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/Infrastructure.kt @@ -4,8 +4,7 @@ import com.atlassian.performance.tools.infrastructure.api.jira.install.TcpHost import com.atlassian.performance.tools.infrastructure.api.jira.node.JiraNode import com.atlassian.performance.tools.infrastructure.api.jira.node.JiraNodePlan -interface Infrastructure { +interface Infrastructure : AutoCloseable { fun serve(jiraNodePlans: List): List fun serve(port: Int, name: String): TcpHost - fun releaseResources() } \ No newline at end of file diff --git a/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/DockerInfrastructure.kt b/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/DockerInfrastructure.kt index 0e7bde15..34ea1563 100644 --- a/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/DockerInfrastructure.kt +++ b/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/DockerInfrastructure.kt @@ -26,7 +26,7 @@ internal class DockerInfrastructure : Infrastructure { it.setPrivilegedMode(true) }) val sshUbuntu = container.start() - allocatedResources.offer(sshUbuntu) + allocatedResources.add(sshUbuntu) return TcpHost( "localhost", port, @@ -35,7 +35,7 @@ internal class DockerInfrastructure : Infrastructure { ) } - override fun releaseResources() { + override fun close() { while (true) { allocatedResources .poll() diff --git a/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/JiraDataCenterPlanIT.kt b/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/JiraDataCenterPlanIT.kt index 21fc7f8c..903913e7 100644 --- a/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/JiraDataCenterPlanIT.kt +++ b/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/JiraDataCenterPlanIT.kt @@ -28,7 +28,7 @@ class JiraDataCenterPlanIT { @After fun tearDown() { - infrastructure.releaseResources() + infrastructure.close() } @Test diff --git a/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/JiraServerPlanIT.kt b/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/JiraServerPlanIT.kt index 584528d1..fdb80901 100644 --- a/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/JiraServerPlanIT.kt +++ b/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/JiraServerPlanIT.kt @@ -27,7 +27,7 @@ class JiraServerPlanIT { @After fun tearDown() { - infrastructure.releaseResources() + infrastructure.close() } @Test From 636525386ec56e47bea38592963e3701eb8e9dbe Mon Sep 17 00:00:00 2001 From: Maciej Kwidzinski Date: Fri, 9 Apr 2021 12:44:54 +0200 Subject: [PATCH 17/52] Fix MySQL unit tests --- .../infrastructure/api/database/LicenseOverridingMysqlTest.kt | 4 ++-- .../tools/infrastructure/database/SshMysqlClientTest.kt | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/database/LicenseOverridingMysqlTest.kt b/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/database/LicenseOverridingMysqlTest.kt index c41a4867..55d23970 100644 --- a/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/database/LicenseOverridingMysqlTest.kt +++ b/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/database/LicenseOverridingMysqlTest.kt @@ -117,13 +117,13 @@ class LicenseOverridingMysqlTest { private fun generateExpectedCommands(ssh: RememberingSshConnection): List { return listOf( // essentially, delete all existing licences - """mysql -h 127.0.0.1 -u root -e "DELETE FROM jiradb.productlicense;"""", + """mysql -h 127.0.0.1 -P 3306 -u root -e "DELETE FROM jiradb.productlicense;"""", // then import the new ones *ssh.uploads .map { listOf( - """mysql -h 127.0.0.1 -u root < ${it.remoteDestination}""", + """mysql -h 127.0.0.1 -P 3306 -u root < ${it.remoteDestination}""", """rm ${it.remoteDestination}""" ) } diff --git a/src/test/kotlin/com/atlassian/performance/tools/infrastructure/database/SshMysqlClientTest.kt b/src/test/kotlin/com/atlassian/performance/tools/infrastructure/database/SshMysqlClientTest.kt index 2773e309..92c9261d 100644 --- a/src/test/kotlin/com/atlassian/performance/tools/infrastructure/database/SshMysqlClientTest.kt +++ b/src/test/kotlin/com/atlassian/performance/tools/infrastructure/database/SshMysqlClientTest.kt @@ -20,7 +20,7 @@ class SshMysqlClientTest { assertThat(ssh.commands) .`as`("SSH commands") .containsExactly( - "mysql -h 127.0.0.1 -u root -e \"$command\"" + "mysql -h 127.0.0.1 -P 3306 -u root -e \"$command\"" ) } @@ -36,7 +36,7 @@ class SshMysqlClientTest { assertThat(ssh.commands) .`as`("SSH commands") .containsExactly( - "mysql -h 127.0.0.1 -u root < ${file.name}", + "mysql -h 127.0.0.1 -P 3306 -u root < ${file.name}", "rm ${file.name}" ) } From f460041fecde956eda860aafe97c7739057cc434 Mon Sep 17 00:00:00 2001 From: Maciej Kwidzinski Date: Fri, 9 Apr 2021 12:55:19 +0200 Subject: [PATCH 18/52] Update dependency locks --- build.gradle.kts | 2 +- .../apiDependenciesMetadata.lockfile | 28 +++++++++---------- .../compileClasspath.lockfile | 17 +++++++---- gradle/dependency-locks/default.lockfile | 28 +++++++++---------- ...mplementationDependenciesMetadata.lockfile | 28 +++++++++---------- .../runtimeClasspath.lockfile | 28 +++++++++---------- .../testApiDependenciesMetadata.lockfile | 28 ++++++++----------- gradle/dependency-locks/testCompile.lockfile | 28 ++++++++----------- .../testCompileClasspath.lockfile | 26 ++++++++--------- ...mplementationDependenciesMetadata.lockfile | 28 +++++++++---------- gradle/dependency-locks/testRuntime.lockfile | 28 ++++++++----------- .../testRuntimeClasspath.lockfile | 28 +++++++++---------- 12 files changed, 143 insertions(+), 154 deletions(-) diff --git a/build.gradle.kts b/build.gradle.kts index 433230df..c474b3e4 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -73,7 +73,7 @@ fun webdriver(): List = listOf( "selenium-support", "selenium-chrome-driver" ).map { module -> - "org.seleniumhq.selenium:$module:3.11.0" + "org.seleniumhq.selenium:$module:[3.11.0, 3.999.999]" } fun log4j( diff --git a/gradle/dependency-locks/apiDependenciesMetadata.lockfile b/gradle/dependency-locks/apiDependenciesMetadata.lockfile index 83ef09cb..595093c5 100644 --- a/gradle/dependency-locks/apiDependenciesMetadata.lockfile +++ b/gradle/dependency-locks/apiDependenciesMetadata.lockfile @@ -1,13 +1,15 @@ # This is a Gradle generated file for dependency locking. # Manual edits can break the build and are not advised. # This file is expected to be part of source control. +com.atlassian.data:random-data:1.4.3 com.atlassian.performance.tools:concurrency:1.1.0 com.atlassian.performance.tools:io:1.2.0 -com.atlassian.performance.tools:jira-actions:3.9.0 -com.atlassian.performance.tools:jira-software-actions:1.3.2 -com.atlassian.performance.tools:jvm-tasks:1.2.0 -com.atlassian.performance.tools:ssh:2.3.0 -com.atlassian.performance.tools:virtual-users:3.10.0 +com.atlassian.performance.tools:jira-actions:3.13.4 +com.atlassian.performance.tools:jira-software-actions:1.3.3 +com.atlassian.performance.tools:jvm-tasks:1.2.1 +com.atlassian.performance.tools:ssh:2.4.0 +com.atlassian.performance.tools:virtual-users:3.13.0 +com.atlassian.performance:selenium-js:1.0.1 com.github.stephenc.jcip:jcip-annotations:1.0-1 com.google.code.findbugs:jsr305:3.0.2 com.google.code.gson:gson:2.8.2 @@ -16,15 +18,15 @@ com.google.guava:guava:23.6-jre com.google.j2objc:j2objc-annotations:1.1 com.hierynomus:sshj:0.23.0 com.jcraft:jzlib:1.1.3 -com.squareup.okhttp3:okhttp:3.9.1 -com.squareup.okio:okio:1.13.0 +com.squareup.okhttp3:okhttp:3.11.0 +com.squareup.okio:okio:1.14.0 com.typesafe:config:1.2.1 commons-cli:commons-cli:1.4 commons-codec:commons-codec:1.10 commons-io:commons-io:2.5 commons-logging:commons-logging:1.2 io.github.bonigarcia:webdrivermanager:1.7.1 -net.bytebuddy:byte-buddy:1.7.9 +net.bytebuddy:byte-buddy:1.8.15 net.i2p.crypto:eddsa:0.2.0 org.apache.commons:commons-compress:1.9 org.apache.commons:commons-csv:1.3 @@ -45,14 +47,12 @@ org.glassfish:javax.json:1.1 org.jetbrains.kotlin:kotlin-stdlib-common:1.2.70 org.jetbrains.kotlin:kotlin-stdlib-jdk7:1.2.70 org.jetbrains.kotlin:kotlin-stdlib-jdk8:1.2.70 -org.jetbrains.kotlin:kotlin-stdlib-jre7:1.2.70 -org.jetbrains.kotlin:kotlin-stdlib-jre8:1.2.70 org.jetbrains.kotlin:kotlin-stdlib:1.2.70 org.jetbrains:annotations:15.0 org.jsoup:jsoup:1.10.2 org.rauschig:jarchivelib:0.7.1 -org.seleniumhq.selenium:selenium-api:3.11.0 -org.seleniumhq.selenium:selenium-chrome-driver:3.11.0 -org.seleniumhq.selenium:selenium-remote-driver:3.11.0 -org.seleniumhq.selenium:selenium-support:3.11.0 +org.seleniumhq.selenium:selenium-api:3.141.59 +org.seleniumhq.selenium:selenium-chrome-driver:3.141.59 +org.seleniumhq.selenium:selenium-remote-driver:3.141.59 +org.seleniumhq.selenium:selenium-support:3.141.59 org.slf4j:slf4j-api:1.8.0-alpha2 diff --git a/gradle/dependency-locks/compileClasspath.lockfile b/gradle/dependency-locks/compileClasspath.lockfile index 5fbd64b4..799e3f69 100644 --- a/gradle/dependency-locks/compileClasspath.lockfile +++ b/gradle/dependency-locks/compileClasspath.lockfile @@ -3,10 +3,11 @@ # This file is expected to be part of source control. com.atlassian.performance.tools:concurrency:1.1.0 com.atlassian.performance.tools:io:1.2.0 -com.atlassian.performance.tools:jira-actions:3.9.0 -com.atlassian.performance.tools:jvm-tasks:1.2.0 -com.atlassian.performance.tools:ssh:2.3.0 -com.atlassian.performance.tools:virtual-users:3.10.0 +com.atlassian.performance.tools:jira-actions:3.13.4 +com.atlassian.performance.tools:jvm-tasks:1.2.1 +com.atlassian.performance.tools:ssh:2.4.0 +com.atlassian.performance.tools:virtual-users:3.13.0 +com.atlassian.performance:selenium-js:1.0.1 com.github.stephenc.jcip:jcip-annotations:1.0-1 com.google.code.findbugs:jsr305:3.0.2 com.google.errorprone:error_prone_annotations:2.1.3 @@ -14,11 +15,15 @@ com.google.guava:guava:23.6-jre com.google.j2objc:j2objc-annotations:1.1 com.hierynomus:sshj:0.23.0 com.jcraft:jzlib:1.1.3 +com.squareup.okhttp3:okhttp:3.11.0 +com.squareup.okio:okio:1.14.0 commons-codec:commons-codec:1.10 commons-io:commons-io:2.5 commons-logging:commons-logging:1.2 javax.inject:javax.inject:1 +net.bytebuddy:byte-buddy:1.8.15 net.i2p.crypto:eddsa:0.2.0 +org.apache.commons:commons-exec:1.3 org.apache.commons:commons-lang3:3.5 org.apache.httpcomponents:httpclient:4.5.5 org.apache.httpcomponents:httpcore:4.4.9 @@ -62,7 +67,9 @@ org.jetbrains.kotlin:kotlin-stdlib-jdk8:1.2.70 org.jetbrains.kotlin:kotlin-stdlib:1.2.70 org.jetbrains:annotations:15.0 org.jsoup:jsoup:1.10.2 -org.seleniumhq.selenium:selenium-api:3.11.0 +org.seleniumhq.selenium:selenium-api:3.141.59 +org.seleniumhq.selenium:selenium-remote-driver:3.141.59 +org.seleniumhq.selenium:selenium-support:3.141.59 org.slf4j:slf4j-api:1.8.0-alpha2 org.sonatype.plexus:plexus-cipher:1.4 org.sonatype.plexus:plexus-sec-dispatcher:1.4 diff --git a/gradle/dependency-locks/default.lockfile b/gradle/dependency-locks/default.lockfile index c0f7c7ab..bc5713af 100644 --- a/gradle/dependency-locks/default.lockfile +++ b/gradle/dependency-locks/default.lockfile @@ -1,13 +1,15 @@ # This is a Gradle generated file for dependency locking. # Manual edits can break the build and are not advised. # This file is expected to be part of source control. +com.atlassian.data:random-data:1.4.3 com.atlassian.performance.tools:concurrency:1.1.0 com.atlassian.performance.tools:io:1.2.0 -com.atlassian.performance.tools:jira-actions:3.9.0 -com.atlassian.performance.tools:jira-software-actions:1.3.2 -com.atlassian.performance.tools:jvm-tasks:1.2.0 -com.atlassian.performance.tools:ssh:2.3.0 -com.atlassian.performance.tools:virtual-users:3.10.0 +com.atlassian.performance.tools:jira-actions:3.13.4 +com.atlassian.performance.tools:jira-software-actions:1.3.3 +com.atlassian.performance.tools:jvm-tasks:1.2.1 +com.atlassian.performance.tools:ssh:2.4.0 +com.atlassian.performance.tools:virtual-users:3.13.0 +com.atlassian.performance:selenium-js:1.0.1 com.github.stephenc.jcip:jcip-annotations:1.0-1 com.google.code.findbugs:jsr305:3.0.2 com.google.code.gson:gson:2.8.2 @@ -16,8 +18,8 @@ com.google.guava:guava:23.6-jre com.google.j2objc:j2objc-annotations:1.1 com.hierynomus:sshj:0.23.0 com.jcraft:jzlib:1.1.3 -com.squareup.okhttp3:okhttp:3.9.1 -com.squareup.okio:okio:1.13.0 +com.squareup.okhttp3:okhttp:3.11.0 +com.squareup.okio:okio:1.14.0 com.typesafe:config:1.2.1 commons-cli:commons-cli:1.4 commons-codec:commons-codec:1.10 @@ -25,7 +27,7 @@ commons-io:commons-io:2.5 commons-logging:commons-logging:1.2 io.github.bonigarcia:webdrivermanager:1.7.1 javax.inject:javax.inject:1 -net.bytebuddy:byte-buddy:1.7.9 +net.bytebuddy:byte-buddy:1.8.15 net.i2p.crypto:eddsa:0.2.0 org.apache.commons:commons-compress:1.9 org.apache.commons:commons-csv:1.3 @@ -72,16 +74,14 @@ org.jboss.shrinkwrap.resolver:shrinkwrap-resolver-spi:3.1.3 org.jetbrains.kotlin:kotlin-stdlib-common:1.2.70 org.jetbrains.kotlin:kotlin-stdlib-jdk7:1.2.70 org.jetbrains.kotlin:kotlin-stdlib-jdk8:1.2.70 -org.jetbrains.kotlin:kotlin-stdlib-jre7:1.2.70 -org.jetbrains.kotlin:kotlin-stdlib-jre8:1.2.70 org.jetbrains.kotlin:kotlin-stdlib:1.2.70 org.jetbrains:annotations:15.0 org.jsoup:jsoup:1.10.2 org.rauschig:jarchivelib:0.7.1 -org.seleniumhq.selenium:selenium-api:3.11.0 -org.seleniumhq.selenium:selenium-chrome-driver:3.11.0 -org.seleniumhq.selenium:selenium-remote-driver:3.11.0 -org.seleniumhq.selenium:selenium-support:3.11.0 +org.seleniumhq.selenium:selenium-api:3.141.59 +org.seleniumhq.selenium:selenium-chrome-driver:3.141.59 +org.seleniumhq.selenium:selenium-remote-driver:3.141.59 +org.seleniumhq.selenium:selenium-support:3.141.59 org.slf4j:slf4j-api:1.8.0-alpha2 org.sonatype.plexus:plexus-cipher:1.4 org.sonatype.plexus:plexus-sec-dispatcher:1.4 diff --git a/gradle/dependency-locks/implementationDependenciesMetadata.lockfile b/gradle/dependency-locks/implementationDependenciesMetadata.lockfile index c0f7c7ab..bc5713af 100644 --- a/gradle/dependency-locks/implementationDependenciesMetadata.lockfile +++ b/gradle/dependency-locks/implementationDependenciesMetadata.lockfile @@ -1,13 +1,15 @@ # This is a Gradle generated file for dependency locking. # Manual edits can break the build and are not advised. # This file is expected to be part of source control. +com.atlassian.data:random-data:1.4.3 com.atlassian.performance.tools:concurrency:1.1.0 com.atlassian.performance.tools:io:1.2.0 -com.atlassian.performance.tools:jira-actions:3.9.0 -com.atlassian.performance.tools:jira-software-actions:1.3.2 -com.atlassian.performance.tools:jvm-tasks:1.2.0 -com.atlassian.performance.tools:ssh:2.3.0 -com.atlassian.performance.tools:virtual-users:3.10.0 +com.atlassian.performance.tools:jira-actions:3.13.4 +com.atlassian.performance.tools:jira-software-actions:1.3.3 +com.atlassian.performance.tools:jvm-tasks:1.2.1 +com.atlassian.performance.tools:ssh:2.4.0 +com.atlassian.performance.tools:virtual-users:3.13.0 +com.atlassian.performance:selenium-js:1.0.1 com.github.stephenc.jcip:jcip-annotations:1.0-1 com.google.code.findbugs:jsr305:3.0.2 com.google.code.gson:gson:2.8.2 @@ -16,8 +18,8 @@ com.google.guava:guava:23.6-jre com.google.j2objc:j2objc-annotations:1.1 com.hierynomus:sshj:0.23.0 com.jcraft:jzlib:1.1.3 -com.squareup.okhttp3:okhttp:3.9.1 -com.squareup.okio:okio:1.13.0 +com.squareup.okhttp3:okhttp:3.11.0 +com.squareup.okio:okio:1.14.0 com.typesafe:config:1.2.1 commons-cli:commons-cli:1.4 commons-codec:commons-codec:1.10 @@ -25,7 +27,7 @@ commons-io:commons-io:2.5 commons-logging:commons-logging:1.2 io.github.bonigarcia:webdrivermanager:1.7.1 javax.inject:javax.inject:1 -net.bytebuddy:byte-buddy:1.7.9 +net.bytebuddy:byte-buddy:1.8.15 net.i2p.crypto:eddsa:0.2.0 org.apache.commons:commons-compress:1.9 org.apache.commons:commons-csv:1.3 @@ -72,16 +74,14 @@ org.jboss.shrinkwrap.resolver:shrinkwrap-resolver-spi:3.1.3 org.jetbrains.kotlin:kotlin-stdlib-common:1.2.70 org.jetbrains.kotlin:kotlin-stdlib-jdk7:1.2.70 org.jetbrains.kotlin:kotlin-stdlib-jdk8:1.2.70 -org.jetbrains.kotlin:kotlin-stdlib-jre7:1.2.70 -org.jetbrains.kotlin:kotlin-stdlib-jre8:1.2.70 org.jetbrains.kotlin:kotlin-stdlib:1.2.70 org.jetbrains:annotations:15.0 org.jsoup:jsoup:1.10.2 org.rauschig:jarchivelib:0.7.1 -org.seleniumhq.selenium:selenium-api:3.11.0 -org.seleniumhq.selenium:selenium-chrome-driver:3.11.0 -org.seleniumhq.selenium:selenium-remote-driver:3.11.0 -org.seleniumhq.selenium:selenium-support:3.11.0 +org.seleniumhq.selenium:selenium-api:3.141.59 +org.seleniumhq.selenium:selenium-chrome-driver:3.141.59 +org.seleniumhq.selenium:selenium-remote-driver:3.141.59 +org.seleniumhq.selenium:selenium-support:3.141.59 org.slf4j:slf4j-api:1.8.0-alpha2 org.sonatype.plexus:plexus-cipher:1.4 org.sonatype.plexus:plexus-sec-dispatcher:1.4 diff --git a/gradle/dependency-locks/runtimeClasspath.lockfile b/gradle/dependency-locks/runtimeClasspath.lockfile index c0f7c7ab..bc5713af 100644 --- a/gradle/dependency-locks/runtimeClasspath.lockfile +++ b/gradle/dependency-locks/runtimeClasspath.lockfile @@ -1,13 +1,15 @@ # This is a Gradle generated file for dependency locking. # Manual edits can break the build and are not advised. # This file is expected to be part of source control. +com.atlassian.data:random-data:1.4.3 com.atlassian.performance.tools:concurrency:1.1.0 com.atlassian.performance.tools:io:1.2.0 -com.atlassian.performance.tools:jira-actions:3.9.0 -com.atlassian.performance.tools:jira-software-actions:1.3.2 -com.atlassian.performance.tools:jvm-tasks:1.2.0 -com.atlassian.performance.tools:ssh:2.3.0 -com.atlassian.performance.tools:virtual-users:3.10.0 +com.atlassian.performance.tools:jira-actions:3.13.4 +com.atlassian.performance.tools:jira-software-actions:1.3.3 +com.atlassian.performance.tools:jvm-tasks:1.2.1 +com.atlassian.performance.tools:ssh:2.4.0 +com.atlassian.performance.tools:virtual-users:3.13.0 +com.atlassian.performance:selenium-js:1.0.1 com.github.stephenc.jcip:jcip-annotations:1.0-1 com.google.code.findbugs:jsr305:3.0.2 com.google.code.gson:gson:2.8.2 @@ -16,8 +18,8 @@ com.google.guava:guava:23.6-jre com.google.j2objc:j2objc-annotations:1.1 com.hierynomus:sshj:0.23.0 com.jcraft:jzlib:1.1.3 -com.squareup.okhttp3:okhttp:3.9.1 -com.squareup.okio:okio:1.13.0 +com.squareup.okhttp3:okhttp:3.11.0 +com.squareup.okio:okio:1.14.0 com.typesafe:config:1.2.1 commons-cli:commons-cli:1.4 commons-codec:commons-codec:1.10 @@ -25,7 +27,7 @@ commons-io:commons-io:2.5 commons-logging:commons-logging:1.2 io.github.bonigarcia:webdrivermanager:1.7.1 javax.inject:javax.inject:1 -net.bytebuddy:byte-buddy:1.7.9 +net.bytebuddy:byte-buddy:1.8.15 net.i2p.crypto:eddsa:0.2.0 org.apache.commons:commons-compress:1.9 org.apache.commons:commons-csv:1.3 @@ -72,16 +74,14 @@ org.jboss.shrinkwrap.resolver:shrinkwrap-resolver-spi:3.1.3 org.jetbrains.kotlin:kotlin-stdlib-common:1.2.70 org.jetbrains.kotlin:kotlin-stdlib-jdk7:1.2.70 org.jetbrains.kotlin:kotlin-stdlib-jdk8:1.2.70 -org.jetbrains.kotlin:kotlin-stdlib-jre7:1.2.70 -org.jetbrains.kotlin:kotlin-stdlib-jre8:1.2.70 org.jetbrains.kotlin:kotlin-stdlib:1.2.70 org.jetbrains:annotations:15.0 org.jsoup:jsoup:1.10.2 org.rauschig:jarchivelib:0.7.1 -org.seleniumhq.selenium:selenium-api:3.11.0 -org.seleniumhq.selenium:selenium-chrome-driver:3.11.0 -org.seleniumhq.selenium:selenium-remote-driver:3.11.0 -org.seleniumhq.selenium:selenium-support:3.11.0 +org.seleniumhq.selenium:selenium-api:3.141.59 +org.seleniumhq.selenium:selenium-chrome-driver:3.141.59 +org.seleniumhq.selenium:selenium-remote-driver:3.141.59 +org.seleniumhq.selenium:selenium-support:3.141.59 org.slf4j:slf4j-api:1.8.0-alpha2 org.sonatype.plexus:plexus-cipher:1.4 org.sonatype.plexus:plexus-sec-dispatcher:1.4 diff --git a/gradle/dependency-locks/testApiDependenciesMetadata.lockfile b/gradle/dependency-locks/testApiDependenciesMetadata.lockfile index 7f43998c..a63a77e9 100644 --- a/gradle/dependency-locks/testApiDependenciesMetadata.lockfile +++ b/gradle/dependency-locks/testApiDependenciesMetadata.lockfile @@ -2,34 +2,30 @@ # Manual edits can break the build and are not advised. # This file is expected to be part of source control. com.atlassian.performance.tools:concurrency:1.1.0 -com.atlassian.performance.tools:jira-actions:3.9.0 -com.atlassian.performance.tools:jira-software-actions:1.3.2 -com.atlassian.performance.tools:jvm-tasks:1.2.0 +com.atlassian.performance.tools:jira-actions:3.13.4 +com.atlassian.performance.tools:jira-software-actions:1.3.3 +com.atlassian.performance.tools:jvm-tasks:1.2.1 com.atlassian.performance.tools:ssh-ubuntu:0.2.0 +com.atlassian.performance:selenium-js:1.0.1 com.github.stephenc.jcip:jcip-annotations:1.0-1 com.google.code.findbugs:jsr305:3.0.2 -com.google.code.gson:gson:2.8.2 com.google.errorprone:error_prone_annotations:2.1.3 com.google.guava:guava:23.6-jre com.google.j2objc:j2objc-annotations:1.1 com.kohlschutter.junixsocket:junixsocket-common:2.0.4 com.kohlschutter.junixsocket:junixsocket-native-common:2.0.4 -com.squareup.okhttp3:okhttp:3.9.1 -com.squareup.okio:okio:1.13.0 -commons-codec:commons-codec:1.10 -commons-logging:commons-logging:1.2 +com.squareup.okhttp3:okhttp:3.11.0 +com.squareup.okio:okio:1.14.0 javax.activation:javax.activation-api:1.2.0 javax.annotation:javax.annotation-api:1.3.2 javax.xml.bind:jaxb-api:2.3.1 junit:junit:4.12 -net.bytebuddy:byte-buddy:1.7.9 +net.bytebuddy:byte-buddy:1.8.15 net.java.dev.jna:jna-platform:5.2.0 net.java.dev.jna:jna:5.2.0 org.apache.commons:commons-compress:1.9 org.apache.commons:commons-exec:1.3 org.apache.commons:commons-math3:3.6.1 -org.apache.httpcomponents:httpclient:4.5.5 -org.apache.httpcomponents:httpcore:4.4.9 org.apache.logging.log4j:log4j-api:2.10.0 org.apache.logging.log4j:log4j-core:2.10.0 org.apache.logging.log4j:log4j-jul:2.10.0 @@ -43,18 +39,16 @@ org.hamcrest:hamcrest-library:1.3 org.jetbrains.kotlin:kotlin-stdlib-common:1.2.70 org.jetbrains.kotlin:kotlin-stdlib-jdk7:1.2.70 org.jetbrains.kotlin:kotlin-stdlib-jdk8:1.2.70 -org.jetbrains.kotlin:kotlin-stdlib-jre7:1.2.70 -org.jetbrains.kotlin:kotlin-stdlib-jre8:1.2.70 org.jetbrains.kotlin:kotlin-stdlib:1.2.70 org.jetbrains:annotations:15.0 org.rnorth.duct-tape:duct-tape:1.0.7 org.rnorth.visible-assertions:visible-assertions:2.1.2 org.rnorth:tcp-unix-socket-proxy:1.0.2 org.scijava:native-lib-loader:2.0.2 -org.seleniumhq.selenium:selenium-api:3.11.0 -org.seleniumhq.selenium:selenium-chrome-driver:3.11.0 -org.seleniumhq.selenium:selenium-remote-driver:3.11.0 -org.seleniumhq.selenium:selenium-support:3.11.0 +org.seleniumhq.selenium:selenium-api:3.141.59 +org.seleniumhq.selenium:selenium-chrome-driver:3.141.59 +org.seleniumhq.selenium:selenium-remote-driver:3.141.59 +org.seleniumhq.selenium:selenium-support:3.141.59 org.slf4j:slf4j-api:1.8.0-alpha2 org.testcontainers:testcontainers:1.10.5 org.threeten:threeten-extra:1.5.0 diff --git a/gradle/dependency-locks/testCompile.lockfile b/gradle/dependency-locks/testCompile.lockfile index 7f43998c..a63a77e9 100644 --- a/gradle/dependency-locks/testCompile.lockfile +++ b/gradle/dependency-locks/testCompile.lockfile @@ -2,34 +2,30 @@ # Manual edits can break the build and are not advised. # This file is expected to be part of source control. com.atlassian.performance.tools:concurrency:1.1.0 -com.atlassian.performance.tools:jira-actions:3.9.0 -com.atlassian.performance.tools:jira-software-actions:1.3.2 -com.atlassian.performance.tools:jvm-tasks:1.2.0 +com.atlassian.performance.tools:jira-actions:3.13.4 +com.atlassian.performance.tools:jira-software-actions:1.3.3 +com.atlassian.performance.tools:jvm-tasks:1.2.1 com.atlassian.performance.tools:ssh-ubuntu:0.2.0 +com.atlassian.performance:selenium-js:1.0.1 com.github.stephenc.jcip:jcip-annotations:1.0-1 com.google.code.findbugs:jsr305:3.0.2 -com.google.code.gson:gson:2.8.2 com.google.errorprone:error_prone_annotations:2.1.3 com.google.guava:guava:23.6-jre com.google.j2objc:j2objc-annotations:1.1 com.kohlschutter.junixsocket:junixsocket-common:2.0.4 com.kohlschutter.junixsocket:junixsocket-native-common:2.0.4 -com.squareup.okhttp3:okhttp:3.9.1 -com.squareup.okio:okio:1.13.0 -commons-codec:commons-codec:1.10 -commons-logging:commons-logging:1.2 +com.squareup.okhttp3:okhttp:3.11.0 +com.squareup.okio:okio:1.14.0 javax.activation:javax.activation-api:1.2.0 javax.annotation:javax.annotation-api:1.3.2 javax.xml.bind:jaxb-api:2.3.1 junit:junit:4.12 -net.bytebuddy:byte-buddy:1.7.9 +net.bytebuddy:byte-buddy:1.8.15 net.java.dev.jna:jna-platform:5.2.0 net.java.dev.jna:jna:5.2.0 org.apache.commons:commons-compress:1.9 org.apache.commons:commons-exec:1.3 org.apache.commons:commons-math3:3.6.1 -org.apache.httpcomponents:httpclient:4.5.5 -org.apache.httpcomponents:httpcore:4.4.9 org.apache.logging.log4j:log4j-api:2.10.0 org.apache.logging.log4j:log4j-core:2.10.0 org.apache.logging.log4j:log4j-jul:2.10.0 @@ -43,18 +39,16 @@ org.hamcrest:hamcrest-library:1.3 org.jetbrains.kotlin:kotlin-stdlib-common:1.2.70 org.jetbrains.kotlin:kotlin-stdlib-jdk7:1.2.70 org.jetbrains.kotlin:kotlin-stdlib-jdk8:1.2.70 -org.jetbrains.kotlin:kotlin-stdlib-jre7:1.2.70 -org.jetbrains.kotlin:kotlin-stdlib-jre8:1.2.70 org.jetbrains.kotlin:kotlin-stdlib:1.2.70 org.jetbrains:annotations:15.0 org.rnorth.duct-tape:duct-tape:1.0.7 org.rnorth.visible-assertions:visible-assertions:2.1.2 org.rnorth:tcp-unix-socket-proxy:1.0.2 org.scijava:native-lib-loader:2.0.2 -org.seleniumhq.selenium:selenium-api:3.11.0 -org.seleniumhq.selenium:selenium-chrome-driver:3.11.0 -org.seleniumhq.selenium:selenium-remote-driver:3.11.0 -org.seleniumhq.selenium:selenium-support:3.11.0 +org.seleniumhq.selenium:selenium-api:3.141.59 +org.seleniumhq.selenium:selenium-chrome-driver:3.141.59 +org.seleniumhq.selenium:selenium-remote-driver:3.141.59 +org.seleniumhq.selenium:selenium-support:3.141.59 org.slf4j:slf4j-api:1.8.0-alpha2 org.testcontainers:testcontainers:1.10.5 org.threeten:threeten-extra:1.5.0 diff --git a/gradle/dependency-locks/testCompileClasspath.lockfile b/gradle/dependency-locks/testCompileClasspath.lockfile index 68e1a517..6984470b 100644 --- a/gradle/dependency-locks/testCompileClasspath.lockfile +++ b/gradle/dependency-locks/testCompileClasspath.lockfile @@ -3,15 +3,15 @@ # This file is expected to be part of source control. com.atlassian.performance.tools:concurrency:1.1.0 com.atlassian.performance.tools:io:1.2.0 -com.atlassian.performance.tools:jira-actions:3.9.0 -com.atlassian.performance.tools:jira-software-actions:1.3.2 -com.atlassian.performance.tools:jvm-tasks:1.2.0 +com.atlassian.performance.tools:jira-actions:3.13.4 +com.atlassian.performance.tools:jira-software-actions:1.3.3 +com.atlassian.performance.tools:jvm-tasks:1.2.1 com.atlassian.performance.tools:ssh-ubuntu:0.2.0 -com.atlassian.performance.tools:ssh:2.3.0 -com.atlassian.performance.tools:virtual-users:3.10.0 +com.atlassian.performance.tools:ssh:2.4.0 +com.atlassian.performance.tools:virtual-users:3.13.0 +com.atlassian.performance:selenium-js:1.0.1 com.github.stephenc.jcip:jcip-annotations:1.0-1 com.google.code.findbugs:jsr305:3.0.2 -com.google.code.gson:gson:2.8.2 com.google.errorprone:error_prone_annotations:2.1.3 com.google.guava:guava:23.6-jre com.google.j2objc:j2objc-annotations:1.1 @@ -19,8 +19,8 @@ com.hierynomus:sshj:0.23.0 com.jcraft:jzlib:1.1.3 com.kohlschutter.junixsocket:junixsocket-common:2.0.4 com.kohlschutter.junixsocket:junixsocket-native-common:2.0.4 -com.squareup.okhttp3:okhttp:3.9.1 -com.squareup.okio:okio:1.13.0 +com.squareup.okhttp3:okhttp:3.11.0 +com.squareup.okio:okio:1.14.0 commons-codec:commons-codec:1.10 commons-io:commons-io:2.5 commons-logging:commons-logging:1.2 @@ -29,7 +29,7 @@ javax.annotation:javax.annotation-api:1.3.2 javax.inject:javax.inject:1 javax.xml.bind:jaxb-api:2.3.1 junit:junit:4.12 -net.bytebuddy:byte-buddy:1.7.9 +net.bytebuddy:byte-buddy:1.8.15 net.i2p.crypto:eddsa:0.2.0 net.java.dev.jna:jna-platform:5.2.0 net.java.dev.jna:jna:5.2.0 @@ -85,10 +85,10 @@ org.rnorth.duct-tape:duct-tape:1.0.7 org.rnorth.visible-assertions:visible-assertions:2.1.2 org.rnorth:tcp-unix-socket-proxy:1.0.2 org.scijava:native-lib-loader:2.0.2 -org.seleniumhq.selenium:selenium-api:3.11.0 -org.seleniumhq.selenium:selenium-chrome-driver:3.11.0 -org.seleniumhq.selenium:selenium-remote-driver:3.11.0 -org.seleniumhq.selenium:selenium-support:3.11.0 +org.seleniumhq.selenium:selenium-api:3.141.59 +org.seleniumhq.selenium:selenium-chrome-driver:3.141.59 +org.seleniumhq.selenium:selenium-remote-driver:3.141.59 +org.seleniumhq.selenium:selenium-support:3.141.59 org.slf4j:slf4j-api:1.8.0-alpha2 org.sonatype.plexus:plexus-cipher:1.4 org.sonatype.plexus:plexus-sec-dispatcher:1.4 diff --git a/gradle/dependency-locks/testImplementationDependenciesMetadata.lockfile b/gradle/dependency-locks/testImplementationDependenciesMetadata.lockfile index 9a7bea8f..fed6f565 100644 --- a/gradle/dependency-locks/testImplementationDependenciesMetadata.lockfile +++ b/gradle/dependency-locks/testImplementationDependenciesMetadata.lockfile @@ -1,14 +1,16 @@ # This is a Gradle generated file for dependency locking. # Manual edits can break the build and are not advised. # This file is expected to be part of source control. +com.atlassian.data:random-data:1.4.3 com.atlassian.performance.tools:concurrency:1.1.0 com.atlassian.performance.tools:io:1.2.0 -com.atlassian.performance.tools:jira-actions:3.9.0 -com.atlassian.performance.tools:jira-software-actions:1.3.2 -com.atlassian.performance.tools:jvm-tasks:1.2.0 +com.atlassian.performance.tools:jira-actions:3.13.4 +com.atlassian.performance.tools:jira-software-actions:1.3.3 +com.atlassian.performance.tools:jvm-tasks:1.2.1 com.atlassian.performance.tools:ssh-ubuntu:0.2.0 -com.atlassian.performance.tools:ssh:2.3.0 -com.atlassian.performance.tools:virtual-users:3.10.0 +com.atlassian.performance.tools:ssh:2.4.0 +com.atlassian.performance.tools:virtual-users:3.13.0 +com.atlassian.performance:selenium-js:1.0.1 com.github.stephenc.jcip:jcip-annotations:1.0-1 com.google.code.findbugs:jsr305:3.0.2 com.google.code.gson:gson:2.8.2 @@ -19,8 +21,8 @@ com.hierynomus:sshj:0.23.0 com.jcraft:jzlib:1.1.3 com.kohlschutter.junixsocket:junixsocket-common:2.0.4 com.kohlschutter.junixsocket:junixsocket-native-common:2.0.4 -com.squareup.okhttp3:okhttp:3.9.1 -com.squareup.okio:okio:1.13.0 +com.squareup.okhttp3:okhttp:3.11.0 +com.squareup.okio:okio:1.14.0 com.typesafe:config:1.2.1 commons-cli:commons-cli:1.4 commons-codec:commons-codec:1.10 @@ -32,7 +34,7 @@ javax.annotation:javax.annotation-api:1.3.2 javax.inject:javax.inject:1 javax.xml.bind:jaxb-api:2.3.1 junit:junit:4.12 -net.bytebuddy:byte-buddy:1.7.9 +net.bytebuddy:byte-buddy:1.8.15 net.i2p.crypto:eddsa:0.2.0 net.java.dev.jna:jna-platform:5.2.0 net.java.dev.jna:jna:5.2.0 @@ -84,8 +86,6 @@ org.jboss.shrinkwrap.resolver:shrinkwrap-resolver-spi:3.1.3 org.jetbrains.kotlin:kotlin-stdlib-common:1.2.70 org.jetbrains.kotlin:kotlin-stdlib-jdk7:1.2.70 org.jetbrains.kotlin:kotlin-stdlib-jdk8:1.2.70 -org.jetbrains.kotlin:kotlin-stdlib-jre7:1.2.70 -org.jetbrains.kotlin:kotlin-stdlib-jre8:1.2.70 org.jetbrains.kotlin:kotlin-stdlib:1.2.70 org.jetbrains:annotations:15.0 org.jsoup:jsoup:1.10.2 @@ -94,10 +94,10 @@ org.rnorth.duct-tape:duct-tape:1.0.7 org.rnorth.visible-assertions:visible-assertions:2.1.2 org.rnorth:tcp-unix-socket-proxy:1.0.2 org.scijava:native-lib-loader:2.0.2 -org.seleniumhq.selenium:selenium-api:3.11.0 -org.seleniumhq.selenium:selenium-chrome-driver:3.11.0 -org.seleniumhq.selenium:selenium-remote-driver:3.11.0 -org.seleniumhq.selenium:selenium-support:3.11.0 +org.seleniumhq.selenium:selenium-api:3.141.59 +org.seleniumhq.selenium:selenium-chrome-driver:3.141.59 +org.seleniumhq.selenium:selenium-remote-driver:3.141.59 +org.seleniumhq.selenium:selenium-support:3.141.59 org.slf4j:slf4j-api:1.8.0-alpha2 org.sonatype.plexus:plexus-cipher:1.4 org.sonatype.plexus:plexus-sec-dispatcher:1.4 diff --git a/gradle/dependency-locks/testRuntime.lockfile b/gradle/dependency-locks/testRuntime.lockfile index 7f43998c..a63a77e9 100644 --- a/gradle/dependency-locks/testRuntime.lockfile +++ b/gradle/dependency-locks/testRuntime.lockfile @@ -2,34 +2,30 @@ # Manual edits can break the build and are not advised. # This file is expected to be part of source control. com.atlassian.performance.tools:concurrency:1.1.0 -com.atlassian.performance.tools:jira-actions:3.9.0 -com.atlassian.performance.tools:jira-software-actions:1.3.2 -com.atlassian.performance.tools:jvm-tasks:1.2.0 +com.atlassian.performance.tools:jira-actions:3.13.4 +com.atlassian.performance.tools:jira-software-actions:1.3.3 +com.atlassian.performance.tools:jvm-tasks:1.2.1 com.atlassian.performance.tools:ssh-ubuntu:0.2.0 +com.atlassian.performance:selenium-js:1.0.1 com.github.stephenc.jcip:jcip-annotations:1.0-1 com.google.code.findbugs:jsr305:3.0.2 -com.google.code.gson:gson:2.8.2 com.google.errorprone:error_prone_annotations:2.1.3 com.google.guava:guava:23.6-jre com.google.j2objc:j2objc-annotations:1.1 com.kohlschutter.junixsocket:junixsocket-common:2.0.4 com.kohlschutter.junixsocket:junixsocket-native-common:2.0.4 -com.squareup.okhttp3:okhttp:3.9.1 -com.squareup.okio:okio:1.13.0 -commons-codec:commons-codec:1.10 -commons-logging:commons-logging:1.2 +com.squareup.okhttp3:okhttp:3.11.0 +com.squareup.okio:okio:1.14.0 javax.activation:javax.activation-api:1.2.0 javax.annotation:javax.annotation-api:1.3.2 javax.xml.bind:jaxb-api:2.3.1 junit:junit:4.12 -net.bytebuddy:byte-buddy:1.7.9 +net.bytebuddy:byte-buddy:1.8.15 net.java.dev.jna:jna-platform:5.2.0 net.java.dev.jna:jna:5.2.0 org.apache.commons:commons-compress:1.9 org.apache.commons:commons-exec:1.3 org.apache.commons:commons-math3:3.6.1 -org.apache.httpcomponents:httpclient:4.5.5 -org.apache.httpcomponents:httpcore:4.4.9 org.apache.logging.log4j:log4j-api:2.10.0 org.apache.logging.log4j:log4j-core:2.10.0 org.apache.logging.log4j:log4j-jul:2.10.0 @@ -43,18 +39,16 @@ org.hamcrest:hamcrest-library:1.3 org.jetbrains.kotlin:kotlin-stdlib-common:1.2.70 org.jetbrains.kotlin:kotlin-stdlib-jdk7:1.2.70 org.jetbrains.kotlin:kotlin-stdlib-jdk8:1.2.70 -org.jetbrains.kotlin:kotlin-stdlib-jre7:1.2.70 -org.jetbrains.kotlin:kotlin-stdlib-jre8:1.2.70 org.jetbrains.kotlin:kotlin-stdlib:1.2.70 org.jetbrains:annotations:15.0 org.rnorth.duct-tape:duct-tape:1.0.7 org.rnorth.visible-assertions:visible-assertions:2.1.2 org.rnorth:tcp-unix-socket-proxy:1.0.2 org.scijava:native-lib-loader:2.0.2 -org.seleniumhq.selenium:selenium-api:3.11.0 -org.seleniumhq.selenium:selenium-chrome-driver:3.11.0 -org.seleniumhq.selenium:selenium-remote-driver:3.11.0 -org.seleniumhq.selenium:selenium-support:3.11.0 +org.seleniumhq.selenium:selenium-api:3.141.59 +org.seleniumhq.selenium:selenium-chrome-driver:3.141.59 +org.seleniumhq.selenium:selenium-remote-driver:3.141.59 +org.seleniumhq.selenium:selenium-support:3.141.59 org.slf4j:slf4j-api:1.8.0-alpha2 org.testcontainers:testcontainers:1.10.5 org.threeten:threeten-extra:1.5.0 diff --git a/gradle/dependency-locks/testRuntimeClasspath.lockfile b/gradle/dependency-locks/testRuntimeClasspath.lockfile index 9a7bea8f..fed6f565 100644 --- a/gradle/dependency-locks/testRuntimeClasspath.lockfile +++ b/gradle/dependency-locks/testRuntimeClasspath.lockfile @@ -1,14 +1,16 @@ # This is a Gradle generated file for dependency locking. # Manual edits can break the build and are not advised. # This file is expected to be part of source control. +com.atlassian.data:random-data:1.4.3 com.atlassian.performance.tools:concurrency:1.1.0 com.atlassian.performance.tools:io:1.2.0 -com.atlassian.performance.tools:jira-actions:3.9.0 -com.atlassian.performance.tools:jira-software-actions:1.3.2 -com.atlassian.performance.tools:jvm-tasks:1.2.0 +com.atlassian.performance.tools:jira-actions:3.13.4 +com.atlassian.performance.tools:jira-software-actions:1.3.3 +com.atlassian.performance.tools:jvm-tasks:1.2.1 com.atlassian.performance.tools:ssh-ubuntu:0.2.0 -com.atlassian.performance.tools:ssh:2.3.0 -com.atlassian.performance.tools:virtual-users:3.10.0 +com.atlassian.performance.tools:ssh:2.4.0 +com.atlassian.performance.tools:virtual-users:3.13.0 +com.atlassian.performance:selenium-js:1.0.1 com.github.stephenc.jcip:jcip-annotations:1.0-1 com.google.code.findbugs:jsr305:3.0.2 com.google.code.gson:gson:2.8.2 @@ -19,8 +21,8 @@ com.hierynomus:sshj:0.23.0 com.jcraft:jzlib:1.1.3 com.kohlschutter.junixsocket:junixsocket-common:2.0.4 com.kohlschutter.junixsocket:junixsocket-native-common:2.0.4 -com.squareup.okhttp3:okhttp:3.9.1 -com.squareup.okio:okio:1.13.0 +com.squareup.okhttp3:okhttp:3.11.0 +com.squareup.okio:okio:1.14.0 com.typesafe:config:1.2.1 commons-cli:commons-cli:1.4 commons-codec:commons-codec:1.10 @@ -32,7 +34,7 @@ javax.annotation:javax.annotation-api:1.3.2 javax.inject:javax.inject:1 javax.xml.bind:jaxb-api:2.3.1 junit:junit:4.12 -net.bytebuddy:byte-buddy:1.7.9 +net.bytebuddy:byte-buddy:1.8.15 net.i2p.crypto:eddsa:0.2.0 net.java.dev.jna:jna-platform:5.2.0 net.java.dev.jna:jna:5.2.0 @@ -84,8 +86,6 @@ org.jboss.shrinkwrap.resolver:shrinkwrap-resolver-spi:3.1.3 org.jetbrains.kotlin:kotlin-stdlib-common:1.2.70 org.jetbrains.kotlin:kotlin-stdlib-jdk7:1.2.70 org.jetbrains.kotlin:kotlin-stdlib-jdk8:1.2.70 -org.jetbrains.kotlin:kotlin-stdlib-jre7:1.2.70 -org.jetbrains.kotlin:kotlin-stdlib-jre8:1.2.70 org.jetbrains.kotlin:kotlin-stdlib:1.2.70 org.jetbrains:annotations:15.0 org.jsoup:jsoup:1.10.2 @@ -94,10 +94,10 @@ org.rnorth.duct-tape:duct-tape:1.0.7 org.rnorth.visible-assertions:visible-assertions:2.1.2 org.rnorth:tcp-unix-socket-proxy:1.0.2 org.scijava:native-lib-loader:2.0.2 -org.seleniumhq.selenium:selenium-api:3.11.0 -org.seleniumhq.selenium:selenium-chrome-driver:3.11.0 -org.seleniumhq.selenium:selenium-remote-driver:3.11.0 -org.seleniumhq.selenium:selenium-support:3.11.0 +org.seleniumhq.selenium:selenium-api:3.141.59 +org.seleniumhq.selenium:selenium-chrome-driver:3.141.59 +org.seleniumhq.selenium:selenium-remote-driver:3.141.59 +org.seleniumhq.selenium:selenium-support:3.141.59 org.slf4j:slf4j-api:1.8.0-alpha2 org.sonatype.plexus:plexus-cipher:1.4 org.sonatype.plexus:plexus-sec-dispatcher:1.4 From 0b74e8e6456fdace93142c0c938d9cc16bb19632 Mon Sep 17 00:00:00 2001 From: Maciej Kwidzinski Date: Fri, 9 Apr 2021 17:13:24 +0200 Subject: [PATCH 19/52] Test on `docker-java` --- build.gradle.kts | 6 +- .../apiDependenciesMetadata.lockfile | 6 +- .../compileClasspath.lockfile | 6 +- gradle/dependency-locks/default.lockfile | 6 +- ...mplementationDependenciesMetadata.lockfile | 6 +- .../runtimeClasspath.lockfile | 6 +- .../testApiDependenciesMetadata.lockfile | 24 +-- gradle/dependency-locks/testCompile.lockfile | 24 +-- .../testCompileClasspath.lockfile | 27 ++-- ...mplementationDependenciesMetadata.lockfile | 27 ++-- gradle/dependency-locks/testRuntime.lockfile | 24 +-- .../testRuntimeClasspath.lockfile | 27 ++-- .../api/database/DockerMysqlServer.kt | 2 +- .../api/database/DockerPostgresServer.kt | 2 +- .../api/jira/install/TcpHost.kt | 3 +- .../api/jira/install/hook/JvmConfig.kt | 2 +- .../api/jira/instance/JiraServerPlan.kt | 2 +- .../infrastructure/SshUbuntuExtensions.kt | 21 --- .../api/DockerInfrastructure.kt | 138 +++++++++++++++--- .../infrastructure/api/browser/ChromeIT.kt | 8 +- .../api/browser/chromium/Chromium69IT.kt | 7 +- .../chromium/PageLoadTimeoutRecoveryTest.kt | 7 +- .../api/dataset/HttpDatasetPackageIT.kt | 7 +- .../PublicJiraServiceDeskDistributionIT.kt | 7 +- .../PublicJiraSoftwareDistributionsIT.kt | 7 +- .../infrastructure/api/docker/DockerIT.kt | 18 +-- .../jira/install/hook/PreInstallHooksTest.kt | 2 +- .../api/jvm/AdoptOpenJdk11IT.kt | 7 +- .../infrastructure/api/jvm/AdoptOpenJdkIT.kt | 7 +- .../infrastructure/api/jvm/JstatSupport.kt | 3 +- .../infrastructure/api/jvm/OpenJdk11IT.kt | 7 +- .../tools/infrastructure/api/jvm/OpenJdkIT.kt | 7 +- .../infrastructure/api/jvm/OracleJdkIT.kt | 7 +- .../tools/infrastructure/api/os/UbuntuIT.kt | 29 ++-- .../lib/docker/ConnectedContainer.kt | 30 ++++ .../lib/docker/CreatedContainer.kt | 18 +++ .../lib/docker/DockerNetwork.kt | 18 +++ .../lib/docker/StartedContainer.kt | 23 +++ 38 files changed, 361 insertions(+), 217 deletions(-) delete mode 100644 src/test/kotlin/com/atlassian/performance/tools/infrastructure/SshUbuntuExtensions.kt create mode 100644 src/test/kotlin/com/atlassian/performance/tools/infrastructure/lib/docker/ConnectedContainer.kt create mode 100644 src/test/kotlin/com/atlassian/performance/tools/infrastructure/lib/docker/CreatedContainer.kt create mode 100644 src/test/kotlin/com/atlassian/performance/tools/infrastructure/lib/docker/DockerNetwork.kt create mode 100644 src/test/kotlin/com/atlassian/performance/tools/infrastructure/lib/docker/StartedContainer.kt diff --git a/build.gradle.kts b/build.gradle.kts index c474b3e4..1d3d01b3 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -32,6 +32,9 @@ configurations.all { "org.jetbrains:annotations" -> useVersion("15.0") "com.google.code.findbugs:jsr305" -> useVersion("3.0.2") "org.apache.commons:commons-compress" -> useVersion("1.9") + "commons-io:commons-io" -> useVersion("2.6") + "org.bouncycastle:bcpkix-jdk15on" -> useVersion("1.64") + "org.bouncycastle:bcprov-jdk15on" -> useVersion("1.64") } when (requested.group) { "org.jetbrains.kotlin" -> useVersion(kotlinVersion) @@ -64,9 +67,10 @@ dependencies { testCompile("com.atlassian.performance.tools:jira-software-actions:[1.0.0,2.0.0)") testCompile("org.hamcrest:hamcrest-library:1.3") testCompile("org.assertj:assertj-core:3.11.1") - testCompile("com.atlassian.performance.tools:ssh-ubuntu:0.2.0") testCompile("org.rnorth.duct-tape:duct-tape:1.0.7") testCompile("org.threeten:threeten-extra:1.5.0") + testCompile("com.github.docker-java:docker-java-core:[3.2.5, 4.0.0)") + testCompile("com.github.docker-java:docker-java-transport-zerodep:[3.2.5, 4.0.0)") } fun webdriver(): List = listOf( diff --git a/gradle/dependency-locks/apiDependenciesMetadata.lockfile b/gradle/dependency-locks/apiDependenciesMetadata.lockfile index 595093c5..f01abbfd 100644 --- a/gradle/dependency-locks/apiDependenciesMetadata.lockfile +++ b/gradle/dependency-locks/apiDependenciesMetadata.lockfile @@ -23,7 +23,7 @@ com.squareup.okio:okio:1.14.0 com.typesafe:config:1.2.1 commons-cli:commons-cli:1.4 commons-codec:commons-codec:1.10 -commons-io:commons-io:2.5 +commons-io:commons-io:2.6 commons-logging:commons-logging:1.2 io.github.bonigarcia:webdrivermanager:1.7.1 net.bytebuddy:byte-buddy:1.8.15 @@ -39,8 +39,8 @@ org.apache.logging.log4j:log4j-api:2.10.0 org.apache.logging.log4j:log4j-core:2.10.0 org.apache.logging.log4j:log4j-jul:2.10.0 org.apache.logging.log4j:log4j-slf4j-impl:2.10.0 -org.bouncycastle:bcpkix-jdk15on:1.56 -org.bouncycastle:bcprov-jdk15on:1.56 +org.bouncycastle:bcpkix-jdk15on:1.64 +org.bouncycastle:bcprov-jdk15on:1.64 org.checkerframework:checker-compat-qual:2.0.0 org.codehaus.mojo:animal-sniffer-annotations:1.14 org.glassfish:javax.json:1.1 diff --git a/gradle/dependency-locks/compileClasspath.lockfile b/gradle/dependency-locks/compileClasspath.lockfile index 799e3f69..ac10f0b8 100644 --- a/gradle/dependency-locks/compileClasspath.lockfile +++ b/gradle/dependency-locks/compileClasspath.lockfile @@ -18,7 +18,7 @@ com.jcraft:jzlib:1.1.3 com.squareup.okhttp3:okhttp:3.11.0 com.squareup.okio:okio:1.14.0 commons-codec:commons-codec:1.10 -commons-io:commons-io:2.5 +commons-io:commons-io:2.6 commons-logging:commons-logging:1.2 javax.inject:javax.inject:1 net.bytebuddy:byte-buddy:1.8.15 @@ -48,8 +48,8 @@ org.apache.maven:maven-repository-metadata:3.5.2 org.apache.maven:maven-resolver-provider:3.5.2 org.apache.maven:maven-settings-builder:3.5.2 org.apache.maven:maven-settings:3.5.2 -org.bouncycastle:bcpkix-jdk15on:1.56 -org.bouncycastle:bcprov-jdk15on:1.56 +org.bouncycastle:bcpkix-jdk15on:1.64 +org.bouncycastle:bcprov-jdk15on:1.64 org.checkerframework:checker-compat-qual:2.0.0 org.codehaus.mojo:animal-sniffer-annotations:1.14 org.codehaus.plexus:plexus-component-annotations:1.7.1 diff --git a/gradle/dependency-locks/default.lockfile b/gradle/dependency-locks/default.lockfile index bc5713af..4674c635 100644 --- a/gradle/dependency-locks/default.lockfile +++ b/gradle/dependency-locks/default.lockfile @@ -23,7 +23,7 @@ com.squareup.okio:okio:1.14.0 com.typesafe:config:1.2.1 commons-cli:commons-cli:1.4 commons-codec:commons-codec:1.10 -commons-io:commons-io:2.5 +commons-io:commons-io:2.6 commons-logging:commons-logging:1.2 io.github.bonigarcia:webdrivermanager:1.7.1 javax.inject:javax.inject:1 @@ -58,8 +58,8 @@ org.apache.maven:maven-repository-metadata:3.5.2 org.apache.maven:maven-resolver-provider:3.5.2 org.apache.maven:maven-settings-builder:3.5.2 org.apache.maven:maven-settings:3.5.2 -org.bouncycastle:bcpkix-jdk15on:1.56 -org.bouncycastle:bcprov-jdk15on:1.56 +org.bouncycastle:bcpkix-jdk15on:1.64 +org.bouncycastle:bcprov-jdk15on:1.64 org.checkerframework:checker-compat-qual:2.0.0 org.codehaus.mojo:animal-sniffer-annotations:1.14 org.codehaus.plexus:plexus-component-annotations:1.7.1 diff --git a/gradle/dependency-locks/implementationDependenciesMetadata.lockfile b/gradle/dependency-locks/implementationDependenciesMetadata.lockfile index bc5713af..4674c635 100644 --- a/gradle/dependency-locks/implementationDependenciesMetadata.lockfile +++ b/gradle/dependency-locks/implementationDependenciesMetadata.lockfile @@ -23,7 +23,7 @@ com.squareup.okio:okio:1.14.0 com.typesafe:config:1.2.1 commons-cli:commons-cli:1.4 commons-codec:commons-codec:1.10 -commons-io:commons-io:2.5 +commons-io:commons-io:2.6 commons-logging:commons-logging:1.2 io.github.bonigarcia:webdrivermanager:1.7.1 javax.inject:javax.inject:1 @@ -58,8 +58,8 @@ org.apache.maven:maven-repository-metadata:3.5.2 org.apache.maven:maven-resolver-provider:3.5.2 org.apache.maven:maven-settings-builder:3.5.2 org.apache.maven:maven-settings:3.5.2 -org.bouncycastle:bcpkix-jdk15on:1.56 -org.bouncycastle:bcprov-jdk15on:1.56 +org.bouncycastle:bcpkix-jdk15on:1.64 +org.bouncycastle:bcprov-jdk15on:1.64 org.checkerframework:checker-compat-qual:2.0.0 org.codehaus.mojo:animal-sniffer-annotations:1.14 org.codehaus.plexus:plexus-component-annotations:1.7.1 diff --git a/gradle/dependency-locks/runtimeClasspath.lockfile b/gradle/dependency-locks/runtimeClasspath.lockfile index bc5713af..4674c635 100644 --- a/gradle/dependency-locks/runtimeClasspath.lockfile +++ b/gradle/dependency-locks/runtimeClasspath.lockfile @@ -23,7 +23,7 @@ com.squareup.okio:okio:1.14.0 com.typesafe:config:1.2.1 commons-cli:commons-cli:1.4 commons-codec:commons-codec:1.10 -commons-io:commons-io:2.5 +commons-io:commons-io:2.6 commons-logging:commons-logging:1.2 io.github.bonigarcia:webdrivermanager:1.7.1 javax.inject:javax.inject:1 @@ -58,8 +58,8 @@ org.apache.maven:maven-repository-metadata:3.5.2 org.apache.maven:maven-resolver-provider:3.5.2 org.apache.maven:maven-settings-builder:3.5.2 org.apache.maven:maven-settings:3.5.2 -org.bouncycastle:bcpkix-jdk15on:1.56 -org.bouncycastle:bcprov-jdk15on:1.56 +org.bouncycastle:bcpkix-jdk15on:1.64 +org.bouncycastle:bcprov-jdk15on:1.64 org.checkerframework:checker-compat-qual:2.0.0 org.codehaus.mojo:animal-sniffer-annotations:1.14 org.codehaus.plexus:plexus-component-annotations:1.7.1 diff --git a/gradle/dependency-locks/testApiDependenciesMetadata.lockfile b/gradle/dependency-locks/testApiDependenciesMetadata.lockfile index a63a77e9..e560bcfb 100644 --- a/gradle/dependency-locks/testApiDependenciesMetadata.lockfile +++ b/gradle/dependency-locks/testApiDependenciesMetadata.lockfile @@ -5,24 +5,26 @@ com.atlassian.performance.tools:concurrency:1.1.0 com.atlassian.performance.tools:jira-actions:3.13.4 com.atlassian.performance.tools:jira-software-actions:1.3.3 com.atlassian.performance.tools:jvm-tasks:1.2.1 -com.atlassian.performance.tools:ssh-ubuntu:0.2.0 com.atlassian.performance:selenium-js:1.0.1 +com.fasterxml.jackson.core:jackson-annotations:2.10.3 +com.fasterxml.jackson.core:jackson-core:2.10.3 +com.fasterxml.jackson.core:jackson-databind:2.10.3 +com.github.docker-java:docker-java-api:3.2.7 +com.github.docker-java:docker-java-core:3.2.7 +com.github.docker-java:docker-java-transport-zerodep:3.2.7 +com.github.docker-java:docker-java-transport:3.2.7 com.github.stephenc.jcip:jcip-annotations:1.0-1 com.google.code.findbugs:jsr305:3.0.2 com.google.errorprone:error_prone_annotations:2.1.3 com.google.guava:guava:23.6-jre com.google.j2objc:j2objc-annotations:1.1 -com.kohlschutter.junixsocket:junixsocket-common:2.0.4 -com.kohlschutter.junixsocket:junixsocket-native-common:2.0.4 com.squareup.okhttp3:okhttp:3.11.0 com.squareup.okio:okio:1.14.0 -javax.activation:javax.activation-api:1.2.0 -javax.annotation:javax.annotation-api:1.3.2 -javax.xml.bind:jaxb-api:2.3.1 +commons-io:commons-io:2.6 +commons-lang:commons-lang:2.6 junit:junit:4.12 net.bytebuddy:byte-buddy:1.8.15 -net.java.dev.jna:jna-platform:5.2.0 -net.java.dev.jna:jna:5.2.0 +net.java.dev.jna:jna:5.5.0 org.apache.commons:commons-compress:1.9 org.apache.commons:commons-exec:1.3 org.apache.commons:commons-math3:3.6.1 @@ -31,6 +33,8 @@ org.apache.logging.log4j:log4j-core:2.10.0 org.apache.logging.log4j:log4j-jul:2.10.0 org.apache.logging.log4j:log4j-slf4j-impl:2.10.0 org.assertj:assertj-core:3.11.1 +org.bouncycastle:bcpkix-jdk15on:1.64 +org.bouncycastle:bcprov-jdk15on:1.64 org.checkerframework:checker-compat-qual:2.0.0 org.codehaus.mojo:animal-sniffer-annotations:1.14 org.glassfish:javax.json:1.1 @@ -42,13 +46,9 @@ org.jetbrains.kotlin:kotlin-stdlib-jdk8:1.2.70 org.jetbrains.kotlin:kotlin-stdlib:1.2.70 org.jetbrains:annotations:15.0 org.rnorth.duct-tape:duct-tape:1.0.7 -org.rnorth.visible-assertions:visible-assertions:2.1.2 -org.rnorth:tcp-unix-socket-proxy:1.0.2 -org.scijava:native-lib-loader:2.0.2 org.seleniumhq.selenium:selenium-api:3.141.59 org.seleniumhq.selenium:selenium-chrome-driver:3.141.59 org.seleniumhq.selenium:selenium-remote-driver:3.141.59 org.seleniumhq.selenium:selenium-support:3.141.59 org.slf4j:slf4j-api:1.8.0-alpha2 -org.testcontainers:testcontainers:1.10.5 org.threeten:threeten-extra:1.5.0 diff --git a/gradle/dependency-locks/testCompile.lockfile b/gradle/dependency-locks/testCompile.lockfile index a63a77e9..e560bcfb 100644 --- a/gradle/dependency-locks/testCompile.lockfile +++ b/gradle/dependency-locks/testCompile.lockfile @@ -5,24 +5,26 @@ com.atlassian.performance.tools:concurrency:1.1.0 com.atlassian.performance.tools:jira-actions:3.13.4 com.atlassian.performance.tools:jira-software-actions:1.3.3 com.atlassian.performance.tools:jvm-tasks:1.2.1 -com.atlassian.performance.tools:ssh-ubuntu:0.2.0 com.atlassian.performance:selenium-js:1.0.1 +com.fasterxml.jackson.core:jackson-annotations:2.10.3 +com.fasterxml.jackson.core:jackson-core:2.10.3 +com.fasterxml.jackson.core:jackson-databind:2.10.3 +com.github.docker-java:docker-java-api:3.2.7 +com.github.docker-java:docker-java-core:3.2.7 +com.github.docker-java:docker-java-transport-zerodep:3.2.7 +com.github.docker-java:docker-java-transport:3.2.7 com.github.stephenc.jcip:jcip-annotations:1.0-1 com.google.code.findbugs:jsr305:3.0.2 com.google.errorprone:error_prone_annotations:2.1.3 com.google.guava:guava:23.6-jre com.google.j2objc:j2objc-annotations:1.1 -com.kohlschutter.junixsocket:junixsocket-common:2.0.4 -com.kohlschutter.junixsocket:junixsocket-native-common:2.0.4 com.squareup.okhttp3:okhttp:3.11.0 com.squareup.okio:okio:1.14.0 -javax.activation:javax.activation-api:1.2.0 -javax.annotation:javax.annotation-api:1.3.2 -javax.xml.bind:jaxb-api:2.3.1 +commons-io:commons-io:2.6 +commons-lang:commons-lang:2.6 junit:junit:4.12 net.bytebuddy:byte-buddy:1.8.15 -net.java.dev.jna:jna-platform:5.2.0 -net.java.dev.jna:jna:5.2.0 +net.java.dev.jna:jna:5.5.0 org.apache.commons:commons-compress:1.9 org.apache.commons:commons-exec:1.3 org.apache.commons:commons-math3:3.6.1 @@ -31,6 +33,8 @@ org.apache.logging.log4j:log4j-core:2.10.0 org.apache.logging.log4j:log4j-jul:2.10.0 org.apache.logging.log4j:log4j-slf4j-impl:2.10.0 org.assertj:assertj-core:3.11.1 +org.bouncycastle:bcpkix-jdk15on:1.64 +org.bouncycastle:bcprov-jdk15on:1.64 org.checkerframework:checker-compat-qual:2.0.0 org.codehaus.mojo:animal-sniffer-annotations:1.14 org.glassfish:javax.json:1.1 @@ -42,13 +46,9 @@ org.jetbrains.kotlin:kotlin-stdlib-jdk8:1.2.70 org.jetbrains.kotlin:kotlin-stdlib:1.2.70 org.jetbrains:annotations:15.0 org.rnorth.duct-tape:duct-tape:1.0.7 -org.rnorth.visible-assertions:visible-assertions:2.1.2 -org.rnorth:tcp-unix-socket-proxy:1.0.2 -org.scijava:native-lib-loader:2.0.2 org.seleniumhq.selenium:selenium-api:3.141.59 org.seleniumhq.selenium:selenium-chrome-driver:3.141.59 org.seleniumhq.selenium:selenium-remote-driver:3.141.59 org.seleniumhq.selenium:selenium-support:3.141.59 org.slf4j:slf4j-api:1.8.0-alpha2 -org.testcontainers:testcontainers:1.10.5 org.threeten:threeten-extra:1.5.0 diff --git a/gradle/dependency-locks/testCompileClasspath.lockfile b/gradle/dependency-locks/testCompileClasspath.lockfile index 6984470b..6cf5d0a9 100644 --- a/gradle/dependency-locks/testCompileClasspath.lockfile +++ b/gradle/dependency-locks/testCompileClasspath.lockfile @@ -6,10 +6,16 @@ com.atlassian.performance.tools:io:1.2.0 com.atlassian.performance.tools:jira-actions:3.13.4 com.atlassian.performance.tools:jira-software-actions:1.3.3 com.atlassian.performance.tools:jvm-tasks:1.2.1 -com.atlassian.performance.tools:ssh-ubuntu:0.2.0 com.atlassian.performance.tools:ssh:2.4.0 com.atlassian.performance.tools:virtual-users:3.13.0 com.atlassian.performance:selenium-js:1.0.1 +com.fasterxml.jackson.core:jackson-annotations:2.10.3 +com.fasterxml.jackson.core:jackson-core:2.10.3 +com.fasterxml.jackson.core:jackson-databind:2.10.3 +com.github.docker-java:docker-java-api:3.2.7 +com.github.docker-java:docker-java-core:3.2.7 +com.github.docker-java:docker-java-transport-zerodep:3.2.7 +com.github.docker-java:docker-java-transport:3.2.7 com.github.stephenc.jcip:jcip-annotations:1.0-1 com.google.code.findbugs:jsr305:3.0.2 com.google.errorprone:error_prone_annotations:2.1.3 @@ -17,22 +23,17 @@ com.google.guava:guava:23.6-jre com.google.j2objc:j2objc-annotations:1.1 com.hierynomus:sshj:0.23.0 com.jcraft:jzlib:1.1.3 -com.kohlschutter.junixsocket:junixsocket-common:2.0.4 -com.kohlschutter.junixsocket:junixsocket-native-common:2.0.4 com.squareup.okhttp3:okhttp:3.11.0 com.squareup.okio:okio:1.14.0 commons-codec:commons-codec:1.10 -commons-io:commons-io:2.5 +commons-io:commons-io:2.6 +commons-lang:commons-lang:2.6 commons-logging:commons-logging:1.2 -javax.activation:javax.activation-api:1.2.0 -javax.annotation:javax.annotation-api:1.3.2 javax.inject:javax.inject:1 -javax.xml.bind:jaxb-api:2.3.1 junit:junit:4.12 net.bytebuddy:byte-buddy:1.8.15 net.i2p.crypto:eddsa:0.2.0 -net.java.dev.jna:jna-platform:5.2.0 -net.java.dev.jna:jna:5.2.0 +net.java.dev.jna:jna:5.5.0 org.apache.commons:commons-compress:1.9 org.apache.commons:commons-exec:1.3 org.apache.commons:commons-lang3:3.5 @@ -60,8 +61,8 @@ org.apache.maven:maven-resolver-provider:3.5.2 org.apache.maven:maven-settings-builder:3.5.2 org.apache.maven:maven-settings:3.5.2 org.assertj:assertj-core:3.11.1 -org.bouncycastle:bcpkix-jdk15on:1.56 -org.bouncycastle:bcprov-jdk15on:1.56 +org.bouncycastle:bcpkix-jdk15on:1.64 +org.bouncycastle:bcprov-jdk15on:1.64 org.checkerframework:checker-compat-qual:2.0.0 org.codehaus.mojo:animal-sniffer-annotations:1.14 org.codehaus.plexus:plexus-component-annotations:1.7.1 @@ -82,9 +83,6 @@ org.jetbrains.kotlin:kotlin-stdlib:1.2.70 org.jetbrains:annotations:15.0 org.jsoup:jsoup:1.10.2 org.rnorth.duct-tape:duct-tape:1.0.7 -org.rnorth.visible-assertions:visible-assertions:2.1.2 -org.rnorth:tcp-unix-socket-proxy:1.0.2 -org.scijava:native-lib-loader:2.0.2 org.seleniumhq.selenium:selenium-api:3.141.59 org.seleniumhq.selenium:selenium-chrome-driver:3.141.59 org.seleniumhq.selenium:selenium-remote-driver:3.141.59 @@ -92,5 +90,4 @@ org.seleniumhq.selenium:selenium-support:3.141.59 org.slf4j:slf4j-api:1.8.0-alpha2 org.sonatype.plexus:plexus-cipher:1.4 org.sonatype.plexus:plexus-sec-dispatcher:1.4 -org.testcontainers:testcontainers:1.10.5 org.threeten:threeten-extra:1.5.0 diff --git a/gradle/dependency-locks/testImplementationDependenciesMetadata.lockfile b/gradle/dependency-locks/testImplementationDependenciesMetadata.lockfile index fed6f565..f6359438 100644 --- a/gradle/dependency-locks/testImplementationDependenciesMetadata.lockfile +++ b/gradle/dependency-locks/testImplementationDependenciesMetadata.lockfile @@ -7,10 +7,16 @@ com.atlassian.performance.tools:io:1.2.0 com.atlassian.performance.tools:jira-actions:3.13.4 com.atlassian.performance.tools:jira-software-actions:1.3.3 com.atlassian.performance.tools:jvm-tasks:1.2.1 -com.atlassian.performance.tools:ssh-ubuntu:0.2.0 com.atlassian.performance.tools:ssh:2.4.0 com.atlassian.performance.tools:virtual-users:3.13.0 com.atlassian.performance:selenium-js:1.0.1 +com.fasterxml.jackson.core:jackson-annotations:2.10.3 +com.fasterxml.jackson.core:jackson-core:2.10.3 +com.fasterxml.jackson.core:jackson-databind:2.10.3 +com.github.docker-java:docker-java-api:3.2.7 +com.github.docker-java:docker-java-core:3.2.7 +com.github.docker-java:docker-java-transport-zerodep:3.2.7 +com.github.docker-java:docker-java-transport:3.2.7 com.github.stephenc.jcip:jcip-annotations:1.0-1 com.google.code.findbugs:jsr305:3.0.2 com.google.code.gson:gson:2.8.2 @@ -19,25 +25,20 @@ com.google.guava:guava:23.6-jre com.google.j2objc:j2objc-annotations:1.1 com.hierynomus:sshj:0.23.0 com.jcraft:jzlib:1.1.3 -com.kohlschutter.junixsocket:junixsocket-common:2.0.4 -com.kohlschutter.junixsocket:junixsocket-native-common:2.0.4 com.squareup.okhttp3:okhttp:3.11.0 com.squareup.okio:okio:1.14.0 com.typesafe:config:1.2.1 commons-cli:commons-cli:1.4 commons-codec:commons-codec:1.10 -commons-io:commons-io:2.5 +commons-io:commons-io:2.6 +commons-lang:commons-lang:2.6 commons-logging:commons-logging:1.2 io.github.bonigarcia:webdrivermanager:1.7.1 -javax.activation:javax.activation-api:1.2.0 -javax.annotation:javax.annotation-api:1.3.2 javax.inject:javax.inject:1 -javax.xml.bind:jaxb-api:2.3.1 junit:junit:4.12 net.bytebuddy:byte-buddy:1.8.15 net.i2p.crypto:eddsa:0.2.0 -net.java.dev.jna:jna-platform:5.2.0 -net.java.dev.jna:jna:5.2.0 +net.java.dev.jna:jna:5.5.0 org.apache.commons:commons-compress:1.9 org.apache.commons:commons-csv:1.3 org.apache.commons:commons-exec:1.3 @@ -68,8 +69,8 @@ org.apache.maven:maven-resolver-provider:3.5.2 org.apache.maven:maven-settings-builder:3.5.2 org.apache.maven:maven-settings:3.5.2 org.assertj:assertj-core:3.11.1 -org.bouncycastle:bcpkix-jdk15on:1.56 -org.bouncycastle:bcprov-jdk15on:1.56 +org.bouncycastle:bcpkix-jdk15on:1.64 +org.bouncycastle:bcprov-jdk15on:1.64 org.checkerframework:checker-compat-qual:2.0.0 org.codehaus.mojo:animal-sniffer-annotations:1.14 org.codehaus.plexus:plexus-component-annotations:1.7.1 @@ -91,9 +92,6 @@ org.jetbrains:annotations:15.0 org.jsoup:jsoup:1.10.2 org.rauschig:jarchivelib:0.7.1 org.rnorth.duct-tape:duct-tape:1.0.7 -org.rnorth.visible-assertions:visible-assertions:2.1.2 -org.rnorth:tcp-unix-socket-proxy:1.0.2 -org.scijava:native-lib-loader:2.0.2 org.seleniumhq.selenium:selenium-api:3.141.59 org.seleniumhq.selenium:selenium-chrome-driver:3.141.59 org.seleniumhq.selenium:selenium-remote-driver:3.141.59 @@ -101,5 +99,4 @@ org.seleniumhq.selenium:selenium-support:3.141.59 org.slf4j:slf4j-api:1.8.0-alpha2 org.sonatype.plexus:plexus-cipher:1.4 org.sonatype.plexus:plexus-sec-dispatcher:1.4 -org.testcontainers:testcontainers:1.10.5 org.threeten:threeten-extra:1.5.0 diff --git a/gradle/dependency-locks/testRuntime.lockfile b/gradle/dependency-locks/testRuntime.lockfile index a63a77e9..e560bcfb 100644 --- a/gradle/dependency-locks/testRuntime.lockfile +++ b/gradle/dependency-locks/testRuntime.lockfile @@ -5,24 +5,26 @@ com.atlassian.performance.tools:concurrency:1.1.0 com.atlassian.performance.tools:jira-actions:3.13.4 com.atlassian.performance.tools:jira-software-actions:1.3.3 com.atlassian.performance.tools:jvm-tasks:1.2.1 -com.atlassian.performance.tools:ssh-ubuntu:0.2.0 com.atlassian.performance:selenium-js:1.0.1 +com.fasterxml.jackson.core:jackson-annotations:2.10.3 +com.fasterxml.jackson.core:jackson-core:2.10.3 +com.fasterxml.jackson.core:jackson-databind:2.10.3 +com.github.docker-java:docker-java-api:3.2.7 +com.github.docker-java:docker-java-core:3.2.7 +com.github.docker-java:docker-java-transport-zerodep:3.2.7 +com.github.docker-java:docker-java-transport:3.2.7 com.github.stephenc.jcip:jcip-annotations:1.0-1 com.google.code.findbugs:jsr305:3.0.2 com.google.errorprone:error_prone_annotations:2.1.3 com.google.guava:guava:23.6-jre com.google.j2objc:j2objc-annotations:1.1 -com.kohlschutter.junixsocket:junixsocket-common:2.0.4 -com.kohlschutter.junixsocket:junixsocket-native-common:2.0.4 com.squareup.okhttp3:okhttp:3.11.0 com.squareup.okio:okio:1.14.0 -javax.activation:javax.activation-api:1.2.0 -javax.annotation:javax.annotation-api:1.3.2 -javax.xml.bind:jaxb-api:2.3.1 +commons-io:commons-io:2.6 +commons-lang:commons-lang:2.6 junit:junit:4.12 net.bytebuddy:byte-buddy:1.8.15 -net.java.dev.jna:jna-platform:5.2.0 -net.java.dev.jna:jna:5.2.0 +net.java.dev.jna:jna:5.5.0 org.apache.commons:commons-compress:1.9 org.apache.commons:commons-exec:1.3 org.apache.commons:commons-math3:3.6.1 @@ -31,6 +33,8 @@ org.apache.logging.log4j:log4j-core:2.10.0 org.apache.logging.log4j:log4j-jul:2.10.0 org.apache.logging.log4j:log4j-slf4j-impl:2.10.0 org.assertj:assertj-core:3.11.1 +org.bouncycastle:bcpkix-jdk15on:1.64 +org.bouncycastle:bcprov-jdk15on:1.64 org.checkerframework:checker-compat-qual:2.0.0 org.codehaus.mojo:animal-sniffer-annotations:1.14 org.glassfish:javax.json:1.1 @@ -42,13 +46,9 @@ org.jetbrains.kotlin:kotlin-stdlib-jdk8:1.2.70 org.jetbrains.kotlin:kotlin-stdlib:1.2.70 org.jetbrains:annotations:15.0 org.rnorth.duct-tape:duct-tape:1.0.7 -org.rnorth.visible-assertions:visible-assertions:2.1.2 -org.rnorth:tcp-unix-socket-proxy:1.0.2 -org.scijava:native-lib-loader:2.0.2 org.seleniumhq.selenium:selenium-api:3.141.59 org.seleniumhq.selenium:selenium-chrome-driver:3.141.59 org.seleniumhq.selenium:selenium-remote-driver:3.141.59 org.seleniumhq.selenium:selenium-support:3.141.59 org.slf4j:slf4j-api:1.8.0-alpha2 -org.testcontainers:testcontainers:1.10.5 org.threeten:threeten-extra:1.5.0 diff --git a/gradle/dependency-locks/testRuntimeClasspath.lockfile b/gradle/dependency-locks/testRuntimeClasspath.lockfile index fed6f565..f6359438 100644 --- a/gradle/dependency-locks/testRuntimeClasspath.lockfile +++ b/gradle/dependency-locks/testRuntimeClasspath.lockfile @@ -7,10 +7,16 @@ com.atlassian.performance.tools:io:1.2.0 com.atlassian.performance.tools:jira-actions:3.13.4 com.atlassian.performance.tools:jira-software-actions:1.3.3 com.atlassian.performance.tools:jvm-tasks:1.2.1 -com.atlassian.performance.tools:ssh-ubuntu:0.2.0 com.atlassian.performance.tools:ssh:2.4.0 com.atlassian.performance.tools:virtual-users:3.13.0 com.atlassian.performance:selenium-js:1.0.1 +com.fasterxml.jackson.core:jackson-annotations:2.10.3 +com.fasterxml.jackson.core:jackson-core:2.10.3 +com.fasterxml.jackson.core:jackson-databind:2.10.3 +com.github.docker-java:docker-java-api:3.2.7 +com.github.docker-java:docker-java-core:3.2.7 +com.github.docker-java:docker-java-transport-zerodep:3.2.7 +com.github.docker-java:docker-java-transport:3.2.7 com.github.stephenc.jcip:jcip-annotations:1.0-1 com.google.code.findbugs:jsr305:3.0.2 com.google.code.gson:gson:2.8.2 @@ -19,25 +25,20 @@ com.google.guava:guava:23.6-jre com.google.j2objc:j2objc-annotations:1.1 com.hierynomus:sshj:0.23.0 com.jcraft:jzlib:1.1.3 -com.kohlschutter.junixsocket:junixsocket-common:2.0.4 -com.kohlschutter.junixsocket:junixsocket-native-common:2.0.4 com.squareup.okhttp3:okhttp:3.11.0 com.squareup.okio:okio:1.14.0 com.typesafe:config:1.2.1 commons-cli:commons-cli:1.4 commons-codec:commons-codec:1.10 -commons-io:commons-io:2.5 +commons-io:commons-io:2.6 +commons-lang:commons-lang:2.6 commons-logging:commons-logging:1.2 io.github.bonigarcia:webdrivermanager:1.7.1 -javax.activation:javax.activation-api:1.2.0 -javax.annotation:javax.annotation-api:1.3.2 javax.inject:javax.inject:1 -javax.xml.bind:jaxb-api:2.3.1 junit:junit:4.12 net.bytebuddy:byte-buddy:1.8.15 net.i2p.crypto:eddsa:0.2.0 -net.java.dev.jna:jna-platform:5.2.0 -net.java.dev.jna:jna:5.2.0 +net.java.dev.jna:jna:5.5.0 org.apache.commons:commons-compress:1.9 org.apache.commons:commons-csv:1.3 org.apache.commons:commons-exec:1.3 @@ -68,8 +69,8 @@ org.apache.maven:maven-resolver-provider:3.5.2 org.apache.maven:maven-settings-builder:3.5.2 org.apache.maven:maven-settings:3.5.2 org.assertj:assertj-core:3.11.1 -org.bouncycastle:bcpkix-jdk15on:1.56 -org.bouncycastle:bcprov-jdk15on:1.56 +org.bouncycastle:bcpkix-jdk15on:1.64 +org.bouncycastle:bcprov-jdk15on:1.64 org.checkerframework:checker-compat-qual:2.0.0 org.codehaus.mojo:animal-sniffer-annotations:1.14 org.codehaus.plexus:plexus-component-annotations:1.7.1 @@ -91,9 +92,6 @@ org.jetbrains:annotations:15.0 org.jsoup:jsoup:1.10.2 org.rauschig:jarchivelib:0.7.1 org.rnorth.duct-tape:duct-tape:1.0.7 -org.rnorth.visible-assertions:visible-assertions:2.1.2 -org.rnorth:tcp-unix-socket-proxy:1.0.2 -org.scijava:native-lib-loader:2.0.2 org.seleniumhq.selenium:selenium-api:3.141.59 org.seleniumhq.selenium:selenium-chrome-driver:3.141.59 org.seleniumhq.selenium:selenium-remote-driver:3.141.59 @@ -101,5 +99,4 @@ org.seleniumhq.selenium:selenium-support:3.141.59 org.slf4j:slf4j-api:1.8.0-alpha2 org.sonatype.plexus:plexus-cipher:1.4 org.sonatype.plexus:plexus-sec-dispatcher:1.4 -org.testcontainers:testcontainers:1.10.5 org.threeten:threeten-extra:1.5.0 diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/database/DockerMysqlServer.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/database/DockerMysqlServer.kt index ace4306a..75cb01d3 100644 --- a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/database/DockerMysqlServer.kt +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/database/DockerMysqlServer.kt @@ -31,7 +31,7 @@ class DockerMysqlServer private constructor( val server = infrastructure.serve(3306, "mysql") val client = server.ssh.newConnection().use { setup(it, server) } nodes.forEach { node -> - node.postInstall.insert(DatabaseIpConfig(server.ip)) + node.postInstall.insert(DatabaseIpConfig(server.privateIp)) node.postInstall.insert(MysqlConnector()) } hooks.postInstance.insert(FixJiraUriViaMysql(server.ssh, client)) diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/database/DockerPostgresServer.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/database/DockerPostgresServer.kt index 269f9a0b..bdf26660 100644 --- a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/database/DockerPostgresServer.kt +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/database/DockerPostgresServer.kt @@ -24,7 +24,7 @@ class DockerPostgresServer private constructor( val server = hostSupplier.get() server.ssh.newConnection().use { setup(it) } nodes.forEach { node -> - node.postInstall.insert(DatabaseIpConfig(server.ip)) + node.postInstall.insert(DatabaseIpConfig(server.publicIp)) } } diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/TcpHost.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/TcpHost.kt index 8fef4548..75862d76 100644 --- a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/TcpHost.kt +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/TcpHost.kt @@ -6,7 +6,8 @@ import com.atlassian.performance.tools.ssh.api.Ssh * Has open TCP sockets. */ class TcpHost( - val ip: String, + val publicIp: String, + val privateIp: String, val port: Int, val name: String, val ssh: Ssh diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/JvmConfig.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/JvmConfig.kt index 9fe8d5a7..88ed1e91 100644 --- a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/JvmConfig.kt +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/JvmConfig.kt @@ -23,7 +23,7 @@ class JvmConfig( connection = ssh, config = config, gcLog = gcLog, - jiraIp = jira.host.ip + jiraIp = jira.host.publicIp ) val report = FileListing(gcLog.path("*")) reports.add(report, jira.host) diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/JiraServerPlan.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/JiraServerPlan.kt index 250c4f55..322c0034 100644 --- a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/JiraServerPlan.kt +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/JiraServerPlan.kt @@ -30,7 +30,7 @@ class JiraServerPlan private constructor( private class JiraServer( node: StartedJira ) : JiraInstance { - override val address: URI = node.installed.host.run { URI("http://$ip:$port/") } + override val address: URI = node.installed.host.run { URI("http://$publicIp:$port/") } override val nodes: List = listOf(node) } diff --git a/src/test/kotlin/com/atlassian/performance/tools/infrastructure/SshUbuntuExtensions.kt b/src/test/kotlin/com/atlassian/performance/tools/infrastructure/SshUbuntuExtensions.kt deleted file mode 100644 index 73d3a946..00000000 --- a/src/test/kotlin/com/atlassian/performance/tools/infrastructure/SshUbuntuExtensions.kt +++ /dev/null @@ -1,21 +0,0 @@ -package com.atlassian.performance.tools.infrastructure - -import com.atlassian.performance.tools.ssh.api.Ssh -import com.atlassian.performance.tools.sshubuntu.api.SshUbuntu -import java.time.Duration - -internal fun SshUbuntu.toSsh(): Ssh { - val ssh = Ssh(with(this.ssh) { - com.atlassian.performance.tools.ssh.api.SshHost( - ipAddress = ipAddress, - userName = userName, - authentication = com.atlassian.performance.tools.ssh.api.auth.PublicKeyAuthentication(privateKey), - port = port - ) - }) - ssh.newConnection().use { connection -> - connection.execute("apt-get update -qq", Duration.ofMinutes(3)) - connection.execute("export DEBIAN_FRONTEND=noninteractive; apt-get install sudo curl screen gnupg2 -y -qq", Duration.ofMinutes(10)) - } - return ssh -} diff --git a/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/DockerInfrastructure.kt b/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/DockerInfrastructure.kt index 34ea1563..d0b7ddd9 100644 --- a/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/DockerInfrastructure.kt +++ b/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/DockerInfrastructure.kt @@ -3,15 +3,37 @@ package com.atlassian.performance.tools.infrastructure.api import com.atlassian.performance.tools.infrastructure.api.jira.install.TcpHost import com.atlassian.performance.tools.infrastructure.api.jira.node.JiraNode import com.atlassian.performance.tools.infrastructure.api.jira.node.JiraNodePlan -import com.atlassian.performance.tools.infrastructure.toSsh -import com.atlassian.performance.tools.sshubuntu.api.SshUbuntuContainer +import com.atlassian.performance.tools.infrastructure.lib.docker.* +import com.atlassian.performance.tools.ssh.api.Ssh +import com.atlassian.performance.tools.ssh.api.SshHost +import com.atlassian.performance.tools.ssh.api.auth.PasswordAuthentication +import com.github.dockerjava.api.DockerClient +import com.github.dockerjava.api.command.PullImageResultCallback +import com.github.dockerjava.api.model.* +import com.github.dockerjava.core.DefaultDockerClientConfig +import com.github.dockerjava.core.DockerClientImpl +import com.github.dockerjava.zerodep.ZerodepDockerHttpClient +import java.time.Duration import java.util.* -import java.util.concurrent.ConcurrentLinkedQueue -import java.util.function.Consumer +import java.util.concurrent.ConcurrentLinkedDeque internal class DockerInfrastructure : Infrastructure { - private val allocatedResources: Queue = ConcurrentLinkedQueue() + private val allocatedResources: Deque = ConcurrentLinkedDeque() + private val docker: DockerClient + private val network: DockerNetwork + + init { + val dockerConfig = DefaultDockerClientConfig.createDefaultConfigBuilder().build() + val dockerHttp = ZerodepDockerHttpClient.Builder().dockerHost(dockerConfig.dockerHost).build() + docker = DockerClientImpl.getInstance(dockerConfig, dockerHttp) + allocatedResources.add(docker) + network = docker + .createNetworkCmd() + .withName(UUID.randomUUID().toString()) + .execAsResource(docker) + allocatedResources.add(network) + } override fun serve(jiraNodePlans: List): List { return jiraNodePlans.mapIndexed { nodeIndex, plan -> @@ -19,26 +41,106 @@ internal class DockerInfrastructure : Infrastructure { } } + fun serve(): Ssh { + return serve("ssh") + } + + fun serve(name: String): Ssh { + return serve(888, name).ssh + } + override fun serve(port: Int, name: String): TcpHost { - val container = SshUbuntuContainer(Consumer { - it.addExposedPort(port) - it.setPortBindings(listOf("$port:$port")) - it.setPrivilegedMode(true) - }) - val sshUbuntu = container.start() - allocatedResources.add(sshUbuntu) - return TcpHost( - "localhost", - port, - name, - sshUbuntu.toSsh() + docker + .pullImageCmd("rastasheep/ubuntu-sshd") + .withTag("18.04") + .exec(PullImageResultCallback()) + .awaitCompletion() + val exposedPort = ExposedPort.tcp(port) + val dockerDaemonSocket = "/var/run/docker.sock" + val createdContainer = docker + .createContainerCmd("rastasheep/ubuntu-sshd:18.04") + .withHostConfig( + HostConfig() + .withPublishAllPorts(true) +// .withPrivileged(true) + .withBinds(Bind(dockerDaemonSocket, Volume(dockerDaemonSocket))) + ) + .withExposedPorts(exposedPort) + .withName(name + "-" + UUID.randomUUID()) + .execAsResource(docker) + allocatedResources.addLast(createdContainer) + return start(createdContainer, exposedPort) + } + + private fun start( + created: CreatedContainer, + port: ExposedPort + ): TcpHost { + val connectedContainer = docker + .connectToNetworkCmd() + .withContainerId(created.response.id) + .withNetworkId(network.response.id) + .execAsResource(docker) + allocatedResources.addLast(connectedContainer) + return start(connectedContainer, port) + } + + private fun start( + connected: ConnectedContainer, + port: ExposedPort + ): TcpHost { + val startedContainer = docker + .startContainerCmd(connected.containerId) + .execAsResource(docker) + allocatedResources.addLast(startedContainer); + return install(startedContainer, connected, port) + } + + private fun install( + started: StartedContainer, + connected: ConnectedContainer, + port: ExposedPort + ): TcpHost { + val networkSettings = docker + .inspectContainerCmd(started.id) + .exec() + .networkSettings + val ip = networkSettings + .networks + .values + .single { it.networkID == connected.networkId } + .ipAddress!! + val portBindings = networkSettings.ports.bindings + val sshPort = getHostPort(portBindings, ExposedPort.tcp(22)) + val tcpPort = getHostPort(portBindings, port) + val sshHost = SshHost( + ipAddress = "localhost", + userName = "root", + authentication = PasswordAuthentication("root"), + port = sshPort ) + val ssh = Ssh(sshHost) + ssh.newConnection().use { + it.execute("apt-get update", Duration.ofMinutes(2)) + it.execute("apt-get -y install sudo gnupg screen") + } + return TcpHost("localhost", ip, tcpPort, started.id, ssh) + } + + private fun getHostPort( + portBindings: MutableMap>, + port: ExposedPort + ): Int { + return portBindings[port]!! + .single() + .hostPortSpec + .toInt() } override fun close() { while (true) { allocatedResources - .poll() + .pollLast() ?.use {} ?: break } diff --git a/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/browser/ChromeIT.kt b/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/browser/ChromeIT.kt index e1891d71..e6f87435 100644 --- a/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/browser/ChromeIT.kt +++ b/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/browser/ChromeIT.kt @@ -1,8 +1,7 @@ package com.atlassian.performance.tools.infrastructure.api.browser -import com.atlassian.performance.tools.infrastructure.toSsh +import com.atlassian.performance.tools.infrastructure.api.DockerInfrastructure import com.atlassian.performance.tools.ssh.api.SshConnection -import com.atlassian.performance.tools.sshubuntu.api.SshUbuntuContainer import org.hamcrest.Matchers import org.junit.Assert import org.junit.Test @@ -12,8 +11,8 @@ class ChromeIT { @Test fun shouldInstallChromeBrowser() { - SshUbuntuContainer().start().use { ssh -> - ssh.toSsh().newConnection().use { connection -> + DockerInfrastructure().use { infra -> + infra.serve(80, "ChromeIT").ssh.newConnection().use { connection -> val wasInstalledBefore = isChromeInstalled(connection) Chrome().install(connection) @@ -24,7 +23,6 @@ class ChromeIT { Assert.assertThat(isInstalledAfter, Matchers.`is`(true)) Assert.assertThat(isChromedriverInstalled(connection), Matchers.`is`(true)) } - } } diff --git a/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/browser/chromium/Chromium69IT.kt b/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/browser/chromium/Chromium69IT.kt index 11b31466..0766022d 100644 --- a/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/browser/chromium/Chromium69IT.kt +++ b/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/browser/chromium/Chromium69IT.kt @@ -1,10 +1,9 @@ package com.atlassian.performance.tools.infrastructure.api.browser.chromium -import com.atlassian.performance.tools.infrastructure.toSsh +import com.atlassian.performance.tools.infrastructure.api.DockerInfrastructure import com.atlassian.performance.tools.jvmtasks.api.IdempotentAction import com.atlassian.performance.tools.jvmtasks.api.StaticBackoff import com.atlassian.performance.tools.ssh.api.SshConnection -import com.atlassian.performance.tools.sshubuntu.api.SshUbuntuContainer import org.hamcrest.Matchers import org.junit.Assert import org.junit.Test @@ -14,8 +13,8 @@ class Chromium69IT { @Test fun shouldInstallBrowser() { - SshUbuntuContainer().start().use { sshUbuntu -> - sshUbuntu.toSsh().newConnection().use { connection -> + DockerInfrastructure().use { infra -> + infra.serve().newConnection().use { connection -> val installedBefore = isChromiumInstalled(connection) Chromium69().install(connection) diff --git a/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/browser/chromium/PageLoadTimeoutRecoveryTest.kt b/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/browser/chromium/PageLoadTimeoutRecoveryTest.kt index 8a8af541..b66f47e2 100644 --- a/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/browser/chromium/PageLoadTimeoutRecoveryTest.kt +++ b/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/browser/chromium/PageLoadTimeoutRecoveryTest.kt @@ -1,10 +1,9 @@ package com.atlassian.performance.tools.infrastructure.api.browser.chromium +import com.atlassian.performance.tools.infrastructure.api.DockerInfrastructure import com.atlassian.performance.tools.infrastructure.api.browser.Browser import com.atlassian.performance.tools.infrastructure.browser.SshChromium import com.atlassian.performance.tools.infrastructure.mock.MockHttpServer -import com.atlassian.performance.tools.infrastructure.toSsh -import com.atlassian.performance.tools.sshubuntu.api.SshUbuntuContainer import com.sun.net.httpserver.HttpExchange import org.assertj.core.api.Assertions import org.openqa.selenium.TimeoutException @@ -20,8 +19,8 @@ internal class PageLoadTimeoutRecoveryTest { MockHttpServer().start().use { httpServer -> val fastResource = httpServer.register(FastResponse()) val slowResource = httpServer.register(SlowResponse()) - SshUbuntuContainer().start().use { sshUbuntu -> - val ssh = sshUbuntu.toSsh() + DockerInfrastructure().use { infra -> + val ssh = infra.serve() ssh.forwardRemotePort(httpServer.getPort(), httpServer.getPort()).use { val localChromedriverPort = findFreePort() ssh.forwardLocalPort(localChromedriverPort, remoteChromedriverPort).use { diff --git a/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/dataset/HttpDatasetPackageIT.kt b/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/dataset/HttpDatasetPackageIT.kt index 6f5f92f7..1c307690 100644 --- a/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/dataset/HttpDatasetPackageIT.kt +++ b/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/dataset/HttpDatasetPackageIT.kt @@ -1,9 +1,8 @@ package com.atlassian.performance.tools.infrastructure.api.dataset import com.atlassian.performance.tools.infrastructure.Ls -import com.atlassian.performance.tools.infrastructure.toSsh +import com.atlassian.performance.tools.infrastructure.api.DockerInfrastructure import com.atlassian.performance.tools.ssh.api.Ssh -import com.atlassian.performance.tools.sshubuntu.api.SshUbuntuContainer import org.assertj.core.api.Assertions import org.junit.Test import java.net.URI @@ -21,8 +20,8 @@ class HttpDatasetPackageIT { downloadTimeout = Duration.ofMinutes(1) ) - val filesInDataset = SshUbuntuContainer().start().use { sshUbuntu -> - val ssh = sshUbuntu.toSsh() + val filesInDataset = DockerInfrastructure().use { infra -> + val ssh = infra.serve(80, "HttpDatasetPackageIT").ssh return@use RandomFilesGenerator(ssh).start().use { ssh.newConnection().use { connection -> val unpackedPath = dataset.download(connection) diff --git a/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/distribution/PublicJiraServiceDeskDistributionIT.kt b/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/distribution/PublicJiraServiceDeskDistributionIT.kt index b6f351bb..c27ec873 100644 --- a/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/distribution/PublicJiraServiceDeskDistributionIT.kt +++ b/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/distribution/PublicJiraServiceDeskDistributionIT.kt @@ -1,7 +1,6 @@ package com.atlassian.performance.tools.infrastructure.api.distribution -import com.atlassian.performance.tools.infrastructure.toSsh -import com.atlassian.performance.tools.sshubuntu.api.SshUbuntuContainer +import com.atlassian.performance.tools.infrastructure.api.DockerInfrastructure import org.assertj.core.api.Assertions.assertThat import org.junit.Test @@ -9,8 +8,8 @@ class PublicJiraServiceDeskDistributionIT { @Test fun shouldDownloadJiraServiceDesk() { - SshUbuntuContainer().start().use { ssh -> - ssh.toSsh().newConnection().use { connection -> + DockerInfrastructure().use { infra -> + infra.serve().newConnection().use { connection -> val serviceDeskDistribution: ProductDistribution = PublicJiraServiceDeskDistribution("4.0.1") val targetFolder = "test" connection.execute("mkdir $targetFolder") diff --git a/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/distribution/PublicJiraSoftwareDistributionsIT.kt b/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/distribution/PublicJiraSoftwareDistributionsIT.kt index a77a70c4..85835229 100644 --- a/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/distribution/PublicJiraSoftwareDistributionsIT.kt +++ b/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/distribution/PublicJiraSoftwareDistributionsIT.kt @@ -1,7 +1,6 @@ package com.atlassian.performance.tools.infrastructure.api.distribution -import com.atlassian.performance.tools.infrastructure.toSsh -import com.atlassian.performance.tools.sshubuntu.api.SshUbuntuContainer +import com.atlassian.performance.tools.infrastructure.api.DockerInfrastructure import org.assertj.core.api.Assertions.assertThat import org.junit.Test @@ -9,8 +8,8 @@ class PublicJiraSoftwareDistributionsIT { @Test fun shouldDownloadJiraSoftware() { - SshUbuntuContainer().start().use { ssh -> - ssh.toSsh().newConnection().use { connection -> + DockerInfrastructure().use { infra -> + infra.serve().newConnection().use { connection -> val jiraDistribution: ProductDistribution = PublicJiraSoftwareDistribution("7.2.0") val targetFolder = "test" connection.execute("mkdir $targetFolder") diff --git a/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/docker/DockerIT.kt b/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/docker/DockerIT.kt index dcb238a4..11d39785 100644 --- a/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/docker/DockerIT.kt +++ b/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/docker/DockerIT.kt @@ -1,21 +1,18 @@ package com.atlassian.performance.tools.infrastructure.api.docker -import com.atlassian.performance.tools.infrastructure.toSsh -import com.atlassian.performance.tools.sshubuntu.api.SshUbuntuContainer -import com.github.dockerjava.api.model.Bind -import com.github.dockerjava.api.model.Volume +import com.atlassian.performance.tools.infrastructure.api.DockerInfrastructure +import com.atlassian.performance.tools.infrastructure.api.os.Ubuntu import org.junit.Test -import org.testcontainers.containers.GenericContainer import java.time.Duration -import java.util.function.Consumer class DockerIT { @Test fun installWorks() { - SshUbuntuContainer(Consumer { enableNestedDocker(it) }).start().use { ssh -> - ssh.toSsh().newConnection().use { connection -> + DockerInfrastructure().use { infra -> + infra.serve().newConnection().use { connection -> //workaround for a bug in Docker download site for bionic val packageFile = "containerd.io_1.2.2-3_amd64.deb" + Ubuntu().install(connection,listOf( "curl")) connection.execute("curl -O https://download.docker.com/linux/ubuntu/dists/bionic/pool/edge/amd64/$packageFile", Duration.ofMinutes(3)) connection.execute("sudo apt install ./$packageFile", Duration.ofMinutes(3)) @@ -24,9 +21,4 @@ class DockerIT { } } } - - private fun enableNestedDocker(container: GenericContainer<*>) { - val dockerDaemonSocket = "/var/run/docker.sock" - container.setBinds(listOf(Bind(dockerDaemonSocket, Volume(dockerDaemonSocket)))) - } } diff --git a/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/PreInstallHooksTest.kt b/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/PreInstallHooksTest.kt index aeb25f99..711dec3b 100644 --- a/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/PreInstallHooksTest.kt +++ b/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/PreInstallHooksTest.kt @@ -13,7 +13,7 @@ import java.nio.file.Paths class PreInstallHooksTest { private val dummySsh = Ssh(SshHost("localhost", "dummyUser", Paths.get("dummyKey"))) - private val dummyServer = TcpHost("doesn't matter", 123, "fake-server", dummySsh) + private val dummyServer = TcpHost("dummyPublicIp", "dummyPrivateIp", 123, "fake-server", dummySsh) @Test fun shouldInsertDuringListing() { diff --git a/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/jvm/AdoptOpenJdk11IT.kt b/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/jvm/AdoptOpenJdk11IT.kt index 205d21c1..6d29a0cc 100644 --- a/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/jvm/AdoptOpenJdk11IT.kt +++ b/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/jvm/AdoptOpenJdk11IT.kt @@ -1,15 +1,14 @@ package com.atlassian.performance.tools.infrastructure.api.jvm -import com.atlassian.performance.tools.infrastructure.toSsh -import com.atlassian.performance.tools.sshubuntu.api.SshUbuntuContainer +import com.atlassian.performance.tools.infrastructure.api.DockerInfrastructure import org.junit.Test class AdoptOpenJdk11IT { @Test fun shouldSupportJstat() { - SshUbuntuContainer().start().use { ssh -> - ssh.toSsh().newConnection().use { connection -> + DockerInfrastructure().use { infra -> + infra.serve().newConnection().use { connection -> JstatSupport(AdoptOpenJDK11()).shouldSupportJstat(connection) } } diff --git a/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/jvm/AdoptOpenJdkIT.kt b/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/jvm/AdoptOpenJdkIT.kt index 6f8aadb8..577de11e 100644 --- a/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/jvm/AdoptOpenJdkIT.kt +++ b/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/jvm/AdoptOpenJdkIT.kt @@ -1,15 +1,14 @@ package com.atlassian.performance.tools.infrastructure.api.jvm -import com.atlassian.performance.tools.infrastructure.toSsh -import com.atlassian.performance.tools.sshubuntu.api.SshUbuntuContainer +import com.atlassian.performance.tools.infrastructure.api.DockerInfrastructure import org.junit.Test class AdoptOpenJdkIT { @Test fun shouldSupportJstat() { - SshUbuntuContainer().start().use { ssh -> - ssh.toSsh().newConnection().use { connection -> + DockerInfrastructure().use { infra -> + infra.serve().newConnection().use { connection -> JstatSupport(AdoptOpenJDK()).shouldSupportJstat(connection) } } diff --git a/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/jvm/JstatSupport.kt b/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/jvm/JstatSupport.kt index 2fdddfeb..fa0c0c0e 100644 --- a/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/jvm/JstatSupport.kt +++ b/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/jvm/JstatSupport.kt @@ -1,5 +1,6 @@ package com.atlassian.performance.tools.infrastructure.api.jvm +import com.atlassian.performance.tools.infrastructure.api.os.Ubuntu import com.atlassian.performance.tools.jvmtasks.api.ExponentialBackoff import com.atlassian.performance.tools.jvmtasks.api.IdempotentAction import com.atlassian.performance.tools.ssh.api.SshConnection @@ -29,7 +30,7 @@ class JstatSupport( private val timestampLength = "2018-12-17T14:10:44+00:00 ".length fun shouldSupportJstat(connection: SshConnection) { - connection.execute("apt-get install curl screen -y -qq", Duration.ofMinutes(2)) + Ubuntu().install(connection, listOf("curl", "screen"), Duration.ofMinutes(2)) connection.upload(File(this.javaClass.getResource(jar).toURI()), jarName) jdk.install(connection) connection.startProcess(jdk.command("-classpath $jarName samples.HelloWorld")) diff --git a/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/jvm/OpenJdk11IT.kt b/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/jvm/OpenJdk11IT.kt index b2fb592e..73ae5e02 100644 --- a/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/jvm/OpenJdk11IT.kt +++ b/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/jvm/OpenJdk11IT.kt @@ -1,15 +1,14 @@ package com.atlassian.performance.tools.infrastructure.api.jvm -import com.atlassian.performance.tools.infrastructure.toSsh -import com.atlassian.performance.tools.sshubuntu.api.SshUbuntuContainer +import com.atlassian.performance.tools.infrastructure.api.DockerInfrastructure import org.junit.Test class OpenJdk11IT { @Test fun shouldSupportJstat() { - SshUbuntuContainer().start().use { ssh -> - ssh.toSsh().newConnection().use { connection -> + DockerInfrastructure().use { infra -> + infra.serve().newConnection().use { connection -> JstatSupport(OpenJDK11()).shouldSupportJstat(connection) } } diff --git a/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/jvm/OpenJdkIT.kt b/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/jvm/OpenJdkIT.kt index e9e5c4c1..ebb23431 100644 --- a/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/jvm/OpenJdkIT.kt +++ b/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/jvm/OpenJdkIT.kt @@ -1,15 +1,14 @@ package com.atlassian.performance.tools.infrastructure.api.jvm -import com.atlassian.performance.tools.infrastructure.toSsh -import com.atlassian.performance.tools.sshubuntu.api.SshUbuntuContainer +import com.atlassian.performance.tools.infrastructure.api.DockerInfrastructure import org.junit.Test class OpenJdkIT { @Test fun shouldSupportJstat() { - SshUbuntuContainer().start().use { ssh -> - ssh.toSsh().newConnection().use { connection -> + DockerInfrastructure().use { infra -> + infra.serve().newConnection().use { connection -> JstatSupport(OpenJDK()).shouldSupportJstat(connection) } } diff --git a/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/jvm/OracleJdkIT.kt b/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/jvm/OracleJdkIT.kt index 4b93b70a..0fa2e116 100644 --- a/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/jvm/OracleJdkIT.kt +++ b/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/jvm/OracleJdkIT.kt @@ -1,15 +1,14 @@ package com.atlassian.performance.tools.infrastructure.api.jvm -import com.atlassian.performance.tools.infrastructure.toSsh -import com.atlassian.performance.tools.sshubuntu.api.SshUbuntuContainer +import com.atlassian.performance.tools.infrastructure.api.DockerInfrastructure import org.junit.Test class OracleJdkIT { @Test fun shouldSupportJstatAndThreadDumps() { - SshUbuntuContainer().start().use { ssh -> - ssh.toSsh().newConnection().use { connection -> + DockerInfrastructure().use { infra -> + infra.serve().newConnection().use { connection -> val jdk = OracleJDK() JstatSupport(jdk).shouldSupportJstat(connection) ThreadDumpTest().shouldGatherThreadDump(jdk, connection) diff --git a/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/os/UbuntuIT.kt b/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/os/UbuntuIT.kt index caea10ee..6721efcf 100644 --- a/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/os/UbuntuIT.kt +++ b/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/os/UbuntuIT.kt @@ -1,10 +1,10 @@ package com.atlassian.performance.tools.infrastructure.api.os -import com.atlassian.performance.tools.infrastructure.toSsh +import com.atlassian.performance.tools.infrastructure.api.DockerInfrastructure +import com.atlassian.performance.tools.infrastructure.api.Infrastructure +import com.atlassian.performance.tools.infrastructure.api.jira.install.TcpHost import com.atlassian.performance.tools.ssh.api.SshConnection import com.atlassian.performance.tools.ssh.api.SshHost -import com.atlassian.performance.tools.sshubuntu.api.SshUbuntu -import com.atlassian.performance.tools.sshubuntu.api.SshUbuntuContainer import org.apache.logging.log4j.Level import org.junit.After import org.junit.Before @@ -17,23 +17,25 @@ import java.util.concurrent.TimeUnit class UbuntuIT { private lateinit var executor: ExecutorService - private lateinit var sshUbuntu: SshUbuntu + private lateinit var infra: Infrastructure + private lateinit var sshUbuntu: TcpHost @Before fun before() { executor = Executors.newCachedThreadPool() - sshUbuntu = SshUbuntuContainer().start() + infra = DockerInfrastructure() + sshUbuntu = infra.serve(80, "UbuntuIT") } @After fun after() { - sshUbuntu.close() + infra.close() executor.shutdownNow() } @Test fun shouldRetry() { - sshUbuntu.toSsh().newConnection().use { connection -> + sshUbuntu.ssh.newConnection().use { connection -> Ubuntu().install( ColdAptSshConnection(connection), listOf("nano"), @@ -81,26 +83,21 @@ class UbuntuIT { @Test fun shouldBeThreadSafe() { - val lock = Object() val concurrency = 5 val latch = CountDownLatch(concurrency) (1..concurrency) .map { - executor.submit { installLftp(lock, latch) } + executor.submit { installLftp(latch) } }.map { it.get(5, TimeUnit.MINUTES) } } - private fun installLftp(lock: Any, latch: CountDownLatch) { - val ssh = synchronized(lock) { - sshUbuntu.toSsh() - } - ssh.newConnection().use { connection -> + private fun installLftp(latch: CountDownLatch) { + sshUbuntu.ssh.newConnection().use { connection -> latch.countDown() latch.await() Ubuntu().install(connection, listOf("lftp")) } } - -} \ No newline at end of file +} diff --git a/src/test/kotlin/com/atlassian/performance/tools/infrastructure/lib/docker/ConnectedContainer.kt b/src/test/kotlin/com/atlassian/performance/tools/infrastructure/lib/docker/ConnectedContainer.kt new file mode 100644 index 00000000..ee5ae626 --- /dev/null +++ b/src/test/kotlin/com/atlassian/performance/tools/infrastructure/lib/docker/ConnectedContainer.kt @@ -0,0 +1,30 @@ +package com.atlassian.performance.tools.infrastructure.lib.docker + +import com.github.dockerjava.api.DockerClient +import com.github.dockerjava.api.command.ConnectToNetworkCmd + +class ConnectedContainer( + private val docker: DockerClient, + val containerId: String, + val networkId: String +) : AutoCloseable { + + override fun close() { + docker + .disconnectFromNetworkCmd() + .withContainerId(containerId) + .withNetworkId(networkId) + .exec() + } +} + +fun ConnectToNetworkCmd.execAsResource( + docker: DockerClient +): ConnectedContainer { + exec() + return ConnectedContainer( + docker, + containerId!!, + networkId!! + ) +} diff --git a/src/test/kotlin/com/atlassian/performance/tools/infrastructure/lib/docker/CreatedContainer.kt b/src/test/kotlin/com/atlassian/performance/tools/infrastructure/lib/docker/CreatedContainer.kt new file mode 100644 index 00000000..b06863ec --- /dev/null +++ b/src/test/kotlin/com/atlassian/performance/tools/infrastructure/lib/docker/CreatedContainer.kt @@ -0,0 +1,18 @@ +package com.atlassian.performance.tools.infrastructure.lib.docker + +import com.github.dockerjava.api.DockerClient +import com.github.dockerjava.api.command.CreateContainerCmd +import com.github.dockerjava.api.command.CreateContainerResponse + +class CreatedContainer( + private val docker: DockerClient, + val response: CreateContainerResponse +) : AutoCloseable { + override fun close() { + docker.removeContainerCmd(response.id).exec() + } +} + +fun CreateContainerCmd.execAsResource( + docker: DockerClient +): CreatedContainer = CreatedContainer(docker, exec()) diff --git a/src/test/kotlin/com/atlassian/performance/tools/infrastructure/lib/docker/DockerNetwork.kt b/src/test/kotlin/com/atlassian/performance/tools/infrastructure/lib/docker/DockerNetwork.kt new file mode 100644 index 00000000..c137d153 --- /dev/null +++ b/src/test/kotlin/com/atlassian/performance/tools/infrastructure/lib/docker/DockerNetwork.kt @@ -0,0 +1,18 @@ +package com.atlassian.performance.tools.infrastructure.lib.docker + +import com.github.dockerjava.api.DockerClient +import com.github.dockerjava.api.command.CreateNetworkCmd +import com.github.dockerjava.api.command.CreateNetworkResponse + +class DockerNetwork( + private val docker: DockerClient, + val response: CreateNetworkResponse +) : AutoCloseable { + override fun close() { + docker.removeNetworkCmd(response.id).exec() + } +} + +fun CreateNetworkCmd.execAsResource( + docker: DockerClient +): DockerNetwork = DockerNetwork(docker, exec()) \ No newline at end of file diff --git a/src/test/kotlin/com/atlassian/performance/tools/infrastructure/lib/docker/StartedContainer.kt b/src/test/kotlin/com/atlassian/performance/tools/infrastructure/lib/docker/StartedContainer.kt new file mode 100644 index 00000000..076a02a7 --- /dev/null +++ b/src/test/kotlin/com/atlassian/performance/tools/infrastructure/lib/docker/StartedContainer.kt @@ -0,0 +1,23 @@ +package com.atlassian.performance.tools.infrastructure.lib.docker + +import com.github.dockerjava.api.DockerClient +import com.github.dockerjava.api.command.StartContainerCmd + +class StartedContainer( + private val docker: DockerClient, + val id: String +) : AutoCloseable { + override fun close() { + docker.stopContainerCmd(id).exec() + } +} + +fun StartContainerCmd.execAsResource( + docker: DockerClient +): StartedContainer { + exec() + return StartedContainer( + docker, + containerId!! + ) +} From 22477411c0d8045bb0d42a8d06f759b90a998d57 Mon Sep 17 00:00:00 2001 From: Maciej Kwidzinski Date: Mon, 12 Apr 2021 10:07:20 +0200 Subject: [PATCH 20/52] Try to fix Docker SSH connectivity To read: * https://danwalsh.livejournal.com/74095.html * https://github.com/bmuschko/gradle-docker-plugin/issues/890 Maybe it's because of multiple port exports? --- .../api/DockerInfrastructure.kt | 51 +++++++++++-------- 1 file changed, 29 insertions(+), 22 deletions(-) diff --git a/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/DockerInfrastructure.kt b/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/DockerInfrastructure.kt index d0b7ddd9..30d15075 100644 --- a/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/DockerInfrastructure.kt +++ b/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/DockerInfrastructure.kt @@ -61,44 +61,51 @@ internal class DockerInfrastructure : Infrastructure { .createContainerCmd("rastasheep/ubuntu-sshd:18.04") .withHostConfig( HostConfig() - .withPublishAllPorts(true) -// .withPrivileged(true) - .withBinds(Bind(dockerDaemonSocket, Volume(dockerDaemonSocket))) + .withPublishAllPorts(false) + .withPortBindings( + PortBinding(Ports.Binding("0.0.0.0", "0"), ExposedPort.tcp(22)), + PortBinding(Ports.Binding("0.0.0.0", "0"), ExposedPort.tcp(port)) + ) + .withPrivileged(false) + .withNetworkMode(network.response.id) +// .withSecurityOpts(emptyList()) +// .withUsernsMode("host") +// .withBinds(Bind(dockerDaemonSocket, Volume(dockerDaemonSocket))) ) - .withExposedPorts(exposedPort) + .withExposedPorts(exposedPort, ExposedPort.tcp(22)) .withName(name + "-" + UUID.randomUUID()) .execAsResource(docker) allocatedResources.addLast(createdContainer) return start(createdContainer, exposedPort) } - private fun start( - created: CreatedContainer, - port: ExposedPort - ): TcpHost { - val connectedContainer = docker - .connectToNetworkCmd() - .withContainerId(created.response.id) - .withNetworkId(network.response.id) - .execAsResource(docker) - allocatedResources.addLast(connectedContainer) - return start(connectedContainer, port) - } +// private fun start( +// created: CreatedContainer, +// port: ExposedPort +// ): TcpHost { +// val connectedContainer = docker // TODO remove? +// .connectToNetworkCmd() +// .withContainerId(created.response.id) +// .withNetworkId(network.response.id) +// .execAsResource(docker) +// allocatedResources.addLast(connectedContainer) +// return start(created, port) +// } private fun start( - connected: ConnectedContainer, + created: CreatedContainer, port: ExposedPort ): TcpHost { val startedContainer = docker - .startContainerCmd(connected.containerId) + .startContainerCmd(created.response.id) .execAsResource(docker) allocatedResources.addLast(startedContainer); - return install(startedContainer, connected, port) + return install(startedContainer, port) } private fun install( started: StartedContainer, - connected: ConnectedContainer, +// connected: ConnectedContainer, port: ExposedPort ): TcpHost { val networkSettings = docker @@ -108,7 +115,7 @@ internal class DockerInfrastructure : Infrastructure { val ip = networkSettings .networks .values - .single { it.networkID == connected.networkId } + .single { it.networkID == network.response.id } .ipAddress!! val portBindings = networkSettings.ports.bindings val sshPort = getHostPort(portBindings, ExposedPort.tcp(22)) @@ -128,7 +135,7 @@ internal class DockerInfrastructure : Infrastructure { } private fun getHostPort( - portBindings: MutableMap>, + portBindings: Map>, port: ExposedPort ): Int { return portBindings[port]!! From d7a3b02eb1f99df4902ec1b4484dacb2b9842f30 Mon Sep 17 00:00:00 2001 From: Maciej Kwidzinski Date: Fri, 23 Apr 2021 13:24:32 +0200 Subject: [PATCH 21/52] Fix Jira-MySQL connectivity across Docker infra --- .../api/jira/install/TcpHost.kt | 2 + .../infrastructure/api/jvm/AdoptOpenJDK.kt | 2 + .../api/DockerInfrastructure.kt | 60 ++++++--------- .../api/jira/instance/JiraDataCenterPlanIT.kt | 73 ++++++++++++++----- 4 files changed, 82 insertions(+), 55 deletions(-) diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/TcpHost.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/TcpHost.kt index 75862d76..5c06e3f8 100644 --- a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/TcpHost.kt +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/TcpHost.kt @@ -4,6 +4,8 @@ import com.atlassian.performance.tools.ssh.api.Ssh /** * Has open TCP sockets. + * @param [ssh] connects to the host via [publicIp] + * @param [port] accepts connections at within the [privateIp] network */ class TcpHost( val publicIp: String, diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jvm/AdoptOpenJDK.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jvm/AdoptOpenJDK.kt index 9675b04a..811fa258 100644 --- a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jvm/AdoptOpenJDK.kt +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jvm/AdoptOpenJDK.kt @@ -1,5 +1,6 @@ package com.atlassian.performance.tools.infrastructure.api.jvm +import com.atlassian.performance.tools.infrastructure.api.os.Ubuntu import com.atlassian.performance.tools.jvmtasks.api.ExponentialBackoff import com.atlassian.performance.tools.jvmtasks.api.IdempotentAction import com.atlassian.performance.tools.ssh.api.SshConnection @@ -17,6 +18,7 @@ class AdoptOpenJDK : VersionedJavaDevelopmentKit { override fun getMajorVersion() = 8 override fun install(connection: SshConnection) { + Ubuntu().install(connection, listOf("curl")) download(connection) connection.execute("tar -xzf $jdkArchive") connection.execute("echo '${use()}' >> ~/.bashrc") diff --git a/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/DockerInfrastructure.kt b/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/DockerInfrastructure.kt index 30d15075..c16032af 100644 --- a/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/DockerInfrastructure.kt +++ b/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/DockerInfrastructure.kt @@ -15,6 +15,7 @@ import com.github.dockerjava.core.DockerClientImpl import com.github.dockerjava.zerodep.ZerodepDockerHttpClient import java.time.Duration import java.util.* +import java.util.UUID.randomUUID import java.util.concurrent.ConcurrentLinkedDeque internal class DockerInfrastructure : Infrastructure { @@ -30,14 +31,15 @@ internal class DockerInfrastructure : Infrastructure { allocatedResources.add(docker) network = docker .createNetworkCmd() - .withName(UUID.randomUUID().toString()) + .withName(randomUUID().toString()) .execAsResource(docker) allocatedResources.add(network) } override fun serve(jiraNodePlans: List): List { return jiraNodePlans.mapIndexed { nodeIndex, plan -> - plan.materialize(serve(8080, "jira-node-$nodeIndex")) + val nodeNumber = nodeIndex + 1 + plan.materialize(serve(8080, "jira-node-$nodeNumber")) } } @@ -56,57 +58,41 @@ internal class DockerInfrastructure : Infrastructure { .exec(PullImageResultCallback()) .awaitCompletion() val exposedPort = ExposedPort.tcp(port) - val dockerDaemonSocket = "/var/run/docker.sock" val createdContainer = docker .createContainerCmd("rastasheep/ubuntu-sshd:18.04") .withHostConfig( HostConfig() - .withPublishAllPorts(false) - .withPortBindings( - PortBinding(Ports.Binding("0.0.0.0", "0"), ExposedPort.tcp(22)), - PortBinding(Ports.Binding("0.0.0.0", "0"), ExposedPort.tcp(port)) - ) - .withPrivileged(false) + .withPublishAllPorts(true) +// .withPortBindings( +// PortBinding(Ports.Binding("0.0.0.0", "0"), ExposedPort.tcp(22)), +// PortBinding(Ports.Binding("0.0.0.0", "0"), exposedPort) +// ) + .withPrivileged(true) .withNetworkMode(network.response.id) -// .withSecurityOpts(emptyList()) -// .withUsernsMode("host") -// .withBinds(Bind(dockerDaemonSocket, Volume(dockerDaemonSocket))) ) .withExposedPorts(exposedPort, ExposedPort.tcp(22)) - .withName(name + "-" + UUID.randomUUID()) + .withName("$name-${randomUUID()}") .execAsResource(docker) allocatedResources.addLast(createdContainer) - return start(createdContainer, exposedPort) + return start(createdContainer, exposedPort, name) } -// private fun start( -// created: CreatedContainer, -// port: ExposedPort -// ): TcpHost { -// val connectedContainer = docker // TODO remove? -// .connectToNetworkCmd() -// .withContainerId(created.response.id) -// .withNetworkId(network.response.id) -// .execAsResource(docker) -// allocatedResources.addLast(connectedContainer) -// return start(created, port) -// } - private fun start( created: CreatedContainer, - port: ExposedPort + port: ExposedPort, + name: String ): TcpHost { val startedContainer = docker .startContainerCmd(created.response.id) .execAsResource(docker) allocatedResources.addLast(startedContainer); - return install(startedContainer, port) + return install(startedContainer, port, name) } private fun install( started: StartedContainer, -// connected: ConnectedContainer, - port: ExposedPort + port: ExposedPort, + name: String ): TcpHost { val networkSettings = docker .inspectContainerCmd(started.id) @@ -117,9 +103,7 @@ internal class DockerInfrastructure : Infrastructure { .values .single { it.networkID == network.response.id } .ipAddress!! - val portBindings = networkSettings.ports.bindings - val sshPort = getHostPort(portBindings, ExposedPort.tcp(22)) - val tcpPort = getHostPort(portBindings, port) + val sshPort = getHostPort(networkSettings, ExposedPort.tcp(22)) val sshHost = SshHost( ipAddress = "localhost", userName = "root", @@ -131,14 +115,16 @@ internal class DockerInfrastructure : Infrastructure { it.execute("apt-get update", Duration.ofMinutes(2)) it.execute("apt-get -y install sudo gnupg screen") } - return TcpHost("localhost", ip, tcpPort, started.id, ssh) + return TcpHost("localhost", ip, port.port, name, ssh) } private fun getHostPort( - portBindings: Map>, + networkSettings: NetworkSettings, port: ExposedPort ): Int { - return portBindings[port]!! + return networkSettings + .ports + .bindings[port]!! .single() .hostPortSpec .toInt() diff --git a/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/JiraDataCenterPlanIT.kt b/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/JiraDataCenterPlanIT.kt index 903913e7..c306e739 100644 --- a/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/JiraDataCenterPlanIT.kt +++ b/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/JiraDataCenterPlanIT.kt @@ -52,30 +52,67 @@ class JiraDataCenterPlanIT { .also { Datasets.JiraSevenDataset.hookMysql(it, infrastructure) } val dcPlan = JiraDataCenterPlan(nodePlans, instanceHooks, Supplier { TODO() }, infrastructure) + // when + val dataCenter = dcPlan.materialize() + + // then + dataCenter.nodes.forEach { node -> + val installed = node.installed + val serverXml = installed + .installation + .resolve("conf/server.xml") + .download(Files.createTempFile("downloaded-config", ".xml")) + assertThat(serverXml.readText()).contains(" - val installed = node.installed - val serverXml = installed - .installation - .resolve("conf/server.xml") - .download(Files.createTempFile("downloaded-config", ".xml")) - assertThat(serverXml.readText()).contains(" Date: Fri, 23 Apr 2021 15:57:06 +0200 Subject: [PATCH 22/52] JCES-273: Add load balancers to Data Center --- .../infrastructure/api/Infrastructure.kt | 7 +- .../api/jira/instance/JiraDataCenterPlan.kt | 22 ++--- .../api/loadbalancer/ApacheProxyPlan.kt | 81 +++++++++++++++++++ .../api/loadbalancer/LoadBalancerPlan.kt | 7 ++ .../api/DockerInfrastructure.kt | 22 ++--- .../api/jira/instance/JiraDataCenterPlanIT.kt | 26 ++++-- 6 files changed, 130 insertions(+), 35 deletions(-) create mode 100644 src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/loadbalancer/ApacheProxyPlan.kt create mode 100644 src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/loadbalancer/LoadBalancerPlan.kt diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/Infrastructure.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/Infrastructure.kt index de596b36..f01ba8b2 100644 --- a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/Infrastructure.kt +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/Infrastructure.kt @@ -1,10 +1,11 @@ package com.atlassian.performance.tools.infrastructure.api import com.atlassian.performance.tools.infrastructure.api.jira.install.TcpHost -import com.atlassian.performance.tools.infrastructure.api.jira.node.JiraNode -import com.atlassian.performance.tools.infrastructure.api.jira.node.JiraNodePlan interface Infrastructure : AutoCloseable { - fun serve(jiraNodePlans: List): List + + /** + * @return can be reached by the caller via [TcpHost.publicIp] and by the rest of the infra via [TcpHost.privateIp] + */ fun serve(port: Int, name: String): TcpHost } \ No newline at end of file diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/JiraDataCenterPlan.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/JiraDataCenterPlan.kt index ac5be1ca..8efbaa8f 100644 --- a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/JiraDataCenterPlan.kt +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/JiraDataCenterPlan.kt @@ -7,17 +7,17 @@ import com.atlassian.performance.tools.infrastructure.api.jira.node.JiraNodePlan import com.atlassian.performance.tools.infrastructure.api.jira.report.Reports import com.atlassian.performance.tools.infrastructure.api.jira.start.StartedJira import com.atlassian.performance.tools.infrastructure.api.loadbalancer.LoadBalancer +import com.atlassian.performance.tools.infrastructure.api.loadbalancer.LoadBalancerPlan import java.net.URI import java.time.Duration -import java.util.function.Supplier import kotlin.streams.asStream import kotlin.streams.toList class JiraDataCenterPlan constructor( - val nodePlans: List, - val instanceHooks: PreInstanceHooks, - val loadBalancerSupplier: Supplier, - val infrastructure: Infrastructure + private val nodePlans: List, + private val instanceHooks: PreInstanceHooks, + private val balancerPlan: LoadBalancerPlan, + private val infrastructure: Infrastructure ) : JiraInstancePlan { private val reports: Reports = Reports() @@ -25,13 +25,17 @@ class JiraDataCenterPlan constructor( override fun materialize(): JiraInstance { instanceHooks.call(nodePlans.map { it.hooks }, reports) - val nodes = infrastructure.serve(nodePlans) + val nodes = nodePlans.mapIndexed { nodeIndex, nodePlan -> + val nodeNumber = nodeIndex + 1 + val host = infrastructure.serve(8080, "jira-node-$nodeNumber") + nodePlan.materialize(host) + } + val balancer = balancerPlan.materialize(nodes) val installed = installInParallel(nodes) val started = installed.map { it.start(reports) } - val loadBalancer = loadBalancerSupplier.get() - val instance = JiraDataCenter(started, loadBalancer) + val instance = JiraDataCenter(started, balancer) instanceHooks.postInstance.call(instance, reports) - loadBalancer.waitUntilHealthy(loadBalancingPatience) + balancer.waitUntilHealthy(loadBalancingPatience) return instance } diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/loadbalancer/ApacheProxyPlan.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/loadbalancer/ApacheProxyPlan.kt new file mode 100644 index 00000000..4d4b9578 --- /dev/null +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/loadbalancer/ApacheProxyPlan.kt @@ -0,0 +1,81 @@ +package com.atlassian.performance.tools.infrastructure.api.loadbalancer + +import com.atlassian.performance.tools.infrastructure.api.Infrastructure +import com.atlassian.performance.tools.infrastructure.api.Sed +import com.atlassian.performance.tools.infrastructure.api.jira.install.InstalledJira +import com.atlassian.performance.tools.infrastructure.api.jira.node.JiraNode +import com.atlassian.performance.tools.infrastructure.api.jira.report.Reports +import com.atlassian.performance.tools.infrastructure.api.jira.start.hook.PreStartHook +import com.atlassian.performance.tools.infrastructure.api.jira.start.hook.PreStartHooks +import com.atlassian.performance.tools.infrastructure.api.os.Ubuntu +import com.atlassian.performance.tools.jvmtasks.api.ExponentialBackoff +import com.atlassian.performance.tools.jvmtasks.api.IdempotentAction +import com.atlassian.performance.tools.ssh.api.SshConnection +import java.net.URI +import java.time.Duration + +class ApacheProxyPlan( + private val httpPort: Int, + private val infrastructure: Infrastructure +) : LoadBalancerPlan { + + private val configPath = "/etc/apache2/sites-enabled/000-default.conf" + + override fun materialize(nodes: List): LoadBalancer { + val proxyNode = infrastructure.serve(httpPort, "apache-proxy") + IdempotentAction("Installing and configuring apache load balancer") { + proxyNode.ssh.newConnection().use { connection -> + tryToProvision(connection, nodes) + } + }.retry(2, ExponentialBackoff(Duration.ofSeconds(5))) + val balancerEndpoint = URI("http://${proxyNode.privateIp}:$httpPort/") + nodes.forEach { it.plan.hooks.preStart.insert(InjectProxy(balancerEndpoint)) } + return ApacheProxy(balancerEndpoint) + } + + private fun tryToProvision(ssh: SshConnection, nodes: List) { + Ubuntu().install(ssh, listOf("apache2")) + ssh.execute("sudo rm $configPath") + ssh.execute("sudo touch $configPath") + val mods = listOf( + "proxy", "proxy_ajp", "proxy_http", "rewrite", "deflate", "headers", "proxy_balancer", "proxy_connect", + "proxy_html", "xml2enc", "lbmethod_byrequests" + ) + ssh.execute("sudo a2enmod ${mods.joinToString(" ")}") + appendConfig( + ssh, + "Header add Set-Cookie \\\"ROUTEID=.%{BALANCER_WORKER_ROUTE}e; path=/\\\" env=BALANCER_ROUTE_CHANGED" + ) + appendConfig(ssh, "") + nodes.forEachIndexed { index, node -> + appendConfig(ssh, "\tBalancerMember http://${node.host.publicIp}:${node.host.port} route=$index") + } + appendConfig(ssh, "\n") + appendConfig(ssh, "ProxyPass / balancer://mycluster/ stickysession=ROUTEID") + appendConfig(ssh, "ProxyPassReverse / balancer://mycluster/ stickysession=ROUTEID") + ssh.execute("sudo service apache2 restart", Duration.ofMinutes(3)) + } + + private fun appendConfig(connection: SshConnection, line: String) { + connection.execute("echo \"$line\" | sudo tee -a $configPath") + } + + private class ApacheProxy( + override val uri: URI + ) : LoadBalancer { + override fun waitUntilHealthy(timeout: Duration) {} + } + + private class InjectProxy( + private val proxy: URI + ) : PreStartHook { + override fun call(ssh: SshConnection, jira: InstalledJira, hooks: PreStartHooks, reports: Reports) { + Sed().replace( + ssh, + "bindOnInit=\"false\"", + "bindOnInit=\"false\" scheme=\"http\" proxyName=\"${proxy.host}\" proxyPort=\"${proxy.port}\"", + "${jira.installation.path}/conf/server.xml" + ) + } + } +} diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/loadbalancer/LoadBalancerPlan.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/loadbalancer/LoadBalancerPlan.kt new file mode 100644 index 00000000..a433661b --- /dev/null +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/loadbalancer/LoadBalancerPlan.kt @@ -0,0 +1,7 @@ +package com.atlassian.performance.tools.infrastructure.api.loadbalancer + +import com.atlassian.performance.tools.infrastructure.api.jira.node.JiraNode + +interface LoadBalancerPlan { + fun materialize(nodes: List): LoadBalancer +} \ No newline at end of file diff --git a/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/DockerInfrastructure.kt b/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/DockerInfrastructure.kt index c16032af..297f12e5 100644 --- a/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/DockerInfrastructure.kt +++ b/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/DockerInfrastructure.kt @@ -1,15 +1,18 @@ package com.atlassian.performance.tools.infrastructure.api import com.atlassian.performance.tools.infrastructure.api.jira.install.TcpHost -import com.atlassian.performance.tools.infrastructure.api.jira.node.JiraNode -import com.atlassian.performance.tools.infrastructure.api.jira.node.JiraNodePlan -import com.atlassian.performance.tools.infrastructure.lib.docker.* +import com.atlassian.performance.tools.infrastructure.lib.docker.CreatedContainer +import com.atlassian.performance.tools.infrastructure.lib.docker.DockerNetwork +import com.atlassian.performance.tools.infrastructure.lib.docker.StartedContainer +import com.atlassian.performance.tools.infrastructure.lib.docker.execAsResource import com.atlassian.performance.tools.ssh.api.Ssh import com.atlassian.performance.tools.ssh.api.SshHost import com.atlassian.performance.tools.ssh.api.auth.PasswordAuthentication import com.github.dockerjava.api.DockerClient import com.github.dockerjava.api.command.PullImageResultCallback -import com.github.dockerjava.api.model.* +import com.github.dockerjava.api.model.ExposedPort +import com.github.dockerjava.api.model.HostConfig +import com.github.dockerjava.api.model.NetworkSettings import com.github.dockerjava.core.DefaultDockerClientConfig import com.github.dockerjava.core.DockerClientImpl import com.github.dockerjava.zerodep.ZerodepDockerHttpClient @@ -36,13 +39,6 @@ internal class DockerInfrastructure : Infrastructure { allocatedResources.add(network) } - override fun serve(jiraNodePlans: List): List { - return jiraNodePlans.mapIndexed { nodeIndex, plan -> - val nodeNumber = nodeIndex + 1 - plan.materialize(serve(8080, "jira-node-$nodeNumber")) - } - } - fun serve(): Ssh { return serve("ssh") } @@ -63,10 +59,6 @@ internal class DockerInfrastructure : Infrastructure { .withHostConfig( HostConfig() .withPublishAllPorts(true) -// .withPortBindings( -// PortBinding(Ports.Binding("0.0.0.0", "0"), ExposedPort.tcp(22)), -// PortBinding(Ports.Binding("0.0.0.0", "0"), exposedPort) -// ) .withPrivileged(true) .withNetworkMode(network.response.id) ) diff --git a/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/JiraDataCenterPlanIT.kt b/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/JiraDataCenterPlanIT.kt index c306e739..690772e4 100644 --- a/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/JiraDataCenterPlanIT.kt +++ b/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/JiraDataCenterPlanIT.kt @@ -5,17 +5,23 @@ import com.atlassian.performance.tools.infrastructure.api.DockerInfrastructure import com.atlassian.performance.tools.infrastructure.api.Infrastructure import com.atlassian.performance.tools.infrastructure.api.distribution.PublicJiraSoftwareDistribution import com.atlassian.performance.tools.infrastructure.api.jira.JiraHomePackage +import com.atlassian.performance.tools.infrastructure.api.jira.install.InstalledJira import com.atlassian.performance.tools.infrastructure.api.jira.install.ParallelInstallation import com.atlassian.performance.tools.infrastructure.api.jira.install.hook.PreInstallHooks import com.atlassian.performance.tools.infrastructure.api.jira.node.JiraNodePlan +import com.atlassian.performance.tools.infrastructure.api.jira.report.Reports import com.atlassian.performance.tools.infrastructure.api.jira.start.JiraLaunchScript +import com.atlassian.performance.tools.infrastructure.api.jira.start.hook.PreStartHook +import com.atlassian.performance.tools.infrastructure.api.jira.start.hook.PreStartHooks import com.atlassian.performance.tools.infrastructure.api.jvm.AdoptOpenJDK +import com.atlassian.performance.tools.infrastructure.api.loadbalancer.ApacheProxyPlan +import com.atlassian.performance.tools.ssh.api.SshConnection import org.assertj.core.api.Assertions.assertThat import org.junit.After import org.junit.Before import org.junit.Test +import java.lang.Exception import java.nio.file.Files -import java.util.function.Supplier class JiraDataCenterPlanIT { @@ -48,9 +54,9 @@ class JiraDataCenterPlanIT { .build() } val instanceHooks = PreInstanceHooks.default() - // TODO this plus `EmptyJiraHome()` = failing `DatabaseIpConfig` - couple them together or stop expecting a preexisting `dbconfig.xml`? but then what about missing lucene indexes? .also { Datasets.JiraSevenDataset.hookMysql(it, infrastructure) } - val dcPlan = JiraDataCenterPlan(nodePlans, instanceHooks, Supplier { TODO() }, infrastructure) + val balancerPlan = ApacheProxyPlan(80, infrastructure) + val dcPlan = JiraDataCenterPlan(nodePlans, instanceHooks, balancerPlan, infrastructure) // when val dataCenter = dcPlan.materialize() @@ -71,6 +77,12 @@ class JiraDataCenterPlanIT { @Test fun shouldProvideLogsToDiagnoseFailure() { // given + class FailingHook : PreStartHook { + override fun call(ssh: SshConnection, jira: InstalledJira, hooks: PreStartHooks, reports: Reports) { + throw Exception("Failing deliberately before Jira starts") + } + } + val nodePlans = listOf(1, 2).map { JiraNodePlan.Builder() .installation( @@ -81,13 +93,11 @@ class JiraDataCenterPlanIT { ) ) .start(JiraLaunchScript()) - .hooks(PreInstallHooks.default().also { Datasets.JiraSevenDataset.hookMysql(it.postStart) }) + .hooks(PreInstallHooks.default().also { it.preStart.insert(FailingHook()) }) .build() } - val instanceHooks = PreInstanceHooks.default() - // TODO this plus `EmptyJiraHome()` = failing `DatabaseIpConfig` - couple them together or stop expecting a preexisting `dbconfig.xml`? but then what about missing lucene indexes? - .also { Datasets.JiraSevenDataset.hookMysql(it, infrastructure) } - val dcPlan = JiraDataCenterPlan(nodePlans, instanceHooks, Supplier { TODO() }, infrastructure) + val balancerPlan = ApacheProxyPlan(80, infrastructure) + val dcPlan = JiraDataCenterPlan(nodePlans, PreInstanceHooks.default(), balancerPlan, infrastructure) try { // when From 1093e78efef4d918a4bf1cca09318218c32ac39e Mon Sep 17 00:00:00 2001 From: Maciej Kwidzinski Date: Fri, 23 Apr 2021 18:47:38 +0200 Subject: [PATCH 23/52] WIP: Try mounting shared home It fails due to: ``` 2021-04-23T16:36:59,942Z DEBUG Test worker [] [com.atlassian.performance.tools.ssh.SshjConnection] root$ sudo service nfs-kernel-server restart 2021-04-23T16:37:00,977Z DEBUG Test worker [] [com.atlassian.performance.tools.ssh.WaitingCommand.Companion] * Stopping NFS kernel daemon ...done. * Unexporting directories for NFS kernel daemon... ...done. * Exporting directories for NFS kernel daemon... ...fail! 2021-04-23T16:37:00,979Z WARN Test worker [] [com.atlassian.performance.tools.ssh.WaitingCommand.Companion] exportfs: /home/ubuntu/jira-shared-home does not support NFS export ``` Options: * keep trying NFS in Docker (in WSL) * try other FS sharing (FUSE? SAN?) * use different FS sharing in Docker (volumes?) and different in AWS (so SPI, but what is default? maybe no default or maybe bound to `Infrastructure` SPI itself) --- .../infrastructure/api/Infrastructure.kt | 10 ++- .../api/database/DockerMysqlServer.kt | 2 +- .../api/jira/install/hook/DataCenterHook.kt | 26 -------- .../api/jira/instance/JiraDataCenterPlan.kt | 2 +- .../api/jira/instance/JiraServerPlan.kt | 2 +- .../api/jira/instance/SharedHomeHook.kt | 62 +++++++++++++++++++ .../infrastructure/api/jira/report/Reports.kt | 2 +- .../api/loadbalancer/ApacheProxyPlan.kt | 5 +- .../api/DockerInfrastructure.kt | 31 ++++++++-- .../infrastructure/api/browser/ChromeIT.kt | 2 +- .../api/browser/chromium/Chromium69IT.kt | 2 +- .../chromium/PageLoadTimeoutRecoveryTest.kt | 2 +- .../api/dataset/HttpDatasetPackageIT.kt | 2 +- .../PublicJiraServiceDeskDistributionIT.kt | 2 +- .../PublicJiraSoftwareDistributionsIT.kt | 2 +- .../infrastructure/api/docker/DockerIT.kt | 2 +- .../api/jira/instance/JiraDataCenterPlanIT.kt | 13 ++-- .../api/jvm/AdoptOpenJdk11IT.kt | 2 +- .../infrastructure/api/jvm/AdoptOpenJdkIT.kt | 2 +- .../infrastructure/api/jvm/OpenJdk11IT.kt | 2 +- .../tools/infrastructure/api/jvm/OpenJdkIT.kt | 2 +- .../infrastructure/api/jvm/OracleJdkIT.kt | 2 +- .../tools/infrastructure/api/os/UbuntuIT.kt | 10 +-- 23 files changed, 126 insertions(+), 63 deletions(-) delete mode 100644 src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/DataCenterHook.kt create mode 100644 src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/SharedHomeHook.kt diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/Infrastructure.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/Infrastructure.kt index f01ba8b2..7fb7c99a 100644 --- a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/Infrastructure.kt +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/Infrastructure.kt @@ -1,11 +1,15 @@ package com.atlassian.performance.tools.infrastructure.api import com.atlassian.performance.tools.infrastructure.api.jira.install.TcpHost +import com.atlassian.performance.tools.ssh.api.Ssh + +interface Infrastructure : AutoCloseable { // TODO rename to ServerRoom + + val subnet: String -interface Infrastructure : AutoCloseable { - /** * @return can be reached by the caller via [TcpHost.publicIp] and by the rest of the infra via [TcpHost.privateIp] */ - fun serve(port: Int, name: String): TcpHost + fun serveTcp(name: String): TcpHost + fun serveSsh(name: String): Ssh } \ No newline at end of file diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/database/DockerMysqlServer.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/database/DockerMysqlServer.kt index 75cb01d3..232ab29e 100644 --- a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/database/DockerMysqlServer.kt +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/database/DockerMysqlServer.kt @@ -28,7 +28,7 @@ class DockerMysqlServer private constructor( hooks: PreInstanceHooks, reports: Reports ) { - val server = infrastructure.serve(3306, "mysql") + val server = infrastructure.serveTcp("mysql") val client = server.ssh.newConnection().use { setup(it, server) } nodes.forEach { node -> node.postInstall.insert(DatabaseIpConfig(server.privateIp)) diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/DataCenterHook.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/DataCenterHook.kt deleted file mode 100644 index 5ec19a6d..00000000 --- a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/DataCenterHook.kt +++ /dev/null @@ -1,26 +0,0 @@ -package com.atlassian.performance.tools.infrastructure.api.jira.install.hook - -import com.atlassian.performance.tools.infrastructure.api.jira.SharedHome -import com.atlassian.performance.tools.infrastructure.api.jira.install.InstalledJira -import com.atlassian.performance.tools.infrastructure.api.jira.report.Reports -import com.atlassian.performance.tools.ssh.api.SshConnection - -class DataCenterHook( - private val nodeId: String, - private val sharedHome: SharedHome -) : PostInstallHook { - - override fun call( - ssh: SshConnection, - jira: InstalledJira, - hooks: PostInstallHooks, - reports: Reports - ) { - val localSharedHome = sharedHome.localSharedHome - sharedHome.mount(ssh) - val jiraHome = jira.home.path // TODO what's the difference between localSharedHome and jiraHome? should both be hookable? - ssh.execute("echo ehcache.object.port = 40011 >> $jiraHome/cluster.properties") - ssh.execute("echo jira.node.id = $nodeId >> $jiraHome/cluster.properties") - ssh.execute("echo jira.shared.home = `realpath $localSharedHome` >> $jiraHome/cluster.properties") - } -} diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/JiraDataCenterPlan.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/JiraDataCenterPlan.kt index 8efbaa8f..0d1d926c 100644 --- a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/JiraDataCenterPlan.kt +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/JiraDataCenterPlan.kt @@ -27,7 +27,7 @@ class JiraDataCenterPlan constructor( instanceHooks.call(nodePlans.map { it.hooks }, reports) val nodes = nodePlans.mapIndexed { nodeIndex, nodePlan -> val nodeNumber = nodeIndex + 1 - val host = infrastructure.serve(8080, "jira-node-$nodeNumber") + val host = infrastructure.serveTcp("jira-node-$nodeNumber") nodePlan.materialize(host) } val balancer = balancerPlan.materialize(nodes) diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/JiraServerPlan.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/JiraServerPlan.kt index 322c0034..c9d7da75 100644 --- a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/JiraServerPlan.kt +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/JiraServerPlan.kt @@ -17,7 +17,7 @@ class JiraServerPlan private constructor( override fun materialize(): JiraInstance { val nodeHooks = listOf(plan).map { it.hooks } hooks.call(nodeHooks, reports) - val jiraNode = infrastructure.serve(8080, "jira-node") + val jiraNode = infrastructure.serveTcp("jira-node") val installed = plan.installation.install(jiraNode, reports) val started = plan.start.start(installed, reports) val instance = JiraServer(started) diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/SharedHomeHook.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/SharedHomeHook.kt new file mode 100644 index 00000000..17f79860 --- /dev/null +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/SharedHomeHook.kt @@ -0,0 +1,62 @@ +package com.atlassian.performance.tools.infrastructure.api.jira.instance + +import com.atlassian.performance.tools.infrastructure.api.Infrastructure +import com.atlassian.performance.tools.infrastructure.api.jira.JiraHomeSource +import com.atlassian.performance.tools.infrastructure.api.jira.install.InstalledJira +import com.atlassian.performance.tools.infrastructure.api.jira.install.hook.PostInstallHook +import com.atlassian.performance.tools.infrastructure.api.jira.install.hook.PostInstallHooks +import com.atlassian.performance.tools.infrastructure.api.jira.install.hook.PreInstallHooks +import com.atlassian.performance.tools.infrastructure.api.jira.report.Reports +import com.atlassian.performance.tools.infrastructure.api.os.RemotePath +import com.atlassian.performance.tools.infrastructure.api.os.Ubuntu +import com.atlassian.performance.tools.ssh.api.SshConnection + +internal class SharedHomeHook( + private val jiraHomeSource: JiraHomeSource, + private val infrastructure: Infrastructure +) : PreInstanceHook { + private val localHome = "/home/ubuntu/jira-shared-home" + + override fun call(nodes: List, hooks: PreInstanceHooks, reports: Reports) { + val server = infrastructure.serveSsh("shared-home") + server.newConnection().use { ssh -> + download(ssh) + export(ssh) + } + val sharedHome = RemotePath(server.host, localHome) + nodes.forEach { it.postInstall.insert(SharedHomeMount(sharedHome)) } + } + + private fun download(ssh: SshConnection) { + ssh.execute("sudo mkdir -p $localHome") + val jiraHome = jiraHomeSource.download(ssh) + ssh.execute("sudo mv $jiraHome/{data,plugins,import,export} $localHome") + ssh.safeExecute("sudo mv $jiraHome/logos $localHome") + } + + private fun export(ssh: SshConnection): SshConnection.SshResult { + Ubuntu().install(ssh, listOf("nfs-kernel-server")) + val options = "rw,sync,no_subtree_check,no_root_squash" + ssh.execute("sudo echo '$localHome ${infrastructure.subnet}($options)' | sudo tee -a /etc/exports") + return ssh.execute("sudo service nfs-kernel-server restart") + } + + private class SharedHomeMount( + private val sharedHome: RemotePath + ) : PostInstallHook { + + override fun call(ssh: SshConnection, jira: InstalledJira, hooks: PostInstallHooks, reports: Reports) { + Ubuntu().install(ssh, listOf("nfs-common")) + val mountSource = "${sharedHome.host.ipAddress}:${sharedHome.path}" + val mountTarget = "mounted-shared-home" + ssh.execute("mkdir -p $mountTarget") + ssh.execute("sudo mount -o soft,intr,rsize=8192,wsize=8192 $mountSource $mountTarget") + ssh.execute("sudo chown ubuntu:ubuntu $mountTarget") + val mountedPath = "`realpath $mountTarget`" + val jiraHome = jira.home.path + ssh.execute("echo ehcache.object.port = 40011 >> $jiraHome/cluster.properties") + ssh.execute("echo jira.node.id = ${jira.host.name} >> $jiraHome/cluster.properties") + ssh.execute("echo jira.shared.home = $mountedPath >> $jiraHome/cluster.properties") + } + } +} diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/report/Reports.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/report/Reports.kt index 1a579e41..84aa3ae9 100644 --- a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/report/Reports.kt +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/report/Reports.kt @@ -10,7 +10,7 @@ import java.nio.file.Paths import java.util.* import java.util.concurrent.ConcurrentLinkedQueue -class Reports private constructor( +class Reports private constructor( // TODO turn into SPI to allow AWS CLI transport (S3) private val hostReports: Queue ) { constructor() : this(ConcurrentLinkedQueue()) diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/loadbalancer/ApacheProxyPlan.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/loadbalancer/ApacheProxyPlan.kt index 4d4b9578..6588dddc 100644 --- a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/loadbalancer/ApacheProxyPlan.kt +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/loadbalancer/ApacheProxyPlan.kt @@ -15,20 +15,19 @@ import java.net.URI import java.time.Duration class ApacheProxyPlan( - private val httpPort: Int, private val infrastructure: Infrastructure ) : LoadBalancerPlan { private val configPath = "/etc/apache2/sites-enabled/000-default.conf" override fun materialize(nodes: List): LoadBalancer { - val proxyNode = infrastructure.serve(httpPort, "apache-proxy") + val proxyNode = infrastructure.serveTcp("apache-proxy") IdempotentAction("Installing and configuring apache load balancer") { proxyNode.ssh.newConnection().use { connection -> tryToProvision(connection, nodes) } }.retry(2, ExponentialBackoff(Duration.ofSeconds(5))) - val balancerEndpoint = URI("http://${proxyNode.privateIp}:$httpPort/") + val balancerEndpoint = URI("http://${proxyNode.privateIp}:${proxyNode.port}/") nodes.forEach { it.plan.hooks.preStart.insert(InjectProxy(balancerEndpoint)) } return ApacheProxy(balancerEndpoint) } diff --git a/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/DockerInfrastructure.kt b/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/DockerInfrastructure.kt index 297f12e5..463979ca 100644 --- a/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/DockerInfrastructure.kt +++ b/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/DockerInfrastructure.kt @@ -26,6 +26,7 @@ internal class DockerInfrastructure : Infrastructure { private val allocatedResources: Deque = ConcurrentLinkedDeque() private val docker: DockerClient private val network: DockerNetwork + override val subnet: String init { val dockerConfig = DefaultDockerClientConfig.createDefaultConfigBuilder().build() @@ -37,17 +38,37 @@ internal class DockerInfrastructure : Infrastructure { .withName(randomUUID().toString()) .execAsResource(docker) allocatedResources.add(network) + subnet = docker + .inspectNetworkCmd() + .withNetworkId(network.response.id) + .exec() + .ipam + .config + .first() + .subnet + } + + fun serveTest(): Ssh { + return serveSsh("ssh") } - fun serve(): Ssh { - return serve("ssh") + override fun serveSsh(name: String): Ssh { + return serveTcp(888, name).ssh } - fun serve(name: String): Ssh { - return serve(888, name).ssh + + override fun serveTcp(name: String): TcpHost { + return when { + name.startsWith("jira-node") -> serveTcp(8080, name) // TODO this is a contract on undocumented behavior + name.startsWith("mysql") -> serveTcp(3306, name) + else -> serveTcp( + 888, + name + ) // TODO pre-provision all the hosts rather than on-demand - unlock batch provisioning (CFN Stack), picking EC2 types, SSD storage, TCP port ranges, subnets, etc. + } } - override fun serve(port: Int, name: String): TcpHost { + private fun serveTcp(port: Int, name: String): TcpHost { docker .pullImageCmd("rastasheep/ubuntu-sshd") .withTag("18.04") diff --git a/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/browser/ChromeIT.kt b/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/browser/ChromeIT.kt index e6f87435..383b8791 100644 --- a/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/browser/ChromeIT.kt +++ b/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/browser/ChromeIT.kt @@ -12,7 +12,7 @@ class ChromeIT { @Test fun shouldInstallChromeBrowser() { DockerInfrastructure().use { infra -> - infra.serve(80, "ChromeIT").ssh.newConnection().use { connection -> + infra.serveSsh("ChromeIT").newConnection().use { connection -> val wasInstalledBefore = isChromeInstalled(connection) Chrome().install(connection) diff --git a/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/browser/chromium/Chromium69IT.kt b/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/browser/chromium/Chromium69IT.kt index 0766022d..331a9c45 100644 --- a/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/browser/chromium/Chromium69IT.kt +++ b/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/browser/chromium/Chromium69IT.kt @@ -14,7 +14,7 @@ class Chromium69IT { @Test fun shouldInstallBrowser() { DockerInfrastructure().use { infra -> - infra.serve().newConnection().use { connection -> + infra.serveTest().newConnection().use { connection -> val installedBefore = isChromiumInstalled(connection) Chromium69().install(connection) diff --git a/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/browser/chromium/PageLoadTimeoutRecoveryTest.kt b/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/browser/chromium/PageLoadTimeoutRecoveryTest.kt index b66f47e2..6ee7cb83 100644 --- a/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/browser/chromium/PageLoadTimeoutRecoveryTest.kt +++ b/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/browser/chromium/PageLoadTimeoutRecoveryTest.kt @@ -20,7 +20,7 @@ internal class PageLoadTimeoutRecoveryTest { val fastResource = httpServer.register(FastResponse()) val slowResource = httpServer.register(SlowResponse()) DockerInfrastructure().use { infra -> - val ssh = infra.serve() + val ssh = infra.serveTest() ssh.forwardRemotePort(httpServer.getPort(), httpServer.getPort()).use { val localChromedriverPort = findFreePort() ssh.forwardLocalPort(localChromedriverPort, remoteChromedriverPort).use { diff --git a/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/dataset/HttpDatasetPackageIT.kt b/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/dataset/HttpDatasetPackageIT.kt index 1c307690..0281cb58 100644 --- a/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/dataset/HttpDatasetPackageIT.kt +++ b/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/dataset/HttpDatasetPackageIT.kt @@ -21,7 +21,7 @@ class HttpDatasetPackageIT { ) val filesInDataset = DockerInfrastructure().use { infra -> - val ssh = infra.serve(80, "HttpDatasetPackageIT").ssh + val ssh = infra.serveSsh("HttpDatasetPackageIT") return@use RandomFilesGenerator(ssh).start().use { ssh.newConnection().use { connection -> val unpackedPath = dataset.download(connection) diff --git a/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/distribution/PublicJiraServiceDeskDistributionIT.kt b/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/distribution/PublicJiraServiceDeskDistributionIT.kt index c27ec873..d27a0666 100644 --- a/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/distribution/PublicJiraServiceDeskDistributionIT.kt +++ b/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/distribution/PublicJiraServiceDeskDistributionIT.kt @@ -9,7 +9,7 @@ class PublicJiraServiceDeskDistributionIT { @Test fun shouldDownloadJiraServiceDesk() { DockerInfrastructure().use { infra -> - infra.serve().newConnection().use { connection -> + infra.serveTest().newConnection().use { connection -> val serviceDeskDistribution: ProductDistribution = PublicJiraServiceDeskDistribution("4.0.1") val targetFolder = "test" connection.execute("mkdir $targetFolder") diff --git a/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/distribution/PublicJiraSoftwareDistributionsIT.kt b/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/distribution/PublicJiraSoftwareDistributionsIT.kt index 85835229..6dd74c85 100644 --- a/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/distribution/PublicJiraSoftwareDistributionsIT.kt +++ b/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/distribution/PublicJiraSoftwareDistributionsIT.kt @@ -9,7 +9,7 @@ class PublicJiraSoftwareDistributionsIT { @Test fun shouldDownloadJiraSoftware() { DockerInfrastructure().use { infra -> - infra.serve().newConnection().use { connection -> + infra.serveTest().newConnection().use { connection -> val jiraDistribution: ProductDistribution = PublicJiraSoftwareDistribution("7.2.0") val targetFolder = "test" connection.execute("mkdir $targetFolder") diff --git a/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/docker/DockerIT.kt b/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/docker/DockerIT.kt index 11d39785..ecd1f8b5 100644 --- a/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/docker/DockerIT.kt +++ b/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/docker/DockerIT.kt @@ -9,7 +9,7 @@ class DockerIT { @Test fun installWorks() { DockerInfrastructure().use { infra -> - infra.serve().newConnection().use { connection -> + infra.serveTest().newConnection().use { connection -> //workaround for a bug in Docker download site for bionic val packageFile = "containerd.io_1.2.2-3_amd64.deb" Ubuntu().install(connection,listOf( "curl")) diff --git a/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/JiraDataCenterPlanIT.kt b/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/JiraDataCenterPlanIT.kt index 690772e4..1be1fc17 100644 --- a/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/JiraDataCenterPlanIT.kt +++ b/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/JiraDataCenterPlanIT.kt @@ -20,7 +20,6 @@ import org.assertj.core.api.Assertions.assertThat import org.junit.After import org.junit.Before import org.junit.Test -import java.lang.Exception import java.nio.file.Files class JiraDataCenterPlanIT { @@ -40,22 +39,26 @@ class JiraDataCenterPlanIT { @Test fun shouldStartDataCenter() { // given + val jiraHomeSource = JiraHomePackage(Datasets.JiraSevenDataset.jiraHome) val nodePlans = listOf(1, 2).map { + val nodeHooks = PreInstallHooks.default() + .also { Datasets.JiraSevenDataset.hookMysql(it.postStart) } JiraNodePlan.Builder() .installation( ParallelInstallation( - jiraHomeSource = JiraHomePackage(Datasets.JiraSevenDataset.jiraHome), + jiraHomeSource = jiraHomeSource, productDistribution = PublicJiraSoftwareDistribution("7.13.0"), jdk = AdoptOpenJDK() ) ) .start(JiraLaunchScript()) - .hooks(PreInstallHooks.default().also { Datasets.JiraSevenDataset.hookMysql(it.postStart) }) + .hooks(nodeHooks) .build() } val instanceHooks = PreInstanceHooks.default() .also { Datasets.JiraSevenDataset.hookMysql(it, infrastructure) } - val balancerPlan = ApacheProxyPlan(80, infrastructure) + .also { it.insert(SharedHomeHook(jiraHomeSource, infrastructure)) } + val balancerPlan = ApacheProxyPlan(infrastructure) val dcPlan = JiraDataCenterPlan(nodePlans, instanceHooks, balancerPlan, infrastructure) // when @@ -96,7 +99,7 @@ class JiraDataCenterPlanIT { .hooks(PreInstallHooks.default().also { it.preStart.insert(FailingHook()) }) .build() } - val balancerPlan = ApacheProxyPlan(80, infrastructure) + val balancerPlan = ApacheProxyPlan(infrastructure) val dcPlan = JiraDataCenterPlan(nodePlans, PreInstanceHooks.default(), balancerPlan, infrastructure) try { diff --git a/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/jvm/AdoptOpenJdk11IT.kt b/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/jvm/AdoptOpenJdk11IT.kt index 6d29a0cc..d80b4819 100644 --- a/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/jvm/AdoptOpenJdk11IT.kt +++ b/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/jvm/AdoptOpenJdk11IT.kt @@ -8,7 +8,7 @@ class AdoptOpenJdk11IT { @Test fun shouldSupportJstat() { DockerInfrastructure().use { infra -> - infra.serve().newConnection().use { connection -> + infra.serveTest().newConnection().use { connection -> JstatSupport(AdoptOpenJDK11()).shouldSupportJstat(connection) } } diff --git a/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/jvm/AdoptOpenJdkIT.kt b/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/jvm/AdoptOpenJdkIT.kt index 577de11e..9859470d 100644 --- a/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/jvm/AdoptOpenJdkIT.kt +++ b/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/jvm/AdoptOpenJdkIT.kt @@ -8,7 +8,7 @@ class AdoptOpenJdkIT { @Test fun shouldSupportJstat() { DockerInfrastructure().use { infra -> - infra.serve().newConnection().use { connection -> + infra.serveTest().newConnection().use { connection -> JstatSupport(AdoptOpenJDK()).shouldSupportJstat(connection) } } diff --git a/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/jvm/OpenJdk11IT.kt b/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/jvm/OpenJdk11IT.kt index 73ae5e02..a3370e7c 100644 --- a/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/jvm/OpenJdk11IT.kt +++ b/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/jvm/OpenJdk11IT.kt @@ -8,7 +8,7 @@ class OpenJdk11IT { @Test fun shouldSupportJstat() { DockerInfrastructure().use { infra -> - infra.serve().newConnection().use { connection -> + infra.serveTest().newConnection().use { connection -> JstatSupport(OpenJDK11()).shouldSupportJstat(connection) } } diff --git a/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/jvm/OpenJdkIT.kt b/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/jvm/OpenJdkIT.kt index ebb23431..7c05be7d 100644 --- a/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/jvm/OpenJdkIT.kt +++ b/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/jvm/OpenJdkIT.kt @@ -8,7 +8,7 @@ class OpenJdkIT { @Test fun shouldSupportJstat() { DockerInfrastructure().use { infra -> - infra.serve().newConnection().use { connection -> + infra.serveTest().newConnection().use { connection -> JstatSupport(OpenJDK()).shouldSupportJstat(connection) } } diff --git a/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/jvm/OracleJdkIT.kt b/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/jvm/OracleJdkIT.kt index 0fa2e116..3ce16533 100644 --- a/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/jvm/OracleJdkIT.kt +++ b/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/jvm/OracleJdkIT.kt @@ -8,7 +8,7 @@ class OracleJdkIT { @Test fun shouldSupportJstatAndThreadDumps() { DockerInfrastructure().use { infra -> - infra.serve().newConnection().use { connection -> + infra.serveTest().newConnection().use { connection -> val jdk = OracleJDK() JstatSupport(jdk).shouldSupportJstat(connection) ThreadDumpTest().shouldGatherThreadDump(jdk, connection) diff --git a/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/os/UbuntuIT.kt b/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/os/UbuntuIT.kt index 6721efcf..78784c32 100644 --- a/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/os/UbuntuIT.kt +++ b/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/os/UbuntuIT.kt @@ -2,7 +2,7 @@ package com.atlassian.performance.tools.infrastructure.api.os import com.atlassian.performance.tools.infrastructure.api.DockerInfrastructure import com.atlassian.performance.tools.infrastructure.api.Infrastructure -import com.atlassian.performance.tools.infrastructure.api.jira.install.TcpHost +import com.atlassian.performance.tools.ssh.api.Ssh import com.atlassian.performance.tools.ssh.api.SshConnection import com.atlassian.performance.tools.ssh.api.SshHost import org.apache.logging.log4j.Level @@ -18,13 +18,13 @@ import java.util.concurrent.TimeUnit class UbuntuIT { private lateinit var executor: ExecutorService private lateinit var infra: Infrastructure - private lateinit var sshUbuntu: TcpHost + private lateinit var sshUbuntu: Ssh @Before fun before() { executor = Executors.newCachedThreadPool() infra = DockerInfrastructure() - sshUbuntu = infra.serve(80, "UbuntuIT") + sshUbuntu = infra.serveSsh("UbuntuIT") } @After @@ -35,7 +35,7 @@ class UbuntuIT { @Test fun shouldRetry() { - sshUbuntu.ssh.newConnection().use { connection -> + sshUbuntu.newConnection().use { connection -> Ubuntu().install( ColdAptSshConnection(connection), listOf("nano"), @@ -94,7 +94,7 @@ class UbuntuIT { } private fun installLftp(latch: CountDownLatch) { - sshUbuntu.ssh.newConnection().use { connection -> + sshUbuntu.newConnection().use { connection -> latch.countDown() latch.await() Ubuntu().install(connection, listOf("lftp")) From 3b52a4cae64edd871f4758c27b36e35b92c584f7 Mon Sep 17 00:00:00 2001 From: Maciej Kwidzinski Date: Fri, 7 May 2021 15:32:57 +0200 Subject: [PATCH 24/52] JPERF-273: Mount shared home via Samba NFS runs in Linux kernel. Docker shares kernel with the host OS. So the host OS might not support NFS (e.g. WSL, CI). Therefore NFS is not portable. However Samba runs in user space. I could emulate NFS with Docker volumes, but it would still leave `NfsSharedHomeHook` untested. --- .../infrastructure/api/Infrastructure.kt | 1 + ...SharedHomeHook.kt => NfsSharedHomeHook.kt} | 6 +- .../api/jira/instance/SambaSharedHomeHook.kt | 86 +++++++++++++++++++ .../api/DockerInfrastructure.kt | 30 ++++--- .../api/jira/instance/JiraDataCenterPlanIT.kt | 2 +- 5 files changed, 109 insertions(+), 16 deletions(-) rename src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/{SharedHomeHook.kt => NfsSharedHomeHook.kt} (95%) create mode 100644 src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/SambaSharedHomeHook.kt diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/Infrastructure.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/Infrastructure.kt index 7fb7c99a..06de898f 100644 --- a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/Infrastructure.kt +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/Infrastructure.kt @@ -11,5 +11,6 @@ interface Infrastructure : AutoCloseable { // TODO rename to ServerRoom * @return can be reached by the caller via [TcpHost.publicIp] and by the rest of the infra via [TcpHost.privateIp] */ fun serveTcp(name: String): TcpHost + fun serve(name: String, tcpPorts: List, udpPorts: List): TcpHost fun serveSsh(name: String): Ssh } \ No newline at end of file diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/SharedHomeHook.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/NfsSharedHomeHook.kt similarity index 95% rename from src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/SharedHomeHook.kt rename to src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/NfsSharedHomeHook.kt index 17f79860..467af37c 100644 --- a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/SharedHomeHook.kt +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/NfsSharedHomeHook.kt @@ -11,7 +11,7 @@ import com.atlassian.performance.tools.infrastructure.api.os.RemotePath import com.atlassian.performance.tools.infrastructure.api.os.Ubuntu import com.atlassian.performance.tools.ssh.api.SshConnection -internal class SharedHomeHook( +internal class NfsSharedHomeHook( private val jiraHomeSource: JiraHomeSource, private val infrastructure: Infrastructure ) : PreInstanceHook { @@ -24,7 +24,7 @@ internal class SharedHomeHook( export(ssh) } val sharedHome = RemotePath(server.host, localHome) - nodes.forEach { it.postInstall.insert(SharedHomeMount(sharedHome)) } + nodes.forEach { it.postInstall.insert(NfsMount(sharedHome)) } } private fun download(ssh: SshConnection) { @@ -41,7 +41,7 @@ internal class SharedHomeHook( return ssh.execute("sudo service nfs-kernel-server restart") } - private class SharedHomeMount( + private class NfsMount( private val sharedHome: RemotePath ) : PostInstallHook { diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/SambaSharedHomeHook.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/SambaSharedHomeHook.kt new file mode 100644 index 00000000..4406fe05 --- /dev/null +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/SambaSharedHomeHook.kt @@ -0,0 +1,86 @@ +package com.atlassian.performance.tools.infrastructure.api.jira.instance + +import com.atlassian.performance.tools.infrastructure.api.Infrastructure +import com.atlassian.performance.tools.infrastructure.api.jira.JiraHomeSource +import com.atlassian.performance.tools.infrastructure.api.jira.install.InstalledJira +import com.atlassian.performance.tools.infrastructure.api.jira.install.TcpHost +import com.atlassian.performance.tools.infrastructure.api.jira.install.hook.PostInstallHook +import com.atlassian.performance.tools.infrastructure.api.jira.install.hook.PostInstallHooks +import com.atlassian.performance.tools.infrastructure.api.jira.install.hook.PreInstallHooks +import com.atlassian.performance.tools.infrastructure.api.jira.report.Reports +import com.atlassian.performance.tools.infrastructure.api.os.Ubuntu +import com.atlassian.performance.tools.ssh.api.SshConnection +import java.util.* + +internal class SambaSharedHomeHook( + private val jiraHomeSource: JiraHomeSource, + private val infrastructure: Infrastructure +) : PreInstanceHook { + + override fun call(nodes: List, hooks: PreInstanceHooks, reports: Reports) { + val server = infrastructure.serve("samba-shared-home", listOf(139, 445), listOf(137, 138)) + val mount = server.ssh.newConnection().use { ssh -> + val sharedHome = download(ssh) + export(ssh, sharedHome, server) + } + nodes.forEach { it.postInstall.insert(mount) } + } + + private fun download(ssh: SshConnection): String { + val sharedHome = "/home/ubuntu/jira-shared-home" + ssh.execute("sudo mkdir -p $sharedHome") + val jiraHome = jiraHomeSource.download(ssh) + ssh.execute("sudo mv $jiraHome/{data,plugins,import,export} $sharedHome") + ssh.safeExecute("sudo mv $jiraHome/logos $sharedHome") + return sharedHome + } + + private fun export(ssh: SshConnection, sharedHome: String, server: TcpHost): SambaMount { + Ubuntu().install(ssh, listOf("samba")) + val shareName = "samba-jira-home" + val share = """ + [$shareName] + comment = shared Jira home + path = $sharedHome + read only = no + browsable = no + """.trimIndent() + ssh.execute("echo '$share' | sudo tee -a /etc/samba/smb.conf") + val user = ssh.execute("whoami").output.trim() + val password = generatePassword() + // could transfer password via file, but it's an ephemeral secret anyway + ssh.execute("echo -e '$password\\n$password\\n' | sudo smbpasswd -s -a $user") + ssh.execute("sudo service smbd restart") + return SambaMount(server.privateIp, shareName, user, password) + } + + private fun generatePassword(): String { + val rng = Random() + val chars = ('a'..'Z') + ('A'..'Z') + ('0'..'9') + return (1..32).map { chars[rng.nextInt(chars.size)] }.joinToString("") + } + + private class SambaMount( + private val ip: String, + private val shareName: String, + private val user: String, + private val password: String + ) : PostInstallHook { + + override fun call(ssh: SshConnection, jira: InstalledJira, hooks: PostInstallHooks, reports: Reports) { + Ubuntu().install(ssh, listOf("cifs-utils")) + val credentials = "username=$user,password=$password" + val mountSource = "//$ip/$shareName" + val mountTarget = "mounted-shared-home" + ssh.execute("mkdir -p $mountTarget") + val localUser = ssh.execute("whoami").output.trim() + ssh.execute("sudo chown $localUser:$localUser $mountTarget") + ssh.execute("sudo mount -t cifs -o $credentials $mountSource $mountTarget") + val sharedHome = "`realpath $mountTarget`" + val nodeHome = jira.home.path + ssh.execute("echo ehcache.object.port = 40011 >> $nodeHome/cluster.properties") + ssh.execute("echo jira.node.id = ${jira.host.name} >> $nodeHome/cluster.properties") + ssh.execute("echo jira.shared.home = $sharedHome >> $nodeHome/cluster.properties") + } + } +} diff --git a/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/DockerInfrastructure.kt b/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/DockerInfrastructure.kt index 463979ca..3292ceac 100644 --- a/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/DockerInfrastructure.kt +++ b/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/DockerInfrastructure.kt @@ -58,23 +58,29 @@ internal class DockerInfrastructure : Infrastructure { override fun serveTcp(name: String): TcpHost { + // TODO pre-provision all the hosts rather than on-demand - unlock batch provisioning (CFN Stack), picking EC2 types, SSD storage, TCP port ranges, subnets, etc. return when { name.startsWith("jira-node") -> serveTcp(8080, name) // TODO this is a contract on undocumented behavior name.startsWith("mysql") -> serveTcp(3306, name) - else -> serveTcp( - 888, - name - ) // TODO pre-provision all the hosts rather than on-demand - unlock batch provisioning (CFN Stack), picking EC2 types, SSD storage, TCP port ranges, subnets, etc. + name.startsWith("samba") -> serveTcp(3306, name) + else -> serveTcp(888, name) } } - private fun serveTcp(port: Int, name: String): TcpHost { + private fun serveTcp(tcpPort: Int, name: String): TcpHost { + return serve(name, listOf(tcpPort), emptyList()) + } + + + override fun serve(name: String, tcpPorts: List, udpPorts: List): TcpHost { + val ports = tcpPorts.map { ExposedPort.tcp(it) } + + udpPorts.map { ExposedPort.udp(it) } + + ExposedPort.tcp(22) docker .pullImageCmd("rastasheep/ubuntu-sshd") .withTag("18.04") .exec(PullImageResultCallback()) .awaitCompletion() - val exposedPort = ExposedPort.tcp(port) val createdContainer = docker .createContainerCmd("rastasheep/ubuntu-sshd:18.04") .withHostConfig( @@ -83,28 +89,28 @@ internal class DockerInfrastructure : Infrastructure { .withPrivileged(true) .withNetworkMode(network.response.id) ) - .withExposedPorts(exposedPort, ExposedPort.tcp(22)) + .withExposedPorts(ports) .withName("$name-${randomUUID()}") .execAsResource(docker) allocatedResources.addLast(createdContainer) - return start(createdContainer, exposedPort, name) + return start(createdContainer, tcpPorts.first(), name) } private fun start( created: CreatedContainer, - port: ExposedPort, + tcpPort: Int, name: String ): TcpHost { val startedContainer = docker .startContainerCmd(created.response.id) .execAsResource(docker) allocatedResources.addLast(startedContainer); - return install(startedContainer, port, name) + return install(startedContainer, tcpPort, name) } private fun install( started: StartedContainer, - port: ExposedPort, + tcpPort: Int, name: String ): TcpHost { val networkSettings = docker @@ -128,7 +134,7 @@ internal class DockerInfrastructure : Infrastructure { it.execute("apt-get update", Duration.ofMinutes(2)) it.execute("apt-get -y install sudo gnupg screen") } - return TcpHost("localhost", ip, port.port, name, ssh) + return TcpHost("localhost", ip, tcpPort, name, ssh) } private fun getHostPort( diff --git a/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/JiraDataCenterPlanIT.kt b/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/JiraDataCenterPlanIT.kt index 1be1fc17..b69dcc7d 100644 --- a/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/JiraDataCenterPlanIT.kt +++ b/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/JiraDataCenterPlanIT.kt @@ -57,7 +57,7 @@ class JiraDataCenterPlanIT { } val instanceHooks = PreInstanceHooks.default() .also { Datasets.JiraSevenDataset.hookMysql(it, infrastructure) } - .also { it.insert(SharedHomeHook(jiraHomeSource, infrastructure)) } + .also { it.insert(SambaSharedHomeHook(jiraHomeSource, infrastructure)) } val balancerPlan = ApacheProxyPlan(infrastructure) val dcPlan = JiraDataCenterPlan(nodePlans, instanceHooks, balancerPlan, infrastructure) From 516f57c116083640368fe41010978492d3e71cca Mon Sep 17 00:00:00 2001 From: Maciej Kwidzinski Date: Fri, 7 May 2021 16:19:21 +0200 Subject: [PATCH 25/52] JPERF-273: Fix `ApacheProxyPlan` --- .../infrastructure/api/loadbalancer/ApacheProxyPlan.kt | 8 +++++--- .../api/jira/instance/JiraDataCenterPlanIT.kt | 3 +++ 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/loadbalancer/ApacheProxyPlan.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/loadbalancer/ApacheProxyPlan.kt index 6588dddc..d0edc217 100644 --- a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/loadbalancer/ApacheProxyPlan.kt +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/loadbalancer/ApacheProxyPlan.kt @@ -3,6 +3,7 @@ package com.atlassian.performance.tools.infrastructure.api.loadbalancer import com.atlassian.performance.tools.infrastructure.api.Infrastructure import com.atlassian.performance.tools.infrastructure.api.Sed import com.atlassian.performance.tools.infrastructure.api.jira.install.InstalledJira +import com.atlassian.performance.tools.infrastructure.api.jira.install.TcpHost import com.atlassian.performance.tools.infrastructure.api.jira.node.JiraNode import com.atlassian.performance.tools.infrastructure.api.jira.report.Reports import com.atlassian.performance.tools.infrastructure.api.jira.start.hook.PreStartHook @@ -24,7 +25,7 @@ class ApacheProxyPlan( val proxyNode = infrastructure.serveTcp("apache-proxy") IdempotentAction("Installing and configuring apache load balancer") { proxyNode.ssh.newConnection().use { connection -> - tryToProvision(connection, nodes) + tryToProvision(connection, nodes, proxyNode) } }.retry(2, ExponentialBackoff(Duration.ofSeconds(5))) val balancerEndpoint = URI("http://${proxyNode.privateIp}:${proxyNode.port}/") @@ -32,8 +33,9 @@ class ApacheProxyPlan( return ApacheProxy(balancerEndpoint) } - private fun tryToProvision(ssh: SshConnection, nodes: List) { + private fun tryToProvision(ssh: SshConnection, nodes: List, proxyNode: TcpHost) { Ubuntu().install(ssh, listOf("apache2")) + Sed().replace(ssh, "Listen 80", "Listen ${proxyNode.port}", "/etc/apache2/ports.conf") ssh.execute("sudo rm $configPath") ssh.execute("sudo touch $configPath") val mods = listOf( @@ -47,7 +49,7 @@ class ApacheProxyPlan( ) appendConfig(ssh, "") nodes.forEachIndexed { index, node -> - appendConfig(ssh, "\tBalancerMember http://${node.host.publicIp}:${node.host.port} route=$index") + appendConfig(ssh, "\tBalancerMember http://${node.host.privateIp}:${node.host.port} route=$index") } appendConfig(ssh, "\n") appendConfig(ssh, "ProxyPass / balancer://mycluster/ stickysession=ROUTEID") diff --git a/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/JiraDataCenterPlanIT.kt b/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/JiraDataCenterPlanIT.kt index b69dcc7d..1c3e031d 100644 --- a/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/JiraDataCenterPlanIT.kt +++ b/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/JiraDataCenterPlanIT.kt @@ -73,6 +73,9 @@ class JiraDataCenterPlanIT { .download(Files.createTempFile("downloaded-config", ".xml")) assertThat(serverXml.readText()).contains(" + ssh.execute("wget ${dataCenter.address}") + } } } From 8094cfe8d9efd446ee21bc9b0a034a703baed382 Mon Sep 17 00:00:00 2001 From: Maciej Kwidzinski Date: Fri, 7 May 2021 16:53:00 +0200 Subject: [PATCH 26/52] Resolve conflicts after rebase --- CHANGELOG.md | 14 ++++++++------ .../api/jira/install/InstalledJira.kt | 2 +- .../api/jira/install/JiraInstallation.kt | 2 +- 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a3806ff1..46a0a81e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -26,12 +26,7 @@ Dropping a requirement of a major version of a dependency is a new contract. ## [Unreleased] [Unreleased]: https://github.com/atlassian/infrastructure/compare/release-4.18.0...master -## [4.18.0] - 2021-04-14 -[4.18.0]: https://github.com/atlassian/infrastructure/compare/release-4.17.5...release-4.18.0 - ### Added -- Point to remote files on SSH hosts via `RemotePath`. - Fix [JPERF-273]: - Allow multiple ways of installing Jira via `JiraInstallation` or starting it via `JiraStart`. - Represent the information required to use an already installed Jira via `InstalledJira` or `JiraStart` if started. @@ -41,11 +36,18 @@ Fix [JPERF-273]: - Let hooks insert new hooks. - Locate and download any logs, charts, profiles and other reports via `Report` (rather than hardcoding the paths). +[JPERF-273]: https://ecosystem.atlassian.net/browse/JPERF-273 + +## [4.18.0] - 2021-04-14 +[4.18.0]: https://github.com/atlassian/infrastructure/compare/release-4.17.5...release-4.18.0 + +### Added +- Point to remote files on SSH hosts via `RemotePath`. + ### Fixed - Increase network-level retries for Jira/browser downloads. Decrease flakiness of such downloads on Ubuntu on WSL2. - Download ChromeDriver version that matches installed Chrome version. Fix [JPERF-732]. -[JPERF-273]: https://ecosystem.atlassian.net/browse/JPERF-273 [JPERF-732]: https://ecosystem.atlassian.net/browse/JPERF-732 ## [4.17.5] - 2020-12-15 diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/InstalledJira.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/InstalledJira.kt index 1845ac86..95c77249 100644 --- a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/InstalledJira.kt +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/InstalledJira.kt @@ -6,7 +6,7 @@ import com.atlassian.performance.tools.infrastructure.api.os.RemotePath /** * Points to an already installed Jira. * - * @since 4.18.0 + * @since 4.19.0 */ class InstalledJira( /** diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/JiraInstallation.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/JiraInstallation.kt index 71c0f488..deb4bfb0 100644 --- a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/JiraInstallation.kt +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/JiraInstallation.kt @@ -4,7 +4,7 @@ import com.atlassian.performance.tools.infrastructure.api.jira.report.Reports import net.jcip.annotations.ThreadSafe /** - * @since 4.18.0 + * @since 4.19.0 */ @ThreadSafe interface JiraInstallation { From 9321ddc43104beba4bbf21dc9f30155f642cb916 Mon Sep 17 00:00:00 2001 From: Maciej Kwidzinski Date: Fri, 14 May 2021 11:33:21 +0200 Subject: [PATCH 27/52] Handle multiple Docker port bindings (IP4, IP6) --- .../tools/infrastructure/api/DockerInfrastructure.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/DockerInfrastructure.kt b/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/DockerInfrastructure.kt index 3292ceac..20791553 100644 --- a/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/DockerInfrastructure.kt +++ b/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/DockerInfrastructure.kt @@ -144,7 +144,7 @@ internal class DockerInfrastructure : Infrastructure { return networkSettings .ports .bindings[port]!! - .single() + .single { it.hostIp == "0.0.0.0" } // include just the IP4 bind .hostPortSpec .toInt() } From 64ab7c191315f9e91caa4f483e3327e98a7ad1cd Mon Sep 17 00:00:00 2001 From: Maciej Kwidzinski Date: Fri, 14 May 2021 13:00:47 +0200 Subject: [PATCH 28/52] Try various DIND on CI --- .../infrastructure/api/DockerInfrastructure.kt | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/DockerInfrastructure.kt b/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/DockerInfrastructure.kt index 20791553..a59ebddc 100644 --- a/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/DockerInfrastructure.kt +++ b/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/DockerInfrastructure.kt @@ -10,9 +10,7 @@ import com.atlassian.performance.tools.ssh.api.SshHost import com.atlassian.performance.tools.ssh.api.auth.PasswordAuthentication import com.github.dockerjava.api.DockerClient import com.github.dockerjava.api.command.PullImageResultCallback -import com.github.dockerjava.api.model.ExposedPort -import com.github.dockerjava.api.model.HostConfig -import com.github.dockerjava.api.model.NetworkSettings +import com.github.dockerjava.api.model.* import com.github.dockerjava.core.DefaultDockerClientConfig import com.github.dockerjava.core.DockerClientImpl import com.github.dockerjava.zerodep.ZerodepDockerHttpClient @@ -86,9 +84,18 @@ internal class DockerInfrastructure : Infrastructure { .withHostConfig( HostConfig() .withPublishAllPorts(true) - .withPrivileged(true) +// .withPrivileged(true) + .withBinds(Bind("/var/run/docker.sock", Volume("/var/run/docker.sock"))) +// .withMounts( +// Mount() +// .withSource("/var/run/docker.sock") +// .withTarget("/var/run/docker.sock") +// .withType(MountType.VOLUME) +// .let { listOf(it) } +// ) .withNetworkMode(network.response.id) ) +// .withVolumes(Volume("/var/run/docker.sock")) .withExposedPorts(ports) .withName("$name-${randomUUID()}") .execAsResource(docker) From 608b5f9f43e839b275ca70e85776c3e662f1bdac Mon Sep 17 00:00:00 2001 From: Maciej Kwidzinski Date: Fri, 14 May 2021 13:53:08 +0200 Subject: [PATCH 29/52] Restore privileged Docker for MySQL --- .../tools/infrastructure/api/DockerInfrastructure.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/DockerInfrastructure.kt b/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/DockerInfrastructure.kt index a59ebddc..8ccb4f11 100644 --- a/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/DockerInfrastructure.kt +++ b/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/DockerInfrastructure.kt @@ -84,7 +84,7 @@ internal class DockerInfrastructure : Infrastructure { .withHostConfig( HostConfig() .withPublishAllPorts(true) -// .withPrivileged(true) + .withPrivileged(true) .withBinds(Bind("/var/run/docker.sock", Volume("/var/run/docker.sock"))) // .withMounts( // Mount() From 1fc72e2ccc42b58b723fa8ae5efa3e99c988e986 Mon Sep 17 00:00:00 2001 From: Maciej Kwidzinski Date: Fri, 14 May 2021 17:14:39 +0200 Subject: [PATCH 30/52] Replace deprecated `startProcess` in tests --- .../tools/infrastructure/SshExtensions.kt | 9 ++++ .../chromium/PageLoadTimeoutRecoveryTest.kt | 4 +- .../api/jvm/AdoptOpenJdk11IT.kt | 6 +-- .../infrastructure/api/jvm/AdoptOpenJdkIT.kt | 4 +- .../infrastructure/api/jvm/JstatSupport.kt | 42 +++++++++---------- .../infrastructure/api/jvm/OpenJdk11IT.kt | 4 +- .../tools/infrastructure/api/jvm/OpenJdkIT.kt | 4 +- .../infrastructure/api/jvm/OracleJdkIT.kt | 9 ++-- .../infrastructure/api/jvm/ThreadDumpIT.kt | 25 +++++++---- .../infrastructure/browser/SshChromium.kt | 24 +++++------ 10 files changed, 67 insertions(+), 64 deletions(-) create mode 100644 src/test/kotlin/com/atlassian/performance/tools/infrastructure/SshExtensions.kt diff --git a/src/test/kotlin/com/atlassian/performance/tools/infrastructure/SshExtensions.kt b/src/test/kotlin/com/atlassian/performance/tools/infrastructure/SshExtensions.kt new file mode 100644 index 00000000..201da516 --- /dev/null +++ b/src/test/kotlin/com/atlassian/performance/tools/infrastructure/SshExtensions.kt @@ -0,0 +1,9 @@ +package com.atlassian.performance.tools.infrastructure + +import com.atlassian.performance.tools.ssh.api.SshConnection + +internal fun SshConnection.SshResult.assertInterruptedJava() { + if (exitStatus !in listOf(0, 130)) { + throw Exception("$this doesn't look like an interrupted Java process") + } +} \ No newline at end of file diff --git a/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/browser/chromium/PageLoadTimeoutRecoveryTest.kt b/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/browser/chromium/PageLoadTimeoutRecoveryTest.kt index 6ee7cb83..40b4db89 100644 --- a/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/browser/chromium/PageLoadTimeoutRecoveryTest.kt +++ b/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/browser/chromium/PageLoadTimeoutRecoveryTest.kt @@ -28,7 +28,7 @@ internal class PageLoadTimeoutRecoveryTest { chromium.install(connection) } val chromedriverUri = URI("http://localhost:$localChromedriverPort") - SshChromium(ssh.newConnection(), chromedriverUri).start().use { sshDriver -> + SshChromium(ssh, chromedriverUri).start().use { sshDriver -> val driver = sshDriver.getDriver() setPageLoadTimeout(driver) @@ -46,7 +46,7 @@ internal class PageLoadTimeoutRecoveryTest { } private fun findFreePort(): Int { - return ServerSocket(0).use { socket -> return socket.localPort } + return ServerSocket(0).use { socket -> socket.localPort } } private class FastResponse : MockHttpServer.RequestHandler { diff --git a/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/jvm/AdoptOpenJdk11IT.kt b/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/jvm/AdoptOpenJdk11IT.kt index d80b4819..8001b8b0 100644 --- a/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/jvm/AdoptOpenJdk11IT.kt +++ b/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/jvm/AdoptOpenJdk11IT.kt @@ -8,9 +8,7 @@ class AdoptOpenJdk11IT { @Test fun shouldSupportJstat() { DockerInfrastructure().use { infra -> - infra.serveTest().newConnection().use { connection -> - JstatSupport(AdoptOpenJDK11()).shouldSupportJstat(connection) - } + JstatSupport(AdoptOpenJDK11(), infra.serveTest()).shouldSupportJstat() } } -} \ No newline at end of file +} diff --git a/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/jvm/AdoptOpenJdkIT.kt b/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/jvm/AdoptOpenJdkIT.kt index 9859470d..fd66fcd1 100644 --- a/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/jvm/AdoptOpenJdkIT.kt +++ b/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/jvm/AdoptOpenJdkIT.kt @@ -8,9 +8,7 @@ class AdoptOpenJdkIT { @Test fun shouldSupportJstat() { DockerInfrastructure().use { infra -> - infra.serveTest().newConnection().use { connection -> - JstatSupport(AdoptOpenJDK()).shouldSupportJstat(connection) - } + JstatSupport(AdoptOpenJDK(), infra.serveTest()).shouldSupportJstat() } } } \ No newline at end of file diff --git a/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/jvm/JstatSupport.kt b/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/jvm/JstatSupport.kt index fa0c0c0e..77b8a80d 100644 --- a/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/jvm/JstatSupport.kt +++ b/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/jvm/JstatSupport.kt @@ -1,39 +1,34 @@ package com.atlassian.performance.tools.infrastructure.api.jvm import com.atlassian.performance.tools.infrastructure.api.os.Ubuntu +import com.atlassian.performance.tools.infrastructure.assertInterruptedJava import com.atlassian.performance.tools.jvmtasks.api.ExponentialBackoff import com.atlassian.performance.tools.jvmtasks.api.IdempotentAction +import com.atlassian.performance.tools.ssh.api.Ssh import com.atlassian.performance.tools.ssh.api.SshConnection -import org.assertj.core.api.Assertions +import org.assertj.core.api.Assertions.assertThat import java.io.File import java.time.Duration class JstatSupport( - private val jdk: VersionedJavaDevelopmentKit + private val jdk: VersionedJavaDevelopmentKit, + private val ssh: Ssh ) { - private val expectedStats: Set = setOf( - "Timestamp", - "S0", - "S1", - "E", - "O", - "M", - "CCS", - "YGC", - "YGCT", - "FGC", - "FGCT", - "GCT" - ) private val jarName = "hello-world-after-1m-wait.jar" - private val jar = "/com/atlassian/performance/tools/infrastructure/api/jvm/$jarName" + private val jarResource = "/com/atlassian/performance/tools/infrastructure/api/jvm/$jarName" private val timestampLength = "2018-12-17T14:10:44+00:00 ".length - fun shouldSupportJstat(connection: SshConnection) { + fun shouldSupportJstat() { + ssh.newConnection().use { connection -> + shouldSupportJstat(connection) + } + } + + private fun shouldSupportJstat(connection: SshConnection) { Ubuntu().install(connection, listOf("curl", "screen"), Duration.ofMinutes(2)) - connection.upload(File(this.javaClass.getResource(jar).toURI()), jarName) + connection.upload(File(javaClass.getResource(jarResource).toURI()), jarName) jdk.install(connection) - connection.startProcess(jdk.command("-classpath $jarName samples.HelloWorld")) + val hello = ssh.runInBackground(jdk.command("-classpath $jarName samples.HelloWorld")) val pid = IdempotentAction( description = "Wait for the Hello, World! process to start.", action = { connection.execute("cat hello-world.pid").output } @@ -44,13 +39,16 @@ class JstatSupport( val jstatMonitoring = jdk.jstatMonitoring.start(connection, pid.toInt()) waitForJstatToCollectSomeData() jstatMonitoring.stop(connection) + hello.stop(Duration.ofSeconds(1)).assertInterruptedJava(); val jstatLog = connection.execute("cat ${jstatMonitoring.getResultPath()}").output val jstatHeader = jstatLog.substring(timestampLength, jstatLog.indexOf('\n')) - Assertions.assertThat(jstatHeader).contains(this.expectedStats) + assertThat(jstatHeader).contains( + "Timestamp", "S0", "S1", "E", "O", "M", "CCS", "YGC", "YGCT", "FGC", "FGCT", "GCT" + ) } private fun waitForJstatToCollectSomeData() { - Thread.sleep(4 * 1000) + Thread.sleep(Duration.ofSeconds(4).toMillis()) } } diff --git a/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/jvm/OpenJdk11IT.kt b/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/jvm/OpenJdk11IT.kt index a3370e7c..a22bcffb 100644 --- a/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/jvm/OpenJdk11IT.kt +++ b/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/jvm/OpenJdk11IT.kt @@ -8,9 +8,7 @@ class OpenJdk11IT { @Test fun shouldSupportJstat() { DockerInfrastructure().use { infra -> - infra.serveTest().newConnection().use { connection -> - JstatSupport(OpenJDK11()).shouldSupportJstat(connection) - } + JstatSupport(OpenJDK11(), infra.serveTest()).shouldSupportJstat() } } } \ No newline at end of file diff --git a/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/jvm/OpenJdkIT.kt b/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/jvm/OpenJdkIT.kt index 7c05be7d..69da4f38 100644 --- a/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/jvm/OpenJdkIT.kt +++ b/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/jvm/OpenJdkIT.kt @@ -8,9 +8,7 @@ class OpenJdkIT { @Test fun shouldSupportJstat() { DockerInfrastructure().use { infra -> - infra.serveTest().newConnection().use { connection -> - JstatSupport(OpenJDK()).shouldSupportJstat(connection) - } + JstatSupport(OpenJDK(), infra.serveTest()).shouldSupportJstat() } } } \ No newline at end of file diff --git a/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/jvm/OracleJdkIT.kt b/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/jvm/OracleJdkIT.kt index 3ce16533..22b267fd 100644 --- a/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/jvm/OracleJdkIT.kt +++ b/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/jvm/OracleJdkIT.kt @@ -7,12 +7,11 @@ class OracleJdkIT { @Test fun shouldSupportJstatAndThreadDumps() { + val jdk = OracleJDK() DockerInfrastructure().use { infra -> - infra.serveTest().newConnection().use { connection -> - val jdk = OracleJDK() - JstatSupport(jdk).shouldSupportJstat(connection) - ThreadDumpTest().shouldGatherThreadDump(jdk, connection) - } + val ssh = infra.serveTest() + JstatSupport(jdk, ssh).shouldSupportJstat() + ThreadDumpTest(jdk, ssh).shouldGatherThreadDump() } } } \ No newline at end of file diff --git a/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/jvm/ThreadDumpIT.kt b/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/jvm/ThreadDumpIT.kt index bd1d0608..fcd0f871 100644 --- a/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/jvm/ThreadDumpIT.kt +++ b/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/jvm/ThreadDumpIT.kt @@ -1,17 +1,28 @@ package com.atlassian.performance.tools.infrastructure.api.jvm +import com.atlassian.performance.tools.infrastructure.assertInterruptedJava import com.atlassian.performance.tools.jvmtasks.api.IdempotentAction import com.atlassian.performance.tools.jvmtasks.api.StaticBackoff +import com.atlassian.performance.tools.ssh.api.Ssh import com.atlassian.performance.tools.ssh.api.SshConnection -import org.assertj.core.api.Assertions +import org.assertj.core.api.Assertions.assertThat import java.time.Duration -class ThreadDumpTest { - fun shouldGatherThreadDump(jdk: JavaDevelopmentKit, connection: SshConnection) { +class ThreadDumpTest( + private val jdk: JavaDevelopmentKit, + private val ssh: Ssh +) { + fun shouldGatherThreadDump() { + ssh.newConnection().use { connection -> + shouldGatherThreadDump(connection) + } + } + + private fun shouldGatherThreadDump(connection: SshConnection) { val destination = "thread-dumps" connection.execute("""echo "public class Test { public static void main(String[] args) { try { Thread.sleep(java.time.Duration.ofMinutes(1).toMillis()); } catch (InterruptedException e) { throw new RuntimeException(e); } }}" > Test.java """.trimIndent()) connection.execute("${jdk.use()}; javac Test.java") - val process = connection.startProcess("${jdk.use()}; java Test") + val process = ssh.runInBackground("${jdk.use()}; java Test") try { val pid = IdempotentAction("Get PID") { getPid(connection, jdk) @@ -21,11 +32,9 @@ class ThreadDumpTest { val threadDumpFile = connection.execute("ls $destination").output val threadDump = connection.execute("cat $destination/$threadDumpFile").output - Assertions.assertThat(threadDump).contains("Full thread dump Java HotSpot") - } catch (e: Exception) { - throw Exception(e) + assertThat(threadDump).contains("Full thread dump Java HotSpot") } finally { - connection.stopProcess(process) + process.stop(Duration.ofSeconds(1)).assertInterruptedJava() } } diff --git a/src/test/kotlin/com/atlassian/performance/tools/infrastructure/browser/SshChromium.kt b/src/test/kotlin/com/atlassian/performance/tools/infrastructure/browser/SshChromium.kt index 5106dd31..17a0bad9 100644 --- a/src/test/kotlin/com/atlassian/performance/tools/infrastructure/browser/SshChromium.kt +++ b/src/test/kotlin/com/atlassian/performance/tools/infrastructure/browser/SshChromium.kt @@ -2,22 +2,21 @@ package com.atlassian.performance.tools.infrastructure.browser import com.atlassian.performance.tools.jvmtasks.api.IdempotentAction import com.atlassian.performance.tools.jvmtasks.api.StaticBackoff -import com.atlassian.performance.tools.ssh.api.DetachedProcess +import com.atlassian.performance.tools.ssh.api.Ssh import com.atlassian.performance.tools.ssh.api.SshConnection import com.atlassian.performance.tools.virtualusers.api.browsers.Browser import com.atlassian.performance.tools.virtualusers.api.browsers.CloseableRemoteWebDriver import org.openqa.selenium.chrome.ChromeOptions import org.openqa.selenium.remote.RemoteWebDriver import java.net.URI -import java.time.Duration +import java.time.Duration.ofSeconds internal class SshChromium( - private val ssh: SshConnection, + private val ssh: Ssh, private val chromedriverUri: URI ) : Browser { override fun start(): CloseableRemoteWebDriver { - val chromedriverProcess: DetachedProcess = ssh.startProcess("./chromedriver --whitelisted-ips") - + ssh.runInBackground("./chromedriver --whitelisted-ips") val chromeOptions = ChromeOptions() .apply { addArguments("--headless") } .apply { addArguments("--no-sandbox") } @@ -28,16 +27,13 @@ internal class SshChromium( "credentials_enable_service" to false ) ) - IdempotentAction("Wait for chrome process") { - waitForChromeProcess(ssh) - }.retry(maxAttempts = 3, backoff = StaticBackoff(Duration.ofSeconds(5L))) - val driver = RemoteWebDriver(chromedriverUri.toURL(), chromeOptions) - return object : CloseableRemoteWebDriver(driver) { - override fun close() { - super.close() - ssh.stopProcess(chromedriverProcess) - } + ssh.newConnection().use { connection -> + IdempotentAction("Wait for chrome process") { + waitForChromeProcess(connection) + }.retry(3, StaticBackoff(ofSeconds(5))) } + val driver = RemoteWebDriver(chromedriverUri.toURL(), chromeOptions) + return CloseableRemoteWebDriver(driver) } private fun waitForChromeProcess(ssh: SshConnection) { From b336d6d63e033ddb7fc3706d140648184cec79a1 Mon Sep 17 00:00:00 2001 From: Maciej Kwidzinski Date: Fri, 14 May 2021 17:15:14 +0200 Subject: [PATCH 31/52] Fail fast when docker-in-docker dies --- .../api/database/DockerMysqlServer.kt | 6 ++++-- .../docker/DeadContainerCheck.kt | 20 +++++++++++++++++++ 2 files changed, 24 insertions(+), 2 deletions(-) create mode 100644 src/main/kotlin/com/atlassian/performance/tools/infrastructure/docker/DeadContainerCheck.kt diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/database/DockerMysqlServer.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/database/DockerMysqlServer.kt index 232ab29e..c597d67a 100644 --- a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/database/DockerMysqlServer.kt +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/database/DockerMysqlServer.kt @@ -10,11 +10,13 @@ import com.atlassian.performance.tools.infrastructure.api.jira.report.Reports import com.atlassian.performance.tools.infrastructure.api.os.Ubuntu import com.atlassian.performance.tools.infrastructure.database.SshMysqlClient import com.atlassian.performance.tools.infrastructure.database.SshSqlClient +import com.atlassian.performance.tools.infrastructure.docker.DeadContainerCheck import com.atlassian.performance.tools.jvmtasks.api.IdempotentAction import com.atlassian.performance.tools.jvmtasks.api.StaticBackoff import com.atlassian.performance.tools.ssh.api.Ssh import com.atlassian.performance.tools.ssh.api.SshConnection import java.time.Duration +import java.time.Duration.ofSeconds class DockerMysqlServer private constructor( private val infrastructure: Infrastructure, @@ -40,7 +42,7 @@ class DockerMysqlServer private constructor( private fun setup(ssh: SshConnection, host: TcpHost): SshSqlClient { val mysqlData = source.download(ssh) val port = host.port - dockerImage.run( + val container = dockerImage.run( ssh = ssh, parameters = "-p $port:$port -v `realpath $mysqlData`:/var/lib/mysql", arguments = "--skip-grant-tables --max_connections=$maxConnections" @@ -48,7 +50,7 @@ class DockerMysqlServer private constructor( Ubuntu().install(ssh, listOf("mysql-client")) val client = SshMysqlClient("127.0.0.1", port, "root") IdempotentAction("wait for MySQL start") { client.runSql(ssh, "select 1;") } - .retry(90, StaticBackoff(Duration.ofSeconds(10))) + .retry(90, DeadContainerCheck(container, ssh, StaticBackoff(ofSeconds(10)))) return client } diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/docker/DeadContainerCheck.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/docker/DeadContainerCheck.kt new file mode 100644 index 00000000..9d222c46 --- /dev/null +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/docker/DeadContainerCheck.kt @@ -0,0 +1,20 @@ +package com.atlassian.performance.tools.infrastructure.docker + +import com.atlassian.performance.tools.jvmtasks.api.Backoff +import com.atlassian.performance.tools.ssh.api.SshConnection +import java.time.Duration + +internal class DeadContainerCheck( + private val container: String, + private val ssh: SshConnection, + private val base: Backoff +) : Backoff { + override fun backOff(attempt: Int): Duration { + val status = ssh.execute("sudo docker inspect --format '{{.State.Status}}' $container").output.trim() + if (status == "exited") { + val logs = ssh.execute("sudo docker logs $container").errorOutput.trim() + throw Exception("$container exited, logs: $logs") + } + return base.backOff(attempt) + } +} \ No newline at end of file From 92a5691eee421bc1011617c203b546a24465f795 Mon Sep 17 00:00:00 2001 From: Maciej Kwidzinski Date: Fri, 14 May 2021 17:17:47 +0200 Subject: [PATCH 32/52] Check Docker info on CI --- .github/workflows/ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 33e97718..5414ff6a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -15,6 +15,8 @@ jobs: build: runs-on: ubuntu-latest steps: + - name: Check Docker + run: docker info - name: Checkout uses: actions/checkout@v2 with: From abc962bb23d9e8192a619aadf6894cce031dc53e Mon Sep 17 00:00:00 2001 From: Maciej Kwidzinski Date: Fri, 21 May 2021 12:36:26 +0200 Subject: [PATCH 33/52] Undo sibling socket-based "DIND" --- .../tools/infrastructure/api/DockerInfrastructure.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/DockerInfrastructure.kt b/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/DockerInfrastructure.kt index 8ccb4f11..9f0ecf6f 100644 --- a/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/DockerInfrastructure.kt +++ b/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/DockerInfrastructure.kt @@ -85,7 +85,7 @@ internal class DockerInfrastructure : Infrastructure { HostConfig() .withPublishAllPorts(true) .withPrivileged(true) - .withBinds(Bind("/var/run/docker.sock", Volume("/var/run/docker.sock"))) +// .withBinds(Bind("/var/run/docker.sock", Volume("/var/run/docker.sock"))) // .withMounts( // Mount() // .withSource("/var/run/docker.sock") From 52e96e40ad86c6d1fcbfa1a5c8616df475553d29 Mon Sep 17 00:00:00 2001 From: Maciej Kwidzinski Date: Fri, 21 May 2021 12:37:24 +0200 Subject: [PATCH 34/52] Force Ubuntu Docker to use OverlayFS --- .../tools/infrastructure/api/docker/Docker.kt | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/docker/Docker.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/docker/Docker.kt index dc79ca6d..533e9bb4 100644 --- a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/docker/Docker.kt +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/docker/Docker.kt @@ -44,9 +44,18 @@ class Docker private constructor( timeout = mainPackageTimeout ) ssh.execute("sudo service docker status || sudo service docker start") + ssh.safeExecute("sudo cat /etc/docker/daemon.json") IdempotentAction("poll docker") { ssh.execute("sudo docker ps") }.retry(2, StaticBackoff(Duration.ofSeconds(1))) + ssh.execute("sudo docker info") + ssh.execute("echo '{\"storage-driver\": \"overlay2\"}' | sudo tee /etc/docker/daemon.json") + ssh.execute("sudo cat /etc/docker/daemon.json") + ssh.execute("sudo service docker restart") + IdempotentAction("poll docker") { + ssh.execute("sudo docker ps") + }.retry(2, StaticBackoff(Duration.ofSeconds(1))) + ssh.execute("sudo docker info") } class Builder { From 2300a41298bf3a68cd7f81631565aa0dfac16678 Mon Sep 17 00:00:00 2001 From: Maciej Kwidzinski Date: Fri, 21 May 2021 14:13:53 +0200 Subject: [PATCH 35/52] Try Docker `vfs` on GHA --- .../performance/tools/infrastructure/api/docker/Docker.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/docker/Docker.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/docker/Docker.kt index 533e9bb4..2d01c0fb 100644 --- a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/docker/Docker.kt +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/docker/Docker.kt @@ -49,7 +49,7 @@ class Docker private constructor( ssh.execute("sudo docker ps") }.retry(2, StaticBackoff(Duration.ofSeconds(1))) ssh.execute("sudo docker info") - ssh.execute("echo '{\"storage-driver\": \"overlay2\"}' | sudo tee /etc/docker/daemon.json") + ssh.execute("echo '{\"storage-driver\": \"vfs\"}' | sudo tee /etc/docker/daemon.json") ssh.execute("sudo cat /etc/docker/daemon.json") ssh.execute("sudo service docker restart") IdempotentAction("poll docker") { From 9d92adfe1e2f5b9c06f21664cad29551d0de493f Mon Sep 17 00:00:00 2001 From: Maciej Kwidzinski Date: Fri, 21 May 2021 14:30:06 +0200 Subject: [PATCH 36/52] Use vfs --- .../tools/infrastructure/api/docker/Docker.kt | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/docker/Docker.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/docker/Docker.kt index 2d01c0fb..70c65ae9 100644 --- a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/docker/Docker.kt +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/docker/Docker.kt @@ -43,19 +43,12 @@ class Docker private constructor( packages = listOf("docker-ce=$version"), timeout = mainPackageTimeout ) - ssh.execute("sudo service docker status || sudo service docker start") - ssh.safeExecute("sudo cat /etc/docker/daemon.json") - IdempotentAction("poll docker") { - ssh.execute("sudo docker ps") - }.retry(2, StaticBackoff(Duration.ofSeconds(1))) - ssh.execute("sudo docker info") + ssh.execute("sudo mkdir -p /etc/docker") ssh.execute("echo '{\"storage-driver\": \"vfs\"}' | sudo tee /etc/docker/daemon.json") - ssh.execute("sudo cat /etc/docker/daemon.json") - ssh.execute("sudo service docker restart") + ssh.execute("sudo service docker status || sudo service docker start") IdempotentAction("poll docker") { ssh.execute("sudo docker ps") }.retry(2, StaticBackoff(Duration.ofSeconds(1))) - ssh.execute("sudo docker info") } class Builder { From 4eb2f3df016e82649f1cb92cdbe18dcb2369b2c4 Mon Sep 17 00:00:00 2001 From: Maciej Kwidzinski Date: Fri, 21 May 2021 14:51:31 +0200 Subject: [PATCH 37/52] Fix `RestUpgrade` --- .../tools/infrastructure/api/jira/start/hook/RestUpgrade.kt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/start/hook/RestUpgrade.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/start/hook/RestUpgrade.kt index 6cff1183..89c12a00 100644 --- a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/start/hook/RestUpgrade.kt +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/start/hook/RestUpgrade.kt @@ -18,8 +18,9 @@ class RestUpgrade( override fun call(ssh: SshConnection, jira: StartedJira, hooks: PostStartHooks, reports: Reports) { val threadDump = ThreadDump(jira.pid, jira.installed.jdk) + val ip = jira.installed.host.privateIp val port = jira.installed.host.port - val upgradesEndpoint = URI("http://$adminUsername:$adminPassword@localhost:$port/rest/api/2/upgrade") + val upgradesEndpoint = URI("http://$adminUsername:$adminPassword@$ip:$port/rest/api/2/upgrade") reports.add(FileListing("thread-dumps/*"), jira.installed.host) waitForStatusToChange( statusQuo = "000", From 766de915b4a2645a9d37c64c36537ba7c4bd7cfd Mon Sep 17 00:00:00 2001 From: Maciej Kwidzinski Date: Fri, 21 May 2021 17:07:03 +0200 Subject: [PATCH 38/52] Fix the `report` assertions --- .../tools/infrastructure/Datasets.kt | 2 +- .../api/jira/instance/JiraDataCenterPlanIT.kt | 63 ++++++++++--------- .../api/jira/instance/JiraServerPlanIT.kt | 25 +++++--- 3 files changed, 52 insertions(+), 38 deletions(-) diff --git a/src/test/kotlin/com/atlassian/performance/tools/infrastructure/Datasets.kt b/src/test/kotlin/com/atlassian/performance/tools/infrastructure/Datasets.kt index 5c475ff2..5c3fb0cb 100644 --- a/src/test/kotlin/com/atlassian/performance/tools/infrastructure/Datasets.kt +++ b/src/test/kotlin/com/atlassian/performance/tools/infrastructure/Datasets.kt @@ -34,7 +34,7 @@ class Datasets { fun hookMysql(postStartHooks: PostStartHooks) { val timeouts = JiraLaunchTimeouts.Builder() - .initTimeout(Duration.ofSeconds(30)) + .initTimeout(Duration.ofSeconds(45)) .build() val dataUpgrade = RestUpgrade(timeouts, "admin", "admin") postStartHooks.insert(dataUpgrade) diff --git a/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/JiraDataCenterPlanIT.kt b/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/JiraDataCenterPlanIT.kt index 1c3e031d..3d9dcca7 100644 --- a/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/JiraDataCenterPlanIT.kt +++ b/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/JiraDataCenterPlanIT.kt @@ -5,22 +5,24 @@ import com.atlassian.performance.tools.infrastructure.api.DockerInfrastructure import com.atlassian.performance.tools.infrastructure.api.Infrastructure import com.atlassian.performance.tools.infrastructure.api.distribution.PublicJiraSoftwareDistribution import com.atlassian.performance.tools.infrastructure.api.jira.JiraHomePackage -import com.atlassian.performance.tools.infrastructure.api.jira.install.InstalledJira import com.atlassian.performance.tools.infrastructure.api.jira.install.ParallelInstallation import com.atlassian.performance.tools.infrastructure.api.jira.install.hook.PreInstallHooks import com.atlassian.performance.tools.infrastructure.api.jira.node.JiraNodePlan import com.atlassian.performance.tools.infrastructure.api.jira.report.Reports import com.atlassian.performance.tools.infrastructure.api.jira.start.JiraLaunchScript -import com.atlassian.performance.tools.infrastructure.api.jira.start.hook.PreStartHook -import com.atlassian.performance.tools.infrastructure.api.jira.start.hook.PreStartHooks +import com.atlassian.performance.tools.infrastructure.api.jira.start.StartedJira +import com.atlassian.performance.tools.infrastructure.api.jira.start.hook.PostStartHook +import com.atlassian.performance.tools.infrastructure.api.jira.start.hook.PostStartHooks import com.atlassian.performance.tools.infrastructure.api.jvm.AdoptOpenJDK import com.atlassian.performance.tools.infrastructure.api.loadbalancer.ApacheProxyPlan import com.atlassian.performance.tools.ssh.api.SshConnection import org.assertj.core.api.Assertions.assertThat +import org.assertj.core.api.Assertions.catchThrowable import org.junit.After import org.junit.Before import org.junit.Test import java.nio.file.Files +import java.time.Duration.ofMinutes class JiraDataCenterPlanIT { @@ -83,9 +85,11 @@ class JiraDataCenterPlanIT { @Test fun shouldProvideLogsToDiagnoseFailure() { // given - class FailingHook : PreStartHook { - override fun call(ssh: SshConnection, jira: InstalledJira, hooks: PreStartHooks, reports: Reports) { - throw Exception("Failing deliberately before Jira starts") + class FailingHook : PostStartHook { + override fun call(ssh: SshConnection, jira: StartedJira, hooks: PostStartHooks, reports: Reports) { + val installed = jira.installed + ssh.execute("${installed.jdk.use()}; ${installed.installation.path}/bin/stop-jira.sh", ofMinutes(1)) + throw Exception("Failing deliberately after Jira started") } } @@ -99,36 +103,35 @@ class JiraDataCenterPlanIT { ) ) .start(JiraLaunchScript()) - .hooks(PreInstallHooks.default().also { it.preStart.insert(FailingHook()) }) + .hooks(PreInstallHooks.default().also { it.postStart.insert(FailingHook()) }) .build() } val balancerPlan = ApacheProxyPlan(infrastructure) val dcPlan = JiraDataCenterPlan(nodePlans, PreInstanceHooks.default(), balancerPlan, infrastructure) - try { - // when + // when + val thrown = catchThrowable { dcPlan.materialize() - } finally { - val reports = dcPlan.report().downloadTo(Files.createTempDirectory("jira-dc-plan-")) - // then - assertThat(reports).isDirectory() - val fileTree = reports - .walkTopDown() - .map { reports.toPath().relativize(it.toPath()) } - .toList() - assertThat(fileTree.map { it.toString() }).contains( - "jira-node-1/root/atlassian-jira-software-7.13.0-standalone/logs/catalina.out", - "jira-node-1/root/thread-dumps", - "jira-node-2/root/~/jpt-jstat.log", - "jira-node-2/root/~/jpt-vmstat.log", - "jira-node-2/root/~/jpt-iostat.log" - ) - assertThat(fileTree.filter { it.fileName.toString().startsWith("access_log") }) - .`as`("access logs") - .isNotEmpty - assertThat(fileTree.filter { it.fileName.toString().startsWith("atlassian-jira-gc") }) - .`as`("GC logs") - .isNotEmpty } + + val reports = dcPlan.report().downloadTo(Files.createTempDirectory("jira-dc-plan-")) + // then + assertThat(thrown).hasMessageStartingWith("Failing deliberately") + assertThat(reports).isDirectory() + val fileTree = reports + .walkTopDown() + .map { reports.toPath().relativize(it.toPath()) } + .toList() + assertThat(fileTree.map { it.toString() }).contains( + "jira-node-1/root/atlassian-jira-software-7.13.0-standalone/logs/catalina.out", + "jira-node-1/root/~/jpt-jstat.log", + "jira-node-2/root/atlassian-jira-software-7.13.0-standalone/logs/catalina.out" + ) + assertThat(fileTree.filter { it.fileName.toString() == "atlassian-jira.log" }) + .`as`("Jira log from $fileTree") + .isNotEmpty + assertThat(fileTree.filter { it.fileName.toString().startsWith("atlassian-jira-gc") }) + .`as`("GC logs from $fileTree") + .isNotEmpty } } diff --git a/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/JiraServerPlanIT.kt b/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/JiraServerPlanIT.kt index fdb80901..dc335b9b 100644 --- a/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/JiraServerPlanIT.kt +++ b/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/JiraServerPlanIT.kt @@ -59,7 +59,7 @@ class JiraServerPlanIT { val theNode = jiraServer.nodes.single() val host = theNode.installed.host - val downloadedReports = jiraServerPlan.report().downloadTo(Files.createTempDirectory("jira-server-plan-")) + val reports = jiraServerPlan.report().downloadTo(Files.createTempDirectory("jira-server-plan-")) // then val serverXml = theNode @@ -69,12 +69,23 @@ class JiraServerPlanIT { .download(Files.createTempFile("downloaded-config", ".xml")) assertThat(serverXml.readText()).contains(" Date: Fri, 11 Jun 2021 13:47:04 +0200 Subject: [PATCH 39/52] Diagnose `RestUpgrade` failures --- .github/workflows/ci.yml | 6 + .../api/jira/install/hook/JiraLogs.kt | 32 ++--- .../api/jira/start/hook/RestUpgrade.kt | 113 ++++++++++-------- .../api/jira/instance/JiraServerPlanIT.kt | 14 ++- 4 files changed, 99 insertions(+), 66 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5414ff6a..c34c1c2d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -34,6 +34,12 @@ jobs: with: name: test-reports path: build/reports/tests + - name: Upload test artifacts + if: always() + uses: actions/upload-artifact@v2 + with: + name: test-artifacts + path: build/test-artifacts - name: Release if: github.event.inputs.release == 'yes' env: diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/JiraLogs.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/JiraLogs.kt index 683f5389..adfc0033 100644 --- a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/JiraLogs.kt +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/JiraLogs.kt @@ -1,8 +1,8 @@ package com.atlassian.performance.tools.infrastructure.api.jira.install.hook import com.atlassian.performance.tools.infrastructure.api.jira.install.InstalledJira +import com.atlassian.performance.tools.infrastructure.api.jira.report.Report import com.atlassian.performance.tools.infrastructure.api.jira.report.Reports -import com.atlassian.performance.tools.infrastructure.api.jira.report.StaticReport import com.atlassian.performance.tools.ssh.api.SshConnection import java.nio.file.Path import java.nio.file.Paths @@ -10,20 +10,24 @@ import java.nio.file.Paths class JiraLogs : PostInstallHook { override fun call(ssh: SshConnection, jira: InstalledJira, hooks: PostInstallHooks, reports: Reports) { - listOf( - "${jira.home.path}/log/atlassian-jira.log", - "${jira.installation.path}/logs/catalina.out" - ) - .onEach { ensureFile(Paths.get(it), ssh) } - .map { StaticReport(it) } - .forEach { reports.add(it, jira.host) } + reports.add(report(jira), jira.host) } - private fun ensureFile( - path: Path, - ssh: SshConnection - ) { - ssh.execute("mkdir -p ${path.parent!!}") - ssh.execute("touch $path") + fun report(jira: InstalledJira): Report { + return JiraLogsReport(jira) + } + + private class JiraLogsReport(private val jira: InstalledJira) : Report { + override fun locate(ssh: SshConnection): List { + return listOf( + "${jira.home.path}/log/atlassian-jira.log", + "${jira.installation.path}/logs/catalina.out" + ).onEach { ensureFile(Paths.get(it), ssh) } + } + + private fun ensureFile(path: Path, ssh: SshConnection) { + ssh.execute("mkdir -p ${path.parent!!}") + ssh.execute("touch $path") + } } } diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/start/hook/RestUpgrade.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/start/hook/RestUpgrade.kt index 89c12a00..d0dcd229 100644 --- a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/start/hook/RestUpgrade.kt +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/start/hook/RestUpgrade.kt @@ -1,6 +1,7 @@ package com.atlassian.performance.tools.infrastructure.api.jira.start.hook import com.atlassian.performance.tools.infrastructure.api.jira.JiraLaunchTimeouts +import com.atlassian.performance.tools.infrastructure.api.jira.install.hook.JiraLogs import com.atlassian.performance.tools.infrastructure.api.jira.report.FileListing import com.atlassian.performance.tools.infrastructure.api.jira.report.Reports import com.atlassian.performance.tools.infrastructure.api.jira.start.StartedJira @@ -18,59 +19,71 @@ class RestUpgrade( override fun call(ssh: SshConnection, jira: StartedJira, hooks: PostStartHooks, reports: Reports) { val threadDump = ThreadDump(jira.pid, jira.installed.jdk) - val ip = jira.installed.host.privateIp - val port = jira.installed.host.port - val upgradesEndpoint = URI("http://$adminUsername:$adminPassword@$ip:$port/rest/api/2/upgrade") - reports.add(FileListing("thread-dumps/*"), jira.installed.host) - waitForStatusToChange( - statusQuo = "000", - timeout = timeouts.offlineTimeout, - ssh = ssh, - uri = upgradesEndpoint, - threadDump = threadDump - ) - waitForStatusToChange( - statusQuo = "503", - timeout = timeouts.initTimeout, - ssh = ssh, - uri = upgradesEndpoint, - threadDump = threadDump - ) - ssh.execute( - cmd = "curl --silent --retry 6 -X POST $upgradesEndpoint", - timeout = Duration.ofSeconds(15) - ) - waitForStatusToChange( - statusQuo = "303", - timeout = timeouts.upgradeTimeout, - ssh = ssh, - uri = upgradesEndpoint, - threadDump = threadDump - ) + val polling = Upgrades(ssh, jira, adminUsername, adminPassword, timeouts, threadDump, reports) + polling.waitUntilOnline() + polling.waitUntilHealthy() + polling.triggerUpgrades() + polling.waitUntilUpgraded() } - private fun waitForStatusToChange( - statusQuo: String, - uri: URI, - timeout: Duration, - ssh: SshConnection, - threadDump: ThreadDump + private class Upgrades( + private val ssh: SshConnection, + private val jira: StartedJira, + adminUsername: String, + adminPassword: String, + private val timeouts: JiraLaunchTimeouts, + private val threadDump: ThreadDump, + private val reports: Reports ) { - val backoff = Duration.ofSeconds(10) - val deadline = Instant.now() + timeout - while (true) { - val currentStatus = ssh.safeExecute( - cmd = "curl --silent --write-out '%{http_code}' --output /dev/null -X GET $uri", - timeout = timeouts.unresponsivenessTimeout - ).output - if (currentStatus != statusQuo) { - break - } - if (deadline < Instant.now()) { - throw Exception("$uri failed to get out of $statusQuo status within $timeout") + private val upgradesEndpoint: URI + + init { + val ip = jira.installed.host.privateIp + val port = jira.installed.host.port + upgradesEndpoint = URI("http://$adminUsername:$adminPassword@$ip:$port/rest/api/2/upgrade") + } + + fun waitUntilOnline() { + waitForStatusToChange("000", timeouts.offlineTimeout) + } + + fun waitUntilHealthy() { + waitForStatusToChange("503", timeouts.initTimeout) + } + + fun waitUntilUpgraded() { + waitForStatusToChange("303", timeouts.upgradeTimeout) + } + + private fun waitForStatusToChange( + statusQuo: String, + timeout: Duration + ) { + val backoff = Duration.ofSeconds(10) + val deadline = Instant.now() + timeout + while (true) { + val currentStatus = ssh.safeExecute( + cmd = "curl --silent --write-out '%{http_code}' --output /dev/null -X GET $upgradesEndpoint", + timeout = timeouts.unresponsivenessTimeout + ).output + if (currentStatus != statusQuo) { + break + } + if (deadline < Instant.now()) { + reports.add(JiraLogs().report(jira.installed), jira.installed.host) + reports.add(FileListing("thread-dumps/*"), jira.installed.host) + throw Exception("$upgradesEndpoint failed to get out of $statusQuo status within $timeout") + } + threadDump.gather(ssh, "thread-dumps") + Thread.sleep(backoff.toMillis()) } - threadDump.gather(ssh, "thread-dumps") - Thread.sleep(backoff.toMillis()) + } + + fun triggerUpgrades() { + ssh.execute( + cmd = "curl --silent --retry 6 -X POST $upgradesEndpoint", + timeout = Duration.ofSeconds(15) + ) } } -} +} \ No newline at end of file diff --git a/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/JiraServerPlanIT.kt b/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/JiraServerPlanIT.kt index dc335b9b..7de06cda 100644 --- a/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/JiraServerPlanIT.kt +++ b/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/JiraServerPlanIT.kt @@ -10,11 +10,14 @@ import com.atlassian.performance.tools.infrastructure.api.jira.install.hook.PreI import com.atlassian.performance.tools.infrastructure.api.jira.node.JiraNodePlan import com.atlassian.performance.tools.infrastructure.api.jira.start.JiraLaunchScript import com.atlassian.performance.tools.infrastructure.api.jvm.AdoptOpenJDK +import com.atlassian.performance.tools.io.api.resolveSafely import org.assertj.core.api.Assertions.assertThat import org.junit.After import org.junit.Before import org.junit.Test import java.nio.file.Files +import java.nio.file.Paths +import java.time.Instant class JiraServerPlanIT { @@ -55,7 +58,15 @@ class JiraServerPlanIT { .build() // when - val jiraServer = jiraServerPlan.materialize() + val jiraServer = try { + jiraServerPlan.materialize() + } catch (e: Exception) { + val debugging = Paths.get("build/test-artifacts/") + .resolveSafely(javaClass.simpleName) + .resolveSafely(Instant.now().toString()) + jiraServerPlan.report().downloadTo(debugging) + throw Exception("Jira Server plan failed to materialize, debugging info available in $debugging", e) + } val theNode = jiraServer.nodes.single() val host = theNode.installed.host @@ -76,7 +87,6 @@ class JiraServerPlanIT { .toList() assertThat(fileTree.map { it.toString() }).contains( "jira-node/root/atlassian-jira-software-7.13.0-standalone/logs/catalina.out", - "jira-node/root/thread-dumps", "jira-node/root/~/jpt-jstat.log", "jira-node/root/~/jpt-vmstat.log", "jira-node/root/~/jpt-iostat.log" From 080f55e2537f906124fd967c58e41342ad0b4617 Mon Sep 17 00:00:00 2001 From: Maciej Kwidzinski Date: Fri, 25 Jun 2021 15:05:05 +0200 Subject: [PATCH 40/52] JPERF-273: Expose NFS or Samba shared homes for DC --- CHANGELOG.md | 1 + .../jira/instance/DefaultClusterProperties.kt | 18 +++++++++++++++ .../api/jira/instance/JiraDataCenterPlan.kt | 23 ++++++++++++++----- .../NfsSharedHome.kt} | 14 +++++------ .../SambaSharedHome.kt} | 14 +++++------ .../jira/instance/ClusterProperties.kt | 13 +++++++++++ .../jira/sharedhome/SharedHomeProperty.kt | 14 +++++++++++ .../api/jira/instance/JiraDataCenterPlanIT.kt | 15 ++++++++---- 8 files changed, 87 insertions(+), 25 deletions(-) create mode 100644 src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/DefaultClusterProperties.kt rename src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/{instance/NfsSharedHomeHook.kt => sharedhome/NfsSharedHome.kt} (86%) rename src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/{instance/SambaSharedHomeHook.kt => sharedhome/SambaSharedHome.kt} (89%) create mode 100644 src/main/kotlin/com/atlassian/performance/tools/infrastructure/jira/instance/ClusterProperties.kt create mode 100644 src/main/kotlin/com/atlassian/performance/tools/infrastructure/jira/sharedhome/SharedHomeProperty.kt diff --git a/CHANGELOG.md b/CHANGELOG.md index 46a0a81e..42eb669f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -35,6 +35,7 @@ Fix [JPERF-273]: - Hook into Jira start via `PreStartHooks` and `PostStartHooks`. - Let hooks insert new hooks. - Locate and download any logs, charts, profiles and other reports via `Report` (rather than hardcoding the paths). +- Expose preset `NfsSharedHome` or `SambaSharedHome` for Data Center. [JPERF-273]: https://ecosystem.atlassian.net/browse/JPERF-273 diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/DefaultClusterProperties.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/DefaultClusterProperties.kt new file mode 100644 index 00000000..1e2bbbb3 --- /dev/null +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/DefaultClusterProperties.kt @@ -0,0 +1,18 @@ +package com.atlassian.performance.tools.infrastructure.api.jira.instance + +import com.atlassian.performance.tools.infrastructure.api.jira.install.InstalledJira +import com.atlassian.performance.tools.infrastructure.api.jira.install.hook.PostInstallHook +import com.atlassian.performance.tools.infrastructure.api.jira.install.hook.PostInstallHooks +import com.atlassian.performance.tools.infrastructure.api.jira.report.Reports +import com.atlassian.performance.tools.infrastructure.jira.instance.ClusterProperties +import com.atlassian.performance.tools.ssh.api.SshConnection + +class DefaultClusterProperties : PostInstallHook { + + override fun call(ssh: SshConnection, jira: InstalledJira, hooks: PostInstallHooks, reports: Reports) { + ClusterProperties(jira).apply { + set("jira.node.id", jira.host.name, ssh) + set("ehcache.object.port", "40011", ssh) + } + } +} diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/JiraDataCenterPlan.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/JiraDataCenterPlan.kt index 0d1d926c..2e2057a2 100644 --- a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/JiraDataCenterPlan.kt +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/JiraDataCenterPlan.kt @@ -2,10 +2,12 @@ package com.atlassian.performance.tools.infrastructure.api.jira.instance import com.atlassian.performance.tools.infrastructure.api.Infrastructure import com.atlassian.performance.tools.infrastructure.api.jira.install.InstalledJira +import com.atlassian.performance.tools.infrastructure.api.jira.install.hook.PreInstallHooks import com.atlassian.performance.tools.infrastructure.api.jira.node.JiraNode import com.atlassian.performance.tools.infrastructure.api.jira.node.JiraNodePlan import com.atlassian.performance.tools.infrastructure.api.jira.report.Reports import com.atlassian.performance.tools.infrastructure.api.jira.start.StartedJira +import com.atlassian.performance.tools.infrastructure.api.loadbalancer.ApacheProxyPlan import com.atlassian.performance.tools.infrastructure.api.loadbalancer.LoadBalancer import com.atlassian.performance.tools.infrastructure.api.loadbalancer.LoadBalancerPlan import java.net.URI @@ -13,7 +15,7 @@ import java.time.Duration import kotlin.streams.asStream import kotlin.streams.toList -class JiraDataCenterPlan constructor( +class JiraDataCenterPlan private constructor( private val nodePlans: List, private val instanceHooks: PreInstanceHooks, private val balancerPlan: LoadBalancerPlan, @@ -74,10 +76,19 @@ class JiraDataCenterPlan constructor( class Builder( private var infrastructure: Infrastructure ) { - private var nodePlans: List = listOf(1, 2).map { JiraNodePlan.Builder().build() } -// TODO private var instanceHooks: PreInstanceHooks = -// PreInstanceHooks(listOf(1..2).map { PreInstallHooks.default() }) // TODO two lists in sync? gotta be a better way -// -// fun build(): Supplier = JiraDataCenterPlan(nodePlans, infrastructure) + private var nodePlans: List = listOf(1, 2).map { + JiraNodePlan.Builder() + .hooks(PreInstallHooks.default().apply { postInstall.insert(DefaultClusterProperties()) }) + .build() + } + private var instanceHooks: PreInstanceHooks = PreInstanceHooks.default() + private var balancerPlan: LoadBalancerPlan = ApacheProxyPlan(infrastructure) + + fun infrastructure(infrastructure: Infrastructure) = apply { this.infrastructure = infrastructure } + fun nodePlans(nodePlans: List) = apply { this.nodePlans = nodePlans } + fun instanceHooks(instanceHooks: PreInstanceHooks) = apply { this.instanceHooks = instanceHooks } + fun balancerPlan(balancerPlan: LoadBalancerPlan) = apply { this.balancerPlan = balancerPlan } + + fun build(): JiraInstancePlan = JiraDataCenterPlan(nodePlans, instanceHooks, balancerPlan, infrastructure) } } \ No newline at end of file diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/NfsSharedHomeHook.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/sharedhome/NfsSharedHome.kt similarity index 86% rename from src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/NfsSharedHomeHook.kt rename to src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/sharedhome/NfsSharedHome.kt index 467af37c..cded91ed 100644 --- a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/NfsSharedHomeHook.kt +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/sharedhome/NfsSharedHome.kt @@ -1,4 +1,4 @@ -package com.atlassian.performance.tools.infrastructure.api.jira.instance +package com.atlassian.performance.tools.infrastructure.api.jira.sharedhome import com.atlassian.performance.tools.infrastructure.api.Infrastructure import com.atlassian.performance.tools.infrastructure.api.jira.JiraHomeSource @@ -6,12 +6,15 @@ import com.atlassian.performance.tools.infrastructure.api.jira.install.Installed import com.atlassian.performance.tools.infrastructure.api.jira.install.hook.PostInstallHook import com.atlassian.performance.tools.infrastructure.api.jira.install.hook.PostInstallHooks import com.atlassian.performance.tools.infrastructure.api.jira.install.hook.PreInstallHooks +import com.atlassian.performance.tools.infrastructure.api.jira.instance.PreInstanceHook +import com.atlassian.performance.tools.infrastructure.api.jira.instance.PreInstanceHooks import com.atlassian.performance.tools.infrastructure.api.jira.report.Reports import com.atlassian.performance.tools.infrastructure.api.os.RemotePath import com.atlassian.performance.tools.infrastructure.api.os.Ubuntu +import com.atlassian.performance.tools.infrastructure.jira.sharedhome.SharedHomeProperty import com.atlassian.performance.tools.ssh.api.SshConnection -internal class NfsSharedHomeHook( +class NfsSharedHome( private val jiraHomeSource: JiraHomeSource, private val infrastructure: Infrastructure ) : PreInstanceHook { @@ -52,11 +55,8 @@ internal class NfsSharedHomeHook( ssh.execute("mkdir -p $mountTarget") ssh.execute("sudo mount -o soft,intr,rsize=8192,wsize=8192 $mountSource $mountTarget") ssh.execute("sudo chown ubuntu:ubuntu $mountTarget") - val mountedPath = "`realpath $mountTarget`" - val jiraHome = jira.home.path - ssh.execute("echo ehcache.object.port = 40011 >> $jiraHome/cluster.properties") - ssh.execute("echo jira.node.id = ${jira.host.name} >> $jiraHome/cluster.properties") - ssh.execute("echo jira.shared.home = $mountedPath >> $jiraHome/cluster.properties") + val mounted = "`realpath $mountTarget`" + SharedHomeProperty(jira).set(mounted, ssh) } } } diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/SambaSharedHomeHook.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/sharedhome/SambaSharedHome.kt similarity index 89% rename from src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/SambaSharedHomeHook.kt rename to src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/sharedhome/SambaSharedHome.kt index 4406fe05..f98a4757 100644 --- a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/SambaSharedHomeHook.kt +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/sharedhome/SambaSharedHome.kt @@ -1,4 +1,4 @@ -package com.atlassian.performance.tools.infrastructure.api.jira.instance +package com.atlassian.performance.tools.infrastructure.api.jira.sharedhome import com.atlassian.performance.tools.infrastructure.api.Infrastructure import com.atlassian.performance.tools.infrastructure.api.jira.JiraHomeSource @@ -7,12 +7,15 @@ import com.atlassian.performance.tools.infrastructure.api.jira.install.TcpHost import com.atlassian.performance.tools.infrastructure.api.jira.install.hook.PostInstallHook import com.atlassian.performance.tools.infrastructure.api.jira.install.hook.PostInstallHooks import com.atlassian.performance.tools.infrastructure.api.jira.install.hook.PreInstallHooks +import com.atlassian.performance.tools.infrastructure.api.jira.instance.PreInstanceHook +import com.atlassian.performance.tools.infrastructure.api.jira.instance.PreInstanceHooks import com.atlassian.performance.tools.infrastructure.api.jira.report.Reports import com.atlassian.performance.tools.infrastructure.api.os.Ubuntu +import com.atlassian.performance.tools.infrastructure.jira.sharedhome.SharedHomeProperty import com.atlassian.performance.tools.ssh.api.SshConnection import java.util.* -internal class SambaSharedHomeHook( +class SambaSharedHome( private val jiraHomeSource: JiraHomeSource, private val infrastructure: Infrastructure ) : PreInstanceHook { @@ -76,11 +79,8 @@ internal class SambaSharedHomeHook( val localUser = ssh.execute("whoami").output.trim() ssh.execute("sudo chown $localUser:$localUser $mountTarget") ssh.execute("sudo mount -t cifs -o $credentials $mountSource $mountTarget") - val sharedHome = "`realpath $mountTarget`" - val nodeHome = jira.home.path - ssh.execute("echo ehcache.object.port = 40011 >> $nodeHome/cluster.properties") - ssh.execute("echo jira.node.id = ${jira.host.name} >> $nodeHome/cluster.properties") - ssh.execute("echo jira.shared.home = $sharedHome >> $nodeHome/cluster.properties") + val mounted = "`realpath $mountTarget`" + SharedHomeProperty(jira).set(mounted, ssh) } } } diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/jira/instance/ClusterProperties.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/jira/instance/ClusterProperties.kt new file mode 100644 index 00000000..f0a4f310 --- /dev/null +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/jira/instance/ClusterProperties.kt @@ -0,0 +1,13 @@ +package com.atlassian.performance.tools.infrastructure.jira.instance + +import com.atlassian.performance.tools.infrastructure.api.jira.install.InstalledJira +import com.atlassian.performance.tools.ssh.api.SshConnection + +internal class ClusterProperties( + private val jira: InstalledJira +) { + + fun set(key: String, value: String, ssh: SshConnection) { + ssh.execute("echo '$key = $value' >> ${jira.home.path}/cluster.properties") + } +} diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/jira/sharedhome/SharedHomeProperty.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/jira/sharedhome/SharedHomeProperty.kt new file mode 100644 index 00000000..d6bda38b --- /dev/null +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/jira/sharedhome/SharedHomeProperty.kt @@ -0,0 +1,14 @@ +package com.atlassian.performance.tools.infrastructure.jira.sharedhome + +import com.atlassian.performance.tools.infrastructure.api.jira.install.InstalledJira +import com.atlassian.performance.tools.infrastructure.jira.instance.ClusterProperties +import com.atlassian.performance.tools.ssh.api.SshConnection + +internal class SharedHomeProperty( + private val jira: InstalledJira +) { + + fun set(mounted: String, ssh: SshConnection) { + ClusterProperties(jira).set("jira.shared.home", mounted, ssh) + } +} diff --git a/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/JiraDataCenterPlanIT.kt b/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/JiraDataCenterPlanIT.kt index 3d9dcca7..834b39d7 100644 --- a/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/JiraDataCenterPlanIT.kt +++ b/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/JiraDataCenterPlanIT.kt @@ -9,6 +9,7 @@ import com.atlassian.performance.tools.infrastructure.api.jira.install.ParallelI import com.atlassian.performance.tools.infrastructure.api.jira.install.hook.PreInstallHooks import com.atlassian.performance.tools.infrastructure.api.jira.node.JiraNodePlan import com.atlassian.performance.tools.infrastructure.api.jira.report.Reports +import com.atlassian.performance.tools.infrastructure.api.jira.sharedhome.SambaSharedHome import com.atlassian.performance.tools.infrastructure.api.jira.start.JiraLaunchScript import com.atlassian.performance.tools.infrastructure.api.jira.start.StartedJira import com.atlassian.performance.tools.infrastructure.api.jira.start.hook.PostStartHook @@ -59,9 +60,12 @@ class JiraDataCenterPlanIT { } val instanceHooks = PreInstanceHooks.default() .also { Datasets.JiraSevenDataset.hookMysql(it, infrastructure) } - .also { it.insert(SambaSharedHomeHook(jiraHomeSource, infrastructure)) } - val balancerPlan = ApacheProxyPlan(infrastructure) - val dcPlan = JiraDataCenterPlan(nodePlans, instanceHooks, balancerPlan, infrastructure) + .also { it.insert(SambaSharedHome(jiraHomeSource, infrastructure)) } + val dcPlan = JiraDataCenterPlan.Builder(infrastructure) + .nodePlans(nodePlans) + .instanceHooks(instanceHooks) + .balancerPlan(ApacheProxyPlan(infrastructure)) + .build() // when val dataCenter = dcPlan.materialize() @@ -106,8 +110,9 @@ class JiraDataCenterPlanIT { .hooks(PreInstallHooks.default().also { it.postStart.insert(FailingHook()) }) .build() } - val balancerPlan = ApacheProxyPlan(infrastructure) - val dcPlan = JiraDataCenterPlan(nodePlans, PreInstanceHooks.default(), balancerPlan, infrastructure) + val dcPlan = JiraDataCenterPlan.Builder(infrastructure) + .nodePlans(nodePlans) + .build() // when val thrown = catchThrowable { From 961107c4dcf3fd84533cd9dd7a8d29713b815f34 Mon Sep 17 00:00:00 2001 From: Maciej Kwidzinski Date: Fri, 25 Jun 2021 18:53:01 +0200 Subject: [PATCH 41/52] JPERF-273: Support Jiras with an HTTP path --- .../api/jira/install/HttpHost.kt | 19 +++++++++++++++++++ .../api/jira/install/InstalledJira.kt | 4 ++-- .../api/jira/install/JiraInstallation.kt | 6 +++--- .../api/jira/install/ParallelInstallation.kt | 6 +++--- .../jira/install/SequentialInstallation.kt | 6 +++--- .../jira/install/hook/AsyncProfilerHook.kt | 4 ++-- .../api/jira/install/hook/JiraLogs.kt | 2 +- .../api/jira/install/hook/JvmConfig.kt | 4 ++-- .../jira/install/hook/LateUbuntuSysstat.kt | 2 +- .../api/jira/install/hook/PreInstallHook.kt | 6 +++--- .../api/jira/install/hook/SystemLog.kt | 4 ++-- .../jira/instance/DefaultClusterProperties.kt | 2 +- .../api/jira/instance/JiraServerPlan.kt | 2 +- .../infrastructure/api/jira/report/Reports.kt | 17 ++++++++++++----- .../api/jira/start/hook/AccessLogs.kt | 2 +- .../api/jira/start/hook/JstatHook.kt | 2 +- .../api/jira/start/hook/RestUpgrade.kt | 16 +++++++--------- .../install/hook/HookedJiraInstallation.kt | 10 +++++----- .../jira/install/hook/ProfilerHook.kt | 2 +- .../jira/install/hook/SplunkForwarderHook.kt | 2 +- .../jira/start/hook/HookedJiraStart.kt | 4 ++-- .../jira/install/hook/PreInstallHooksTest.kt | 4 ++-- .../api/jira/instance/JiraDataCenterPlanIT.kt | 4 ++-- .../api/jira/instance/JiraServerPlanIT.kt | 6 ++---- 24 files changed, 79 insertions(+), 57 deletions(-) create mode 100644 src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/HttpHost.kt diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/HttpHost.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/HttpHost.kt new file mode 100644 index 00000000..429e51cc --- /dev/null +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/HttpHost.kt @@ -0,0 +1,19 @@ +package com.atlassian.performance.tools.infrastructure.api.jira.install + +import java.net.URI + +class HttpHost internal constructor( + val tcp: TcpHost, + private val basePath: String, + private val supportsTls: Boolean +) { + + fun addressPublicly(): URI = address(tcp.publicIp) + fun addressPrivately(): URI = address(tcp.privateIp) + fun addressPrivately(userName: String, password: String): URI = address(tcp.privateIp, "$userName:$password@") + + private fun address(ip: String, userInfo: String = ""): URI { + val scheme = if (supportsTls) "https" else "http" + return URI("$scheme://$userInfo$ip:${tcp.port}$basePath/") + } +} diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/InstalledJira.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/InstalledJira.kt index 95c77249..4927bc38 100644 --- a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/InstalledJira.kt +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/InstalledJira.kt @@ -22,7 +22,7 @@ class InstalledJira( */ val jdk: JavaDevelopmentKit, /** - * Hosts Jira. Specifies sockets used by Jira to handle requests. + * Connects to Jira on HTTP level or below. */ - val host: TcpHost + val http: HttpHost ) diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/JiraInstallation.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/JiraInstallation.kt index deb4bfb0..299a6882 100644 --- a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/JiraInstallation.kt +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/JiraInstallation.kt @@ -10,13 +10,13 @@ import net.jcip.annotations.ThreadSafe interface JiraInstallation { /** - * Installs Jira on [host]. + * Installs Jira on [tcp]. * - * @param [host] will host the Jira + * @param [tcp] will host the Jira * @param [reports] accumulates reports */ fun install( - host: TcpHost, + tcp: TcpHost, reports: Reports ): InstalledJira } diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/ParallelInstallation.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/ParallelInstallation.kt index 452f1bea..7bb6a7dd 100644 --- a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/ParallelInstallation.kt +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/ParallelInstallation.kt @@ -16,10 +16,10 @@ class ParallelInstallation( ) : JiraInstallation { override fun install( - host: TcpHost, + tcp: TcpHost, reports: Reports ): InstalledJira { - host.ssh.newConnection().use { ssh -> + tcp.ssh.newConnection().use { ssh -> val pool = Executors.newCachedThreadPool { runnable -> Thread(runnable, "jira-installation-${runnable.hashCode()}") } @@ -32,7 +32,7 @@ class ParallelInstallation( val java = pool.submitWithLogContext("java") { jdk.also { it.install(ssh) } } - val jira = InstalledJira(home.get(), product.get(), java.get(), host) + val jira = InstalledJira(home.get(), product.get(), java.get(), HttpHost(tcp, "", false)) pool.shutdownNow() return jira } diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/SequentialInstallation.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/SequentialInstallation.kt index c5d124d3..3fa86766 100644 --- a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/SequentialInstallation.kt +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/SequentialInstallation.kt @@ -14,14 +14,14 @@ class SequentialInstallation( ) : JiraInstallation { override fun install( - host: TcpHost, + tcp: TcpHost, reports: Reports ): InstalledJira { - host.ssh.newConnection().use { ssh -> + tcp.ssh.newConnection().use { ssh -> val installation = productDistribution.installRemotely(ssh, ".") val home = jiraHomeSource.downloadRemotely(ssh) jdk.install(ssh) - return InstalledJira(home, installation, jdk, host) + return InstalledJira(home, installation, jdk, HttpHost(tcp, "", false)) } } } diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/AsyncProfilerHook.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/AsyncProfilerHook.kt index 186453e7..ef86eafc 100644 --- a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/AsyncProfilerHook.kt +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/AsyncProfilerHook.kt @@ -13,7 +13,7 @@ class AsyncProfilerHook : PreInstallHook { override fun call( ssh: SshConnection, - host: TcpHost, + tcp: TcpHost, hooks: PreInstallHooks, reports: Reports ) { @@ -43,7 +43,7 @@ private class InstalledAsyncProfiler( ) { ssh.execute("$profilerPath -b 20000000 start ${jira.pid}") val profiler = StartedAsyncProfiler(jira.pid, profilerPath) - reports.add(profiler, jira.installed.host) + reports.add(profiler, jira) } } diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/JiraLogs.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/JiraLogs.kt index adfc0033..7d544f19 100644 --- a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/JiraLogs.kt +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/JiraLogs.kt @@ -10,7 +10,7 @@ import java.nio.file.Paths class JiraLogs : PostInstallHook { override fun call(ssh: SshConnection, jira: InstalledJira, hooks: PostInstallHooks, reports: Reports) { - reports.add(report(jira), jira.host) + reports.add(report(jira), jira) } fun report(jira: InstalledJira): Report { diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/JvmConfig.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/JvmConfig.kt index 88ed1e91..3fe4ab97 100644 --- a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/JvmConfig.kt +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/JvmConfig.kt @@ -23,9 +23,9 @@ class JvmConfig( connection = ssh, config = config, gcLog = gcLog, - jiraIp = jira.host.publicIp + jiraIp = jira.http.tcp.publicIp ) val report = FileListing(gcLog.path("*")) - reports.add(report, jira.host) + reports.add(report, jira) } } diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/LateUbuntuSysstat.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/LateUbuntuSysstat.kt index 26f327e2..2404e567 100644 --- a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/LateUbuntuSysstat.kt +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/LateUbuntuSysstat.kt @@ -32,6 +32,6 @@ private class PostStartOsMetric( ) : PostStartHook { override fun call(ssh: SshConnection, jira: StartedJira, hooks: PostStartHooks, reports: Reports) { val process = metric.start(ssh) - reports.add(RemoteMonitoringProcessReport(process), jira.installed.host) + reports.add(RemoteMonitoringProcessReport(process), jira) } } diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/PreInstallHook.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/PreInstallHook.kt index b9be4a39..ec5f329f 100644 --- a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/PreInstallHook.kt +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/PreInstallHook.kt @@ -10,14 +10,14 @@ import com.atlassian.performance.tools.ssh.api.SshConnection interface PreInstallHook { /** - * @param [ssh] connects to the [host] - * @param [host] will install Jira + * @param [ssh] connects to the [tcp] + * @param [tcp] will install Jira * @param [hooks] inserts future hooks * @param [reports] accumulates reports */ fun call( ssh: SshConnection, - host: TcpHost, + tcp: TcpHost, hooks: PreInstallHooks, reports: Reports ) diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/SystemLog.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/SystemLog.kt index 235e41d2..28084dfa 100644 --- a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/SystemLog.kt +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/SystemLog.kt @@ -7,7 +7,7 @@ import com.atlassian.performance.tools.ssh.api.SshConnection class SystemLog : PreInstallHook { - override fun call(ssh: SshConnection, host: TcpHost, hooks: PreInstallHooks, reports: Reports) { - reports.add(FileListing("/var/log/syslog"), host) + override fun call(ssh: SshConnection, tcp: TcpHost, hooks: PreInstallHooks, reports: Reports) { + reports.add(FileListing("/var/log/syslog"), tcp) } } diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/DefaultClusterProperties.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/DefaultClusterProperties.kt index 1e2bbbb3..ab342f07 100644 --- a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/DefaultClusterProperties.kt +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/DefaultClusterProperties.kt @@ -11,7 +11,7 @@ class DefaultClusterProperties : PostInstallHook { override fun call(ssh: SshConnection, jira: InstalledJira, hooks: PostInstallHooks, reports: Reports) { ClusterProperties(jira).apply { - set("jira.node.id", jira.host.name, ssh) + set("jira.node.id", jira.http.tcp.name, ssh) set("ehcache.object.port", "40011", ssh) } } diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/JiraServerPlan.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/JiraServerPlan.kt index c9d7da75..d15eaa0a 100644 --- a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/JiraServerPlan.kt +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/JiraServerPlan.kt @@ -30,7 +30,7 @@ class JiraServerPlan private constructor( private class JiraServer( node: StartedJira ) : JiraInstance { - override val address: URI = node.installed.host.run { URI("http://$publicIp:$port/") } + override val address: URI = node.installed.http.addressPublicly() override val nodes: List = listOf(node) } diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/report/Reports.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/report/Reports.kt index 84aa3ae9..76a7b1c4 100644 --- a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/report/Reports.kt +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/report/Reports.kt @@ -1,6 +1,8 @@ package com.atlassian.performance.tools.infrastructure.api.jira.report +import com.atlassian.performance.tools.infrastructure.api.jira.install.InstalledJira import com.atlassian.performance.tools.infrastructure.api.jira.install.TcpHost +import com.atlassian.performance.tools.infrastructure.api.jira.start.StartedJira import com.atlassian.performance.tools.infrastructure.api.os.RemotePath import com.atlassian.performance.tools.io.api.ensureDirectory import com.atlassian.performance.tools.io.api.resolveSafely @@ -15,11 +17,16 @@ class Reports private constructor( // TODO turn into SPI to allow AWS CLI transp ) { constructor() : this(ConcurrentLinkedQueue()) - fun add( - report: Report, - host: TcpHost - ) { - hostReports.add(HostReport(host, report)) + fun add(report: Report, started: StartedJira) { + add(report, started.installed) + } + + fun add(report: Report, installed: InstalledJira) { + add(report, installed.http.tcp) + } + + fun add(report: Report, tcp: TcpHost) { + hostReports.add(HostReport(tcp, report)) } fun downloadTo( diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/start/hook/AccessLogs.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/start/hook/AccessLogs.kt index f6f35c5d..c2c30a78 100644 --- a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/start/hook/AccessLogs.kt +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/start/hook/AccessLogs.kt @@ -8,6 +8,6 @@ import com.atlassian.performance.tools.ssh.api.SshConnection class AccessLogs : PreStartHook { override fun call(ssh: SshConnection, jira: InstalledJira, hooks: PreStartHooks, reports: Reports) { - reports.add(FileListing("${jira.installation.path}/logs/*access*"), jira.host) + reports.add(FileListing("${jira.installation.path}/logs/*access*"), jira) } } diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/start/hook/JstatHook.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/start/hook/JstatHook.kt index 547e51af..f1e15be5 100644 --- a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/start/hook/JstatHook.kt +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/start/hook/JstatHook.kt @@ -14,6 +14,6 @@ class JstatHook : PostStartHook { reports: Reports ) { val process = jira.installed.jdk.jstatMonitoring.start(ssh, jira.pid) - reports.add(RemoteMonitoringProcessReport(process), jira.installed.host) + reports.add(RemoteMonitoringProcessReport(process), jira) } } diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/start/hook/RestUpgrade.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/start/hook/RestUpgrade.kt index d0dcd229..e9e3b9b5 100644 --- a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/start/hook/RestUpgrade.kt +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/start/hook/RestUpgrade.kt @@ -35,13 +35,11 @@ class RestUpgrade( private val threadDump: ThreadDump, private val reports: Reports ) { - private val upgradesEndpoint: URI - - init { - val ip = jira.installed.host.privateIp - val port = jira.installed.host.port - upgradesEndpoint = URI("http://$adminUsername:$adminPassword@$ip:$port/rest/api/2/upgrade") - } + private val upgradesEndpoint: URI = jira + .installed + .http + .addressPrivately(adminUsername, adminPassword) + .resolve("rest/api/2/upgrade") fun waitUntilOnline() { waitForStatusToChange("000", timeouts.offlineTimeout) @@ -70,8 +68,8 @@ class RestUpgrade( break } if (deadline < Instant.now()) { - reports.add(JiraLogs().report(jira.installed), jira.installed.host) - reports.add(FileListing("thread-dumps/*"), jira.installed.host) + reports.add(JiraLogs().report(jira.installed), jira) + reports.add(FileListing("thread-dumps/*"), jira) throw Exception("$upgradesEndpoint failed to get out of $statusQuo status within $timeout") } threadDump.gather(ssh, "thread-dumps") diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/jira/install/hook/HookedJiraInstallation.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/jira/install/hook/HookedJiraInstallation.kt index 153fafe8..ba8b8cc7 100644 --- a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/jira/install/hook/HookedJiraInstallation.kt +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/jira/install/hook/HookedJiraInstallation.kt @@ -12,14 +12,14 @@ class HookedJiraInstallation( ) : JiraInstallation { override fun install( - host: TcpHost, + tcp: TcpHost, reports: Reports ): InstalledJira { - host.ssh.newConnection().use { ssh -> - hooks.call(ssh, host, reports) + tcp.ssh.newConnection().use { ssh -> + hooks.call(ssh, tcp, reports) } - val installed = installation.install(host, reports) - host.ssh.newConnection().use { ssh -> + val installed = installation.install(tcp, reports) + tcp.ssh.newConnection().use { ssh -> hooks.postInstall.call(ssh, installed, reports) } return installed diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/jira/install/hook/ProfilerHook.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/jira/install/hook/ProfilerHook.kt index 294ab3e5..3ecd6098 100644 --- a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/jira/install/hook/ProfilerHook.kt +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/jira/install/hook/ProfilerHook.kt @@ -37,7 +37,7 @@ private class InstalledProfiler( ) { val process = profiler.start(ssh, jira.pid) if (process != null) { - reports.add(RemoteMonitoringProcessReport(process), jira.installed.host) + reports.add(RemoteMonitoringProcessReport(process), jira) } } } diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/jira/install/hook/SplunkForwarderHook.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/jira/install/hook/SplunkForwarderHook.kt index 1e27f25b..0baf1797 100644 --- a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/jira/install/hook/SplunkForwarderHook.kt +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/jira/install/hook/SplunkForwarderHook.kt @@ -18,6 +18,6 @@ internal class SplunkForwarderHook( reports: Reports ) { splunk.jsonifyLog4j(ssh, "${jira.installation.path}/atlassian-jira/WEB-INF/classes/log4j.properties") - splunk.run(ssh, jira.host.name, "/home/ubuntu/jirahome/log") + splunk.run(ssh, jira.http.tcp.name, "/home/ubuntu/jirahome/log") } } diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/jira/start/hook/HookedJiraStart.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/jira/start/hook/HookedJiraStart.kt index e23395f9..73d02676 100644 --- a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/jira/start/hook/HookedJiraStart.kt +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/jira/start/hook/HookedJiraStart.kt @@ -15,11 +15,11 @@ class HookedJiraStart( installed: InstalledJira, reports: Reports ): StartedJira { - installed.host.ssh.newConnection().use { ssh -> + installed.http.tcp.ssh.newConnection().use { ssh -> hooks.call(ssh, installed, reports) } val started = start.start(installed, reports) - installed.host.ssh.newConnection().use { ssh -> + installed.http.tcp.ssh.newConnection().use { ssh -> hooks.postStart.call(ssh, started, reports) } return started diff --git a/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/PreInstallHooksTest.kt b/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/PreInstallHooksTest.kt index 711dec3b..2ddfb10c 100644 --- a/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/PreInstallHooksTest.kt +++ b/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/PreInstallHooksTest.kt @@ -48,7 +48,7 @@ private class CountingHook : PreInstallHook { var count = 0 - override fun call(ssh: SshConnection, host: TcpHost, hooks: PreInstallHooks, reports: Reports) { + override fun call(ssh: SshConnection, tcp: TcpHost, hooks: PreInstallHooks, reports: Reports) { count++ } } @@ -56,7 +56,7 @@ private class CountingHook : PreInstallHook { private class InsertingHook( private val hook: PreInstallHook ) : PreInstallHook { - override fun call(ssh: SshConnection, host: TcpHost, hooks: PreInstallHooks, reports: Reports) { + override fun call(ssh: SshConnection, tcp: TcpHost, hooks: PreInstallHooks, reports: Reports) { hooks.insert(hook) } } diff --git a/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/JiraDataCenterPlanIT.kt b/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/JiraDataCenterPlanIT.kt index 834b39d7..50e91343 100644 --- a/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/JiraDataCenterPlanIT.kt +++ b/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/JiraDataCenterPlanIT.kt @@ -77,9 +77,9 @@ class JiraDataCenterPlanIT { .installation .resolve("conf/server.xml") .download(Files.createTempFile("downloaded-config", ".xml")) - assertThat(serverXml.readText()).contains(" + installed.http.tcp.ssh.newConnection().use { ssh -> ssh.execute("wget ${dataCenter.address}") } } diff --git a/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/JiraServerPlanIT.kt b/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/JiraServerPlanIT.kt index 7de06cda..3c944099 100644 --- a/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/JiraServerPlanIT.kt +++ b/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/JiraServerPlanIT.kt @@ -67,18 +67,16 @@ class JiraServerPlanIT { jiraServerPlan.report().downloadTo(debugging) throw Exception("Jira Server plan failed to materialize, debugging info available in $debugging", e) } - - val theNode = jiraServer.nodes.single() - val host = theNode.installed.host val reports = jiraServerPlan.report().downloadTo(Files.createTempDirectory("jira-server-plan-")) // then + val theNode = jiraServer.nodes.single() val serverXml = theNode .installed .installation .resolve("conf/server.xml") .download(Files.createTempFile("downloaded-config", ".xml")) - assertThat(serverXml.readText()).contains(" Date: Fri, 25 Jun 2021 18:53:40 +0200 Subject: [PATCH 42/52] JPERF-273: Check Jira HTML when upgrade fails --- .../api/jira/start/hook/RestUpgrade.kt | 2 ++ .../jira/report/JiraLandingPage.kt | 19 +++++++++++++++++++ 2 files changed, 21 insertions(+) create mode 100644 src/main/kotlin/com/atlassian/performance/tools/infrastructure/jira/report/JiraLandingPage.kt diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/start/hook/RestUpgrade.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/start/hook/RestUpgrade.kt index e9e3b9b5..3762f7d2 100644 --- a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/start/hook/RestUpgrade.kt +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/start/hook/RestUpgrade.kt @@ -6,6 +6,7 @@ import com.atlassian.performance.tools.infrastructure.api.jira.report.FileListin import com.atlassian.performance.tools.infrastructure.api.jira.report.Reports import com.atlassian.performance.tools.infrastructure.api.jira.start.StartedJira import com.atlassian.performance.tools.infrastructure.api.jvm.ThreadDump +import com.atlassian.performance.tools.infrastructure.jira.report.JiraLandingPage import com.atlassian.performance.tools.ssh.api.SshConnection import java.net.URI import java.time.Duration @@ -70,6 +71,7 @@ class RestUpgrade( if (deadline < Instant.now()) { reports.add(JiraLogs().report(jira.installed), jira) reports.add(FileListing("thread-dumps/*"), jira) + reports.add(JiraLandingPage(jira), jira) throw Exception("$upgradesEndpoint failed to get out of $statusQuo status within $timeout") } threadDump.gather(ssh, "thread-dumps") diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/jira/report/JiraLandingPage.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/jira/report/JiraLandingPage.kt new file mode 100644 index 00000000..8d0b1302 --- /dev/null +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/jira/report/JiraLandingPage.kt @@ -0,0 +1,19 @@ +package com.atlassian.performance.tools.infrastructure.jira.report + +import com.atlassian.performance.tools.infrastructure.api.jira.report.Report +import com.atlassian.performance.tools.infrastructure.api.jira.start.StartedJira +import com.atlassian.performance.tools.infrastructure.api.os.Ubuntu +import com.atlassian.performance.tools.ssh.api.SshConnection + +internal class JiraLandingPage( + private val started: StartedJira +) : Report { + override fun locate(ssh: SshConnection): List { + Ubuntu().install(ssh, listOf("curl")) + val landingPage = started.installed.http.addressPrivately() + val html = "jira-landing-page.html" + val headers = "jira-landing-page-headers.txt" + ssh.execute("curl $landingPage --location --output $html --dump-header $headers --silent") + return listOf(html, headers) + } +} From 064b1b9701477334162f8b5de19044d43a172a12 Mon Sep 17 00:00:00 2001 From: Maciej Kwidzinski Date: Mon, 28 Jun 2021 10:09:25 +0200 Subject: [PATCH 43/52] More time for Jira plugins on GHA --- .../com/atlassian/performance/tools/infrastructure/Datasets.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/kotlin/com/atlassian/performance/tools/infrastructure/Datasets.kt b/src/test/kotlin/com/atlassian/performance/tools/infrastructure/Datasets.kt index 5c3fb0cb..ed65cf1f 100644 --- a/src/test/kotlin/com/atlassian/performance/tools/infrastructure/Datasets.kt +++ b/src/test/kotlin/com/atlassian/performance/tools/infrastructure/Datasets.kt @@ -34,7 +34,7 @@ class Datasets { fun hookMysql(postStartHooks: PostStartHooks) { val timeouts = JiraLaunchTimeouts.Builder() - .initTimeout(Duration.ofSeconds(45)) + .initTimeout(Duration.ofMinutes(2)) .build() val dataUpgrade = RestUpgrade(timeouts, "admin", "admin") postStartHooks.insert(dataUpgrade) From 97442c334d999dc51b02cabe4b240ddcd807d727 Mon Sep 17 00:00:00 2001 From: Maciej Kwidzinski Date: Mon, 28 Jun 2021 12:32:53 +0200 Subject: [PATCH 44/52] Give even more time for Jira plugins on GHA --- .../com/atlassian/performance/tools/infrastructure/Datasets.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/kotlin/com/atlassian/performance/tools/infrastructure/Datasets.kt b/src/test/kotlin/com/atlassian/performance/tools/infrastructure/Datasets.kt index ed65cf1f..0363e416 100644 --- a/src/test/kotlin/com/atlassian/performance/tools/infrastructure/Datasets.kt +++ b/src/test/kotlin/com/atlassian/performance/tools/infrastructure/Datasets.kt @@ -34,7 +34,7 @@ class Datasets { fun hookMysql(postStartHooks: PostStartHooks) { val timeouts = JiraLaunchTimeouts.Builder() - .initTimeout(Duration.ofMinutes(2)) + .initTimeout(Duration.ofMinutes(4)) .build() val dataUpgrade = RestUpgrade(timeouts, "admin", "admin") postStartHooks.insert(dataUpgrade) From c9927146e1e0f9ac4f9e487f663a6d350517beb1 Mon Sep 17 00:00:00 2001 From: Maciej Kwidzinski Date: Fri, 2 Jul 2021 17:44:46 +0200 Subject: [PATCH 45/52] Clarify what `JiraNodePlan` does and doesn't do Some architectural answers: * `JiraInstancePlan`, `JiraNodePlan`, `JiraInstallation`, `JiraStart`, are all lightweight plans without any resources * `HttpNode`, `InstalledJira`, `StartedJira`, `JiraInstance` are all resources without any plans * plans are independent from each other and don't talk, even through resources Open questions: * same as previous commit: who controls the ports? `ParallelInstallation` has run into trouble when it ignored the factual `HttpNode` it received --- .../infrastructure/api/Infrastructure.kt | 10 ++-- .../api/database/DockerMysqlServer.kt | 4 +- .../api/database/DockerPostgresServer.kt | 8 +-- .../jira/install/{HttpHost.kt => HttpNode.kt} | 4 +- .../api/jira/install/InstalledJira.kt | 2 +- .../api/jira/install/JiraInstallation.kt | 6 +-- .../api/jira/install/ParallelInstallation.kt | 7 +-- .../jira/install/SequentialInstallation.kt | 6 +-- .../jira/install/{TcpHost.kt => TcpNode.kt} | 2 +- .../jira/install/hook/AsyncProfilerHook.kt | 4 +- .../api/jira/install/hook/PreInstallHook.kt | 9 ++-- .../api/jira/install/hook/PreInstallHooks.kt | 6 +-- .../api/jira/install/hook/SystemLog.kt | 6 +-- .../api/jira/instance/JiraDataCenterPlan.kt | 50 +++++++++---------- .../jira/{node => instance}/JiraNodePlan.kt | 26 +++++++--- .../api/jira/instance/JiraServerPlan.kt | 13 ++--- .../infrastructure/api/jira/node/JiraNode.kt | 8 --- .../infrastructure/api/jira/report/Reports.kt | 13 +++-- .../api/jira/sharedhome/SambaSharedHome.kt | 4 +- .../api/jira/start/JiraStart.kt | 3 ++ .../api/loadbalancer/ApacheProxyPlan.kt | 22 ++++---- .../api/loadbalancer/LoadBalancerPlan.kt | 5 +- .../install/hook/HookedJiraInstallation.kt | 12 ++--- .../api/DockerInfrastructure.kt | 23 ++++++--- .../jira/install/hook/PreInstallHooksTest.kt | 17 ++++--- .../api/jira/instance/JiraDataCenterPlanIT.kt | 5 +- .../api/jira/instance/JiraServerPlanIT.kt | 25 +++++++--- 27 files changed, 169 insertions(+), 131 deletions(-) rename src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/{HttpHost.kt => HttpNode.kt} (91%) rename src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/{TcpHost.kt => TcpNode.kt} (96%) rename src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/{node => instance}/JiraNodePlan.kt (70%) delete mode 100644 src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/node/JiraNode.kt diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/Infrastructure.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/Infrastructure.kt index 06de898f..82b1fdfe 100644 --- a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/Infrastructure.kt +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/Infrastructure.kt @@ -1,6 +1,7 @@ package com.atlassian.performance.tools.infrastructure.api -import com.atlassian.performance.tools.infrastructure.api.jira.install.TcpHost +import com.atlassian.performance.tools.infrastructure.api.jira.install.HttpNode +import com.atlassian.performance.tools.infrastructure.api.jira.install.TcpNode import com.atlassian.performance.tools.ssh.api.Ssh interface Infrastructure : AutoCloseable { // TODO rename to ServerRoom @@ -8,9 +9,10 @@ interface Infrastructure : AutoCloseable { // TODO rename to ServerRoom val subnet: String /** - * @return can be reached by the caller via [TcpHost.publicIp] and by the rest of the infra via [TcpHost.privateIp] + * @return can be reached by the caller via [TcpNode.publicIp] and by the rest of the infra via [TcpNode.privateIp] */ - fun serveTcp(name: String): TcpHost - fun serve(name: String, tcpPorts: List, udpPorts: List): TcpHost + fun serveTcp(name: String): TcpNode + fun serveHttp(name: String): HttpNode + fun serve(name: String, tcpPorts: List, udpPorts: List): TcpNode fun serveSsh(name: String): Ssh } \ No newline at end of file diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/database/DockerMysqlServer.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/database/DockerMysqlServer.kt index c597d67a..a8f65e6c 100644 --- a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/database/DockerMysqlServer.kt +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/database/DockerMysqlServer.kt @@ -3,7 +3,7 @@ package com.atlassian.performance.tools.infrastructure.api.database import com.atlassian.performance.tools.infrastructure.api.Infrastructure import com.atlassian.performance.tools.infrastructure.api.dataset.DatasetPackage import com.atlassian.performance.tools.infrastructure.api.docker.DockerImage -import com.atlassian.performance.tools.infrastructure.api.jira.install.TcpHost +import com.atlassian.performance.tools.infrastructure.api.jira.install.TcpNode import com.atlassian.performance.tools.infrastructure.api.jira.install.hook.PreInstallHooks import com.atlassian.performance.tools.infrastructure.api.jira.instance.* import com.atlassian.performance.tools.infrastructure.api.jira.report.Reports @@ -39,7 +39,7 @@ class DockerMysqlServer private constructor( hooks.postInstance.insert(FixJiraUriViaMysql(server.ssh, client)) } - private fun setup(ssh: SshConnection, host: TcpHost): SshSqlClient { + private fun setup(ssh: SshConnection, host: TcpNode): SshSqlClient { val mysqlData = source.download(ssh) val port = host.port val container = dockerImage.run( diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/database/DockerPostgresServer.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/database/DockerPostgresServer.kt index bdf26660..7e5ce97d 100644 --- a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/database/DockerPostgresServer.kt +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/database/DockerPostgresServer.kt @@ -1,7 +1,7 @@ package com.atlassian.performance.tools.infrastructure.api.database import com.atlassian.performance.tools.infrastructure.api.docker.DockerImage -import com.atlassian.performance.tools.infrastructure.api.jira.install.TcpHost +import com.atlassian.performance.tools.infrastructure.api.jira.install.TcpNode import com.atlassian.performance.tools.infrastructure.api.jira.install.hook.PreInstallHooks import com.atlassian.performance.tools.infrastructure.api.jira.instance.PreInstanceHook import com.atlassian.performance.tools.infrastructure.api.jira.instance.PreInstanceHooks @@ -13,7 +13,7 @@ import java.time.Duration import java.util.function.Supplier class DockerPostgresServer private constructor( - private val hostSupplier: Supplier, + private val hostSupplier: Supplier, private val dockerImage: DockerImage, private val maxConnections: Int ) : PreInstanceHook { @@ -48,7 +48,7 @@ class DockerPostgresServer private constructor( } class Builder( - private var hostSupplier: Supplier + private var hostSupplier: Supplier ) { private var dockerImage = DockerImage.Builder("postgres:9.6.15") @@ -56,7 +56,7 @@ class DockerPostgresServer private constructor( .build() private var maxConnections: Int = 200 - fun serverSupplier(hostSupplier: Supplier) = apply { this.hostSupplier = hostSupplier } + fun serverSupplier(hostSupplier: Supplier) = apply { this.hostSupplier = hostSupplier } fun dockerImage(dockerImage: DockerImage) = apply { this.dockerImage = dockerImage } fun maxConnections(maxConnections: Int) = apply { this.maxConnections = maxConnections } diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/HttpHost.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/HttpNode.kt similarity index 91% rename from src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/HttpHost.kt rename to src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/HttpNode.kt index 429e51cc..d4636e21 100644 --- a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/HttpHost.kt +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/HttpNode.kt @@ -2,8 +2,8 @@ package com.atlassian.performance.tools.infrastructure.api.jira.install import java.net.URI -class HttpHost internal constructor( - val tcp: TcpHost, +class HttpNode internal constructor( + val tcp: TcpNode, private val basePath: String, private val supportsTls: Boolean ) { diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/InstalledJira.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/InstalledJira.kt index 4927bc38..92dd76c3 100644 --- a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/InstalledJira.kt +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/InstalledJira.kt @@ -24,5 +24,5 @@ class InstalledJira( /** * Connects to Jira on HTTP level or below. */ - val http: HttpHost + val http: HttpNode ) diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/JiraInstallation.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/JiraInstallation.kt index 299a6882..d34f1a76 100644 --- a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/JiraInstallation.kt +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/JiraInstallation.kt @@ -10,13 +10,13 @@ import net.jcip.annotations.ThreadSafe interface JiraInstallation { /** - * Installs Jira on [tcp]. + * Installs Jira on [http] node. * - * @param [tcp] will host the Jira + * @param [http] will host the Jira * @param [reports] accumulates reports */ fun install( - tcp: TcpHost, + http: HttpNode, reports: Reports ): InstalledJira } diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/ParallelInstallation.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/ParallelInstallation.kt index 7bb6a7dd..ec81eb86 100644 --- a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/ParallelInstallation.kt +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/ParallelInstallation.kt @@ -16,10 +16,10 @@ class ParallelInstallation( ) : JiraInstallation { override fun install( - tcp: TcpHost, + http: HttpNode, reports: Reports ): InstalledJira { - tcp.ssh.newConnection().use { ssh -> + http.tcp.ssh.newConnection().use { ssh -> val pool = Executors.newCachedThreadPool { runnable -> Thread(runnable, "jira-installation-${runnable.hashCode()}") } @@ -32,7 +32,8 @@ class ParallelInstallation( val java = pool.submitWithLogContext("java") { jdk.also { it.install(ssh) } } - val jira = InstalledJira(home.get(), product.get(), java.get(), HttpHost(tcp, "", false)) + TODO("${http.tcp.privateIp} is ignored and `InstalledJira.installation.resolve('server.xml')` still hardcodes ` + http.tcp.ssh.newConnection().use { ssh -> val installation = productDistribution.installRemotely(ssh, ".") val home = jiraHomeSource.downloadRemotely(ssh) jdk.install(ssh) - return InstalledJira(home, installation, jdk, HttpHost(tcp, "", false)) + return InstalledJira(home, installation, jdk, http) } } } diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/TcpHost.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/TcpNode.kt similarity index 96% rename from src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/TcpHost.kt rename to src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/TcpNode.kt index 5c06e3f8..a69e756a 100644 --- a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/TcpHost.kt +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/TcpNode.kt @@ -7,7 +7,7 @@ import com.atlassian.performance.tools.ssh.api.Ssh * @param [ssh] connects to the host via [publicIp] * @param [port] accepts connections at within the [privateIp] network */ -class TcpHost( +class TcpNode( val publicIp: String, val privateIp: String, val port: Int, diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/AsyncProfilerHook.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/AsyncProfilerHook.kt index ef86eafc..453ba963 100644 --- a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/AsyncProfilerHook.kt +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/AsyncProfilerHook.kt @@ -1,6 +1,6 @@ package com.atlassian.performance.tools.infrastructure.api.jira.install.hook -import com.atlassian.performance.tools.infrastructure.api.jira.install.TcpHost +import com.atlassian.performance.tools.infrastructure.api.jira.install.HttpNode import com.atlassian.performance.tools.infrastructure.api.jira.report.Report import com.atlassian.performance.tools.infrastructure.api.jira.report.Reports import com.atlassian.performance.tools.infrastructure.api.jira.start.StartedJira @@ -13,7 +13,7 @@ class AsyncProfilerHook : PreInstallHook { override fun call( ssh: SshConnection, - tcp: TcpHost, + http: HttpNode, hooks: PreInstallHooks, reports: Reports ) { diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/PreInstallHook.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/PreInstallHook.kt index ec5f329f..6998f149 100644 --- a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/PreInstallHook.kt +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/PreInstallHook.kt @@ -1,8 +1,9 @@ package com.atlassian.performance.tools.infrastructure.api.jira.install.hook -import com.atlassian.performance.tools.infrastructure.api.jira.install.TcpHost +import com.atlassian.performance.tools.infrastructure.api.jira.install.HttpNode import com.atlassian.performance.tools.infrastructure.api.jira.report.Reports import com.atlassian.performance.tools.ssh.api.SshConnection +import org.apache.http.HttpHost /** * Intercepts a call before Jira is installed. @@ -10,14 +11,14 @@ import com.atlassian.performance.tools.ssh.api.SshConnection interface PreInstallHook { /** - * @param [ssh] connects to the [tcp] - * @param [tcp] will install Jira + * @param [ssh] connects to the [http] host + * @param [http] will install Jira * @param [hooks] inserts future hooks * @param [reports] accumulates reports */ fun call( ssh: SshConnection, - tcp: TcpHost, + http: HttpNode, hooks: PreInstallHooks, reports: Reports ) diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/PreInstallHooks.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/PreInstallHooks.kt index 58e256fa..d748580f 100644 --- a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/PreInstallHooks.kt +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/PreInstallHooks.kt @@ -1,6 +1,6 @@ package com.atlassian.performance.tools.infrastructure.api.jira.install.hook -import com.atlassian.performance.tools.infrastructure.api.jira.install.TcpHost +import com.atlassian.performance.tools.infrastructure.api.jira.install.HttpNode import com.atlassian.performance.tools.infrastructure.api.jira.report.Reports import com.atlassian.performance.tools.ssh.api.SshConnection import java.util.* @@ -22,13 +22,13 @@ class PreInstallHooks private constructor( internal fun call( ssh: SshConnection, - host: TcpHost, + http: HttpNode, reports: Reports ) { while (true) { hooks .poll() - ?.call(ssh, host, this, reports) + ?.call(ssh, http, this, reports) ?: break } } diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/SystemLog.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/SystemLog.kt index 28084dfa..21904606 100644 --- a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/SystemLog.kt +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/SystemLog.kt @@ -1,13 +1,13 @@ package com.atlassian.performance.tools.infrastructure.api.jira.install.hook -import com.atlassian.performance.tools.infrastructure.api.jira.install.TcpHost +import com.atlassian.performance.tools.infrastructure.api.jira.install.HttpNode import com.atlassian.performance.tools.infrastructure.api.jira.report.FileListing import com.atlassian.performance.tools.infrastructure.api.jira.report.Reports import com.atlassian.performance.tools.ssh.api.SshConnection class SystemLog : PreInstallHook { - override fun call(ssh: SshConnection, tcp: TcpHost, hooks: PreInstallHooks, reports: Reports) { - reports.add(FileListing("/var/log/syslog"), tcp) + override fun call(ssh: SshConnection, http: HttpNode, hooks: PreInstallHooks, reports: Reports) { + reports.add(FileListing("/var/log/syslog"), http) } } diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/JiraDataCenterPlan.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/JiraDataCenterPlan.kt index 2e2057a2..0783d011 100644 --- a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/JiraDataCenterPlan.kt +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/JiraDataCenterPlan.kt @@ -1,10 +1,9 @@ package com.atlassian.performance.tools.infrastructure.api.jira.instance import com.atlassian.performance.tools.infrastructure.api.Infrastructure +import com.atlassian.performance.tools.infrastructure.api.jira.install.HttpNode import com.atlassian.performance.tools.infrastructure.api.jira.install.InstalledJira import com.atlassian.performance.tools.infrastructure.api.jira.install.hook.PreInstallHooks -import com.atlassian.performance.tools.infrastructure.api.jira.node.JiraNode -import com.atlassian.performance.tools.infrastructure.api.jira.node.JiraNodePlan import com.atlassian.performance.tools.infrastructure.api.jira.report.Reports import com.atlassian.performance.tools.infrastructure.api.jira.start.StartedJira import com.atlassian.performance.tools.infrastructure.api.loadbalancer.ApacheProxyPlan @@ -18,8 +17,7 @@ import kotlin.streams.toList class JiraDataCenterPlan private constructor( private val nodePlans: List, private val instanceHooks: PreInstanceHooks, - private val balancerPlan: LoadBalancerPlan, - private val infrastructure: Infrastructure + private val balancerPlan: LoadBalancerPlan ) : JiraInstancePlan { private val reports: Reports = Reports() @@ -27,13 +25,13 @@ class JiraDataCenterPlan private constructor( override fun materialize(): JiraInstance { instanceHooks.call(nodePlans.map { it.hooks }, reports) - val nodes = nodePlans.mapIndexed { nodeIndex, nodePlan -> + val http = nodePlans.mapIndexed { nodeIndex, plan -> val nodeNumber = nodeIndex + 1 - val host = infrastructure.serveTcp("jira-node-$nodeNumber") - nodePlan.materialize(host) + val http = plan.infrastructure.serveHttp("jira-node-$nodeNumber") + PlannedHttpNode(http, plan) } - val balancer = balancerPlan.materialize(nodes) - val installed = installInParallel(nodes) + val balancer = balancerPlan.materialize(http.map { it.http }, nodePlans.map { it.hooks.preStart }) + val installed = installInParallel(http) val started = installed.map { it.start(reports) } val instance = JiraDataCenter(started, balancer) instanceHooks.postInstance.call(instance, reports) @@ -43,25 +41,28 @@ class JiraDataCenterPlan private constructor( override fun report(): Reports = reports.copy() - private fun installInParallel(nodes: List) = nodes + private fun installInParallel(nodes: Collection): List = nodes .asSequence() .asStream() .parallel() - .map { jiraNode -> - jiraNode - .plan - .installation - .install(jiraNode.host, reports) - .let { installedJira -> InstalledJiraNode(installedJira, jiraNode.plan) } - } + .map { it.install(reports) } .toList() - private class InstalledJiraNode( - private val installedJira: InstalledJira, - private val nodePlan: JiraNodePlan + private class PlannedHttpNode( + val http: HttpNode, + val plan: JiraNodePlan + ) { + fun install(reports: Reports): PlannedInstalledJira { + return plan.installation.install(http, reports).let { PlannedInstalledJira(it, plan) } + } + } + + private class PlannedInstalledJira( + val installed: InstalledJira, + val plan: JiraNodePlan ) { fun start(reports: Reports): StartedJira { - return nodePlan.start.start(installedJira, reports) + return plan.start.start(installed, reports) } } @@ -74,21 +75,20 @@ class JiraDataCenterPlan private constructor( } class Builder( - private var infrastructure: Infrastructure + infrastructure: Infrastructure ) { private var nodePlans: List = listOf(1, 2).map { - JiraNodePlan.Builder() + JiraNodePlan.Builder(infrastructure) .hooks(PreInstallHooks.default().apply { postInstall.insert(DefaultClusterProperties()) }) .build() } private var instanceHooks: PreInstanceHooks = PreInstanceHooks.default() private var balancerPlan: LoadBalancerPlan = ApacheProxyPlan(infrastructure) - fun infrastructure(infrastructure: Infrastructure) = apply { this.infrastructure = infrastructure } fun nodePlans(nodePlans: List) = apply { this.nodePlans = nodePlans } fun instanceHooks(instanceHooks: PreInstanceHooks) = apply { this.instanceHooks = instanceHooks } fun balancerPlan(balancerPlan: LoadBalancerPlan) = apply { this.balancerPlan = balancerPlan } - fun build(): JiraInstancePlan = JiraDataCenterPlan(nodePlans, instanceHooks, balancerPlan, infrastructure) + fun build(): JiraInstancePlan = JiraDataCenterPlan(nodePlans, instanceHooks, balancerPlan) } } \ No newline at end of file diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/node/JiraNodePlan.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/JiraNodePlan.kt similarity index 70% rename from src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/node/JiraNodePlan.kt rename to src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/JiraNodePlan.kt index d4664efe..594e4696 100644 --- a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/node/JiraNodePlan.kt +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/JiraNodePlan.kt @@ -1,10 +1,10 @@ -package com.atlassian.performance.tools.infrastructure.api.jira.node +package com.atlassian.performance.tools.infrastructure.api.jira.instance +import com.atlassian.performance.tools.infrastructure.api.Infrastructure import com.atlassian.performance.tools.infrastructure.api.distribution.PublicJiraSoftwareDistribution import com.atlassian.performance.tools.infrastructure.api.jira.EmptyJiraHome import com.atlassian.performance.tools.infrastructure.api.jira.install.JiraInstallation import com.atlassian.performance.tools.infrastructure.api.jira.install.ParallelInstallation -import com.atlassian.performance.tools.infrastructure.api.jira.install.TcpHost import com.atlassian.performance.tools.infrastructure.api.jira.install.hook.PreInstallHooks import com.atlassian.performance.tools.infrastructure.api.jira.start.JiraLaunchScript import com.atlassian.performance.tools.infrastructure.api.jira.start.JiraStart @@ -13,16 +13,26 @@ import com.atlassian.performance.tools.infrastructure.jira.install.hook.HookedJi import com.atlassian.performance.tools.infrastructure.jira.start.hook.HookedJiraStart import net.jcip.annotations.NotThreadSafe +/** + * Specifies how a Jira node should be built. + * Does not build the node, due to various possible ordering, concurrency and optimizations. + * Actual provisioning is reserved for a [JiraInstancePlan]. + * + * @constructor specifies the plan, but doesn't hold any resources + * @see JiraInstancePlan + * @since 4.19.0 + */ class JiraNodePlan private constructor( - val installation: JiraInstallation, - val start: JiraStart, + internal val infrastructure: Infrastructure, + internal val installation: JiraInstallation, + internal val start: JiraStart, internal val hooks: PreInstallHooks ) { - fun materialize(host: TcpHost) = JiraNode(host, this) - @NotThreadSafe - class Builder { + class Builder( + private var infrastructure: Infrastructure + ) { private var installation: JiraInstallation = ParallelInstallation( EmptyJiraHome(), PublicJiraSoftwareDistribution("7.13.0"), @@ -31,11 +41,13 @@ class JiraNodePlan private constructor( private var start: JiraStart = JiraLaunchScript() private var hooks: PreInstallHooks = PreInstallHooks.default() + fun infrastructure(infrastructure: Infrastructure) = apply { this.infrastructure = infrastructure } fun installation(installation: JiraInstallation) = apply { this.installation = installation } fun start(start: JiraStart) = apply { this.start = start } fun hooks(hooks: PreInstallHooks) = apply { this.hooks = hooks } fun build() = JiraNodePlan( + infrastructure, HookedJiraInstallation(installation, hooks), HookedJiraStart(start, hooks.preStart), hooks diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/JiraServerPlan.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/JiraServerPlan.kt index d15eaa0a..1f819e6d 100644 --- a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/JiraServerPlan.kt +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/JiraServerPlan.kt @@ -1,14 +1,12 @@ package com.atlassian.performance.tools.infrastructure.api.jira.instance import com.atlassian.performance.tools.infrastructure.api.Infrastructure -import com.atlassian.performance.tools.infrastructure.api.jira.node.JiraNodePlan import com.atlassian.performance.tools.infrastructure.api.jira.report.Reports import com.atlassian.performance.tools.infrastructure.api.jira.start.StartedJira import java.net.URI class JiraServerPlan private constructor( private val plan: JiraNodePlan, - private val infrastructure: Infrastructure, private val hooks: PreInstanceHooks ) : JiraInstancePlan { @@ -17,8 +15,8 @@ class JiraServerPlan private constructor( override fun materialize(): JiraInstance { val nodeHooks = listOf(plan).map { it.hooks } hooks.call(nodeHooks, reports) - val jiraNode = infrastructure.serveTcp("jira-node") - val installed = plan.installation.install(jiraNode, reports) + val http = plan.infrastructure.serveHttp("jira-node") + val installed = plan.installation.install(http, reports) val started = plan.start.start(installed, reports) val instance = JiraServer(started) hooks.postInstance.call(instance, reports) @@ -35,15 +33,14 @@ class JiraServerPlan private constructor( } class Builder( - private var infrastructure: Infrastructure + infrastructure: Infrastructure ) { - private var plan: JiraNodePlan = JiraNodePlan.Builder().build() + private var plan: JiraNodePlan = JiraNodePlan.Builder(infrastructure).build() private var hooks: PreInstanceHooks = PreInstanceHooks.default() fun plan(plan: JiraNodePlan) = apply { this.plan = plan } - fun infrastructure(infrastructure: Infrastructure) = apply { this.infrastructure = infrastructure } fun hooks(hooks: PreInstanceHooks) = apply { this.hooks = hooks } - fun build(): JiraInstancePlan = JiraServerPlan(plan, infrastructure, hooks) + fun build(): JiraInstancePlan = JiraServerPlan(plan, hooks) } } \ No newline at end of file diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/node/JiraNode.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/node/JiraNode.kt deleted file mode 100644 index 3e0edb01..00000000 --- a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/node/JiraNode.kt +++ /dev/null @@ -1,8 +0,0 @@ -package com.atlassian.performance.tools.infrastructure.api.jira.node - -import com.atlassian.performance.tools.infrastructure.api.jira.install.TcpHost - -class JiraNode constructor( - val host: TcpHost, - val plan: JiraNodePlan -) \ No newline at end of file diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/report/Reports.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/report/Reports.kt index 76a7b1c4..4feca137 100644 --- a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/report/Reports.kt +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/report/Reports.kt @@ -1,7 +1,8 @@ package com.atlassian.performance.tools.infrastructure.api.jira.report +import com.atlassian.performance.tools.infrastructure.api.jira.install.HttpNode import com.atlassian.performance.tools.infrastructure.api.jira.install.InstalledJira -import com.atlassian.performance.tools.infrastructure.api.jira.install.TcpHost +import com.atlassian.performance.tools.infrastructure.api.jira.install.TcpNode import com.atlassian.performance.tools.infrastructure.api.jira.start.StartedJira import com.atlassian.performance.tools.infrastructure.api.os.RemotePath import com.atlassian.performance.tools.io.api.ensureDirectory @@ -22,10 +23,14 @@ class Reports private constructor( // TODO turn into SPI to allow AWS CLI transp } fun add(report: Report, installed: InstalledJira) { - add(report, installed.http.tcp) + add(report, installed.http) } - fun add(report: Report, tcp: TcpHost) { + fun add(report: Report, http: HttpNode) { + hostReports.add(HostReport(http.tcp, report)) + } + + fun add(report: Report, tcp: TcpNode) { hostReports.add(HostReport(tcp, report)) } @@ -60,7 +65,7 @@ class Reports private constructor( // TODO turn into SPI to allow AWS CLI transp } private class HostReport( - val host: TcpHost, + val host: TcpNode, val report: Report ) } diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/sharedhome/SambaSharedHome.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/sharedhome/SambaSharedHome.kt index f98a4757..7d364381 100644 --- a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/sharedhome/SambaSharedHome.kt +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/sharedhome/SambaSharedHome.kt @@ -3,7 +3,7 @@ package com.atlassian.performance.tools.infrastructure.api.jira.sharedhome import com.atlassian.performance.tools.infrastructure.api.Infrastructure import com.atlassian.performance.tools.infrastructure.api.jira.JiraHomeSource import com.atlassian.performance.tools.infrastructure.api.jira.install.InstalledJira -import com.atlassian.performance.tools.infrastructure.api.jira.install.TcpHost +import com.atlassian.performance.tools.infrastructure.api.jira.install.TcpNode import com.atlassian.performance.tools.infrastructure.api.jira.install.hook.PostInstallHook import com.atlassian.performance.tools.infrastructure.api.jira.install.hook.PostInstallHooks import com.atlassian.performance.tools.infrastructure.api.jira.install.hook.PreInstallHooks @@ -38,7 +38,7 @@ class SambaSharedHome( return sharedHome } - private fun export(ssh: SshConnection, sharedHome: String, server: TcpHost): SambaMount { + private fun export(ssh: SshConnection, sharedHome: String, server: TcpNode): SambaMount { Ubuntu().install(ssh, listOf("samba")) val shareName = "samba-jira-home" val share = """ diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/start/JiraStart.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/start/JiraStart.kt index 706e01e2..891e000c 100644 --- a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/start/JiraStart.kt +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/start/JiraStart.kt @@ -4,6 +4,9 @@ import com.atlassian.performance.tools.infrastructure.api.jira.install.Installed import com.atlassian.performance.tools.infrastructure.api.jira.report.Reports import net.jcip.annotations.ThreadSafe +/** + * @since 4.19.0 + */ @ThreadSafe interface JiraStart { diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/loadbalancer/ApacheProxyPlan.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/loadbalancer/ApacheProxyPlan.kt index d0edc217..5704ea7f 100644 --- a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/loadbalancer/ApacheProxyPlan.kt +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/loadbalancer/ApacheProxyPlan.kt @@ -2,9 +2,9 @@ package com.atlassian.performance.tools.infrastructure.api.loadbalancer import com.atlassian.performance.tools.infrastructure.api.Infrastructure import com.atlassian.performance.tools.infrastructure.api.Sed +import com.atlassian.performance.tools.infrastructure.api.jira.install.HttpNode import com.atlassian.performance.tools.infrastructure.api.jira.install.InstalledJira -import com.atlassian.performance.tools.infrastructure.api.jira.install.TcpHost -import com.atlassian.performance.tools.infrastructure.api.jira.node.JiraNode +import com.atlassian.performance.tools.infrastructure.api.jira.install.hook.PreInstallHooks import com.atlassian.performance.tools.infrastructure.api.jira.report.Reports import com.atlassian.performance.tools.infrastructure.api.jira.start.hook.PreStartHook import com.atlassian.performance.tools.infrastructure.api.jira.start.hook.PreStartHooks @@ -21,21 +21,21 @@ class ApacheProxyPlan( private val configPath = "/etc/apache2/sites-enabled/000-default.conf" - override fun materialize(nodes: List): LoadBalancer { - val proxyNode = infrastructure.serveTcp("apache-proxy") + override fun materialize(nodes: List, hooks: List): LoadBalancer { + val proxyNode = infrastructure.serveHttp("apache-proxy") IdempotentAction("Installing and configuring apache load balancer") { - proxyNode.ssh.newConnection().use { connection -> + proxyNode.tcp.ssh.newConnection().use { connection -> tryToProvision(connection, nodes, proxyNode) } }.retry(2, ExponentialBackoff(Duration.ofSeconds(5))) - val balancerEndpoint = URI("http://${proxyNode.privateIp}:${proxyNode.port}/") - nodes.forEach { it.plan.hooks.preStart.insert(InjectProxy(balancerEndpoint)) } + val balancerEndpoint = proxyNode.addressPrivately() + hooks.forEach { it.insert(InjectProxy(balancerEndpoint)) } return ApacheProxy(balancerEndpoint) } - private fun tryToProvision(ssh: SshConnection, nodes: List, proxyNode: TcpHost) { + private fun tryToProvision(ssh: SshConnection, nodes: List, proxyNode: HttpNode) { Ubuntu().install(ssh, listOf("apache2")) - Sed().replace(ssh, "Listen 80", "Listen ${proxyNode.port}", "/etc/apache2/ports.conf") + Sed().replace(ssh, "Listen 80", "Listen ${proxyNode.tcp.port}", "/etc/apache2/ports.conf") ssh.execute("sudo rm $configPath") ssh.execute("sudo touch $configPath") val mods = listOf( @@ -48,8 +48,8 @@ class ApacheProxyPlan( "Header add Set-Cookie \\\"ROUTEID=.%{BALANCER_WORKER_ROUTE}e; path=/\\\" env=BALANCER_ROUTE_CHANGED" ) appendConfig(ssh, "") - nodes.forEachIndexed { index, node -> - appendConfig(ssh, "\tBalancerMember http://${node.host.privateIp}:${node.host.port} route=$index") + nodes.forEachIndexed { index, http -> + appendConfig(ssh, "\tBalancerMember ${http.addressPrivately()} route=$index") } appendConfig(ssh, "\n") appendConfig(ssh, "ProxyPass / balancer://mycluster/ stickysession=ROUTEID") diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/loadbalancer/LoadBalancerPlan.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/loadbalancer/LoadBalancerPlan.kt index a433661b..c0d9cca6 100644 --- a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/loadbalancer/LoadBalancerPlan.kt +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/loadbalancer/LoadBalancerPlan.kt @@ -1,7 +1,8 @@ package com.atlassian.performance.tools.infrastructure.api.loadbalancer -import com.atlassian.performance.tools.infrastructure.api.jira.node.JiraNode +import com.atlassian.performance.tools.infrastructure.api.jira.install.HttpNode +import com.atlassian.performance.tools.infrastructure.api.jira.start.hook.PreStartHooks interface LoadBalancerPlan { - fun materialize(nodes: List): LoadBalancer + fun materialize(nodes: List, hooks: List): LoadBalancer } \ No newline at end of file diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/jira/install/hook/HookedJiraInstallation.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/jira/install/hook/HookedJiraInstallation.kt index ba8b8cc7..89c2ac7d 100644 --- a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/jira/install/hook/HookedJiraInstallation.kt +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/jira/install/hook/HookedJiraInstallation.kt @@ -1,8 +1,8 @@ package com.atlassian.performance.tools.infrastructure.jira.install.hook +import com.atlassian.performance.tools.infrastructure.api.jira.install.HttpNode import com.atlassian.performance.tools.infrastructure.api.jira.install.InstalledJira import com.atlassian.performance.tools.infrastructure.api.jira.install.JiraInstallation -import com.atlassian.performance.tools.infrastructure.api.jira.install.TcpHost import com.atlassian.performance.tools.infrastructure.api.jira.install.hook.PreInstallHooks import com.atlassian.performance.tools.infrastructure.api.jira.report.Reports @@ -12,14 +12,14 @@ class HookedJiraInstallation( ) : JiraInstallation { override fun install( - tcp: TcpHost, + http: HttpNode, reports: Reports ): InstalledJira { - tcp.ssh.newConnection().use { ssh -> - hooks.call(ssh, tcp, reports) + http.tcp.ssh.newConnection().use { ssh -> + hooks.call(ssh, http, reports) } - val installed = installation.install(tcp, reports) - tcp.ssh.newConnection().use { ssh -> + val installed = installation.install(http, reports) + http.tcp.ssh.newConnection().use { ssh -> hooks.postInstall.call(ssh, installed, reports) } return installed diff --git a/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/DockerInfrastructure.kt b/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/DockerInfrastructure.kt index 9f0ecf6f..11ad4cd3 100644 --- a/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/DockerInfrastructure.kt +++ b/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/DockerInfrastructure.kt @@ -1,6 +1,7 @@ package com.atlassian.performance.tools.infrastructure.api -import com.atlassian.performance.tools.infrastructure.api.jira.install.TcpHost +import com.atlassian.performance.tools.infrastructure.api.jira.install.HttpNode +import com.atlassian.performance.tools.infrastructure.api.jira.install.TcpNode import com.atlassian.performance.tools.infrastructure.lib.docker.CreatedContainer import com.atlassian.performance.tools.infrastructure.lib.docker.DockerNetwork import com.atlassian.performance.tools.infrastructure.lib.docker.StartedContainer @@ -55,7 +56,7 @@ internal class DockerInfrastructure : Infrastructure { } - override fun serveTcp(name: String): TcpHost { + override fun serveTcp(name: String): TcpNode { // TODO pre-provision all the hosts rather than on-demand - unlock batch provisioning (CFN Stack), picking EC2 types, SSD storage, TCP port ranges, subnets, etc. return when { name.startsWith("jira-node") -> serveTcp(8080, name) // TODO this is a contract on undocumented behavior @@ -65,12 +66,20 @@ internal class DockerInfrastructure : Infrastructure { } } - private fun serveTcp(tcpPort: Int, name: String): TcpHost { + override fun serveHttp(name: String): HttpNode { + return HttpNode( + serveTcp(80, name), + "/", + false + ) + } + + private fun serveTcp(tcpPort: Int, name: String): TcpNode { return serve(name, listOf(tcpPort), emptyList()) } - override fun serve(name: String, tcpPorts: List, udpPorts: List): TcpHost { + override fun serve(name: String, tcpPorts: List, udpPorts: List): TcpNode { val ports = tcpPorts.map { ExposedPort.tcp(it) } + udpPorts.map { ExposedPort.udp(it) } + ExposedPort.tcp(22) @@ -107,7 +116,7 @@ internal class DockerInfrastructure : Infrastructure { created: CreatedContainer, tcpPort: Int, name: String - ): TcpHost { + ): TcpNode { val startedContainer = docker .startContainerCmd(created.response.id) .execAsResource(docker) @@ -119,7 +128,7 @@ internal class DockerInfrastructure : Infrastructure { started: StartedContainer, tcpPort: Int, name: String - ): TcpHost { + ): TcpNode { val networkSettings = docker .inspectContainerCmd(started.id) .exec() @@ -141,7 +150,7 @@ internal class DockerInfrastructure : Infrastructure { it.execute("apt-get update", Duration.ofMinutes(2)) it.execute("apt-get -y install sudo gnupg screen") } - return TcpHost("localhost", ip, tcpPort, name, ssh) + return TcpNode("localhost", ip, tcpPort, name, ssh) } private fun getHostPort( diff --git a/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/PreInstallHooksTest.kt b/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/PreInstallHooksTest.kt index 2ddfb10c..b94cfca2 100644 --- a/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/PreInstallHooksTest.kt +++ b/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/PreInstallHooksTest.kt @@ -1,6 +1,7 @@ package com.atlassian.performance.tools.infrastructure.api.jira.install.hook -import com.atlassian.performance.tools.infrastructure.api.jira.install.TcpHost +import com.atlassian.performance.tools.infrastructure.api.jira.install.HttpNode +import com.atlassian.performance.tools.infrastructure.api.jira.install.TcpNode import com.atlassian.performance.tools.infrastructure.api.jira.report.Reports import com.atlassian.performance.tools.infrastructure.mock.UnimplementedSshConnection import com.atlassian.performance.tools.ssh.api.Ssh @@ -13,7 +14,11 @@ import java.nio.file.Paths class PreInstallHooksTest { private val dummySsh = Ssh(SshHost("localhost", "dummyUser", Paths.get("dummyKey"))) - private val dummyServer = TcpHost("dummyPublicIp", "dummyPrivateIp", 123, "fake-server", dummySsh) + private val dummyHttp = HttpNode( + TcpNode("dummyPublicIp", "dummyPrivateIp", 123, "fake-server", dummySsh), + "/", + false + ) @Test fun shouldInsertDuringListing() { @@ -24,7 +29,7 @@ class PreInstallHooksTest { insert(counter) } - hooks.call(UnimplementedSshConnection(), dummyServer, Reports()) + hooks.call(UnimplementedSshConnection(), dummyHttp, Reports()) assertThat(counter.count).isEqualTo(3) } @@ -38,7 +43,7 @@ class PreInstallHooksTest { insert(InsertingHook(counter)) } - hooks.call(UnimplementedSshConnection(), dummyServer, Reports()) + hooks.call(UnimplementedSshConnection(), dummyHttp, Reports()) assertThat(counter.count).isEqualTo(3) } @@ -48,7 +53,7 @@ private class CountingHook : PreInstallHook { var count = 0 - override fun call(ssh: SshConnection, tcp: TcpHost, hooks: PreInstallHooks, reports: Reports) { + override fun call(ssh: SshConnection, http: HttpNode, hooks: PreInstallHooks, reports: Reports) { count++ } } @@ -56,7 +61,7 @@ private class CountingHook : PreInstallHook { private class InsertingHook( private val hook: PreInstallHook ) : PreInstallHook { - override fun call(ssh: SshConnection, tcp: TcpHost, hooks: PreInstallHooks, reports: Reports) { + override fun call(ssh: SshConnection, http: HttpNode, hooks: PreInstallHooks, reports: Reports) { hooks.insert(hook) } } diff --git a/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/JiraDataCenterPlanIT.kt b/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/JiraDataCenterPlanIT.kt index 50e91343..8171c21b 100644 --- a/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/JiraDataCenterPlanIT.kt +++ b/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/JiraDataCenterPlanIT.kt @@ -7,7 +7,6 @@ import com.atlassian.performance.tools.infrastructure.api.distribution.PublicJir import com.atlassian.performance.tools.infrastructure.api.jira.JiraHomePackage import com.atlassian.performance.tools.infrastructure.api.jira.install.ParallelInstallation import com.atlassian.performance.tools.infrastructure.api.jira.install.hook.PreInstallHooks -import com.atlassian.performance.tools.infrastructure.api.jira.node.JiraNodePlan import com.atlassian.performance.tools.infrastructure.api.jira.report.Reports import com.atlassian.performance.tools.infrastructure.api.jira.sharedhome.SambaSharedHome import com.atlassian.performance.tools.infrastructure.api.jira.start.JiraLaunchScript @@ -46,7 +45,7 @@ class JiraDataCenterPlanIT { val nodePlans = listOf(1, 2).map { val nodeHooks = PreInstallHooks.default() .also { Datasets.JiraSevenDataset.hookMysql(it.postStart) } - JiraNodePlan.Builder() + JiraNodePlan.Builder(infrastructure) .installation( ParallelInstallation( jiraHomeSource = jiraHomeSource, @@ -98,7 +97,7 @@ class JiraDataCenterPlanIT { } val nodePlans = listOf(1, 2).map { - JiraNodePlan.Builder() + JiraNodePlan.Builder(infrastructure) .installation( ParallelInstallation( jiraHomeSource = JiraHomePackage(Datasets.JiraSevenDataset.jiraHome), diff --git a/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/JiraServerPlanIT.kt b/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/JiraServerPlanIT.kt index 3c944099..4cd9c275 100644 --- a/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/JiraServerPlanIT.kt +++ b/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/JiraServerPlanIT.kt @@ -7,7 +7,6 @@ import com.atlassian.performance.tools.infrastructure.api.distribution.PublicJir import com.atlassian.performance.tools.infrastructure.api.jira.JiraHomePackage import com.atlassian.performance.tools.infrastructure.api.jira.install.ParallelInstallation import com.atlassian.performance.tools.infrastructure.api.jira.install.hook.PreInstallHooks -import com.atlassian.performance.tools.infrastructure.api.jira.node.JiraNodePlan import com.atlassian.performance.tools.infrastructure.api.jira.start.JiraLaunchScript import com.atlassian.performance.tools.infrastructure.api.jvm.AdoptOpenJDK import com.atlassian.performance.tools.io.api.resolveSafely @@ -38,7 +37,7 @@ class JiraServerPlanIT { // given val hooks = PreInstallHooks.default() .also { Datasets.JiraSevenDataset.hookMysql(it.postStart) } - val nodePlan = JiraNodePlan.Builder() + val nodePlan = JiraNodePlan.Builder(infrastructure) .hooks(hooks) .installation( ParallelInstallation( @@ -61,11 +60,7 @@ class JiraServerPlanIT { val jiraServer = try { jiraServerPlan.materialize() } catch (e: Exception) { - val debugging = Paths.get("build/test-artifacts/") - .resolveSafely(javaClass.simpleName) - .resolveSafely(Instant.now().toString()) - jiraServerPlan.report().downloadTo(debugging) - throw Exception("Jira Server plan failed to materialize, debugging info available in $debugging", e) + debug(jiraServerPlan, e) } val reports = jiraServerPlan.report().downloadTo(Files.createTempDirectory("jira-server-plan-")) @@ -96,4 +91,20 @@ class JiraServerPlanIT { .`as`("GC logs from $fileTree") .isNotEmpty } + + private fun debug( + jiraServerPlan: JiraInstancePlan, + e: Exception + ) : Nothing { + val debugging = Paths.get("build/test-artifacts/") + .resolveSafely(javaClass.simpleName) + .resolveSafely(Instant.now().toString()) + try { + jiraServerPlan.report().downloadTo(debugging) + } catch (debuggingException: Exception) { + debuggingException.addSuppressed(e) + throw debuggingException + } + throw Exception("Jira Server plan failed to materialize, debugging info available in $debugging", e) + } } From 290dd1bfea2291cbf05e88f47610c15ba4528c72 Mon Sep 17 00:00:00 2001 From: Maciej Kwidzinski Date: Fri, 9 Jul 2021 13:20:24 +0200 Subject: [PATCH 46/52] Fix Tomcat config --- .../api/jira/install/ParallelInstallation.kt | 3 ++- .../jira/install/SequentialInstallation.kt | 5 +++- .../jira/install/TomcatConfig.kt | 26 +++++++++++++++++++ 3 files changed, 32 insertions(+), 2 deletions(-) create mode 100644 src/main/kotlin/com/atlassian/performance/tools/infrastructure/jira/install/TomcatConfig.kt diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/ParallelInstallation.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/ParallelInstallation.kt index ec81eb86..a6d9b29c 100644 --- a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/ParallelInstallation.kt +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/ParallelInstallation.kt @@ -7,6 +7,7 @@ import com.atlassian.performance.tools.infrastructure.api.jira.report.Reports import com.atlassian.performance.tools.infrastructure.api.jvm.JavaDevelopmentKit import com.atlassian.performance.tools.infrastructure.downloadRemotely import com.atlassian.performance.tools.infrastructure.installRemotely +import com.atlassian.performance.tools.infrastructure.jira.install.TomcatConfig import java.util.concurrent.Executors class ParallelInstallation( @@ -32,9 +33,9 @@ class ParallelInstallation( val java = pool.submitWithLogContext("java") { jdk.also { it.install(ssh) } } - TODO("${http.tcp.privateIp} is ignored and `InstalledJira.installation.resolve('server.xml')` still hardcodes ` Date: Fri, 23 Jul 2021 11:39:35 +0200 Subject: [PATCH 47/52] expose HttpNode --- .../tools/infrastructure/api/jira/install/HttpNode.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/HttpNode.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/HttpNode.kt index d4636e21..e7219f85 100644 --- a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/HttpNode.kt +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/HttpNode.kt @@ -2,7 +2,7 @@ package com.atlassian.performance.tools.infrastructure.api.jira.install import java.net.URI -class HttpNode internal constructor( +class HttpNode( val tcp: TcpNode, private val basePath: String, private val supportsTls: Boolean From ad8db21365f76fab3b87a5ffbbbad1fb22d12e32 Mon Sep 17 00:00:00 2001 From: Maciej Kwidzinski Date: Fri, 23 Jul 2021 14:22:32 +0200 Subject: [PATCH 48/52] Split infra into server rooms --- .../infrastructure/api/Infrastructure.kt | 18 ------------- .../api/database/DockerMysqlServer.kt | 12 ++++----- .../api/jira/instance/JiraDataCenterPlan.kt | 25 ++++++++++++------- .../api/jira/instance/JiraNodePlan.kt | 19 ++++++++++---- .../api/jira/instance/JiraServerPlan.kt | 12 ++++++--- .../api/jira/sharedhome/NfsSharedHome.kt | 8 +++--- .../api/jira/sharedhome/SambaSharedHome.kt | 6 ++--- .../api/loadbalancer/ApacheProxyPlan.kt | 7 +++--- .../api/network/HttpServerRoom.kt | 8 ++++++ .../infrastructure/api/network/Networked.kt | 6 +++++ .../api/network/SshServerRoom.kt | 7 ++++++ .../api/network/TcpServerRoom.kt | 12 +++++++++ .../tools/infrastructure/Datasets.kt | 6 ++--- .../api/DockerInfrastructure.kt | 14 +++++++---- .../api/jira/instance/JiraDataCenterPlanIT.kt | 3 +-- .../api/jira/instance/JiraServerPlanIT.kt | 3 +-- .../tools/infrastructure/api/os/UbuntuIT.kt | 3 +-- 17 files changed, 103 insertions(+), 66 deletions(-) delete mode 100644 src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/Infrastructure.kt create mode 100644 src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/network/HttpServerRoom.kt create mode 100644 src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/network/Networked.kt create mode 100644 src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/network/SshServerRoom.kt create mode 100644 src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/network/TcpServerRoom.kt diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/Infrastructure.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/Infrastructure.kt deleted file mode 100644 index 82b1fdfe..00000000 --- a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/Infrastructure.kt +++ /dev/null @@ -1,18 +0,0 @@ -package com.atlassian.performance.tools.infrastructure.api - -import com.atlassian.performance.tools.infrastructure.api.jira.install.HttpNode -import com.atlassian.performance.tools.infrastructure.api.jira.install.TcpNode -import com.atlassian.performance.tools.ssh.api.Ssh - -interface Infrastructure : AutoCloseable { // TODO rename to ServerRoom - - val subnet: String - - /** - * @return can be reached by the caller via [TcpNode.publicIp] and by the rest of the infra via [TcpNode.privateIp] - */ - fun serveTcp(name: String): TcpNode - fun serveHttp(name: String): HttpNode - fun serve(name: String, tcpPorts: List, udpPorts: List): TcpNode - fun serveSsh(name: String): Ssh -} \ No newline at end of file diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/database/DockerMysqlServer.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/database/DockerMysqlServer.kt index a8f65e6c..790f770e 100644 --- a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/database/DockerMysqlServer.kt +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/database/DockerMysqlServer.kt @@ -1,12 +1,12 @@ package com.atlassian.performance.tools.infrastructure.api.database -import com.atlassian.performance.tools.infrastructure.api.Infrastructure import com.atlassian.performance.tools.infrastructure.api.dataset.DatasetPackage import com.atlassian.performance.tools.infrastructure.api.docker.DockerImage import com.atlassian.performance.tools.infrastructure.api.jira.install.TcpNode import com.atlassian.performance.tools.infrastructure.api.jira.install.hook.PreInstallHooks import com.atlassian.performance.tools.infrastructure.api.jira.instance.* import com.atlassian.performance.tools.infrastructure.api.jira.report.Reports +import com.atlassian.performance.tools.infrastructure.api.network.TcpServerRoom import com.atlassian.performance.tools.infrastructure.api.os.Ubuntu import com.atlassian.performance.tools.infrastructure.database.SshMysqlClient import com.atlassian.performance.tools.infrastructure.database.SshSqlClient @@ -19,7 +19,7 @@ import java.time.Duration import java.time.Duration.ofSeconds class DockerMysqlServer private constructor( - private val infrastructure: Infrastructure, + private val serverRoom: TcpServerRoom, private val source: DatasetPackage, private val dockerImage: DockerImage, private val maxConnections: Int @@ -30,7 +30,7 @@ class DockerMysqlServer private constructor( hooks: PreInstanceHooks, reports: Reports ) { - val server = infrastructure.serveTcp("mysql") + val server = serverRoom.serveTcp("mysql") val client = server.ssh.newConnection().use { setup(it, server) } nodes.forEach { node -> node.postInstall.insert(DatabaseIpConfig(server.privateIp)) @@ -55,7 +55,7 @@ class DockerMysqlServer private constructor( } class Builder( - private var infrastructure: Infrastructure, + private var serverRoom: TcpServerRoom, private var source: DatasetPackage ) { @@ -64,13 +64,13 @@ class DockerMysqlServer private constructor( .build() private var maxConnections: Int = 151 - fun infrastructure(infrastructure: Infrastructure) = apply { this.infrastructure = infrastructure } + fun serverRoom(serverRoom: TcpServerRoom) = apply { this.serverRoom = serverRoom } fun source(source: DatasetPackage) = apply { this.source = source } fun dockerImage(dockerImage: DockerImage) = apply { this.dockerImage = dockerImage } fun maxConnections(maxConnections: Int) = apply { this.maxConnections = maxConnections } fun build(): DockerMysqlServer = DockerMysqlServer( - infrastructure, + serverRoom, source, dockerImage, maxConnections diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/JiraDataCenterPlan.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/JiraDataCenterPlan.kt index 0783d011..47710fa7 100644 --- a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/JiraDataCenterPlan.kt +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/JiraDataCenterPlan.kt @@ -1,6 +1,5 @@ package com.atlassian.performance.tools.infrastructure.api.jira.instance -import com.atlassian.performance.tools.infrastructure.api.Infrastructure import com.atlassian.performance.tools.infrastructure.api.jira.install.HttpNode import com.atlassian.performance.tools.infrastructure.api.jira.install.InstalledJira import com.atlassian.performance.tools.infrastructure.api.jira.install.hook.PreInstallHooks @@ -9,8 +8,10 @@ import com.atlassian.performance.tools.infrastructure.api.jira.start.StartedJira import com.atlassian.performance.tools.infrastructure.api.loadbalancer.ApacheProxyPlan import com.atlassian.performance.tools.infrastructure.api.loadbalancer.LoadBalancer import com.atlassian.performance.tools.infrastructure.api.loadbalancer.LoadBalancerPlan +import com.atlassian.performance.tools.infrastructure.api.network.HttpServerRoom import java.net.URI import java.time.Duration +import java.util.function.Consumer import kotlin.streams.asStream import kotlin.streams.toList @@ -27,7 +28,7 @@ class JiraDataCenterPlan private constructor( instanceHooks.call(nodePlans.map { it.hooks }, reports) val http = nodePlans.mapIndexed { nodeIndex, plan -> val nodeNumber = nodeIndex + 1 - val http = plan.infrastructure.serveHttp("jira-node-$nodeNumber") + val http = plan.serverRoom.serveHttp("jira-node-$nodeNumber") PlannedHttpNode(http, plan) } val balancer = balancerPlan.materialize(http.map { it.http }, nodePlans.map { it.hooks.preStart }) @@ -75,15 +76,21 @@ class JiraDataCenterPlan private constructor( } class Builder( - infrastructure: Infrastructure + private var nodePlans: List, + private var balancerPlan: LoadBalancerPlan ) { - private var nodePlans: List = listOf(1, 2).map { - JiraNodePlan.Builder(infrastructure) - .hooks(PreInstallHooks.default().apply { postInstall.insert(DefaultClusterProperties()) }) - .build() - } private var instanceHooks: PreInstanceHooks = PreInstanceHooks.default() - private var balancerPlan: LoadBalancerPlan = ApacheProxyPlan(infrastructure) + + constructor( + serverRoom: HttpServerRoom + ) : this( + nodePlans = listOf(1, 2).map { + JiraNodePlan.Builder(serverRoom) + .dataCenter() + .build() + }, + balancerPlan = ApacheProxyPlan(serverRoom) + ) fun nodePlans(nodePlans: List) = apply { this.nodePlans = nodePlans } fun instanceHooks(instanceHooks: PreInstanceHooks) = apply { this.instanceHooks = instanceHooks } diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/JiraNodePlan.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/JiraNodePlan.kt index 594e4696..922ddd28 100644 --- a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/JiraNodePlan.kt +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/JiraNodePlan.kt @@ -1,6 +1,5 @@ package com.atlassian.performance.tools.infrastructure.api.jira.instance -import com.atlassian.performance.tools.infrastructure.api.Infrastructure import com.atlassian.performance.tools.infrastructure.api.distribution.PublicJiraSoftwareDistribution import com.atlassian.performance.tools.infrastructure.api.jira.EmptyJiraHome import com.atlassian.performance.tools.infrastructure.api.jira.install.JiraInstallation @@ -9,6 +8,7 @@ import com.atlassian.performance.tools.infrastructure.api.jira.install.hook.PreI import com.atlassian.performance.tools.infrastructure.api.jira.start.JiraLaunchScript import com.atlassian.performance.tools.infrastructure.api.jira.start.JiraStart import com.atlassian.performance.tools.infrastructure.api.jvm.OracleJDK +import com.atlassian.performance.tools.infrastructure.api.network.HttpServerRoom import com.atlassian.performance.tools.infrastructure.jira.install.hook.HookedJiraInstallation import com.atlassian.performance.tools.infrastructure.jira.start.hook.HookedJiraStart import net.jcip.annotations.NotThreadSafe @@ -23,7 +23,7 @@ import net.jcip.annotations.NotThreadSafe * @since 4.19.0 */ class JiraNodePlan private constructor( - internal val infrastructure: Infrastructure, + internal val serverRoom: HttpServerRoom, internal val installation: JiraInstallation, internal val start: JiraStart, internal val hooks: PreInstallHooks @@ -31,7 +31,7 @@ class JiraNodePlan private constructor( @NotThreadSafe class Builder( - private var infrastructure: Infrastructure + private var serverRoom: HttpServerRoom ) { private var installation: JiraInstallation = ParallelInstallation( EmptyJiraHome(), @@ -41,13 +41,22 @@ class JiraNodePlan private constructor( private var start: JiraStart = JiraLaunchScript() private var hooks: PreInstallHooks = PreInstallHooks.default() - fun infrastructure(infrastructure: Infrastructure) = apply { this.infrastructure = infrastructure } + constructor(plan: JiraNodePlan) : this( + plan.serverRoom + ) { + this.installation = plan.installation + this.start = plan.start + this.hooks = plan.hooks + } + + fun serverRoom(serverRoom: HttpServerRoom) = apply { this.serverRoom = serverRoom } fun installation(installation: JiraInstallation) = apply { this.installation = installation } fun start(start: JiraStart) = apply { this.start = start } fun hooks(hooks: PreInstallHooks) = apply { this.hooks = hooks } + fun dataCenter() = hooks(PreInstallHooks.default().apply { postInstall.insert(DefaultClusterProperties()) }) fun build() = JiraNodePlan( - infrastructure, + serverRoom, HookedJiraInstallation(installation, hooks), HookedJiraStart(start, hooks.preStart), hooks diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/JiraServerPlan.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/JiraServerPlan.kt index 1f819e6d..284f160c 100644 --- a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/JiraServerPlan.kt +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/JiraServerPlan.kt @@ -1,8 +1,8 @@ package com.atlassian.performance.tools.infrastructure.api.jira.instance -import com.atlassian.performance.tools.infrastructure.api.Infrastructure import com.atlassian.performance.tools.infrastructure.api.jira.report.Reports import com.atlassian.performance.tools.infrastructure.api.jira.start.StartedJira +import com.atlassian.performance.tools.infrastructure.api.network.HttpServerRoom import java.net.URI class JiraServerPlan private constructor( @@ -15,7 +15,7 @@ class JiraServerPlan private constructor( override fun materialize(): JiraInstance { val nodeHooks = listOf(plan).map { it.hooks } hooks.call(nodeHooks, reports) - val http = plan.infrastructure.serveHttp("jira-node") + val http = plan.serverRoom.serveHttp("jira-node") val installed = plan.installation.install(http, reports) val started = plan.start.start(installed, reports) val instance = JiraServer(started) @@ -33,11 +33,15 @@ class JiraServerPlan private constructor( } class Builder( - infrastructure: Infrastructure + private var plan: JiraNodePlan + ) { - private var plan: JiraNodePlan = JiraNodePlan.Builder(infrastructure).build() private var hooks: PreInstanceHooks = PreInstanceHooks.default() + constructor(serverRoom: HttpServerRoom) : this( + plan = JiraNodePlan.Builder(serverRoom).build() + ) + fun plan(plan: JiraNodePlan) = apply { this.plan = plan } fun hooks(hooks: PreInstanceHooks) = apply { this.hooks = hooks } diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/sharedhome/NfsSharedHome.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/sharedhome/NfsSharedHome.kt index cded91ed..67cb611d 100644 --- a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/sharedhome/NfsSharedHome.kt +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/sharedhome/NfsSharedHome.kt @@ -1,6 +1,5 @@ package com.atlassian.performance.tools.infrastructure.api.jira.sharedhome -import com.atlassian.performance.tools.infrastructure.api.Infrastructure import com.atlassian.performance.tools.infrastructure.api.jira.JiraHomeSource import com.atlassian.performance.tools.infrastructure.api.jira.install.InstalledJira import com.atlassian.performance.tools.infrastructure.api.jira.install.hook.PostInstallHook @@ -9,6 +8,8 @@ import com.atlassian.performance.tools.infrastructure.api.jira.install.hook.PreI import com.atlassian.performance.tools.infrastructure.api.jira.instance.PreInstanceHook import com.atlassian.performance.tools.infrastructure.api.jira.instance.PreInstanceHooks import com.atlassian.performance.tools.infrastructure.api.jira.report.Reports +import com.atlassian.performance.tools.infrastructure.api.network.Networked +import com.atlassian.performance.tools.infrastructure.api.network.SshServerRoom import com.atlassian.performance.tools.infrastructure.api.os.RemotePath import com.atlassian.performance.tools.infrastructure.api.os.Ubuntu import com.atlassian.performance.tools.infrastructure.jira.sharedhome.SharedHomeProperty @@ -16,7 +17,8 @@ import com.atlassian.performance.tools.ssh.api.SshConnection class NfsSharedHome( private val jiraHomeSource: JiraHomeSource, - private val infrastructure: Infrastructure + private val infrastructure: SshServerRoom, + private val networked: Networked ) : PreInstanceHook { private val localHome = "/home/ubuntu/jira-shared-home" @@ -40,7 +42,7 @@ class NfsSharedHome( private fun export(ssh: SshConnection): SshConnection.SshResult { Ubuntu().install(ssh, listOf("nfs-kernel-server")) val options = "rw,sync,no_subtree_check,no_root_squash" - ssh.execute("sudo echo '$localHome ${infrastructure.subnet}($options)' | sudo tee -a /etc/exports") + ssh.execute("sudo echo '$localHome ${networked.subnetCidr}($options)' | sudo tee -a /etc/exports") return ssh.execute("sudo service nfs-kernel-server restart") } diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/sharedhome/SambaSharedHome.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/sharedhome/SambaSharedHome.kt index 7d364381..3d744054 100644 --- a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/sharedhome/SambaSharedHome.kt +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/sharedhome/SambaSharedHome.kt @@ -1,6 +1,5 @@ package com.atlassian.performance.tools.infrastructure.api.jira.sharedhome -import com.atlassian.performance.tools.infrastructure.api.Infrastructure import com.atlassian.performance.tools.infrastructure.api.jira.JiraHomeSource import com.atlassian.performance.tools.infrastructure.api.jira.install.InstalledJira import com.atlassian.performance.tools.infrastructure.api.jira.install.TcpNode @@ -10,6 +9,7 @@ import com.atlassian.performance.tools.infrastructure.api.jira.install.hook.PreI import com.atlassian.performance.tools.infrastructure.api.jira.instance.PreInstanceHook import com.atlassian.performance.tools.infrastructure.api.jira.instance.PreInstanceHooks import com.atlassian.performance.tools.infrastructure.api.jira.report.Reports +import com.atlassian.performance.tools.infrastructure.api.network.TcpServerRoom import com.atlassian.performance.tools.infrastructure.api.os.Ubuntu import com.atlassian.performance.tools.infrastructure.jira.sharedhome.SharedHomeProperty import com.atlassian.performance.tools.ssh.api.SshConnection @@ -17,11 +17,11 @@ import java.util.* class SambaSharedHome( private val jiraHomeSource: JiraHomeSource, - private val infrastructure: Infrastructure + private val serverRoom: TcpServerRoom ) : PreInstanceHook { override fun call(nodes: List, hooks: PreInstanceHooks, reports: Reports) { - val server = infrastructure.serve("samba-shared-home", listOf(139, 445), listOf(137, 138)) + val server = serverRoom.serveTcp("samba-shared-home", listOf(139, 445), listOf(137, 138)) val mount = server.ssh.newConnection().use { ssh -> val sharedHome = download(ssh) export(ssh, sharedHome, server) diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/loadbalancer/ApacheProxyPlan.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/loadbalancer/ApacheProxyPlan.kt index 5704ea7f..c3c498aa 100644 --- a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/loadbalancer/ApacheProxyPlan.kt +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/loadbalancer/ApacheProxyPlan.kt @@ -1,13 +1,12 @@ package com.atlassian.performance.tools.infrastructure.api.loadbalancer -import com.atlassian.performance.tools.infrastructure.api.Infrastructure import com.atlassian.performance.tools.infrastructure.api.Sed import com.atlassian.performance.tools.infrastructure.api.jira.install.HttpNode import com.atlassian.performance.tools.infrastructure.api.jira.install.InstalledJira -import com.atlassian.performance.tools.infrastructure.api.jira.install.hook.PreInstallHooks import com.atlassian.performance.tools.infrastructure.api.jira.report.Reports import com.atlassian.performance.tools.infrastructure.api.jira.start.hook.PreStartHook import com.atlassian.performance.tools.infrastructure.api.jira.start.hook.PreStartHooks +import com.atlassian.performance.tools.infrastructure.api.network.HttpServerRoom import com.atlassian.performance.tools.infrastructure.api.os.Ubuntu import com.atlassian.performance.tools.jvmtasks.api.ExponentialBackoff import com.atlassian.performance.tools.jvmtasks.api.IdempotentAction @@ -16,13 +15,13 @@ import java.net.URI import java.time.Duration class ApacheProxyPlan( - private val infrastructure: Infrastructure + private val serverRoom: HttpServerRoom ) : LoadBalancerPlan { private val configPath = "/etc/apache2/sites-enabled/000-default.conf" override fun materialize(nodes: List, hooks: List): LoadBalancer { - val proxyNode = infrastructure.serveHttp("apache-proxy") + val proxyNode = serverRoom.serveHttp("apache-proxy") IdempotentAction("Installing and configuring apache load balancer") { proxyNode.tcp.ssh.newConnection().use { connection -> tryToProvision(connection, nodes, proxyNode) diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/network/HttpServerRoom.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/network/HttpServerRoom.kt new file mode 100644 index 00000000..c6701304 --- /dev/null +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/network/HttpServerRoom.kt @@ -0,0 +1,8 @@ +package com.atlassian.performance.tools.infrastructure.api.network + +import com.atlassian.performance.tools.infrastructure.api.jira.install.HttpNode + +interface HttpServerRoom : AutoCloseable { + + fun serveHttp(name: String): HttpNode +} \ No newline at end of file diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/network/Networked.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/network/Networked.kt new file mode 100644 index 00000000..bc85cc71 --- /dev/null +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/network/Networked.kt @@ -0,0 +1,6 @@ +package com.atlassian.performance.tools.infrastructure.api.network + +interface Networked { + + val subnetCidr: String +} \ No newline at end of file diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/network/SshServerRoom.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/network/SshServerRoom.kt new file mode 100644 index 00000000..fcace766 --- /dev/null +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/network/SshServerRoom.kt @@ -0,0 +1,7 @@ +package com.atlassian.performance.tools.infrastructure.api.network + +import com.atlassian.performance.tools.ssh.api.Ssh + +interface SshServerRoom : AutoCloseable { + fun serveSsh(name: String): Ssh +} \ No newline at end of file diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/network/TcpServerRoom.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/network/TcpServerRoom.kt new file mode 100644 index 00000000..3ff31a6f --- /dev/null +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/network/TcpServerRoom.kt @@ -0,0 +1,12 @@ +package com.atlassian.performance.tools.infrastructure.api.network + +import com.atlassian.performance.tools.infrastructure.api.jira.install.TcpNode + +interface TcpServerRoom : AutoCloseable { + + /** + * @return reachable by the caller via [TcpNode.publicIp] and by the rest of the network via [TcpNode.privateIp] + */ + fun serveTcp(name: String): TcpNode + fun serveTcp(name: String, tcpPorts: List, udpPorts: List): TcpNode +} \ No newline at end of file diff --git a/src/test/kotlin/com/atlassian/performance/tools/infrastructure/Datasets.kt b/src/test/kotlin/com/atlassian/performance/tools/infrastructure/Datasets.kt index 0363e416..b4e78b39 100644 --- a/src/test/kotlin/com/atlassian/performance/tools/infrastructure/Datasets.kt +++ b/src/test/kotlin/com/atlassian/performance/tools/infrastructure/Datasets.kt @@ -1,12 +1,12 @@ package com.atlassian.performance.tools.infrastructure -import com.atlassian.performance.tools.infrastructure.api.Infrastructure import com.atlassian.performance.tools.infrastructure.api.database.DockerMysqlServer import com.atlassian.performance.tools.infrastructure.api.dataset.HttpDatasetPackage import com.atlassian.performance.tools.infrastructure.api.jira.JiraLaunchTimeouts import com.atlassian.performance.tools.infrastructure.api.jira.instance.PreInstanceHooks import com.atlassian.performance.tools.infrastructure.api.jira.start.hook.PostStartHooks import com.atlassian.performance.tools.infrastructure.api.jira.start.hook.RestUpgrade +import com.atlassian.performance.tools.infrastructure.api.network.TcpServerRoom import java.net.URI import java.time.Duration @@ -27,8 +27,8 @@ class Datasets { downloadTimeout = Duration.ofMinutes(6) ) - fun hookMysql(preInstanceHooks: PreInstanceHooks, infrastructure: Infrastructure) { - val mysqlServer = DockerMysqlServer.Builder(infrastructure, mysql).build() + fun hookMysql(preInstanceHooks: PreInstanceHooks, serverRoom: TcpServerRoom) { + val mysqlServer = DockerMysqlServer.Builder(serverRoom, mysql).build() preInstanceHooks.insert(mysqlServer) } diff --git a/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/DockerInfrastructure.kt b/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/DockerInfrastructure.kt index 11ad4cd3..06cba3d6 100644 --- a/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/DockerInfrastructure.kt +++ b/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/DockerInfrastructure.kt @@ -2,6 +2,10 @@ package com.atlassian.performance.tools.infrastructure.api import com.atlassian.performance.tools.infrastructure.api.jira.install.HttpNode import com.atlassian.performance.tools.infrastructure.api.jira.install.TcpNode +import com.atlassian.performance.tools.infrastructure.api.network.HttpServerRoom +import com.atlassian.performance.tools.infrastructure.api.network.Networked +import com.atlassian.performance.tools.infrastructure.api.network.SshServerRoom +import com.atlassian.performance.tools.infrastructure.api.network.TcpServerRoom import com.atlassian.performance.tools.infrastructure.lib.docker.CreatedContainer import com.atlassian.performance.tools.infrastructure.lib.docker.DockerNetwork import com.atlassian.performance.tools.infrastructure.lib.docker.StartedContainer @@ -20,12 +24,12 @@ import java.util.* import java.util.UUID.randomUUID import java.util.concurrent.ConcurrentLinkedDeque -internal class DockerInfrastructure : Infrastructure { +internal class DockerInfrastructure : SshServerRoom, TcpServerRoom, HttpServerRoom, Networked { private val allocatedResources: Deque = ConcurrentLinkedDeque() private val docker: DockerClient private val network: DockerNetwork - override val subnet: String + override val subnetCidr: String init { val dockerConfig = DefaultDockerClientConfig.createDefaultConfigBuilder().build() @@ -37,7 +41,7 @@ internal class DockerInfrastructure : Infrastructure { .withName(randomUUID().toString()) .execAsResource(docker) allocatedResources.add(network) - subnet = docker + subnetCidr = docker .inspectNetworkCmd() .withNetworkId(network.response.id) .exec() @@ -75,11 +79,11 @@ internal class DockerInfrastructure : Infrastructure { } private fun serveTcp(tcpPort: Int, name: String): TcpNode { - return serve(name, listOf(tcpPort), emptyList()) + return serveTcp(name, listOf(tcpPort), emptyList()) } - override fun serve(name: String, tcpPorts: List, udpPorts: List): TcpNode { + override fun serveTcp(name: String, tcpPorts: List, udpPorts: List): TcpNode { val ports = tcpPorts.map { ExposedPort.tcp(it) } + udpPorts.map { ExposedPort.udp(it) } + ExposedPort.tcp(22) diff --git a/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/JiraDataCenterPlanIT.kt b/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/JiraDataCenterPlanIT.kt index 8171c21b..104e61fa 100644 --- a/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/JiraDataCenterPlanIT.kt +++ b/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/JiraDataCenterPlanIT.kt @@ -2,7 +2,6 @@ package com.atlassian.performance.tools.infrastructure.api.jira.instance import com.atlassian.performance.tools.infrastructure.Datasets import com.atlassian.performance.tools.infrastructure.api.DockerInfrastructure -import com.atlassian.performance.tools.infrastructure.api.Infrastructure import com.atlassian.performance.tools.infrastructure.api.distribution.PublicJiraSoftwareDistribution import com.atlassian.performance.tools.infrastructure.api.jira.JiraHomePackage import com.atlassian.performance.tools.infrastructure.api.jira.install.ParallelInstallation @@ -26,7 +25,7 @@ import java.time.Duration.ofMinutes class JiraDataCenterPlanIT { - private lateinit var infrastructure: Infrastructure + private lateinit var infrastructure: DockerInfrastructure @Before fun setUp() { diff --git a/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/JiraServerPlanIT.kt b/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/JiraServerPlanIT.kt index 4cd9c275..84fb6c33 100644 --- a/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/JiraServerPlanIT.kt +++ b/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/JiraServerPlanIT.kt @@ -2,7 +2,6 @@ package com.atlassian.performance.tools.infrastructure.api.jira.instance import com.atlassian.performance.tools.infrastructure.Datasets import com.atlassian.performance.tools.infrastructure.api.DockerInfrastructure -import com.atlassian.performance.tools.infrastructure.api.Infrastructure import com.atlassian.performance.tools.infrastructure.api.distribution.PublicJiraSoftwareDistribution import com.atlassian.performance.tools.infrastructure.api.jira.JiraHomePackage import com.atlassian.performance.tools.infrastructure.api.jira.install.ParallelInstallation @@ -20,7 +19,7 @@ import java.time.Instant class JiraServerPlanIT { - private lateinit var infrastructure: Infrastructure + private lateinit var infrastructure: DockerInfrastructure @Before fun setUp() { diff --git a/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/os/UbuntuIT.kt b/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/os/UbuntuIT.kt index 78784c32..aa05264b 100644 --- a/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/os/UbuntuIT.kt +++ b/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/os/UbuntuIT.kt @@ -1,7 +1,6 @@ package com.atlassian.performance.tools.infrastructure.api.os import com.atlassian.performance.tools.infrastructure.api.DockerInfrastructure -import com.atlassian.performance.tools.infrastructure.api.Infrastructure import com.atlassian.performance.tools.ssh.api.Ssh import com.atlassian.performance.tools.ssh.api.SshConnection import com.atlassian.performance.tools.ssh.api.SshHost @@ -17,7 +16,7 @@ import java.util.concurrent.TimeUnit class UbuntuIT { private lateinit var executor: ExecutorService - private lateinit var infra: Infrastructure + private lateinit var infra: DockerInfrastructure private lateinit var sshUbuntu: Ssh @Before From 79b89cfb841dec34ce66e31b03ba6dbd477d2084 Mon Sep 17 00:00:00 2001 From: Maciej Kwidzinski Date: Fri, 23 Jul 2021 14:38:07 +0200 Subject: [PATCH 49/52] Server rooms dont need to be closeable --- .../infrastructure/api/jira/sharedhome/NfsSharedHome.kt | 2 +- .../tools/infrastructure/api/network/HttpServerRoom.kt | 2 +- .../tools/infrastructure/api/network/Networked.kt | 5 ++++- .../tools/infrastructure/api/network/SshServerRoom.kt | 2 +- .../tools/infrastructure/api/network/TcpServerRoom.kt | 2 +- .../tools/infrastructure/api/DockerInfrastructure.kt | 2 +- 6 files changed, 9 insertions(+), 6 deletions(-) diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/sharedhome/NfsSharedHome.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/sharedhome/NfsSharedHome.kt index 67cb611d..eaff9ec1 100644 --- a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/sharedhome/NfsSharedHome.kt +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/sharedhome/NfsSharedHome.kt @@ -42,7 +42,7 @@ class NfsSharedHome( private fun export(ssh: SshConnection): SshConnection.SshResult { Ubuntu().install(ssh, listOf("nfs-kernel-server")) val options = "rw,sync,no_subtree_check,no_root_squash" - ssh.execute("sudo echo '$localHome ${networked.subnetCidr}($options)' | sudo tee -a /etc/exports") + ssh.execute("sudo echo '$localHome ${networked.subnet()}($options)' | sudo tee -a /etc/exports") return ssh.execute("sudo service nfs-kernel-server restart") } diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/network/HttpServerRoom.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/network/HttpServerRoom.kt index c6701304..89316ba6 100644 --- a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/network/HttpServerRoom.kt +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/network/HttpServerRoom.kt @@ -2,7 +2,7 @@ package com.atlassian.performance.tools.infrastructure.api.network import com.atlassian.performance.tools.infrastructure.api.jira.install.HttpNode -interface HttpServerRoom : AutoCloseable { +interface HttpServerRoom { fun serveHttp(name: String): HttpNode } \ No newline at end of file diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/network/Networked.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/network/Networked.kt index bc85cc71..27a9ee33 100644 --- a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/network/Networked.kt +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/network/Networked.kt @@ -2,5 +2,8 @@ package com.atlassian.performance.tools.infrastructure.api.network interface Networked { - val subnetCidr: String + /** + * @return CIDR + */ + fun subnet(): String } \ No newline at end of file diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/network/SshServerRoom.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/network/SshServerRoom.kt index fcace766..90ae5632 100644 --- a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/network/SshServerRoom.kt +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/network/SshServerRoom.kt @@ -2,6 +2,6 @@ package com.atlassian.performance.tools.infrastructure.api.network import com.atlassian.performance.tools.ssh.api.Ssh -interface SshServerRoom : AutoCloseable { +interface SshServerRoom { fun serveSsh(name: String): Ssh } \ No newline at end of file diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/network/TcpServerRoom.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/network/TcpServerRoom.kt index 3ff31a6f..a8c786e6 100644 --- a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/network/TcpServerRoom.kt +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/network/TcpServerRoom.kt @@ -2,7 +2,7 @@ package com.atlassian.performance.tools.infrastructure.api.network import com.atlassian.performance.tools.infrastructure.api.jira.install.TcpNode -interface TcpServerRoom : AutoCloseable { +interface TcpServerRoom { /** * @return reachable by the caller via [TcpNode.publicIp] and by the rest of the network via [TcpNode.privateIp] diff --git a/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/DockerInfrastructure.kt b/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/DockerInfrastructure.kt index 06cba3d6..d1dd5f1f 100644 --- a/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/DockerInfrastructure.kt +++ b/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/DockerInfrastructure.kt @@ -24,7 +24,7 @@ import java.util.* import java.util.UUID.randomUUID import java.util.concurrent.ConcurrentLinkedDeque -internal class DockerInfrastructure : SshServerRoom, TcpServerRoom, HttpServerRoom, Networked { +internal class DockerInfrastructure : SshServerRoom, TcpServerRoom, HttpServerRoom, Networked, AutoCloseable { private val allocatedResources: Deque = ConcurrentLinkedDeque() private val docker: DockerClient From f98570edd6d6d7c20fc2de14a2c9235f07f360b6 Mon Sep 17 00:00:00 2001 From: Maciej Kwidzinski Date: Fri, 30 Jul 2021 15:41:00 +0200 Subject: [PATCH 50/52] Mount TCP shared home in private network --- .../api/jira/sharedhome/NfsSharedHome.kt | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/sharedhome/NfsSharedHome.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/sharedhome/NfsSharedHome.kt index eaff9ec1..66c80bff 100644 --- a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/sharedhome/NfsSharedHome.kt +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/sharedhome/NfsSharedHome.kt @@ -2,6 +2,7 @@ package com.atlassian.performance.tools.infrastructure.api.jira.sharedhome import com.atlassian.performance.tools.infrastructure.api.jira.JiraHomeSource import com.atlassian.performance.tools.infrastructure.api.jira.install.InstalledJira +import com.atlassian.performance.tools.infrastructure.api.jira.install.TcpNode import com.atlassian.performance.tools.infrastructure.api.jira.install.hook.PostInstallHook import com.atlassian.performance.tools.infrastructure.api.jira.install.hook.PostInstallHooks import com.atlassian.performance.tools.infrastructure.api.jira.install.hook.PreInstallHooks @@ -9,27 +10,25 @@ import com.atlassian.performance.tools.infrastructure.api.jira.instance.PreInsta import com.atlassian.performance.tools.infrastructure.api.jira.instance.PreInstanceHooks import com.atlassian.performance.tools.infrastructure.api.jira.report.Reports import com.atlassian.performance.tools.infrastructure.api.network.Networked -import com.atlassian.performance.tools.infrastructure.api.network.SshServerRoom -import com.atlassian.performance.tools.infrastructure.api.os.RemotePath +import com.atlassian.performance.tools.infrastructure.api.network.TcpServerRoom import com.atlassian.performance.tools.infrastructure.api.os.Ubuntu import com.atlassian.performance.tools.infrastructure.jira.sharedhome.SharedHomeProperty import com.atlassian.performance.tools.ssh.api.SshConnection class NfsSharedHome( private val jiraHomeSource: JiraHomeSource, - private val infrastructure: SshServerRoom, + private val infrastructure: TcpServerRoom, private val networked: Networked ) : PreInstanceHook { private val localHome = "/home/ubuntu/jira-shared-home" override fun call(nodes: List, hooks: PreInstanceHooks, reports: Reports) { - val server = infrastructure.serveSsh("shared-home") - server.newConnection().use { ssh -> + val tcp = infrastructure.serveTcp("shared-home") + tcp.ssh.newConnection().use { ssh -> download(ssh) export(ssh) } - val sharedHome = RemotePath(server.host, localHome) - nodes.forEach { it.postInstall.insert(NfsMount(sharedHome)) } + nodes.forEach { it.postInstall.insert(NfsMount(tcp, localHome)) } } private fun download(ssh: SshConnection) { @@ -47,12 +46,13 @@ class NfsSharedHome( } private class NfsMount( - private val sharedHome: RemotePath + private val tcp: TcpNode, + private val sharedHome: String ) : PostInstallHook { override fun call(ssh: SshConnection, jira: InstalledJira, hooks: PostInstallHooks, reports: Reports) { Ubuntu().install(ssh, listOf("nfs-common")) - val mountSource = "${sharedHome.host.ipAddress}:${sharedHome.path}" + val mountSource = "${tcp.privateIp}:$sharedHome" val mountTarget = "mounted-shared-home" ssh.execute("mkdir -p $mountTarget") ssh.execute("sudo mount -o soft,intr,rsize=8192,wsize=8192 $mountSource $mountTarget") From b6a4576d5059c323ad71ba55f9cb5a6151e097e3 Mon Sep 17 00:00:00 2001 From: Maciej Kwidzinski Date: Fri, 30 Jul 2021 16:50:14 +0200 Subject: [PATCH 51/52] Call pre-instance hooks in parallel --- .../infrastructure/api/database/DockerMysqlServer.kt | 4 ++-- .../infrastructure/api/database/DockerPostgresServer.kt | 3 ++- .../infrastructure/api/jira/instance/PreInstanceHook.kt | 2 +- .../infrastructure/api/jira/instance/PreInstanceHooks.kt | 9 +++------ .../infrastructure/api/jira/sharedhome/NfsSharedHome.kt | 4 ++-- .../api/jira/sharedhome/SambaSharedHome.kt | 3 ++- .../tools/infrastructure/api/DockerInfrastructure.kt | 4 +++- 7 files changed, 15 insertions(+), 14 deletions(-) diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/database/DockerMysqlServer.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/database/DockerMysqlServer.kt index 790f770e..78fa1f6e 100644 --- a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/database/DockerMysqlServer.kt +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/database/DockerMysqlServer.kt @@ -27,7 +27,7 @@ class DockerMysqlServer private constructor( override fun call( nodes: List, - hooks: PreInstanceHooks, + hooks: PostInstanceHooks, reports: Reports ) { val server = serverRoom.serveTcp("mysql") @@ -36,7 +36,7 @@ class DockerMysqlServer private constructor( node.postInstall.insert(DatabaseIpConfig(server.privateIp)) node.postInstall.insert(MysqlConnector()) } - hooks.postInstance.insert(FixJiraUriViaMysql(server.ssh, client)) + hooks.insert(FixJiraUriViaMysql(server.ssh, client)) } private fun setup(ssh: SshConnection, host: TcpNode): SshSqlClient { diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/database/DockerPostgresServer.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/database/DockerPostgresServer.kt index 7e5ce97d..8b38eab8 100644 --- a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/database/DockerPostgresServer.kt +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/database/DockerPostgresServer.kt @@ -3,6 +3,7 @@ package com.atlassian.performance.tools.infrastructure.api.database import com.atlassian.performance.tools.infrastructure.api.docker.DockerImage import com.atlassian.performance.tools.infrastructure.api.jira.install.TcpNode import com.atlassian.performance.tools.infrastructure.api.jira.install.hook.PreInstallHooks +import com.atlassian.performance.tools.infrastructure.api.jira.instance.PostInstanceHooks import com.atlassian.performance.tools.infrastructure.api.jira.instance.PreInstanceHook import com.atlassian.performance.tools.infrastructure.api.jira.instance.PreInstanceHooks import com.atlassian.performance.tools.infrastructure.api.jira.report.Reports @@ -20,7 +21,7 @@ class DockerPostgresServer private constructor( private val logger: Logger = LogManager.getLogger(this::class.java) - override fun call(nodes: List, hooks: PreInstanceHooks, reports: Reports) { + override fun call(nodes: List, hooks: PostInstanceHooks, reports: Reports) { val server = hostSupplier.get() server.ssh.newConnection().use { setup(it) } nodes.forEach { node -> diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/PreInstanceHook.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/PreInstanceHook.kt index be674f40..10de8f1e 100644 --- a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/PreInstanceHook.kt +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/PreInstanceHook.kt @@ -12,7 +12,7 @@ interface PreInstanceHook { */ fun call( nodes: List, - hooks: PreInstanceHooks, + hooks: PostInstanceHooks, reports: Reports ) } diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/PreInstanceHooks.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/PreInstanceHooks.kt index 5643ca27..f732f6df 100644 --- a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/PreInstanceHooks.kt +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/PreInstanceHooks.kt @@ -16,12 +16,9 @@ class PreInstanceHooks private constructor( } internal fun call(nodes: List, reports: Reports) { - while (true) { - hooks - .poll() - ?.call(nodes, this, reports) - ?: break - } + hooks + .parallelStream() + .forEach { it.call(nodes, postInstance, reports) } } companion object Factory { diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/sharedhome/NfsSharedHome.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/sharedhome/NfsSharedHome.kt index 66c80bff..99f7072e 100644 --- a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/sharedhome/NfsSharedHome.kt +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/sharedhome/NfsSharedHome.kt @@ -6,8 +6,8 @@ import com.atlassian.performance.tools.infrastructure.api.jira.install.TcpNode import com.atlassian.performance.tools.infrastructure.api.jira.install.hook.PostInstallHook import com.atlassian.performance.tools.infrastructure.api.jira.install.hook.PostInstallHooks import com.atlassian.performance.tools.infrastructure.api.jira.install.hook.PreInstallHooks +import com.atlassian.performance.tools.infrastructure.api.jira.instance.PostInstanceHooks import com.atlassian.performance.tools.infrastructure.api.jira.instance.PreInstanceHook -import com.atlassian.performance.tools.infrastructure.api.jira.instance.PreInstanceHooks import com.atlassian.performance.tools.infrastructure.api.jira.report.Reports import com.atlassian.performance.tools.infrastructure.api.network.Networked import com.atlassian.performance.tools.infrastructure.api.network.TcpServerRoom @@ -22,7 +22,7 @@ class NfsSharedHome( ) : PreInstanceHook { private val localHome = "/home/ubuntu/jira-shared-home" - override fun call(nodes: List, hooks: PreInstanceHooks, reports: Reports) { + override fun call(nodes: List, hooks: PostInstanceHooks, reports: Reports) { val tcp = infrastructure.serveTcp("shared-home") tcp.ssh.newConnection().use { ssh -> download(ssh) diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/sharedhome/SambaSharedHome.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/sharedhome/SambaSharedHome.kt index 3d744054..da4d8c29 100644 --- a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/sharedhome/SambaSharedHome.kt +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/sharedhome/SambaSharedHome.kt @@ -6,6 +6,7 @@ import com.atlassian.performance.tools.infrastructure.api.jira.install.TcpNode import com.atlassian.performance.tools.infrastructure.api.jira.install.hook.PostInstallHook import com.atlassian.performance.tools.infrastructure.api.jira.install.hook.PostInstallHooks import com.atlassian.performance.tools.infrastructure.api.jira.install.hook.PreInstallHooks +import com.atlassian.performance.tools.infrastructure.api.jira.instance.PostInstanceHooks import com.atlassian.performance.tools.infrastructure.api.jira.instance.PreInstanceHook import com.atlassian.performance.tools.infrastructure.api.jira.instance.PreInstanceHooks import com.atlassian.performance.tools.infrastructure.api.jira.report.Reports @@ -20,7 +21,7 @@ class SambaSharedHome( private val serverRoom: TcpServerRoom ) : PreInstanceHook { - override fun call(nodes: List, hooks: PreInstanceHooks, reports: Reports) { + override fun call(nodes: List, hooks: PostInstanceHooks, reports: Reports) { val server = serverRoom.serveTcp("samba-shared-home", listOf(139, 445), listOf(137, 138)) val mount = server.ssh.newConnection().use { ssh -> val sharedHome = download(ssh) diff --git a/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/DockerInfrastructure.kt b/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/DockerInfrastructure.kt index d1dd5f1f..eb64a04b 100644 --- a/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/DockerInfrastructure.kt +++ b/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/DockerInfrastructure.kt @@ -29,7 +29,7 @@ internal class DockerInfrastructure : SshServerRoom, TcpServerRoom, HttpServerRo private val allocatedResources: Deque = ConcurrentLinkedDeque() private val docker: DockerClient private val network: DockerNetwork - override val subnetCidr: String + private val subnetCidr: String init { val dockerConfig = DefaultDockerClientConfig.createDefaultConfigBuilder().build() @@ -177,4 +177,6 @@ internal class DockerInfrastructure : SshServerRoom, TcpServerRoom, HttpServerRo ?: break } } + + override fun subnet(): String = subnetCidr } \ No newline at end of file From ce8c26cd471ffe981bfb791be7fc218acf948e51 Mon Sep 17 00:00:00 2001 From: Maciej Kwidzinski Date: Fri, 6 Aug 2021 12:14:07 +0200 Subject: [PATCH 52/52] Stop trying to fix relative remote paths --- .../performance/tools/infrastructure/api/jira/report/Reports.kt | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/report/Reports.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/report/Reports.kt index 4feca137..b9737f42 100644 --- a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/report/Reports.kt +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/report/Reports.kt @@ -40,7 +40,6 @@ class Reports private constructor( // TODO turn into SPI to allow AWS CLI transp localDirectory.ensureDirectory() hostReports.groupBy { it.host }.map { (host, reports) -> host.ssh.newConnection().use { ssh -> - val remoteBase = RemotePath(ssh.getHost(), ssh.execute("pwd").output.trim()) reports .flatMap { report -> report.report.locate(ssh).map { path -> RemotePath(host.ssh.host, path) } @@ -48,7 +47,6 @@ class Reports private constructor( // TODO turn into SPI to allow AWS CLI transp .forEach { remotePath -> localDirectory .resolveSafely(host.name) - .resolve(remoteBase.toLocalRelativePath()) .resolve(remotePath.toLocalRelativePath()) .normalize() .let { remotePath.download(it) }