From a052757de27108e238a0dd11535bca157ab2f0cc Mon Sep 17 00:00:00 2001 From: Maciej Kwidzinski Date: Fri, 11 Jun 2021 13:47:04 +0200 Subject: [PATCH] Diagnose `RestUpgrade` failures --- .github/workflows/ci.yml | 6 + .../api/jira/install/hook/JiraLogs.kt | 32 ++--- .../api/jira/start/hook/RestUpgrade.kt | 113 ++++++++++-------- .../api/jira/instance/JiraServerPlanIT.kt | 14 ++- 4 files changed, 99 insertions(+), 66 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5414ff6a..c34c1c2d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -34,6 +34,12 @@ jobs: with: name: test-reports path: build/reports/tests + - name: Upload test artifacts + if: always() + uses: actions/upload-artifact@v2 + with: + name: test-artifacts + path: build/test-artifacts - name: Release if: github.event.inputs.release == 'yes' env: diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/JiraLogs.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/JiraLogs.kt index 683f5389..adfc0033 100644 --- a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/JiraLogs.kt +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/JiraLogs.kt @@ -1,8 +1,8 @@ package com.atlassian.performance.tools.infrastructure.api.jira.install.hook import com.atlassian.performance.tools.infrastructure.api.jira.install.InstalledJira +import com.atlassian.performance.tools.infrastructure.api.jira.report.Report import com.atlassian.performance.tools.infrastructure.api.jira.report.Reports -import com.atlassian.performance.tools.infrastructure.api.jira.report.StaticReport import com.atlassian.performance.tools.ssh.api.SshConnection import java.nio.file.Path import java.nio.file.Paths @@ -10,20 +10,24 @@ import java.nio.file.Paths class JiraLogs : PostInstallHook { override fun call(ssh: SshConnection, jira: InstalledJira, hooks: PostInstallHooks, reports: Reports) { - listOf( - "${jira.home.path}/log/atlassian-jira.log", - "${jira.installation.path}/logs/catalina.out" - ) - .onEach { ensureFile(Paths.get(it), ssh) } - .map { StaticReport(it) } - .forEach { reports.add(it, jira.host) } + reports.add(report(jira), jira.host) } - private fun ensureFile( - path: Path, - ssh: SshConnection - ) { - ssh.execute("mkdir -p ${path.parent!!}") - ssh.execute("touch $path") + fun report(jira: InstalledJira): Report { + return JiraLogsReport(jira) + } + + private class JiraLogsReport(private val jira: InstalledJira) : Report { + override fun locate(ssh: SshConnection): List { + return listOf( + "${jira.home.path}/log/atlassian-jira.log", + "${jira.installation.path}/logs/catalina.out" + ).onEach { ensureFile(Paths.get(it), ssh) } + } + + private fun ensureFile(path: Path, ssh: SshConnection) { + ssh.execute("mkdir -p ${path.parent!!}") + ssh.execute("touch $path") + } } } diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/start/hook/RestUpgrade.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/start/hook/RestUpgrade.kt index 89c12a00..65ae076d 100644 --- a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/start/hook/RestUpgrade.kt +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/start/hook/RestUpgrade.kt @@ -1,6 +1,7 @@ package com.atlassian.performance.tools.infrastructure.api.jira.start.hook import com.atlassian.performance.tools.infrastructure.api.jira.JiraLaunchTimeouts +import com.atlassian.performance.tools.infrastructure.api.jira.install.hook.JiraLogs import com.atlassian.performance.tools.infrastructure.api.jira.report.FileListing import com.atlassian.performance.tools.infrastructure.api.jira.report.Reports import com.atlassian.performance.tools.infrastructure.api.jira.start.StartedJira @@ -18,59 +19,71 @@ class RestUpgrade( override fun call(ssh: SshConnection, jira: StartedJira, hooks: PostStartHooks, reports: Reports) { val threadDump = ThreadDump(jira.pid, jira.installed.jdk) - val ip = jira.installed.host.privateIp - val port = jira.installed.host.port - val upgradesEndpoint = URI("http://$adminUsername:$adminPassword@$ip:$port/rest/api/2/upgrade") - reports.add(FileListing("thread-dumps/*"), jira.installed.host) - waitForStatusToChange( - statusQuo = "000", - timeout = timeouts.offlineTimeout, - ssh = ssh, - uri = upgradesEndpoint, - threadDump = threadDump - ) - waitForStatusToChange( - statusQuo = "503", - timeout = timeouts.initTimeout, - ssh = ssh, - uri = upgradesEndpoint, - threadDump = threadDump - ) - ssh.execute( - cmd = "curl --silent --retry 6 -X POST $upgradesEndpoint", - timeout = Duration.ofSeconds(15) - ) - waitForStatusToChange( - statusQuo = "303", - timeout = timeouts.upgradeTimeout, - ssh = ssh, - uri = upgradesEndpoint, - threadDump = threadDump - ) + val polling = Upgrades(ssh, jira, adminUsername, adminPassword, timeouts, threadDump, reports) + polling.waitUntilOnline() + polling.waitUntilHealthy() + polling.triggerUpgrades() + polling.waitUntilUpgraded() } - private fun waitForStatusToChange( - statusQuo: String, - uri: URI, - timeout: Duration, - ssh: SshConnection, - threadDump: ThreadDump + private class Upgrades( + private val ssh: SshConnection, + private val jira: StartedJira, + adminUsername: String, + adminPassword: String, + private val timeouts: JiraLaunchTimeouts, + private val threadDump: ThreadDump, + private val reports: Reports ) { - val backoff = Duration.ofSeconds(10) - val deadline = Instant.now() + timeout - while (true) { - val currentStatus = ssh.safeExecute( - cmd = "curl --silent --write-out '%{http_code}' --output /dev/null -X GET $uri", - timeout = timeouts.unresponsivenessTimeout - ).output - if (currentStatus != statusQuo) { - break - } - if (deadline < Instant.now()) { - throw Exception("$uri failed to get out of $statusQuo status within $timeout") + private val upgradesEndpoint: URI + + init { + val ip = jira.installed.host.privateIp + val port = jira.installed.host.port + upgradesEndpoint = URI("http://$adminUsername:$adminPassword@$ip:$port/rest/api/2/upgrade") + } + + fun waitUntilOnline() { + waitForStatusToChange("000", timeouts.offlineTimeout) + } + + fun waitUntilHealthy() { + waitForStatusToChange("503", timeouts.initTimeout) + } + + fun waitUntilUpgraded() { + waitForStatusToChange("303", timeouts.upgradeTimeout) + } + + private fun waitForStatusToChange( + statusQuo: String, + timeout: Duration + ) { + val backoff = Duration.ofSeconds(10) + val deadline = Instant.now() + timeout + while (true) { + val currentStatus = ssh.safeExecute( + cmd = "curl --silent --write-out '%{http_code}' --output /dev/null -X GET $upgradesEndpoint", + timeout = timeouts.unresponsivenessTimeout + ).output + if (currentStatus != statusQuo) { + break + } + threadDump.gather(ssh, "thread-dumps") + if (deadline < Instant.now()) { + reports.add(JiraLogs().report(jira.installed), jira.installed.host) + reports.add(FileListing("thread-dumps/*"), jira.installed.host) + throw Exception("$upgradesEndpoint failed to get out of $statusQuo status within $timeout") + } + Thread.sleep(backoff.toMillis()) } - threadDump.gather(ssh, "thread-dumps") - Thread.sleep(backoff.toMillis()) + } + + fun triggerUpgrades() { + ssh.execute( + cmd = "curl --silent --retry 6 -X POST $upgradesEndpoint", + timeout = Duration.ofSeconds(15) + ) } } -} +} \ No newline at end of file diff --git a/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/JiraServerPlanIT.kt b/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/JiraServerPlanIT.kt index dc335b9b..7de06cda 100644 --- a/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/JiraServerPlanIT.kt +++ b/src/test/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/JiraServerPlanIT.kt @@ -10,11 +10,14 @@ import com.atlassian.performance.tools.infrastructure.api.jira.install.hook.PreI import com.atlassian.performance.tools.infrastructure.api.jira.node.JiraNodePlan import com.atlassian.performance.tools.infrastructure.api.jira.start.JiraLaunchScript import com.atlassian.performance.tools.infrastructure.api.jvm.AdoptOpenJDK +import com.atlassian.performance.tools.io.api.resolveSafely import org.assertj.core.api.Assertions.assertThat import org.junit.After import org.junit.Before import org.junit.Test import java.nio.file.Files +import java.nio.file.Paths +import java.time.Instant class JiraServerPlanIT { @@ -55,7 +58,15 @@ class JiraServerPlanIT { .build() // when - val jiraServer = jiraServerPlan.materialize() + val jiraServer = try { + jiraServerPlan.materialize() + } catch (e: Exception) { + val debugging = Paths.get("build/test-artifacts/") + .resolveSafely(javaClass.simpleName) + .resolveSafely(Instant.now().toString()) + jiraServerPlan.report().downloadTo(debugging) + throw Exception("Jira Server plan failed to materialize, debugging info available in $debugging", e) + } val theNode = jiraServer.nodes.single() val host = theNode.installed.host @@ -76,7 +87,6 @@ class JiraServerPlanIT { .toList() assertThat(fileTree.map { it.toString() }).contains( "jira-node/root/atlassian-jira-software-7.13.0-standalone/logs/catalina.out", - "jira-node/root/thread-dumps", "jira-node/root/~/jpt-jstat.log", "jira-node/root/~/jpt-vmstat.log", "jira-node/root/~/jpt-iostat.log"