Diagnose RestUpgrade failures

atlassian · Jun 25, 2021 · a052757 · a052757
1 parent 766de91
commit a052757
Show file tree

Hide file tree

Showing 4 changed files with 99 additions and 66 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -34,6 +34,12 @@ jobs:
       with:
         name: test-reports
         path: build/reports/tests
+    - name: Upload test artifacts
+      if: always()
+      uses: actions/upload-artifact@v2
+      with:
+        name: test-artifacts
+        path: build/test-artifacts
     - name: Release
       if: github.event.inputs.release == 'yes'
       env:

diff --git a/...n/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/JiraLogs.kt b/...n/kotlin/com/atlassian/performance/tools/infrastructure/api/jira/install/hook/JiraLogs.kt
@@ -1,29 +1,33 @@
 package com.atlassian.performance.tools.infrastructure.api.jira.install.hook
 
 import com.atlassian.performance.tools.infrastructure.api.jira.install.InstalledJira
+import com.atlassian.performance.tools.infrastructure.api.jira.report.Report
 import com.atlassian.performance.tools.infrastructure.api.jira.report.Reports
-import com.atlassian.performance.tools.infrastructure.api.jira.report.StaticReport
 import com.atlassian.performance.tools.ssh.api.SshConnection
 import java.nio.file.Path
 import java.nio.file.Paths
 
 class JiraLogs : PostInstallHook {
 
     override fun call(ssh: SshConnection, jira: InstalledJira, hooks: PostInstallHooks, reports: Reports) {
-        listOf(
-            "${jira.home.path}/log/atlassian-jira.log",
-            "${jira.installation.path}/logs/catalina.out"
-        )
-            .onEach { ensureFile(Paths.get(it), ssh) }
-            .map { StaticReport(it) }
-            .forEach { reports.add(it, jira.host) }
+        reports.add(report(jira), jira.host)
     }
 
-    private fun ensureFile(
-        path: Path,
-        ssh: SshConnection
-    ) {
-        ssh.execute("mkdir -p ${path.parent!!}")
-        ssh.execute("touch $path")
+    fun report(jira: InstalledJira): Report {
+        return JiraLogsReport(jira)
+    }
+
+    private class JiraLogsReport(private val jira: InstalledJira) : Report {
+        override fun locate(ssh: SshConnection): List<String> {
+            return listOf(
+                "${jira.home.path}/log/atlassian-jira.log",
+                "${jira.installation.path}/logs/catalina.out"
+            ).onEach { ensureFile(Paths.get(it), ssh) }
+        }
+
+        private fun ensureFile(path: Path, ssh: SshConnection) {
+            ssh.execute("mkdir -p ${path.parent!!}")
+            ssh.execute("touch $path")
+        }
     }
 }
diff --git a/.../kotlin/com/atlassian/performance/tools/infrastructure/api/jira/start/hook/RestUpgrade.kt b/.../kotlin/com/atlassian/performance/tools/infrastructure/api/jira/start/hook/RestUpgrade.kt
@@ -1,6 +1,7 @@
 package com.atlassian.performance.tools.infrastructure.api.jira.start.hook
 
 import com.atlassian.performance.tools.infrastructure.api.jira.JiraLaunchTimeouts
+import com.atlassian.performance.tools.infrastructure.api.jira.install.hook.JiraLogs
 import com.atlassian.performance.tools.infrastructure.api.jira.report.FileListing
 import com.atlassian.performance.tools.infrastructure.api.jira.report.Reports
 import com.atlassian.performance.tools.infrastructure.api.jira.start.StartedJira
@@ -18,59 +19,71 @@ class RestUpgrade(
 
     override fun call(ssh: SshConnection, jira: StartedJira, hooks: PostStartHooks, reports: Reports) {
         val threadDump = ThreadDump(jira.pid, jira.installed.jdk)
-        val ip = jira.installed.host.privateIp
-        val port = jira.installed.host.port
-        val upgradesEndpoint = URI("http://$adminUsername:$adminPassword@$ip:$port/rest/api/2/upgrade")
-        reports.add(FileListing("thread-dumps/*"), jira.installed.host)
-        waitForStatusToChange(
-            statusQuo = "000",
-            timeout = timeouts.offlineTimeout,
-            ssh = ssh,
-            uri = upgradesEndpoint,
-            threadDump = threadDump
-        )
-        waitForStatusToChange(
-            statusQuo = "503",
-            timeout = timeouts.initTimeout,
-            ssh = ssh,
-            uri = upgradesEndpoint,
-            threadDump = threadDump
-        )
-        ssh.execute(
-            cmd = "curl --silent --retry 6 -X POST $upgradesEndpoint",
-            timeout = Duration.ofSeconds(15)
-        )
-        waitForStatusToChange(
-            statusQuo = "303",
-            timeout = timeouts.upgradeTimeout,
-            ssh = ssh,
-            uri = upgradesEndpoint,
-            threadDump = threadDump
-        )
+        val polling = Upgrades(ssh, jira, adminUsername, adminPassword, timeouts, threadDump, reports)
+        polling.waitUntilOnline()
+        polling.waitUntilHealthy()
+        polling.triggerUpgrades()
+        polling.waitUntilUpgraded()
     }
 
-    private fun waitForStatusToChange(
-        statusQuo: String,
-        uri: URI,
-        timeout: Duration,
-        ssh: SshConnection,
-        threadDump: ThreadDump
+    private class Upgrades(
+        private val ssh: SshConnection,
+        private val jira: StartedJira,
+        adminUsername: String,
+        adminPassword: String,
+        private val timeouts: JiraLaunchTimeouts,
+        private val threadDump: ThreadDump,
+        private val reports: Reports
     ) {
-        val backoff = Duration.ofSeconds(10)
-        val deadline = Instant.now() + timeout
-        while (true) {
-            val currentStatus = ssh.safeExecute(
-                cmd = "curl --silent --write-out '%{http_code}' --output /dev/null -X GET $uri",
-                timeout = timeouts.unresponsivenessTimeout
-            ).output
-            if (currentStatus != statusQuo) {
-                break
-            }
-            if (deadline < Instant.now()) {
-                throw Exception("$uri failed to get out of $statusQuo status within $timeout")
+        private val upgradesEndpoint: URI
+
+        init {
+            val ip = jira.installed.host.privateIp
+            val port = jira.installed.host.port
+            upgradesEndpoint = URI("http://$adminUsername:$adminPassword@$ip:$port/rest/api/2/upgrade")
+        }
+
+        fun waitUntilOnline() {
+            waitForStatusToChange("000", timeouts.offlineTimeout)
+        }
+
+        fun waitUntilHealthy() {
+            waitForStatusToChange("503", timeouts.initTimeout)
+        }
+
+        fun waitUntilUpgraded() {
+            waitForStatusToChange("303", timeouts.upgradeTimeout)
+        }
+
+        private fun waitForStatusToChange(
+            statusQuo: String,
+            timeout: Duration
+        ) {
+            val backoff = Duration.ofSeconds(10)
+            val deadline = Instant.now() + timeout
+            while (true) {
+                val currentStatus = ssh.safeExecute(
+                    cmd = "curl --silent --write-out '%{http_code}' --output /dev/null -X GET $upgradesEndpoint",
+                    timeout = timeouts.unresponsivenessTimeout
+                ).output
+                if (currentStatus != statusQuo) {
+                    break
+                }
+                threadDump.gather(ssh, "thread-dumps")
+                if (deadline < Instant.now()) {
+                    reports.add(JiraLogs().report(jira.installed), jira.installed.host)
+                    reports.add(FileListing("thread-dumps/*"), jira.installed.host)
+                    throw Exception("$upgradesEndpoint failed to get out of $statusQuo status within $timeout")
+                }
+                Thread.sleep(backoff.toMillis())
             }
-            threadDump.gather(ssh, "thread-dumps")
-            Thread.sleep(backoff.toMillis())
+        }
+
+        fun triggerUpgrades() {
+            ssh.execute(
+                cmd = "curl --silent --retry 6 -X POST $upgradesEndpoint",
+                timeout = Duration.ofSeconds(15)
+            )
         }
     }
-}
+}
diff --git a/...tlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/JiraServerPlanIT.kt b/...tlin/com/atlassian/performance/tools/infrastructure/api/jira/instance/JiraServerPlanIT.kt
@@ -10,11 +10,14 @@ import com.atlassian.performance.tools.infrastructure.api.jira.install.hook.PreI
 import com.atlassian.performance.tools.infrastructure.api.jira.node.JiraNodePlan
 import com.atlassian.performance.tools.infrastructure.api.jira.start.JiraLaunchScript
 import com.atlassian.performance.tools.infrastructure.api.jvm.AdoptOpenJDK
+import com.atlassian.performance.tools.io.api.resolveSafely
 import org.assertj.core.api.Assertions.assertThat
 import org.junit.After
 import org.junit.Before
 import org.junit.Test
 import java.nio.file.Files
+import java.nio.file.Paths
+import java.time.Instant
 
 class JiraServerPlanIT {
 
@@ -55,7 +58,15 @@ class JiraServerPlanIT {
             .build()
 
         // when
-        val jiraServer = jiraServerPlan.materialize()
+        val jiraServer = try {
+            jiraServerPlan.materialize()
+        } catch (e: Exception) {
+            val debugging = Paths.get("build/test-artifacts/")
+                .resolveSafely(javaClass.simpleName)
+                .resolveSafely(Instant.now().toString())
+            jiraServerPlan.report().downloadTo(debugging)
+            throw Exception("Jira Server plan failed to materialize, debugging info available in $debugging", e)
+        }
 
         val theNode = jiraServer.nodes.single()
         val host = theNode.installed.host
@@ -76,7 +87,6 @@ class JiraServerPlanIT {
             .toList()
         assertThat(fileTree.map { it.toString() }).contains(
             "jira-node/root/atlassian-jira-software-7.13.0-standalone/logs/catalina.out",
-            "jira-node/root/thread-dumps",
             "jira-node/root/~/jpt-jstat.log",
             "jira-node/root/~/jpt-vmstat.log",
             "jira-node/root/~/jpt-iostat.log"