try to debug why can't reconnect to clickhouse-server

Altinity · Aug 29, 2024 · 00845bb · 00845bb
1 parent 55c7b4f
commit 00845bb
Show file tree

Hide file tree

Showing 2 changed files with 32 additions and 8 deletions.
diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml
@@ -329,13 +329,25 @@ jobs:
             if wait "$pid"; then
                 echo "$pid docker compose up successful"
             else
-                docker compose -f ${CUR_DIR}/${COMPOSE_FILE} --project-name project${i} --progress plain logs
+                docker compose -f ${CUR_DIR}/${COMPOSE_FILE} --project-name ${project_id} --progress plain logs
                 echo "$pid the docker compose up failed."
                 exit 1  # Exit with an error code if any command fails
             fi
           done
-          
-          go test -parallel ${RUN_PARALLEL} -timeout 60m -failfast -tags=integration -run "${RUN_TESTS:-.+}" -v test/integration/integration_test.go
+          set +e
+          go test -parallel ${RUN_PARALLEL} -timeout 90m -failfast -tags=integration -run "${RUN_TESTS:-.+}" -v test/integration/integration_test.go
+          TEST_FAILED=$?
+          set -e
+          if [[ "0" != "${TEST_FAILED}" ]]; then
+            for project_id in "${project_ids[@]}"; do
+                echo "=== docker ${project_id} state ==="
+                docker compose -f ${CUR_DIR}/${COMPOSE_FILE} --project-name ${project_id} --progress plain ps -a
+                echo "=== docker ${project_id} logs ==="
+                docker compose -f ${CUR_DIR}/${COMPOSE_FILE} --project-name ${project_id} --progress plain logs
+            done
+            exit 1
+          fi      
+
       - name: Format integration coverage
         env:
           GOROOT: ${{ env.GOROOT_1_22_X64 }}

diff --git a/test/integration/integration_test.go b/test/integration/integration_test.go
@@ -2933,15 +2933,27 @@ func (env *TestEnvironment) connectWithWait(r *require.Assertions, sleepBefore,
 }
 
 func (env *TestEnvironment) connect(timeOut string) error {
-	portOut, err := utils.ExecCmdOut(context.Background(), 10*time.Second, "docker", append(env.GetDefaultComposeCommand(), "port", "clickhouse", "9000")...)
-	if err != nil {
+	for i := 0; i < 10; i++ {
+		statusOut, statusErr := utils.ExecCmdOut(context.Background(), 10*time.Second, "docker", append(env.GetDefaultComposeCommand(), "ps", "--status", "running", "clickhouse")...)
+		if statusErr == nil {
+			break
+		}
+		log.Warn().Msg(statusOut)
+		level := zerolog.WarnLevel
+		if i == 9 {
+			level = zerolog.FatalLevel
+		}
+		log.WithLevel(level).Msgf("can't ps --status running clickhouse: %v", statusErr)
+		time.Sleep(1 * time.Second)
+	}
+	portOut, portErr := utils.ExecCmdOut(context.Background(), 10*time.Second, "docker", append(env.GetDefaultComposeCommand(), "port", "clickhouse", "9000")...)
+	if portErr != nil {
 		log.Error().Msg(portOut)
-		log.Fatal().Msgf("can't get port for clickhouse: %v", err)
+		log.Fatal().Msgf("can't get port for clickhouse: %v", portErr)
 	}
 	hostAndPort := strings.Split(strings.Trim(portOut, " \r\n\t"), ":")
 	if len(hostAndPort) < 1 {
-		log.Error().Msg(portOut)
-		log.Fatal().Msgf("invalid port for clickhouse: %v", err)
+		log.Fatal().Msgf("invalid port for clickhouse: %v", portOut)
 	}
 	port, err := strconv.Atoi(hostAndPort[1])
 	if err != nil {