mcs: make scheduling server test stable

.github/workflows/pd-tests.yaml

@@ -55,24 +55,24 @@ jobs:
         with:
           name: cover-reports
           path: covprofile_${{ matrix.worker_id }}
-  report-coverage:
-    needs: chunks
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@v3
-      - name: Download chunk report
-        uses: actions/download-artifact@v2
-        with:
-          name: cover-reports
-      - name: Merge
-        env:
-          TOTAL_JOBS: ${{needs.chunks.outputs.job-total}}
-        run: for i in $(seq 1 $TOTAL_JOBS); do cat covprofile_$i >> covprofile; done
-      - name: Send coverage
-        uses: codecov/codecov-action@v1
-        with:
-          token: ${{ secrets.CODECOV }}
-          file: ./covprofile
-          flags: unittests
-          name: codecov-umbrella
+  # report-coverage:
+  #   needs: chunks
+  #   runs-on: ubuntu-latest
+  #   steps:
+  #     - name: Checkout code
+  #       uses: actions/checkout@v3
+  #     - name: Download chunk report
+  #       uses: actions/download-artifact@v2
+  #       with:
+  #         name: cover-reports
+  #     - name: Merge
+  #       env:
+  #         TOTAL_JOBS: ${{needs.chunks.outputs.job-total}}
+  #       run: for i in $(seq 1 $TOTAL_JOBS); do cat covprofile_$i >> covprofile; done
+  #     - name: Send coverage
+  #       uses: codecov/codecov-action@v1
+  #       with:
+  #         token: ${{ secrets.CODECOV }}
+  #         file: ./covprofile
+  #         flags: unittests
+  #         name: codecov-umbrella

pkg/utils/testutil/api_check.go

@@ -114,6 +114,21 @@ func CheckGetJSON(client *http.Client, url string, data []byte, checkOpts ...fun
 	return checkResp(resp, checkOpts...)
 }
 
+// CheckGetUntilStatusCode is used to do get request and do check options.
+func CheckGetUntilStatusCode(re *require.Assertions, client *http.Client, url string, code int) error {
+	var err error
+	Eventually(re, func() bool {
+		resp, err2 := apiutil.GetJSON(client, url, nil)
+		if err2 != nil {
+			err = err2
+			return true
+		}
+		defer resp.Body.Close()
+		return resp.StatusCode == code
+	})
+	return err
+}
+
 // CheckPatchJSON is used to do patch request and do check options.
 func CheckPatchJSON(client *http.Client, url string, data []byte, checkOpts ...func([]byte, int, http.Header)) error {
 	resp, err := apiutil.PatchJSON(client, url, data)

server/api/admin_test.go

@@ -18,6 +18,7 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
+	"io"
 	"net/http"
 	"testing"
 	"time"
@@ -27,6 +28,7 @@ import (
 	"github.com/stretchr/testify/suite"
 	"github.com/tikv/pd/pkg/core"
 	"github.com/tikv/pd/pkg/replication"
+	"github.com/tikv/pd/pkg/utils/apiutil"
 	tu "github.com/tikv/pd/pkg/utils/testutil"
 	"github.com/tikv/pd/server"
 )
@@ -188,9 +190,24 @@ func (suite *adminTestSuite) TestResetTS() {
 	values, err := json.Marshal(args)
 	suite.NoError(err)
 	re := suite.Require()
-	err = tu.CheckPostJSON(testDialClient, url, values,
-		tu.StatusOK(re),
-		tu.StringEqual(re, "\"Reset ts successfully.\"\n"))
+	tu.Eventually(re, func() bool {
+		resp, err := apiutil.PostJSON(testDialClient, url, values)
+		re.NoError(err)
+		defer resp.Body.Close()
+		b, err := io.ReadAll(resp.Body)
+		re.NoError(err)
+		switch resp.StatusCode {
+		case http.StatusOK:
+			re.Contains(string(b), "Reset ts successfully.")
+			return true
+		case http.StatusForbidden:
+			re.Contains(string(b), "[PD:etcd:ErrEtcdTxnConflict]etcd transaction failed, conflicted and rolled back")
+			return false
+		default:
+			re.FailNow("unexpected status code %d", resp.StatusCode)
+			return false
+		}
+	})
 	suite.NoError(err)
 	t2 := makeTS(32 * time.Hour)
 	args["tso"] = fmt.Sprintf("%d", t2)

server/api/diagnostic_test.go

@@ -17,6 +17,7 @@ package api
 import (
 	"encoding/json"
 	"fmt"
+	"net/http"
 	"testing"
 	"time"
 
@@ -63,6 +64,8 @@ func (suite *diagnosticTestSuite) TearDownSuite() {
 
 func (suite *diagnosticTestSuite) checkStatus(status string, url string) {
 	re := suite.Require()
+	err := tu.CheckGetUntilStatusCode(re, testDialClient, url, http.StatusOK)
+	suite.NoError(err)
 	suite.Eventually(func() bool {
 		result := &schedulers.DiagnosticResult{}
 		err := tu.ReadGetJSON(re, testDialClient, url, result)

tests/integrations/mcs/scheduling/config_test.go

@@ -93,6 +93,9 @@ func (suite *configTestSuite) TestConfigWatch() {
 	re.Equal(sc.DefaultSplitMergeInterval, watcher.GetScheduleConfig().SplitMergeInterval.Duration)
 	re.Equal("0.0.0", watcher.GetClusterVersion().String())
 	// Update the config and check if the scheduling config watcher can get the latest value.
+	testutil.Eventually(re, func() bool {
+		return watcher.GetReplicationConfig().MaxReplicas == 3
+	})
 	persistOpts := suite.pdLeaderServer.GetPersistOptions()
 	persistOpts.SetMaxReplicas(5)
 	persistConfig(re, suite.pdLeaderServer)