diff --git a/.buildkite/test-e2e.yml b/.buildkite/test-e2e.yml
index c736f4fa19..d94dcc1361 100644
--- a/.buildkite/test-e2e.yml
+++ b/.buildkite/test-e2e.yml
@@ -1,3 +1,21 @@
+- label: 'kuberay-2654 TestAutoscalingRayService flaky test'
+  instance_size: large
+  image: golang:1.22
+  commands:
+    - source .buildkite/setup-env.sh
+    - kind create cluster --wait 900s --config ./tests/framework/config/kind-config-buildkite.yml
+    - kubectl config set clusters.kind-kind.server https://docker:6443
+    # Build nightly KubeRay operator image
+    - pushd ray-operator
+    - IMG=kuberay/operator:nightly make docker-image
+    - kind load docker-image kuberay/operator:nightly
+    - IMG=kuberay/operator:nightly make deploy
+    - kubectl wait --timeout=90s --for=condition=Available=true deployment kuberay-operator
+    # Run e2e tests
+    - KUBERAY_TEST_TIMEOUT_SHORT=1m KUBERAY_TEST_TIMEOUT_MEDIUM=5m KUBERAY_TEST_TIMEOUT_LONG=10m go test -count=1 -timeout 30m -run TestAutoscalingRayService -v ./test/e2erayservice
+    # Printing KubeRay operator logs
+    - kubectl logs --tail -1 -l app.kubernetes.io/name=kuberay
+
 - label: 'Test E2E (nightly operator)'
   instance_size: large
   image: golang:1.22