Merge commit '7021b10356069cf424556f1a5683c5f270a87e5b' into releases…

…/2.24.0

fast forward

.bazelrc

@@ -167,6 +167,7 @@ test:ci --flaky_test_attempts=3
 test:ci --nocache_test_results
 test:ci --spawn_strategy=local
 test:ci --test_output=errors
+test:ci --experimental_ui_max_stdouterr_bytes=-1
 test:ci --test_verbose_timeout_warnings
 test:ci-debug -c dbg
 test:ci-debug --copt="-g"

.buildkite/build.rayci.yml

@@ -28,6 +28,7 @@ steps:
       - forge
 
   - label: ":tapioca: build: jar"
+    key: java_wheels
     tags:
       - java
       - oss

.buildkite/core.rayci.yml

@@ -142,10 +142,10 @@ steps:
     commands:
       - bazel run //ci/ray_ci:test_in_docker --
         python/ray/util/dask/... python/ray/tests/modin/... core
-        --build-name data15build
+        --build-name data16build
         --parallelism-per-worker 2
     depends_on:
-      - data15build
+      - data16build
       - forge
 
   - label: ":ray: core: dashboard tests"
@@ -315,6 +315,7 @@ steps:
     tags:
       - python
       - docker
+      - oss
     instance_type: medium
     commands:
       - bazel run //ci/ray_ci:build_in_docker -- docker --platform cpu

.buildkite/data.rayci.yml

@@ -7,8 +7,8 @@ steps:
   - name: data6build
     wanda: ci/docker/data6.build.wanda.yaml
 
-  - name: data15build
-    wanda: ci/docker/data15.build.wanda.yaml
+  - name: data16build
+    wanda: ci/docker/data16.build.wanda.yaml
 
   - name: databuild-multipy
     label: "wanda: databuild-py{{matrix}}"
@@ -42,7 +42,7 @@ steps:
         --except-tags data_integration,doctest
     depends_on: data6build
 
-  - label: ":database: data: arrow 15 tests"
+  - label: ":database: data: arrow 16 tests"
     tags: 
       - python
       - data
@@ -52,15 +52,15 @@ steps:
       - bazel run //ci/ray_ci:test_in_docker -- //python/ray/data/... //python/ray/air/... data
         --workers "$${BUILDKITE_PARALLEL_JOB_COUNT}" 
         --worker-id "$${BUILDKITE_PARALLEL_JOB}" --parallelism-per-worker 3
-        --build-name data15build
+        --build-name data16build
         --except-tags data_integration,doctest
-    depends_on: data15build
+    depends_on: data16build
 
   - label: ":database: data: arrow nightly tests"
+    if: pipeline.id == "0189e759-8c96-4302-b6b5-b4274406bf89" || pipeline.id == "018f4f1e-1b73-4906-9802-92422e3badaa"
     tags: 
       - python
       - data
-      - skip-on-premerge
     instance_type: medium
     parallelism: 2
     commands:
@@ -91,17 +91,17 @@ steps:
     commands:
       # doc tests
       - bazel run //ci/ray_ci:test_in_docker -- python/ray/... //doc/... data 
-        --build-name data15build
+        --build-name data16build
         --except-tags gpu
         --only-tags doctest
         --parallelism-per-worker 2
       # doc examples
       - bazel run //ci/ray_ci:test_in_docker -- //doc/... data 
-        --build-name data15build
+        --build-name data16build
         --except-tags gpu,post_wheel_build,doctest
         --parallelism-per-worker 2
         --skip-ray-installation
-    depends_on: data15build
+    depends_on: data16build
 
   - label: ":database: data: doc gpu tests"
     tags: 
@@ -144,9 +144,9 @@ steps:
     instance_type: small
     commands:
       - bazel run //ci/ray_ci:test_in_docker -- python/ray/dashboard/... data
-        --build-name data15build
+        --build-name data16build
         --parallelism-per-worker 3
-    depends_on: data15build
+    depends_on: data16build
 
   - label: ":database: data: flaky tests"
     tags: 
@@ -158,9 +158,9 @@ steps:
     commands:
       - bazel run //ci/ray_ci:test_in_docker -- //... data --run-flaky-tests 
         --parallelism-per-worker 3
-        --build-name data15build
+        --build-name data16build
         --except-tags gpu_only,gpu
-    depends_on: data15build
+    depends_on: data16build
 
   - label: ":database: data: flaky gpu tests"
     tags: 

.buildkite/kuberay.rayci.yml

@@ -20,6 +20,7 @@ steps:
       - raycpubase
 
   - label: ":kubernetes: chaos {{matrix.workload}} under {{matrix.fault}}"
+    key: kuberay_tests
     tags:
       - python
       - docker

.buildkite/lint.rayci.yml

@@ -24,9 +24,10 @@ steps:
     key: lint-medium
     instance_type: medium
     depends_on:
-      - forge
+      - oss-ci-base_build
     commands:
       - ./ci/lint/lint.sh {{matrix}}
+    job_env: oss-ci-base_build
     matrix:
       - api_annotations
       - api_discrepancy

.buildkite/serve.rayci.yml

@@ -57,7 +57,7 @@ steps:
     depends_on: servepydantic1build
 
   - label: ":ray-serve: serve: python {{matrix.python}} tests ({{matrix.worker_id}})"
-    if: build.pull_request.labels includes "continuous-build" || pipeline.id == "0189e759-8c96-4302-b6b5-b4274406bf89"
+    if: build.pull_request.labels includes "continuous-build" || pipeline.id == "0189e759-8c96-4302-b6b5-b4274406bf89" || pipeline.id == "018f4f1e-1b73-4906-9802-92422e3badaa"
     tags: 
       - serve
       - python

.rayciversion

@@ -1 +1 @@
-0.6.0
+0.7.0

ci/ci.sh

@@ -106,7 +106,8 @@ compile_pip_dependencies() {
     "${WORKSPACE_DIR}/python/requirements/ml/train-requirements.txt" \
     "${WORKSPACE_DIR}/python/requirements/ml/train-test-requirements.txt" \
     "${WORKSPACE_DIR}/python/requirements/ml/tune-requirements.txt" \
-    "${WORKSPACE_DIR}/python/requirements/ml/tune-test-requirements.txt"
+    "${WORKSPACE_DIR}/python/requirements/ml/tune-test-requirements.txt" \
+    "${WORKSPACE_DIR}/python/requirements/security-requirements.txt"
 
   # Remove some pins from upstream dependencies:
   # ray, xgboost-ray, lightgbm-ray, tune-sklearn

ci/docker/data15.build.wanda.yaml → ci/docker/data16.build.wanda.yaml

@@ -1,4 +1,4 @@
-name: "data15build"
+name: "data16build"
 froms: ["cr.ray.io/rayproject/oss-ci-base_ml"]
 dockerfile: ci/docker/data.build.Dockerfile
 srcs:
@@ -10,6 +10,6 @@ srcs:
   - python/requirements/ml/data-requirements.txt
   - python/requirements/ml/data-test-requirements.txt
 build_args:
-  - ARROW_VERSION=15.*
+  - ARROW_VERSION=16.*
 tags:
-  - cr.ray.io/rayproject/data15build
+  - cr.ray.io/rayproject/data16build

ci/lint/lint.sh

@@ -64,7 +64,8 @@ api_annotations() {
 api_discrepancy() {
   # shellcheck disable=SC2102
   RAY_DISABLE_EXTRA_CPP=1 pip install -e python/[all]
-  ./ci/lint/check_api_discrepancy.py
+  # TODO(can): run this check with other ray packages
+  bazel run //ci/ray_ci/doc:cmd_check_api_discrepancy -- ray.data
 }
 
 documentation_style() {

ci/ray_ci/automation/determine_microcheck_step_ids.py

@@ -1,4 +1,5 @@
 import click
+import os
 
 from ci.ray_ci.utils import ci_init
 from ray_release.test import (
@@ -8,6 +9,8 @@
     MACOS_TEST_PREFIX,
 )
 
+BAZEL_WORKSPACE_DIR = os.environ.get("BUILD_WORKSPACE_DIRECTORY", "")
+
 
 @click.command()
 def main() -> None:
@@ -16,9 +19,9 @@ def main() -> None:
     """
     ci_init()
     steps = (
-        list(Test.gen_high_impact_tests(LINUX_TEST_PREFIX).keys())
-        + list(Test.gen_high_impact_tests(WINDOWS_TEST_PREFIX).keys())
-        + list(Test.gen_high_impact_tests(MACOS_TEST_PREFIX).keys())
+        Test.gen_microcheck_step_ids(LINUX_TEST_PREFIX, BAZEL_WORKSPACE_DIR)
+        .union(Test.gen_microcheck_step_ids(WINDOWS_TEST_PREFIX, BAZEL_WORKSPACE_DIR))
+        .union(Test.gen_microcheck_step_ids(MACOS_TEST_PREFIX, BAZEL_WORKSPACE_DIR))
     )
 
     print(",".join(steps))

ci/ray_ci/doc/BUILD.bazel

@@ -0,0 +1,36 @@
+load("@rules_python//python:defs.bzl", "py_library", "py_test")
+load("@py_deps_buildkite//:requirements.bzl", ci_require = "requirement")
+
+py_binary(
+    name = "cmd_check_api_discrepancy",
+    srcs = ["cmd_check_api_discrepancy.py"],
+    deps = [":doc"],
+)
+
+py_library(
+    name = "doc",
+    srcs = glob(
+        ["*.py"],
+        exclude = [
+            "test_*.py",
+            "cmd_*.py",
+            "mock_module.py",
+        ],
+    ),
+    visibility = ["//ci/ray_ci/doc:__subpackages__"],
+)
+
+py_test(
+    name = "test_module",
+    size = "small",
+    srcs = ["test_module.py", "mock_module.py"],
+    exec_compatible_with = ["//:hermetic_python"],
+    tags = [
+        "ci_unit",
+        "team:ci",
+    ],
+    deps = [
+        ":doc",
+        ci_require("pytest"),
+    ],
+)

ci/ray_ci/doc/api.py

@@ -0,0 +1,21 @@
+from enum import Enum
+from dataclasses import dataclass
+
+
+class AnnotationType(Enum):
+    PUBLIC_API = "PublicAPI"
+    DEVELOPER_API = "DeveloperAPI"
+    DEPRECATED = "Deprecated"
+    UNKNOWN = "Unknown"
+
+
+class CodeType(Enum):
+    CLASS = "Class"
+    FUNCTION = "Function"
+
+
+@dataclass
+class API:
+    name: str
+    annotation_type: AnnotationType
+    code_type: CodeType

ci/ray_ci/doc/cmd_check_api_discrepancy.py

@@ -0,0 +1,23 @@
+import click
+
+from ci.ray_ci.doc.module import Module
+
+
+@click.command()
+@click.argument("module", required=True, type=str)
+def main(module: str) -> None:
+    """
+    This script checks for annotated classes and functions in a module, and finds
+    discrepancies between the annotations and the documentation.
+    """
+    module = Module(module)
+    for api in module.get_apis():
+        print(
+            f"API: {api.name}, "
+            f"Annotation Type: {api.annotation_type}, "
+            f"Code Type: {api.code_type}"
+        )
+
+
+if __name__ == "__main__":
+    main()