Merge branch 'opensearch-project:main' into 532_publish_proto_files

Author: amberzsy

.cspell

@@ -63,6 +63,7 @@ geotile
 gsub
 Gsub
 haasephonetik
+hashcode
 heteroscedastic
 hnsw
 homoscedastic
@@ -134,6 +135,7 @@ Oversample
 performanceanalyzer
 permissionsinfo
 pipefail
+pipenv
 preconfigure
 preconfigured
 prefilter

.github/workflows/test-spec.yml

@@ -27,26 +27,28 @@ jobs:
             admin_password: admin
           - version: 2.0.0
             admin_password: admin
-          - version: 2.16.0
-          - version: 2.16.0
+          - version: 2.18.0
+          - version: 2.18.0
             tests: plugins/index_state_management
-          - version: 2.16.0
+          - version: 2.18.0
             tests: plugins/ml
-          - version: 2.16.0
+          - version: 2.18.0
             tests: routing
-          - version: 2.16.0
+          - version: 2.18.0
             tests: snapshot
-          - version: 2.17.0
+          - version: 2.18.0
             tests: plugins/streaming
-          - version: 2.17.0
+          - version: 2.18.0
             tests: plugins/notifications
-          - version: 2.17.0
+          - version: 2.18.0
             tests: plugins/query_insights
-          - version: 2.17.0
+          - version: 2.18.0
             tests: plugins/workload-management
           - version: 2.18.0
+            tests: plugins/analysis
+          - version: 2.19.0
             hub: opensearchstaging
-            ref: '@sha256:57a1cd1142d68c203e2e4aa0666d9a691e1e409a5d14aa4a8f5044036f05cf06'
+            ref: '@sha256:4da23e0137b2b67206d23b36fcf0914cc39b3bf19310c782f536e4934b86f6cc'
           - version: 3.0.0
             hub: opensearchstaging
             ref: '@sha256:727643acdfebed77bfdb26362dbcff536b7ea02a0cc4ae2da2521729171333de'
@@ -77,12 +79,16 @@ jobs:
         working-directory: tests/${{ matrix.entry.tests || 'default' }}
         run: docker compose up -d
 
+      - name: Generate Test Files Hash
+        id: tests
+        run: echo "hash=${{ matrix.entry.version }}-${{ hashFiles(format('tests/{0}', matrix.entry.tests || 'default')) }}" >> $GITHUB_OUTPUT
+
       - name: Run Tests
         run: |
           npm run test:spec -- \
             --opensearch-insecure \
             --opensearch-version=${{ matrix.entry.version }} \
-            --coverage coverage/test-spec-coverage-${{ matrix.entry.version }}-${{ matrix.entry.tests || 'default' }}.json \
+            --coverage coverage/test-spec-coverage-${{ steps.tests.outputs.hash }}.json \
             --tests=tests/${{ matrix.entry.tests || 'default' }}
 
       - name: Get Container Logs
@@ -96,12 +102,11 @@ jobs:
       - name: Upload Test Coverage Results
         uses: actions/upload-artifact@v4
         with:
-          name: coverage-${{ matrix.entry.version }}-${{ hashFiles(format('tests/{0}', matrix.entry.tests || 'default')) }}
-          path: coverage/test-spec-coverage-${{ matrix.entry.version }}-${{ matrix.entry.tests || 'default' }}.json
+          name: coverage-${{ matrix.entry.version }}-${{ steps.tests.outputs.hash }}
+          path: coverage/test-spec-coverage-${{ steps.tests.outputs.hash }}.json
 
   merge-coverage:
     runs-on: ubuntu-latest
-    if: github.event_name == 'pull_request'
     needs: test-opensearch-spec
     steps:
       - uses: actions/checkout@v4
@@ -111,6 +116,17 @@ jobs:
         with:
           path: coverage
 
+      - name: Display Missing Test Paths
+        run: |
+          jq -sc '
+            (map(.operations) | add | unique) as $all |
+            (map(.evaluated_operations) | add | unique) as $evaluated |
+            $all-$evaluated |
+            sort_by(.path) |
+            .[] |
+            "\(.method) \(.path)"
+          ' $(find ./ -name "test-spec-coverage-*.json")
+
       - name: Combine Test Coverage Data
         shell: bash -eo pipefail {0}
         run: |
@@ -126,6 +142,7 @@ jobs:
           cat ./coverage/coverage.json
 
       - name: Construct Comment Data Payload
+        if: github.event_name == 'pull_request'
         shell: bash -eo pipefail {0}
         run: |
           jq \
@@ -146,6 +163,7 @@ jobs:
 
       - name: Upload PR Comment Payload
         uses: actions/upload-artifact@v4
+        if: github.event_name == 'pull_request'
         with:
           name: pr-comment
           path: pr-comment.json

.github/workflows/validate-spec.yml .github/workflows/validate-spec-lint.yml

@@ -0,0 +1,34 @@
+name: Validate Spec (Python)
+
+on: [pull_request,push]
+
+jobs:
+  validate-spec-py:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout the repo
+        uses: actions/checkout@v4
+
+      - name: Setup Node.js
+        uses: actions/setup-node@v3
+        with:
+          node-version: '20'
+
+      - name: Build
+        run: npm ci && npm run merge
+
+      - name: Set Up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.12'
+
+      - name: Install Dependencies
+        working-directory: tools/src/validate-spec-py
+        run: |
+          pip install --user pipenv
+          pipenv install
+
+      - name: Validate Spec
+        working-directory: tools/src/validate-spec-py
+        run: |
+          pipenv run python validate.py ../../../build/opensearch-openapi.yaml

.github/workflows/validate-spec-py.yml

@@ -0,0 +1,30 @@
+name: Validate Spec (Ruby)
+
+on: [pull_request, push]
+
+jobs:
+  validate-spec-ruby:
+    runs-on: ubuntu-latest
+    env:
+      BUNDLE_GEMFILE: ${{ github.workspace }}/tools/src/validate-spec-ruby/Gemfile
+    steps:
+      - name: Checkout the repo
+        uses: actions/checkout@v4
+
+      - name: Setup Node.js
+        uses: actions/setup-node@v3
+        with:
+          node-version: '20'
+
+      - name: Build
+        run: npm ci && npm run merge
+
+      - uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: '3.3'
+          bundler-cache: true
+
+      - name: Validate Spec
+        working-directory: tools/src/validate-spec-ruby
+        run: |
+          bundle exec ruby validate.rb ../../../build/opensearch-openapi.yaml

.github/workflows/validate-spec-ruby.yml

@@ -6,9 +6,33 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 
 ### Added
 - Added API specs for query groups lifecycle APIs ([#649](https://github.com/opensearch-project/opensearch-api-specification/pull/649))
+- Added Python and Ruby spec validators ([#646](https://github.com/opensearch-project/opensearch-api-specification/pull/646))
+- Added verbose output of the story being evaluated ([#646](https://github.com/opensearch-project/opensearch-api-specification/pull/646))
+- Added `_search` with `sort: direction` ([#658](https://github.com/opensearch-project/opensearch-api-specification/pull/658))
+- Added `_common.mapping:FlatObjectProperty` ([#661](https://github.com/opensearch-project/opensearch-api-specification/pull/661)) 
+- Added `HEAD /{index}/_doc/{id}` returning `404` ([#670](https://github.com/opensearch-project/opensearch-api-specification/pull/670))
+- Added `_common.mapping:IcuCollationKeywordProperty` ([#666](https://github.com/opensearch-project/opensearch-api-specification/pull/666))
+- Added `/_cluster/stats/{metric}/nodes/{node_id}` and `/_cluster/stats/{metric}/{index_metric}/nodes/{node_id}` ([#639](https://github.com/opensearch-project/opensearch-api-specification/pull/639))
+- Added `PhoneAnalyzer` from `analysis-phonenumber` plugin ([#609](https://github.com/opensearch-project/opensearch-api-specification/pull/609))
+- Added `/_list/indices` & `/_list/shards` api specs ([#613](https://github.com/opensearch-project/opensearch-api-specification/pull/613))
 
 ### Removed
 - Removed unsupported `_common.mapping:SourceField`'s `mode` field and associated `_common.mapping:SourceFieldMode` enum ([#652](https://github.com/opensearch-project/opensearch-api-specification/pull/652))
+- Removed unsupported `_common.mapping:DenseVectorProperty`, `_common.mapping:SparseVectorProperty`, and `_common.mapping:FlattenedProperty` ([#666](https://github.com/opensearch-project/opensearch-api-specification/pull/666))
+- Removed unsupported time series dimension properties and types ([#666](https://github.com/opensearch-project/opensearch-api-specification/pull/666))
+- Removed unsupported runtime field properties (`script`/`on_script_error`) on `NumberPropertyBase` ([#666](https://github.com/opensearch-project/opensearch-api-specification/pull/666)) 
+
+### Fixed
+- Spec passes OpenAPI 3.1.0 validations ([#646](https://github.com/opensearch-project/opensearch-api-specification/pull/646))
+- Disallowed characters (`::`, `@`, and `:`) in key names are replaced with `___` on merge ([#646](https://github.com/opensearch-project/opensearch-api-specification/pull/646))
+- Added missing `required` to `path` parameters ([#646](https://github.com/opensearch-project/opensearch-api-specification/pull/646))
+- Added missing `schema` parent to response types ([#646](https://github.com/opensearch-project/opensearch-api-specification/pull/646))
+- Removed invalid `externalDocs` from `flow_framework.create/update::query.use_case` ([#646](https://github.com/opensearch-project/opensearch-api-specification/pull/646))
+- Fixed incorrect `style` in `indices.get_mapping::query.index` ([#646](https://github.com/opensearch-project/opensearch-api-specification/pull/646))
+- Removed invalid `required` from `ppl` responses ([#646](https://github.com/opensearch-project/opensearch-api-specification/pull/646))
+- Added schema for security API error responses ([#646](https://github.com/opensearch-project/opensearch-api-specification/pull/646))
+- Fixed `/{index}/_create/{id}` returning `201` ([#669](https://github.com/opensearch-project/opensearch-api-specification/pull/669))
+- Fixed `ml._common.yaml#SearchModelsResponse` and `SearchModelsHitsHit` ([#672](https://github.com/opensearch-project/opensearch-api-specification/pull/672))
 
 ## [0.1.0] - 2024-10-25
 

CHANGELOG.md

@@ -26,7 +26,9 @@
     - [Comment on PR](#comment-on-pr)
     - [Test Tools (Unit)](#test-tools-unit)
     - [Test Tools (Integration)](#test-tools-integration)
-    - [Validate Spec](#validate-spec)
+    - [Validate Spec (Lint)](#validate-spec-lint)
+    - [Validate Spec (Python)](#validate-spec-python)
+    - [Validate Spec (Ruby)](#validate-spec-ruby)
 <!-- TOC -->
 
 # Developer Guide
@@ -375,6 +377,30 @@ This workflow runs on PRs to invoke the [tools' unit tests](tools/tests), upload
 
 This workflow runs on PRs to invoke the [tools' integration tests](tools/tests) that require a running instance of OpenSearch to ensure there are no breakages in behavior.
 
-### [Validate Spec](.github/workflows/validate-spec.yml)
+### [Validate Spec (Lint)](.github/workflows/validate-spec-lint.yml)
 
 This workflow runs on PRs to invoke the [spec linter](#spec-linter) and ensure the multi-file spec is correct and follows the design guidelines.
+
+### [Validate Spec (Python)](.github/workflows/validate-spec-py.yml)
+
+This workflow runs on PRs to invoke the [Python openapi-spec-validator](https://pypi.org/project/openapi-spec-validator/) to ensure that the resulting spec can be loaded by Python tools.
+
+You can run the validator locally as follows after installing [pipenv](https://pipenv.pypa.io/en/latest/installation.html).
+
+```
+cd tools/src/validate-spec-py
+pipenv install
+npm run merge ; pipenv run python validate.py ../../../build/opensearch-openapi.yaml
+```
+
+### [Validate Spec (Ruby)](.github/workflows/validate-spec-ruby.yml)
+
+This workflow runs on PRs to invoke the Ruby [Json Schemer](https://github.com/davishmcclurg/json_schemer/) to ensure that the resulting spec can be loaded by Ruby tools.
+
+You can run the validator locally as follows.
+
+```
+cd tools/src/validate-spec-ruby
+bundle install
+npm run merge ; bundle exec ruby validate.rb ../../../build/opensearch-openapi.yaml
+```

DEVELOPER_GUIDE.md

@@ -6,10 +6,12 @@
 [![Test Tools (Unit)](https://github.com/opensearch-project/opensearch-api-specification/actions/workflows/test-tools-unit.yml/badge.svg)](https://github.com/opensearch-project/opensearch-api-specification/actions/workflows/test-tools-unit.yml)
 [![Test Tools (Integration)](https://github.com/opensearch-project/opensearch-api-specification/actions/workflows/test-tools-integ.yml/badge.svg)](https://github.com/opensearch-project/opensearch-api-specification/actions/workflows/test-tools-integ.yml)
 [![Test Spec](https://github.com/opensearch-project/opensearch-api-specification/actions/workflows/test-spec.yml/badge.svg)](https://github.com/opensearch-project/opensearch-api-specification/actions/workflows/test-spec.yml)
-[![Validate Spec](https://github.com/opensearch-project/opensearch-api-specification/actions/workflows/validate-spec.yml/badge.svg)](https://github.com/opensearch-project/opensearch-api-specification/actions/workflows/validate-spec.yml)
+[![Validate Spec](https://github.com/opensearch-project/opensearch-api-specification/actions/workflows/validate-spec-lint.yml/badge.svg)](https://github.com/opensearch-project/opensearch-api-specification/actions/workflows/validate-spec-lint.yml)
 
 - [OpenSearch API Specification](#opensearch-api-specification)
 - [Welcome!](#welcome)
+  - [OpenSearch API Source of Truth](#opensearch-api-source-of-truth)
+  - [Working in this Repo](#working-in-this-repo)
 - [Project Resources](#project-resources)
 - [Code of Conduct](#code-of-conduct)
 - [Security](#security)
@@ -18,10 +20,15 @@
 
 ## Welcome!
 
-The `opensearch-api-specification` is an open source, community-driven collection of API model specifications for [OpenSearch](https://github.com/opensearch-project/OpenSearch) APIs. The API models are written in OpenAPI format and are used to generate client libraries and documentation. You can find the latest version of the API specification [here](https://github.com/opensearch-project/opensearch-api-specification/releases/download/main-latest/opensearch-openapi.yaml).
+The `opensearch-api-specification` is an open source, community-driven collection of API model specifications for [OpenSearch](https://github.com/opensearch-project/OpenSearch) APIs. The API models are written in OpenAPI format and are used to generate client libraries and documentation. You can find the latest release of the API specification [here](https://github.com/opensearch-project/opensearch-api-specification/releases).
 
-To contribute to this project or to track the developments head over to [Projects](https://github.com/opensearch-project/opensearch-api-specification/projects) board. Follow the [Developer guide](DEVELOPER_GUIDE.md) and [Contributing guidelines](CONTRIBUTING.md) for instructions
-on building and contributing to opensearch-api-specification.
+### OpenSearch API Source of Truth
+
+This repo aims to be the complete source of truth for OpenSearch and OpenSearch Dashboards REST APIs, including plugins. To be the source of truth, this repo [accurately represents APIs in YAML](spec/), and [publishes](https://github.com/opensearch-project/opensearch-api-specification/releases) a single-file OpenAPI 3.1.0 spec. The latter is used to [generate OpenSearch language clients](https://github.com/opensearch-project/opensearch-clients/issues/19), [generate mechanical parts of the OpenSearch documentation](https://github.com/opensearch-project/documentation-website/issues/7700), and will be used to [generate the OpenSearch server API itself](https://github.com/opensearch-project/OpenSearch/issues/3090).
+
+### Working in this Repo
+
+Because of our legacy, the specification is produced by reading the [OpenSearch documentation](https://opensearch.org/docs/latest/) and reverse-engineering code. Thus, a good place to start contributing to this repo is to [identify a missing API and to add it](https://github.com/opensearch-project/opensearch-api-specification/issues/168) by following the [developer guide](DEVELOPER_GUIDE.md). This repo also contains a set of [test tools](TESTING_GUIDE.md) that ensure the correctness of this API and infrastructure that evaluates the gap between the OpenSearch server and this API spec by comparing REST routes registered in a running OpenSearch to the list of the APIs in this repo. You can see the latest API and test coverage numbers in the comments automatically added to any [recently merged pull requests](https://github.com/opensearch-project/opensearch-api-specification/pulls?q=is%3Apr+is%3Aclosed).
 
 ## Project Resources
 

README.md

@@ -97,7 +97,7 @@ Remember to set the `OPENSEARCH_PASSWORD` or `AWS_ACCESS_KEY_ID` and `AWS_SECRET
 
 #### FORBIDDEN/10/cluster create-index blocked (api)
 
-The cluster is most likely hitting a disk watermark threshold. This example sets the disk watermark thresholds to 1500MB low, 100MB high, and 500MB flood stage, allowing the cluster to create indices even if the disk is almost full.
+The cluster is most likely hitting a disk watermark threshold. This example sets the disk watermark thresholds to 1500MB low, 100MB high, and 500MB flood stage, allowing the cluster to create indexes even if the disk is almost full.
 
 ```bash
 curl -k -X PUT --user "admin:${OPENSEARCH_PASSWORD}" https://localhost:9200/_cluster/settings -H 'Content-Type: application/json' -d'

TESTING_GUIDE.md

@@ -10,21 +10,21 @@ components:
       description: Whether to pretty format the returned JSON response.
       schema:
         type: boolean
-      default: false
+        default: false
     human:
       name: human
       in: query
       description: Whether to return human readable values for statistics.
       schema:
         type: boolean
-      default: true
+        default: true
     error_trace:
       name: error_trace
       in: query
       description: Whether to include the stack trace of returned errors.
       schema:
         type: boolean
-      default: false
+        default: false
     source:
       name: source
       in: query