Add new release notes scripts for consolidated release notes (opensea…

…rch-project#3601)

Signed-off-by: Peter Zhu <[email protected]>

.shellcheckrc

@@ -0,0 +1 @@
+disable=SC2006,SC2086,SC2207,SC2129

scripts/release-notes/release-notes.sh

@@ -0,0 +1,234 @@
+#!/bin/bash
+
+# Copyright OpenSearch Contributors
+# SPDX-License-Identifier: Apache-2.0
+#
+# The OpenSearch Contributors require contributions made to
+# this file be licensed under the Apache-2.0 license or a
+# compatible open source license.
+
+# This script is to automate the opensearch-build consolidated release notes generation
+
+# On darwin and bsd the given grep/sed is lacking functionalities in gnu grep/gnu sed
+# Will exit if no ggrep/gsed found on the host
+
+### Variables ###
+shopt -s expand_aliases
+ROOT=`dirname "$(realpath $0)"`; echo $ROOT
+INPUT_MANIFEST=$1
+SKIP_ORIG_URLS=$2
+ORIG_URLS_TXT="$ROOT/release-notes-orig-urls.txt"
+ORIG_URLS_ARRAY=()
+RELEASE_NOTE_MD="$ROOT/release-notes-draft.md"
+OLD_IFS=$IFS
+RELEASENOTES_CATEGORIES="BREAKING,FEATURE#ADD,ENHANCE,BUG FIX,INFRASTRUCTURE,DOCUMENT,MAINT,REFACTOR"
+
+### Pre-steps ###
+
+if echo $OSTYPE | grep -qi darwin || echo $OSTYPE | grep -qi bsd; then
+    echo "System is $OSTYPE"
+
+    if ! command -v ggrep > /dev/null; then
+        echo "No ggrep found for darwin/bsd system, exit 1"
+        exit 1
+    fi
+
+    if ! command -v gsed > /dev/null; then
+        echo "No gsed found for darwin/bsd system, exit 1"
+        exit 1
+    fi
+
+    alias grep='ggrep'
+    alias sed='gsed'
+
+    echo "Switch grep to ggrep, sed to gsed on macOS / BSD servers"
+
+fi
+
+if ! command -v yq > /dev/null; then
+    echo "No yq found on the system, exit 1"
+    exit 1
+else
+    yq_version=`yq --version | grep -Eo 'v[0-9.]+' | cut -d. -f1`
+    if [ "$yq_version" != "v4" ]; then
+        echo "You must install v4+ version of yq to run this script, exit 1"
+        exit 1
+    else
+        echo "yq verified: $yq_version"
+    fi
+fi
+
+if [ -z "$INPUT_MANIFEST" ]; then
+    echo "User should provide manifest file as the input, e.g. ../../manifests/2.7.0/opensearch-2.7.0.yml, exit 1"
+    exit 1
+fi
+
+if ! yq -v "$INPUT_MANIFEST" > /dev/null 2>&1; then
+    echo "$INPUT_MANIFEST is either not found, or is not a valid yaml file, exit 1"
+    exit 1
+fi
+
+
+### Main Steps: Get Links ###
+
+# Get all release notes links for all repos
+REPO_NAMES_ARRAY=( `yq -r '.components.[].repository' $INPUT_MANIFEST` )
+REPO_REFS_ARRAY=( `yq -r '.components.[].ref' $INPUT_MANIFEST` )
+COMPONENT_NAMES_ARRAY=( `yq -r '.components.[].name' $INPUT_MANIFEST` )
+BUILD_VERSION=`yq -r '.build.version' $INPUT_MANIFEST`
+NOT_FOUND=0
+
+
+if [ "$SKIP_ORIG_URLS" != 'true' ]; then
+
+    echo -e "\nRetrieve release notes links for all components\n"
+    echo -n > $ORIG_URLS_TXT
+
+    for index in "${!REPO_NAMES_ARRAY[@]}"; do
+        repo_name=`echo ${REPO_NAMES_ARRAY[$index]} | rev | cut -d/ -f1 | rev | sed 's/.git//g'`
+        repo_ref=`echo ${REPO_REFS_ARRAY[$index]} | sed 's/tags\///g'`
+        component_name=`echo ${COMPONENT_NAMES_ARRAY[$index]} | tr '[:upper:]' '[:lower:]'`
+        component_name_prefix=""
+        build_version="$BUILD_VERSION.0"
+
+        # OS / OSD core
+        if [ "$component_name" = "opensearch" ] || [ "$component_name" = "opensearch-dashboards" ]; then
+            build_version=$BUILD_VERSION
+            continue
+        fi
+
+        # OS defaults
+        if (echo $INPUT_MANIFEST | grep -q "opensearch-$BUILD_VERSION") && ( ! echo $component_name | grep -q "opensearch-"); then
+            component_name_prefix="opensearch-"
+        fi
+
+        # OSD defaults
+        if (echo $INPUT_MANIFEST | grep -q "opensearch-dashboards-$BUILD_VERSION") && [ "$component_name" != "opensearch-dashboards" ]; then
+            component_name="opensearch-${repo_name}"
+        fi
+
+        # OS plugins
+        if [ "$component_name" = "opensearch" ]; then
+            component_name_prefix=""
+        fi
+        if [ "$component_name" = "job-scheduler" ]; then
+            component_name_prefix="opensearch."
+        fi
+        if [ "$component_name" = "k-nn" ]; then
+            component_name="knn"
+        fi
+        if [ "$component_name" = "ml-commons" ]; then
+            component_name="ml-common"
+        fi
+        if [ "$component_name" = "opensearch-reports" ]; then
+            component_name="opensearch-reporting"
+        fi
+        if [ "$repo_name" = "opensearch-dashboards-functional-test" ] || [ "$component_name" = "notifications-core" ]; then
+            continue
+        fi
+
+        # OSD plugins
+        if [ "$repo_name" = "dashboards-observability" ]; then
+            component_name=$repo_name
+            if [ "$BUILD_VERSION" = "2.8.0" ]; then
+                component_name=${repo_name//dashboards/opensearch}
+            fi
+        fi
+        if [ "$repo_name" = "dashboards-query-workbench" ] || [ "$repo_name" = "index-management-dashboards-plugin" ]; then
+            component_name="${repo_name//dashboards/opensearch}"
+        fi
+        if [ "$repo_name" = "anomaly-detection-dashboards-plugin" ] || [ "$repo_name" = "security-analytics-dashboards-plugin" ]; then
+            component_name="opensearch-${repo_name//-plugin/}"
+        fi
+        if [ "$repo_name" = "index-management-dashboards-plugin" ]; then
+            component_name="opensearch-$repo_name"
+        fi
+
+        release_note_url="https://raw.githubusercontent.com/opensearch-project/$repo_name/$repo_ref/release-notes/$component_name_prefix$component_name.release-notes-$build_version.md" 
+
+        if curl --output /dev/null --silent --fail -r 0-0 "$release_note_url"; then
+            echo "Release notes found: $release_note_url"
+            ORIG_URLS_ARRAY+=("$release_note_url")
+        else
+            NOT_FOUND=$(( NOT_FOUND + 1 ))
+            echo "Release notes NOT found: $release_note_url"
+        fi
+
+
+    done
+
+
+    # Results
+    if [ "$NOT_FOUND" -eq 0 ]; then
+        echo -e "\nRelease Note found for all components"
+        echo "Write all linkes to local file: $ORIG_URLS_TXT"
+        echo "${ORIG_URLS_ARRAY[@]}" | tr ' ' '\n' > $ORIG_URLS_TXT
+    else
+        echo -e "\nRelease Note NOT FOUND for $NOT_FOUND components, exit 1"
+        exit 1
+    fi
+
+else
+    echo -e "\nSkip release notes link retrieval and use the existing links in $ORIG_URLS_TXT\n"
+
+fi
+
+### Main Stage: Analyze and Re-order the Release Notes ###
+
+echo -e "Clean up urls in the file\n"
+# Removen empty space and newlines here and sort
+sed -e 's/^[ \t]*//' -e '/^$/d' $ORIG_URLS_TXT | sort | uniq > $ORIG_URLS_TXT.tweaks
+echo -e "Release Notes $BUILD_VERSION\n\n" > $RELEASE_NOTE_MD 
+
+IFS=","
+for category in $RELEASENOTES_CATEGORIES; do
+    echo $category
+    cat_main=`echo $category | cut -d '#' -f 1`
+    echo "## $cat_main" >> $RELEASE_NOTE_MD
+    echo "" >> $RELEASE_NOTE_MD
+
+    IFS="#"
+    cat_temp_complete=0
+    for cat_temp in $category; do
+        echo $cat_temp
+        if [ "$cat_temp_complete" -eq 1 ]; then
+            continue
+        fi
+
+        while read -r url; do
+            if [ "${url::1}" != "#" ]; then
+                echo $url
+                curl -sSLo $ORIG_URLS_TXT.plugin_temp $url
+                cat_temp_num=`grep -ni "### $cat_temp" $ORIG_URLS_TXT.plugin_temp | cut -d: -f1`
+                if [ -n "$cat_temp_num" ]; then
+                    # Upper Case every word of the line
+                    echo -e "\n### `echo $url | cut -d/ -f5 | sed -e 's/\b\(.\)/\u\1/g;s/-Plugin//g'`" >> $RELEASE_NOTE_MD
+                    # Upper Case first letter of each line
+                    sed -n "$(( cat_temp_num + 1)),/^$/p" $ORIG_URLS_TXT.plugin_temp | sed 's/\w/\u&/' >> $RELEASE_NOTE_MD
+                    echo "" >> $RELEASE_NOTE_MD
+                    #cat_temp_complete=1
+                fi
+            fi
+        done < $ORIG_URLS_TXT.tweaks
+
+    done
+    IFS=","
+done
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+