From 0e749a2c259561b1bee5b6a384dd73cb97cbee5c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=81lex=20Ruiz?= Date: Wed, 27 Dec 2023 19:07:15 +0100 Subject: [PATCH 1/9] Add tools to assemble DEB packages --- distribution/packages/src/deb/Makefile | 19 ++ .../packages/src/deb/debmake_install.sh | 46 +++ scripts/README.md | 87 +++++- scripts/assemble.sh | 292 ++++++++++++------ 4 files changed, 341 insertions(+), 103 deletions(-) create mode 100644 distribution/packages/src/deb/Makefile create mode 100644 distribution/packages/src/deb/debmake_install.sh diff --git a/distribution/packages/src/deb/Makefile b/distribution/packages/src/deb/Makefile new file mode 100644 index 0000000000000..14e4dbd7efc82 --- /dev/null +++ b/distribution/packages/src/deb/Makefile @@ -0,0 +1,19 @@ +# Copyright OpenSearch Contributors +# SPDX-License-Identifier: Apache-2.0 +# +# The OpenSearch Contributors require contributions made to +# this file be licensed under the Apache-2.0 license or a +# compatible open source license. + +# deb opensearch Makefile + +all: install + +install: + ./debmake_install.sh $(CURDIR) + +clean: ; + +distclean: clean + +.PHONY: all clean distclean install \ No newline at end of file diff --git a/distribution/packages/src/deb/debmake_install.sh b/distribution/packages/src/deb/debmake_install.sh new file mode 100644 index 0000000000000..4b10a36e001c4 --- /dev/null +++ b/distribution/packages/src/deb/debmake_install.sh @@ -0,0 +1,46 @@ +#!/bin/bash + +# Copyright OpenSearch Contributors +# SPDX-License-Identifier: Apache-2.0 +# +# The OpenSearch Contributors require contributions made to +# this file be licensed under the Apache-2.0 license or a +# compatible open source license. + +# debmake opensearch install script + +set -ex + +if [ -z "$1" ]; then + echo "Missing curdir path" + exit 1 +fi + +curdir=$1 +product_dir=/usr/share/wazuh-indexer +# config_dir=/etc/wazuh-indexer +data_dir=/var/lib/wazuh-indexer +log_dir=/var/log/wazuh-indexer +pid_dir=/var/run/wazuh-indexer +buildroot=${curdir}/debian/wazuh-indexer + +# Create necessary directories +mkdir -p "${buildroot}" +mkdir -p "${buildroot}${pid_dir}" +mkdir -p "${buildroot}${product_dir}/plugins" + +# Install directories/files +cp -a "${curdir}"/etc "${curdir}"/usr "${curdir}"/var "${buildroot}"/ +chmod -c 0755 "${buildroot}${product_dir}"/bin/* +if [ -d "${buildroot}${product_dir}"/plugins/opensearch-security ]; then + chmod -c 0755 "${buildroot}${product_dir}"/plugins/opensearch-security/tools/* +fi + +# Symlinks (do not symlink config dir as security demo installer has dependency, if no presense it will switch to rpm/deb mode) +ln -s ${data_dir} "${buildroot}${product_dir}/data" +ln -s ${log_dir} "${buildroot}${product_dir}/logs" + +# Change Permissions +chmod -Rf a+rX,u+w,g-w,o-w "${buildroot}"/* + +exit 0 diff --git a/scripts/README.md b/scripts/README.md index e71941bed4085..22fd7f08de532 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -43,8 +43,93 @@ The generated package is sent to `artifacts/` +### DEB + +The script will: + +- Extract the deb package using `ar` and `tar` tools. + + > By default, `ar` and `tar` tools expect the package to be in `wazuh-indexer/artifacts/tmp/deb`. The script takes care of creating the required folder structure, copying also the min package and the Makefile. + + Current folder loadout at this stage: + ``` + /deb/$ARCH + /etc + /usr + /var + data.tar.gz + debmake_install + Makefile + wazuh-indexer-min-*.deb + ``` + + `usr`, `etc` and `var` folders contain `wazuh-indexer` files, extracted from `wazuh-indexer-min-*.deb`. + `Makefile` and the `debmake_install` are copied over from `wazuh-indexer/distribution/packages/src/deb`. + The `wazuh-indexer-performance-analyzer.service` file is also copied from the same folder. It is a dependency of the SPEC file. + +- Install the plugins using the `opensearch-plugin` CLI tool. +- Set up configuration files. + + > Included in `min-package`. Default files are overwritten. + +- Bundle a DEB file with `debmake` and the `Makefile`. + + > `rpmbuild` is invoked from `wazuh-indexer/artifacts/tmp/rpm`. It creates the {BUILD,RPMS,SOURCES,SRPMS,SPECS,TMP} folders and applies the rules in the SPEC file. If successful, `rpmbuild` will generate the package in the `RPMS/` folder. The script will copy it to `wazuh-indexer/artifacts/dist` and clean: remove the `tmp\` folder and its contents. + + Current folder loadout at this stage: + ``` + /deb/$ARCH + /{BUILD,RPMS,SOURCES,SRPMS,SPECS,TMP} + /etc + /usr + /var + wazuh-indexer-min-*.deb + wazuh-indexer.rpm.spec + ``` + +### Running in Act + +```console +act -j assemble -W .github/workflows/build.yml --artifact-server-path ./artifacts --matrix distribution:deb --matrix architecture:x64 --var OPENSEARCH_VERSION=2.11.0 + +[Build slim packages/build] 🏁 Job succeeded +``` + +#### Running in Docker + +Pre-requisites: + +* Current directory: `wazuh-indexer/` +* Existing deb package in `wazuh-indexer/artifacts/dist/deb`, as a result of the _Build_ stage. + +```console +MIN_PKG_PATH="./artifacts" +docker run --rm \ + -v ./scripts/:/home/wazuh-indexer/scripts \ + -v $MIN_PKG_PATH:/home/wazuh-indexer/artifacts \ + -v ./distribution/packages/src:/home/wazuh-indexer/distribution/packages/src \ + -w /home/wazuh-indexer \ + -it ubuntu:jammy /bin/bash + +# https://github.com/opensearch-project/opensearch-build/blob/2.11.1/docker/ci/dockerfiles/current/build.ubuntu2004.opensearch.x64.arm64.dockerfile + +# Install necessary packages +apt-get update -y && apt-get upgrade -y && apt-get install -y curl build-essential curl && \ + apt-get install -y debmake debhelper-compat && \ + apt-get install -y libxrender1 libxtst6 libasound2 libxi6 libgconf-2-4 && \ + apt-get install -y libatk1.0-0 libatk-bridge2.0-0 libcups2 libdrm2 libatspi2.0-dev libxcomposite-dev libxdamage1 libxfixes3 libxfixes-dev libxrandr2 libgbm-dev libxkbcommon-x11-0 libpangocairo-1.0-0 libcairo2 libcairo2-dev libnss3 libnspr4 libnspr4-dev freeglut3 && \ + apt-get clean -y + +# Install aptly and required changes to debmake +# Remove lintian for now due to it takes nearly 20 minutes for OpenSearch as well as nearly an hour for OpenSearch-Dashboards during debmake +curl -o- https://www.aptly.info/pubkey.txt | apt-key add - && \ + echo "deb http://repo.aptly.info/ squeeze main" | tee -a /etc/apt/sources.list.d/aptly.list && \ + apt-get update -y && apt-get install -y aptly && apt-get clean -y && \ + dpkg -r lintian + +bash scripts/assemble.sh -v 2.11.0 -p linux -a x64 -d deb +``` ### RPM diff --git a/scripts/assemble.sh b/scripts/assemble.sh index dba52e4181d8c..99c553e69d88e 100755 --- a/scripts/assemble.sh +++ b/scripts/assemble.sh @@ -9,27 +9,36 @@ set -ex +# Minimum required plugins plugins=( - "alerting" # "opensearch-alerting" - "opensearch-job-scheduler" - "opensearch-anomaly-detection" # requires "opensearch-job-scheduler" - "asynchronous-search" # "opensearch-asynchronous-search" - "opensearch-cross-cluster-replication" - "geospatial" # "opensearch-geospatial" - "opensearch-index-management" - "opensearch-knn" - "opensearch-ml-plugin" # "opensearch-ml" - "neural-search" # "opensearch-neural-search" - "opensearch-notifications-core" - "notifications" # "opensearch-notifications" requires "opensearch-notifications-core" - "opensearch-observability" - "performance-analyzer" # "opensearch-performance-analyzer" - "opensearch-reports-scheduler" + "performance-analyzer" "opensearch-security" - "opensearch-security-analytics" - "opensearch-sql-plugin" # "opensearch-sql" ) +# plugins=( +# "alerting" # "opensearch-alerting" +# "opensearch-job-scheduler" +# "opensearch-anomaly-detection" # requires "opensearch-job-scheduler" +# "asynchronous-search" # "opensearch-asynchronous-search" +# "opensearch-cross-cluster-replication" +# "geospatial" # "opensearch-geospatial" +# "opensearch-index-management" +# "opensearch-knn" +# "opensearch-ml-plugin" # "opensearch-ml" +# "neural-search" # "opensearch-neural-search" +# "opensearch-notifications-core" +# "notifications" # "opensearch-notifications" requires "opensearch-notifications-core" +# "opensearch-observability" +# "performance-analyzer" # "opensearch-performance-analyzer" +# "opensearch-reports-scheduler" +# "opensearch-security" +# "opensearch-security-analytics" +# "opensearch-sql-plugin" # "opensearch-sql" +# ) + +# ==== +# Usage +# ==== function usage() { echo "Usage: $0 [args]" echo "" @@ -43,129 +52,109 @@ function usage() { echo -e "-h help" } -while getopts ":h:v:q:o:p:a:d:" arg; do - case $arg in - h) - usage - exit 1 - ;; - v) - VERSION=$OPTARG - ;; - q) - QUALIFIER=$OPTARG - ;; - o) - OUTPUT=$OPTARG - ;; - p) - PLATFORM=$OPTARG - ;; - a) - ARCHITECTURE=$OPTARG - ;; - d) - DISTRIBUTION=$OPTARG - ;; - :) - echo "Error: -${OPTARG} requires an argument" +# ==== +# Parse arguments +# ==== +function parse_args() { + + while getopts ":h:v:q:o:p:a:d:" arg; do + case $arg in + h) + usage + exit 1 + ;; + v) + VERSION=$OPTARG + ;; + q) + QUALIFIER=$OPTARG + ;; + o) + OUTPUT=$OPTARG + ;; + p) + PLATFORM=$OPTARG + ;; + a) + ARCHITECTURE=$OPTARG + ;; + d) + DISTRIBUTION=$OPTARG + ;; + :) + echo "Error: -${OPTARG} requires an argument" + usage + exit 1 + ;; + ?) + echo "Invalid option: -${arg}" + exit 1 + ;; + esac + done + + if [ -z "$VERSION" ]; then + echo "Error: You must specify the OpenSearch version" usage exit 1 - ;; - ?) - echo "Invalid option: -${arg}" - exit 1 - ;; - esac -done + fi -if [ -z "$VERSION" ]; then - echo "Error: You must specify the OpenSearch version" - usage - exit 1 -fi + [ -z "$OUTPUT" ] && OUTPUT=artifacts -[ -z "$OUTPUT" ] && OUTPUT=artifacts + # Assemble distribution artifact + # see https://github.com/opensearch-project/OpenSearch/blob/main/settings.gradle#L34 for other distribution targets -# Assemble distribution artifact -# see https://github.com/opensearch-project/OpenSearch/blob/main/settings.gradle#L34 for other distribution targets + [ -z "$PLATFORM" ] && PLATFORM=$(uname -s | awk '{print tolower($0)}') + [ -z "$ARCHITECTURE" ] && ARCHITECTURE=$(uname -m) + [ -z "$DISTRIBUTION" ] && DISTRIBUTION="tar" -[ -z "$PLATFORM" ] && PLATFORM=$(uname -s | awk '{print tolower($0)}') -[ -z "$ARCHITECTURE" ] && ARCHITECTURE=$(uname -m) -[ -z "$DISTRIBUTION" ] && DISTRIBUTION="tar" - -case $PLATFORM-$DISTRIBUTION-$ARCHITECTURE in + case $PLATFORM-$DISTRIBUTION-$ARCHITECTURE in linux-tar-x64 | darwin-tar-x64) PACKAGE="tar" EXT="tar.gz" - # TYPE="archives" TARGET="$PLATFORM-$PACKAGE" SUFFIX="$PLATFORM-x64" ;; linux-tar-arm64 | darwin-tar-arm64) PACKAGE="tar" EXT="tar.gz" - # TYPE="archives" TARGET="$PLATFORM-arm64-$PACKAGE" SUFFIX="$PLATFORM-arm64" ;; linux-deb-x64) PACKAGE="deb" EXT="deb" - # TYPE="packages" TARGET="deb" SUFFIX="amd64" ;; linux-deb-arm64) PACKAGE="deb" EXT="deb" - # TYPE="packages" TARGET="arm64-deb" SUFFIX="arm64" ;; linux-rpm-x64) PACKAGE="rpm" EXT="rpm" - # TYPE="packages" TARGET="rpm" SUFFIX="x86_64" ;; linux-rpm-arm64) PACKAGE="rpm" EXT="rpm" - # TYPE="packages" TARGET="arm64-rpm" SUFFIX="aarch64" ;; - # windows-zip-x64) - # PACKAGE="zip" - # EXT="zip" - # # TYPE="archives" - # TARGET="$PLATFORM-$PACKAGE" - # SUFFIX="$PLATFORM-x64" - # ;; - # windows-zip-arm64) - # PACKAGE="zip" - # EXT="zip" - # # TYPE="archives" - # TARGET="$PLATFORM-arm64-$PACKAGE" - # SUFFIX="$PLATFORM-arm64" - # ;; *) echo "Unsupported platform-distribution-architecture combination: $PLATFORM-$DISTRIBUTION-$ARCHITECTURE" exit 1 ;; -esac - -echo "Assembling OpenSearch for $PLATFORM-$DISTRIBUTION-$ARCHITECTURE" -# wazuh-indexer-min_4.9.0-1-x64_78fcc3db6a5b470294319e48b58c3d715bee39d1.rpm -ARTIFACT_BUILD_NAME=$(ls "${OUTPUT}/dist/" | grep "wazuh-indexer-min.*.$EXT") - -# Create temporal directory and copy the min package there for extraction -TMP_DIR="${OUTPUT}/tmp/${TARGET}" -mkdir -p "$TMP_DIR" -cp "${OUTPUT}/dist/$ARTIFACT_BUILD_NAME" "${TMP_DIR}" + esac +} +# ==== +# Tar assemble +# ==== function assemble_tar() { cd "${TMP_DIR}" PATH_CONF="./config" @@ -203,7 +192,9 @@ function assemble_tar() { echo "After execution, shell path is $(pwd)" } - +# ==== +# RPM assemble +# ==== function assemble_rpm() { # Copy spec cp "distribution/packages/src/rpm/wazuh-indexer.rpm.spec" "${TMP_DIR}" @@ -212,12 +203,13 @@ function assemble_rpm() { cp "distribution/packages/src/rpm/wazuh-indexer-performance-analyzer.service" "${TMP_DIR}"/usr/lib/systemd/system cd "${TMP_DIR}" + local src_path="./usr/share/wazuh-indexer" PATH_CONF="./etc/wazuh-indexer" - PATH_BIN="./usr/share/wazuh-indexer/bin" + PATH_BIN="${src_path}/bin" # Extract min-package. Creates usr/, etc/ and var/ in the current directory echo "Extract ${ARTIFACT_BUILD_NAME} archive" - rpm2cpio "${ARTIFACT_BUILD_NAME}" | cpio -imdv + rpm2cpio "${ARTIFACT_BUILD_NAME}" | cpio -imdv # Install plugins from Maven repository echo "Install plugins" @@ -226,6 +218,11 @@ function assemble_rpm() { OPENSEARCH_PATH_CONF=$PATH_CONF "${PATH_BIN}/opensearch-plugin" install --batch --verbose "${plugin_from_maven}" done + # Move performance-analyzer-rca to its final location + local rca_src="${src_path}/plugins/opensearch-performance-analyzer/performance-analyzer-rca" + local rca_dest="${src_path}" + mv "${rca_src}" "${rca_dest}" + # Set up configuration files cp $PATH_CONF/security/* $PATH_CONF/opensearch-security/ cp $PATH_CONF/jvm.prod.options $PATH_CONF/jvm.options @@ -251,19 +248,105 @@ function assemble_rpm() { --define "_architecture ${SUFFIX}" \ ${spec_file} - # Move to the root folder, copy the package and clean. + # Move to the root folder, copy the package and clean. cd ../../.. package_name="wazuh-indexer-${version}-1.${SUFFIX}.${EXT}" cp "${TMP_DIR}/RPMS/${SUFFIX}/${package_name}" "${OUTPUT}/dist/" - + echo "Cleaning temporary ${TMP_DIR} folder" - rm -r "${TMP_DIR}" + rm -r "artifacts/tmp" + echo "After execution, shell path is $(pwd)" + # Store package's name to file. Used by GH Action. + echo "${package_name}" >"${OUTPUT}/artifact_name.txt" +} + +# ==== +# DEB assemble +# ==== +function assemble_deb() { + # Copy spec + cp "distribution/packages/src/deb/Makefile" "${TMP_DIR}" + cp "distribution/packages/src/deb/debmake_install.sh" "${TMP_DIR}" + chmod a+x "${TMP_DIR}/debmake_install.sh" + # Copy performance analyzer service file + mkdir -p "${TMP_DIR}"/usr/lib/systemd/system + cp "distribution/packages/src/rpm/wazuh-indexer-performance-analyzer.service" "${TMP_DIR}"/usr/lib/systemd/system + + cd "${TMP_DIR}" + local src_path="./usr/share/wazuh-indexer" + PATH_CONF="./etc/wazuh-indexer" + PATH_BIN="${src_path}/bin" + + # Extract min-package. Creates usr/, etc/ and var/ in the current directory + echo "Extract ${ARTIFACT_BUILD_NAME} archive" + ar xf "${ARTIFACT_BUILD_NAME}" data.tar.gz + tar zvxf data.tar.gz + + # Install plugins from Maven repository + echo "Install plugins" + for plugin in "${plugins[@]}"; do + plugin_from_maven="org.opensearch.plugin:${plugin}:$VERSION.0" + OPENSEARCH_PATH_CONF=$PATH_CONF "${PATH_BIN}/opensearch-plugin" install --batch --verbose "${plugin_from_maven}" + done + + # Move performance-analyzer-rca to its final location + local rca_src="${src_path}/plugins/opensearch-performance-analyzer/performance-analyzer-rca" + local rca_dest="${src_path}" + mv "${rca_src}" "${rca_dest}" + + # Set up configuration files + cp $PATH_CONF/security/* $PATH_CONF/opensearch-security/ + cp $PATH_CONF/jvm.prod.options $PATH_CONF/jvm.options + cp $PATH_CONF/opensearch.prod.yml $PATH_CONF/opensearch.yml + + rm -r $PATH_CONF/security + rm $PATH_CONF/jvm.prod.options $PATH_CONF/opensearch.prod.yml + + # Remove symbolic links and bat files + find . -type l -exec rm -rf {} \; + find . -name "*.bat" -exec rm -rf {} \; + + # Generate final package + local version + version=$(cat ./usr/share/wazuh-indexer/VERSION) + debmake \ + --fullname "Wazuh Team" \ + --email "hello@wazuh.com" \ + --invoke debuild \ + --package wazuh-indexer \ + --native \ + --revision 1 \ + --upstreamversion "${version}" + + # Move to the root folder, copy the package and clean. + cd ../../.. + package_name="wazuh-indexer_${version}_${SUFFIX}.${EXT}" + # debmake creates the package one level up + cp "${TMP_DIR}/../${package_name}" "${OUTPUT}/dist/" + + echo "Cleaning temporary ${TMP_DIR} folder" + rm -r "artifacts/tmp" echo "After execution, shell path is $(pwd)" # Store package's name to file. Used by GH Action. - echo "${package_name}" > "${OUTPUT}/artifact_name.txt" + echo "${package_name}" >"${OUTPUT}/artifact_name.txt" } -case $SUFFIX.$EXT in +# ==== +# Main function +# ==== +function main() { + parse_args "${@}" + + echo "Assembling OpenSearch for $PLATFORM-$DISTRIBUTION-$ARCHITECTURE" + # wazuh-indexer-min_4.9.0-1-x64_78fcc3db6a5b470294319e48b58c3d715bee39d1.rpm + ARTIFACT_BUILD_NAME=$(ls "${OUTPUT}/dist/" | grep "wazuh-indexer-min.*.$EXT") + + # Create temporal directory and copy the min package there for extraction + TMP_DIR="${OUTPUT}/tmp/${TARGET}" + mkdir -p "$TMP_DIR" + cp "${OUTPUT}/dist/$ARTIFACT_BUILD_NAME" "${TMP_DIR}" + + case $SUFFIX.$EXT in linux-arm64.tar.gz) assemble_tar ;; @@ -277,7 +360,12 @@ case $SUFFIX.$EXT in assemble_rpm ;; amd64.deb) + assemble_deb ;; arm64.deb) + assemble_deb ;; -esac + esac +} + +main "${@}" From 1c358da06f737b3e5a5166068bf05d0ac98146f9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=81lex=20Ruiz?= Date: Wed, 27 Dec 2023 19:11:26 +0100 Subject: [PATCH 2/9] Move wazuh-indexer-performance-analyzer.service to common --- .../wazuh-indexer-performance-analyzer.service | 0 scripts/assemble.sh | 6 +++--- 2 files changed, 3 insertions(+), 3 deletions(-) rename distribution/packages/src/{rpm => common}/wazuh-indexer-performance-analyzer.service (100%) diff --git a/distribution/packages/src/rpm/wazuh-indexer-performance-analyzer.service b/distribution/packages/src/common/wazuh-indexer-performance-analyzer.service similarity index 100% rename from distribution/packages/src/rpm/wazuh-indexer-performance-analyzer.service rename to distribution/packages/src/common/wazuh-indexer-performance-analyzer.service diff --git a/scripts/assemble.sh b/scripts/assemble.sh index 99c553e69d88e..8846524d464e5 100755 --- a/scripts/assemble.sh +++ b/scripts/assemble.sh @@ -200,7 +200,7 @@ function assemble_rpm() { cp "distribution/packages/src/rpm/wazuh-indexer.rpm.spec" "${TMP_DIR}" # Copy performance analyzer service file mkdir -p "${TMP_DIR}"/usr/lib/systemd/system - cp "distribution/packages/src/rpm/wazuh-indexer-performance-analyzer.service" "${TMP_DIR}"/usr/lib/systemd/system + cp "distribution/packages/src/common/wazuh-indexer-performance-analyzer.service" "${TMP_DIR}"/usr/lib/systemd/system cd "${TMP_DIR}" local src_path="./usr/share/wazuh-indexer" @@ -270,7 +270,7 @@ function assemble_deb() { chmod a+x "${TMP_DIR}/debmake_install.sh" # Copy performance analyzer service file mkdir -p "${TMP_DIR}"/usr/lib/systemd/system - cp "distribution/packages/src/rpm/wazuh-indexer-performance-analyzer.service" "${TMP_DIR}"/usr/lib/systemd/system + cp "distribution/packages/src/common/wazuh-indexer-performance-analyzer.service" "${TMP_DIR}"/usr/lib/systemd/system cd "${TMP_DIR}" local src_path="./usr/share/wazuh-indexer" @@ -321,7 +321,7 @@ function assemble_deb() { # Move to the root folder, copy the package and clean. cd ../../.. package_name="wazuh-indexer_${version}_${SUFFIX}.${EXT}" - # debmake creates the package one level up + # debmake creates the package one level above cp "${TMP_DIR}/../${package_name}" "${OUTPUT}/dist/" echo "Cleaning temporary ${TMP_DIR} folder" From d22440ec598e9770860d8adc41fe4a79905303e8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=81lex=20Ruiz?= Date: Wed, 27 Dec 2023 19:13:12 +0100 Subject: [PATCH 3/9] Enable assembly of DEB packages --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 1483793de3f68..0789b0e2bd5af 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -50,7 +50,7 @@ jobs: exclude: # skip arm64 until we have arm runners - architecture: arm64 - - distribution: [tar, deb] # Exclude deb assembly until it's implemented + - distribution: [tar] uses: ./.github/workflows/r_assemble.yml with: From c6c74582060d0bc4f945e12a0e5dbbf22969d548 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=81lex=20Ruiz?= Date: Wed, 27 Dec 2023 19:17:49 +0100 Subject: [PATCH 4/9] Enable full set of plugins --- scripts/assemble.sh | 44 ++++++++++++++++++++++---------------------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/scripts/assemble.sh b/scripts/assemble.sh index 8846524d464e5..de417d76e972f 100755 --- a/scripts/assemble.sh +++ b/scripts/assemble.sh @@ -10,32 +10,32 @@ set -ex # Minimum required plugins -plugins=( - "performance-analyzer" - "opensearch-security" -) - # plugins=( -# "alerting" # "opensearch-alerting" -# "opensearch-job-scheduler" -# "opensearch-anomaly-detection" # requires "opensearch-job-scheduler" -# "asynchronous-search" # "opensearch-asynchronous-search" -# "opensearch-cross-cluster-replication" -# "geospatial" # "opensearch-geospatial" -# "opensearch-index-management" -# "opensearch-knn" -# "opensearch-ml-plugin" # "opensearch-ml" -# "neural-search" # "opensearch-neural-search" -# "opensearch-notifications-core" -# "notifications" # "opensearch-notifications" requires "opensearch-notifications-core" -# "opensearch-observability" -# "performance-analyzer" # "opensearch-performance-analyzer" -# "opensearch-reports-scheduler" +# "performance-analyzer" # "opensearch-security" -# "opensearch-security-analytics" -# "opensearch-sql-plugin" # "opensearch-sql" # ) +plugins=( + "alerting" # "opensearch-alerting" + "opensearch-job-scheduler" + "opensearch-anomaly-detection" # requires "opensearch-job-scheduler" + "asynchronous-search" # "opensearch-asynchronous-search" + "opensearch-cross-cluster-replication" + "geospatial" # "opensearch-geospatial" + "opensearch-index-management" + "opensearch-knn" + "opensearch-ml-plugin" # "opensearch-ml" + "neural-search" # "opensearch-neural-search" + "opensearch-notifications-core" + "notifications" # "opensearch-notifications" requires "opensearch-notifications-core" + "opensearch-observability" + "performance-analyzer" # "opensearch-performance-analyzer" + "opensearch-reports-scheduler" + "opensearch-security" + "opensearch-security-analytics" + "opensearch-sql-plugin" # "opensearch-sql" +) + # ==== # Usage # ==== From 246f2d57b0c266a0b96da61e1379e82bdce225ca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=81lex=20Ruiz?= Date: Wed, 27 Dec 2023 19:31:53 +0100 Subject: [PATCH 5/9] Actually skip tar assembly --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 0789b0e2bd5af..4b286884541c7 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -50,7 +50,7 @@ jobs: exclude: # skip arm64 until we have arm runners - architecture: arm64 - - distribution: [tar] + - distribution: tar uses: ./.github/workflows/r_assemble.yml with: From eeb2c425a8f727bdf9cd2adca3fdb018392c4c06 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=81lex=20Ruiz?= Date: Wed, 3 Jan 2024 16:28:55 +0100 Subject: [PATCH 6/9] Add installation of dependencies for DEB assembly --- .github/workflows/r_assemble.yml | 5 +++++ scripts/README.md | 4 +++- scripts/provision.sh | 10 ++++++++++ 3 files changed, 18 insertions(+), 1 deletion(-) create mode 100644 scripts/provision.sh diff --git a/.github/workflows/r_assemble.yml b/.github/workflows/r_assemble.yml index c807a58a4f781..2cbf33290620f 100644 --- a/.github/workflows/r_assemble.yml +++ b/.github/workflows/r_assemble.yml @@ -39,6 +39,11 @@ jobs: name: ${{ inputs.min }} path: artifacts/dist + - name: Provision + if: ${{ inputs.distribution == 'deb' }} + run: + bash scripts/provision.sh + - name: Run `assemble.sh` run: | bash scripts/assemble.sh -v ${{ vars.OPENSEARCH_VERSION }} -p linux -a ${{ inputs.architecture }} -d ${{ inputs.distribution }} diff --git a/scripts/README.md b/scripts/README.md index 22fd7f08de532..9546bee1ee456 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -75,8 +75,10 @@ The script will: - Bundle a DEB file with `debmake` and the `Makefile`. - > `rpmbuild` is invoked from `wazuh-indexer/artifacts/tmp/rpm`. It creates the {BUILD,RPMS,SOURCES,SRPMS,SPECS,TMP} folders and applies the rules in the SPEC file. If successful, `rpmbuild` will generate the package in the `RPMS/` folder. The script will copy it to `wazuh-indexer/artifacts/dist` and clean: remove the `tmp\` folder and its contents. + > `debmake` and other dependencies can be installed using the provision.sh script. The + > script is invoked by the GitHub Workflow. + Current folder loadout at this stage: ``` /deb/$ARCH diff --git a/scripts/provision.sh b/scripts/provision.sh new file mode 100644 index 0000000000000..2d424931071d0 --- /dev/null +++ b/scripts/provision.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +# Provision script for assembly of DEB packages + +# Install necessary packages +apt-get update -y && apt-get upgrade -y && apt-get install -y curl build-essential curl && \ + apt-get install -y debmake debhelper-compat && \ + apt-get install -y libxrender1 libxtst6 libasound2 libxi6 libgconf-2-4 && \ + apt-get install -y libatk1.0-0 libatk-bridge2.0-0 libcups2 libdrm2 libatspi2.0-dev libxcomposite-dev libxdamage1 libxfixes3 libxfixes-dev libxrandr2 libgbm-dev libxkbcommon-x11-0 libpangocairo-1.0-0 libcairo2 libcairo2-dev libnss3 libnspr4 libnspr4-dev freeglut3 && \ + apt-get clean -y \ No newline at end of file From c6ff8347bb73cae9d9a979a5aa9291e14e825ce0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=81lex=20Ruiz?= Date: Wed, 3 Jan 2024 17:39:14 +0100 Subject: [PATCH 7/9] Install dependencies using sudo --- .github/workflows/r_assemble.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/r_assemble.yml b/.github/workflows/r_assemble.yml index 2cbf33290620f..7dc7133117318 100644 --- a/.github/workflows/r_assemble.yml +++ b/.github/workflows/r_assemble.yml @@ -42,7 +42,7 @@ jobs: - name: Provision if: ${{ inputs.distribution == 'deb' }} run: - bash scripts/provision.sh + sudo bash scripts/provision.sh - name: Run `assemble.sh` run: | From 4ca29438b674e1e2534c634153b8236a729ed6f2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=81lex=20Ruiz?= Date: Thu, 4 Jan 2024 12:57:19 +0100 Subject: [PATCH 8/9] Format files --- .github/workflows/r_assemble.yml | 3 +- docker/README.md | 2 +- scripts/README.md | 170 ++++++++++++++++--------------- scripts/provision.sh | 10 +- 4 files changed, 94 insertions(+), 91 deletions(-) diff --git a/.github/workflows/r_assemble.yml b/.github/workflows/r_assemble.yml index 7dc7133117318..6414729a1a635 100644 --- a/.github/workflows/r_assemble.yml +++ b/.github/workflows/r_assemble.yml @@ -41,7 +41,7 @@ jobs: - name: Provision if: ${{ inputs.distribution == 'deb' }} - run: + run: | sudo bash scripts/provision.sh - name: Run `assemble.sh` @@ -60,4 +60,3 @@ jobs: name: ${{ inputs.name }} path: artifacts/dist/${{ steps.get_name.outputs.name }} if-no-files-found: error - diff --git a/docker/README.md b/docker/README.md index 28a58329ab7d7..4b41a42e0e9c9 100644 --- a/docker/README.md +++ b/docker/README.md @@ -1,4 +1,4 @@ -# Frontend development environments +# Indexer development environments Install [Docker Desktop][docker-desktop] as per its instructions, available for Windows, Mac and Linux (Ubuntu, Debian & Fedora). diff --git a/scripts/README.md b/scripts/README.md index 9546bee1ee456..7a7f001ab18db 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -2,9 +2,9 @@ The packages' generation process consists on 2 steps: -* **Build**: compiles the Java application and bundles it into a package. -* **Assembly**: uses the package from the previous step and inflates it with plugins and -configuration files, ready for production deployment. +- **Build**: compiles the Java application and bundles it into a package. +- **Assembly**: uses the package from the previous step and inflates it with plugins and + configuration files, ready for production deployment. We usually generate the packages using GitHub Actions, however, the process is designed to be independent enough for maximum portability. GitHub Actions provides infrastructure, while @@ -27,7 +27,6 @@ act -j build -W .github/workflows/build.yml --artifact-server-path ./artifacts [Build slim packages/build] 🏁 Job succeeded ``` - #### Running in Docker Within the [Docker environment](../docker): @@ -38,57 +37,60 @@ bash scripts/build.sh -v 2.11.0 -s false -p linux -a {x64|arm64} -d {rpm|deb|tar The generated package is sent to `artifacts/` - ## Assemble - + ### DEB The script will: - Extract the deb package using `ar` and `tar` tools. - - > By default, `ar` and `tar` tools expect the package to be in `wazuh-indexer/artifacts/tmp/deb`. The script takes care of creating the required folder structure, copying also the min package and the Makefile. - - Current folder loadout at this stage: - ``` - /deb/$ARCH - /etc - /usr - /var - data.tar.gz - debmake_install - Makefile - wazuh-indexer-min-*.deb - ``` - - `usr`, `etc` and `var` folders contain `wazuh-indexer` files, extracted from `wazuh-indexer-min-*.deb`. - `Makefile` and the `debmake_install` are copied over from `wazuh-indexer/distribution/packages/src/deb`. - The `wazuh-indexer-performance-analyzer.service` file is also copied from the same folder. It is a dependency of the SPEC file. + + > By default, `ar` and `tar` tools expect the package to be in `wazuh-indexer/artifacts/tmp/deb`. The script takes care of creating the required folder structure, copying also the min package and the Makefile. + + Current folder loadout at this stage: + + ``` + /deb/$ARCH + /etc + /usr + /var + data.tar.gz + debmake_install + Makefile + wazuh-indexer-min-*.deb + ``` + + `usr`, `etc` and `var` folders contain `wazuh-indexer` files, extracted from `wazuh-indexer-min-*.deb`. + `Makefile` and the `debmake_install` are copied over from `wazuh-indexer/distribution/packages/src/deb`. + The `wazuh-indexer-performance-analyzer.service` file is also copied from the same folder. It is a dependency of the SPEC file. - Install the plugins using the `opensearch-plugin` CLI tool. - Set up configuration files. - > Included in `min-package`. Default files are overwritten. - + > Included in `min-package`. Default files are overwritten. + - Bundle a DEB file with `debmake` and the `Makefile`. - > `debmake` and other dependencies can be installed using the provision.sh script. The - > script is invoked by the GitHub Workflow. + > `debmake` and other dependencies can be installed using the provision.sh script. The + > script is invoked by the GitHub Workflow. - Current folder loadout at this stage: - ``` - /deb/$ARCH - /{BUILD,RPMS,SOURCES,SRPMS,SPECS,TMP} - /etc - /usr - /var - wazuh-indexer-min-*.deb - wazuh-indexer.rpm.spec - ``` + + Current folder loadout at this stage: + + ``` + /deb/$ARCH + /{BUILD,RPMS,SOURCES,SRPMS,SPECS,TMP} + /etc + /usr + /var + wazuh-indexer-min-*.deb + wazuh-indexer.rpm.spec + ``` ### Running in Act @@ -102,8 +104,8 @@ act -j assemble -W .github/workflows/build.yml --artifact-server-path ./artifact Pre-requisites: -* Current directory: `wazuh-indexer/` -* Existing deb package in `wazuh-indexer/artifacts/dist/deb`, as a result of the _Build_ stage. +- Current directory: `wazuh-indexer/` +- Existing deb package in `wazuh-indexer/artifacts/dist/deb`, as a result of the _Build_ stage. ```console MIN_PKG_PATH="./artifacts" @@ -117,17 +119,17 @@ docker run --rm \ # https://github.com/opensearch-project/opensearch-build/blob/2.11.1/docker/ci/dockerfiles/current/build.ubuntu2004.opensearch.x64.arm64.dockerfile # Install necessary packages -apt-get update -y && apt-get upgrade -y && apt-get install -y curl build-essential curl && \ - apt-get install -y debmake debhelper-compat && \ - apt-get install -y libxrender1 libxtst6 libasound2 libxi6 libgconf-2-4 && \ - apt-get install -y libatk1.0-0 libatk-bridge2.0-0 libcups2 libdrm2 libatspi2.0-dev libxcomposite-dev libxdamage1 libxfixes3 libxfixes-dev libxrandr2 libgbm-dev libxkbcommon-x11-0 libpangocairo-1.0-0 libcairo2 libcairo2-dev libnss3 libnspr4 libnspr4-dev freeglut3 && \ - apt-get clean -y +apt-get update -y && apt-get upgrade -y && apt-get install -y curl build-essential curl && + apt-get install -y debmake debhelper-compat && + apt-get install -y libxrender1 libxtst6 libasound2 libxi6 libgconf-2-4 && + apt-get install -y libatk1.0-0 libatk-bridge2.0-0 libcups2 libdrm2 libatspi2.0-dev libxcomposite-dev libxdamage1 libxfixes3 libxfixes-dev libxrandr2 libgbm-dev libxkbcommon-x11-0 libpangocairo-1.0-0 libcairo2 libcairo2-dev libnss3 libnspr4 libnspr4-dev freeglut3 && + apt-get clean -y # Install aptly and required changes to debmake # Remove lintian for now due to it takes nearly 20 minutes for OpenSearch as well as nearly an hour for OpenSearch-Dashboards during debmake -curl -o- https://www.aptly.info/pubkey.txt | apt-key add - && \ - echo "deb http://repo.aptly.info/ squeeze main" | tee -a /etc/apt/sources.list.d/aptly.list && \ - apt-get update -y && apt-get install -y aptly && apt-get clean -y && \ +curl -o- https://www.aptly.info/pubkey.txt | apt-key add - && + echo "deb http://repo.aptly.info/ squeeze main" | tee -a /etc/apt/sources.list.d/aptly.list && + apt-get update -y && apt-get install -y aptly && apt-get clean -y && dpkg -r lintian bash scripts/assemble.sh -v 2.11.0 -p linux -a x64 -d deb @@ -135,50 +137,53 @@ bash scripts/assemble.sh -v 2.11.0 -p linux -a x64 -d deb ### RPM -The `assemble.sh` script will use the output from the `build.sh` script and use it as a -base to bundle together a final package containing the plugins, the production configuration +The `assemble.sh` script will use the output from the `build.sh` script and use it as a +base to bundle together a final package containing the plugins, the production configuration and the service files. The script will: - Extract the rpm package using `rpm2cpio` and `cpio` tools. - - > By default, `rpm2cpio` and `cpio` tools expect the package to be in `wazuh-indexer/artifacts/tmp/rpm`. The script takes care of creating the required folder structure, copying also the min package and the SPEC file. - - Current folder loadout at this stage: - ``` - /rpm/$ARCH - /etc - /usr - /var - wazuh-indexer-min-*.rpm - wazuh-indexer.rpm.spec - ``` - - `usr`, `etc` and `var` folders contain `wazuh-indexer` files, extracted from `wazuh-indexer-min-*.rpm`. - `wazuh-indexer.rpm.spec` is copied over from `wazuh-indexer/distribution/packages/src/rpm/wazuh-indexer.rpm.spec`. - The `wazuh-indexer-performance-analyzer.service` file is also copied from the same folder. It is a dependency of the SPEC file. + + > By default, `rpm2cpio` and `cpio` tools expect the package to be in `wazuh-indexer/artifacts/tmp/rpm`. The script takes care of creating the required folder structure, copying also the min package and the SPEC file. + + Current folder loadout at this stage: + + ``` + /rpm/$ARCH + /etc + /usr + /var + wazuh-indexer-min-*.rpm + wazuh-indexer.rpm.spec + ``` + + `usr`, `etc` and `var` folders contain `wazuh-indexer` files, extracted from `wazuh-indexer-min-*.rpm`. + `wazuh-indexer.rpm.spec` is copied over from `wazuh-indexer/distribution/packages/src/rpm/wazuh-indexer.rpm.spec`. + The `wazuh-indexer-performance-analyzer.service` file is also copied from the same folder. It is a dependency of the SPEC file. - Install the plugins using the `opensearch-plugin` CLI tool. - Set up configuration files. - > Included in `min-package`. Default files are overwritten. - + > Included in `min-package`. Default files are overwritten. + - Bundle an RPM file with `rpmbuild` and the SPEC file `wazuh-indexer.rpm.spec`. - - `rpmbuild` is part of the `rpm` OS package. - > `rpmbuild` is invoked from `wazuh-indexer/artifacts/tmp/rpm`. It creates the {BUILD,RPMS,SOURCES,SRPMS,SPECS,TMP} folders and applies the rules in the SPEC file. If successful, `rpmbuild` will generate the package in the `RPMS/` folder. The script will copy it to `wazuh-indexer/artifacts/dist` and clean: remove the `tmp\` folder and its contents. + - `rpmbuild` is part of the `rpm` OS package. + + > `rpmbuild` is invoked from `wazuh-indexer/artifacts/tmp/rpm`. It creates the {BUILD,RPMS,SOURCES,SRPMS,SPECS,TMP} folders and applies the rules in the SPEC file. If successful, `rpmbuild` will generate the package in the `RPMS/` folder. The script will copy it to `wazuh-indexer/artifacts/dist` and clean: remove the `tmp\` folder and its contents. - Current folder loadout at this stage: - ``` - /rpm/$ARCH - /{BUILD,RPMS,SOURCES,SRPMS,SPECS,TMP} - /etc - /usr - /var - wazuh-indexer-min-*.rpm - wazuh-indexer.rpm.spec - ``` + Current folder loadout at this stage: + + ``` + /rpm/$ARCH + /{BUILD,RPMS,SOURCES,SRPMS,SPECS,TMP} + /etc + /usr + /var + wazuh-indexer-min-*.rpm + wazuh-indexer.rpm.spec + ``` ### Running in Act @@ -192,8 +197,8 @@ act -j assemble -W .github/workflows/build.yml --artifact-server-path ./artifact Pre-requisites: -* Current directory: `wazuh-indexer/` -* Existing rpm package in `wazuh-indexer/artifacts/dist/rpm`, as a result of the _Build_ stage. +- Current directory: `wazuh-indexer/` +- Existing rpm package in `wazuh-indexer/artifacts/dist/rpm`, as a result of the _Build_ stage. ```console MIN_PKG_PATH="./artifacts" @@ -208,4 +213,3 @@ apt-get update apt-get install -y rpm2cpio rpm cpio bash scripts/assemble.sh -v 2.11.0 -p linux -a x64 -d rpm ``` - diff --git a/scripts/provision.sh b/scripts/provision.sh index 2d424931071d0..3cd155dc44fb5 100644 --- a/scripts/provision.sh +++ b/scripts/provision.sh @@ -3,8 +3,8 @@ # Provision script for assembly of DEB packages # Install necessary packages -apt-get update -y && apt-get upgrade -y && apt-get install -y curl build-essential curl && \ - apt-get install -y debmake debhelper-compat && \ - apt-get install -y libxrender1 libxtst6 libasound2 libxi6 libgconf-2-4 && \ - apt-get install -y libatk1.0-0 libatk-bridge2.0-0 libcups2 libdrm2 libatspi2.0-dev libxcomposite-dev libxdamage1 libxfixes3 libxfixes-dev libxrandr2 libgbm-dev libxkbcommon-x11-0 libpangocairo-1.0-0 libcairo2 libcairo2-dev libnss3 libnspr4 libnspr4-dev freeglut3 && \ - apt-get clean -y \ No newline at end of file +apt-get update -y && apt-get upgrade -y && apt-get install -y curl build-essential curl && + apt-get install -y debmake debhelper-compat && + apt-get install -y libxrender1 libxtst6 libasound2 libxi6 libgconf-2-4 && + apt-get install -y libatk1.0-0 libatk-bridge2.0-0 libcups2 libdrm2 libatspi2.0-dev libxcomposite-dev libxdamage1 libxfixes3 libxfixes-dev libxrandr2 libgbm-dev libxkbcommon-x11-0 libpangocairo-1.0-0 libcairo2 libcairo2-dev libnss3 libnspr4 libnspr4-dev freeglut3 && + apt-get clean -y From 6d30b1196fd49131b7f6199a3e63514532794097 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=81lex=20Ruiz?= Date: Thu, 4 Jan 2024 14:06:08 +0100 Subject: [PATCH 9/9] Refactor assemble script --- scripts/README.md | 43 ++++++---- scripts/assemble.sh | 190 ++++++++++++++++++++------------------------ 2 files changed, 114 insertions(+), 119 deletions(-) diff --git a/scripts/README.md b/scripts/README.md index 7a7f001ab18db..8c98f8206cab5 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -54,14 +54,18 @@ The script will: Current folder loadout at this stage: ``` - /deb/$ARCH - /etc - /usr - /var - data.tar.gz - debmake_install - Makefile - wazuh-indexer-min-*.deb + artifacts/ + |-- dist + | |-- wazuh-indexer-min_4.9.0_amd64.deb + `-- tmp + `-- deb + |-- Makefile + |-- data.tar.gz + |-- debmake_install.sh + |-- etc + |-- usr + |-- var + `-- wazuh-indexer-min_4.9.0_amd64.deb ``` `usr`, `etc` and `var` folders contain `wazuh-indexer` files, extracted from `wazuh-indexer-min-*.deb`. @@ -78,18 +82,23 @@ The script will: > `debmake` and other dependencies can be installed using the provision.sh script. The > script is invoked by the GitHub Workflow. - - Current folder loadout at this stage: ``` - /deb/$ARCH - /{BUILD,RPMS,SOURCES,SRPMS,SPECS,TMP} - /etc - /usr - /var - wazuh-indexer-min-*.deb - wazuh-indexer.rpm.spec + artifacts/ + |-- artifact_name.txt + |-- dist + | |-- wazuh-indexer-min_4.9.0_amd64.deb + | `-- wazuh-indexer_4.9.0_amd64.deb + `-- tmp + `-- deb + |-- Makefile + |-- data.tar.gz + |-- debmake_install.sh + |-- etc + |-- usr + |-- var + `-- wazuh-indexer-min_4.9.0_amd64.deb ``` ### Running in Act diff --git a/scripts/assemble.sh b/scripts/assemble.sh index de417d76e972f..fa9205b00f931 100755 --- a/scripts/assemble.sh +++ b/scripts/assemble.sh @@ -18,16 +18,16 @@ set -ex plugins=( "alerting" # "opensearch-alerting" "opensearch-job-scheduler" - "opensearch-anomaly-detection" # requires "opensearch-job-scheduler" - "asynchronous-search" # "opensearch-asynchronous-search" + "opensearch-anomaly-detection" # Requires "opensearch-job-scheduler" + "asynchronous-search" # "opensearch-asynchronous-search" "opensearch-cross-cluster-replication" "geospatial" # "opensearch-geospatial" "opensearch-index-management" "opensearch-knn" "opensearch-ml-plugin" # "opensearch-ml" - "neural-search" # "opensearch-neural-search" + "neural-search" # "opensearch-neural-search" "opensearch-notifications-core" - "notifications" # "opensearch-notifications" requires "opensearch-notifications-core" + "notifications" # "opensearch-notifications". Requires "opensearch-notifications-core" "opensearch-observability" "performance-analyzer" # "opensearch-performance-analyzer" "opensearch-reports-scheduler" @@ -44,7 +44,6 @@ function usage() { echo "" echo "Arguments:" echo -e "-v VERSION\t[Required] OpenSearch version." - echo -e "-q QUALIFIER\t[Optional] Version qualifier." echo -e "-p PLATFORM\t[Optional] Platform, default is 'uname -s'." echo -e "-a ARCHITECTURE\t[Optional] Build architecture, default is 'uname -m'." echo -e "-d DISTRIBUTION\t[Optional] Distribution, default is 'tar'." @@ -57,7 +56,7 @@ function usage() { # ==== function parse_args() { - while getopts ":h:v:q:o:p:a:d:" arg; do + while getopts ":h:v:o:p:a:d:" arg; do case $arg in h) usage @@ -66,9 +65,6 @@ function parse_args() { v) VERSION=$OPTARG ;; - q) - QUALIFIER=$OPTARG - ;; o) OUTPUT=$OPTARG ;; @@ -152,6 +148,63 @@ function parse_args() { esac } +# ==== +# Set up configuration files +# ==== +function add_configuration_files() { + # swap configuration files + cp $PATH_CONF/security/* $PATH_CONF/opensearch-security/ + cp $PATH_CONF/jvm.prod.options $PATH_CONF/jvm.options + cp $PATH_CONF/opensearch.prod.yml $PATH_CONF/opensearch.yml + + rm -r $PATH_CONF/security + rm $PATH_CONF/jvm.prod.options $PATH_CONF/opensearch.prod.yml + + # Remove symbolic links and bat files + find . -type l -exec rm -rf {} \; + find . -name "*.bat" -exec rm -rf {} \; +} + +# ==== +# Copy performance analyzer service file +# ==== +function enable_performance_analyzer() { + mkdir -p "${TMP_DIR}"/usr/lib/systemd/system + cp "distribution/packages/src/common/wazuh-indexer-performance-analyzer.service" "${TMP_DIR}"/usr/lib/systemd/system +} + +# ==== +# Move performance-analyzer-rca to its final location +# ==== +function enable_performance_analyzer_rca() { + local rca_src="${1}/plugins/opensearch-performance-analyzer/performance-analyzer-rca" + local rca_dest="${1}" + mv "${rca_src}" "${rca_dest}" +} + +# ==== +# Install plugins +# ==== +function install_plugins() { + # Install plugins from Maven repository + echo "Install plugins" + for plugin in "${plugins[@]}"; do + plugin_from_maven="org.opensearch.plugin:${plugin}:$VERSION.0" + OPENSEARCH_PATH_CONF=$PATH_CONF "${PATH_BIN}/opensearch-plugin" install --batch --verbose "${plugin_from_maven}" + done +} + +# ==== +# Clean +# ==== +function clean() { + echo "Cleaning temporary ${TMP_DIR} folder" + rm -r "${OUTPUT}/tmp" + echo "After execution, shell path is $(pwd)" + # Store package's name to file. Used by GH Action. + echo "${package_name}" >"${OUTPUT}/artifact_name.txt" +} + # ==== # Tar assemble # ==== @@ -160,36 +213,24 @@ function assemble_tar() { PATH_CONF="./config" PATH_BIN="./bin" - # Step 1: extract + # Extract echo "Extract ${ARTIFACT_BUILD_NAME} archive" tar -zvxf "${ARTIFACT_BUILD_NAME}" cd "$(ls -d wazuh-indexer-*/)" - # Step 2: install plugins - echo "Install plugins" - for plugin in "${plugins[@]}"; do - plugin_from_maven="org.opensearch.plugin:${plugin}:$VERSION.0" - "${PATH_BIN}/opensearch-plugin" install --batch --verbose "${plugin_from_maven}" - done - - # Step 3: swap configuration files - cp $PATH_CONF/security/* $PATH_CONF/opensearch-security/ - cp $PATH_CONF/jvm.prod.options $PATH_CONF/jvm.options - cp $PATH_CONF/opensearch.prod.yml $PATH_CONF/opensearch.yml - - rm -r $PATH_CONF/security - rm $PATH_CONF/jvm.prod.options $PATH_CONF/opensearch.prod.yml + # Install plugins + install_plugins + # Swap configuration files + add_configuration_files - # Step 4: pack + # Pack archive_name="wazuh-indexer-$(cat VERSION)" cd .. tar -cvf "${archive_name}-${SUFFIX}.${EXT}" "${archive_name}" cd ../../.. cp "${TMP_DIR}/${archive_name}-${SUFFIX}.${EXT}" "${OUTPUT}/dist/" - echo "Cleaning temporary ${TMP_DIR} folder" - rm -r "${TMP_DIR}" - echo "After execution, shell path is $(pwd)" + clean } # ==== @@ -199,8 +240,7 @@ function assemble_rpm() { # Copy spec cp "distribution/packages/src/rpm/wazuh-indexer.rpm.spec" "${TMP_DIR}" # Copy performance analyzer service file - mkdir -p "${TMP_DIR}"/usr/lib/systemd/system - cp "distribution/packages/src/common/wazuh-indexer-performance-analyzer.service" "${TMP_DIR}"/usr/lib/systemd/system + enable_performance_analyzer cd "${TMP_DIR}" local src_path="./usr/share/wazuh-indexer" @@ -211,29 +251,11 @@ function assemble_rpm() { echo "Extract ${ARTIFACT_BUILD_NAME} archive" rpm2cpio "${ARTIFACT_BUILD_NAME}" | cpio -imdv - # Install plugins from Maven repository - echo "Install plugins" - for plugin in "${plugins[@]}"; do - plugin_from_maven="org.opensearch.plugin:${plugin}:$VERSION.0" - OPENSEARCH_PATH_CONF=$PATH_CONF "${PATH_BIN}/opensearch-plugin" install --batch --verbose "${plugin_from_maven}" - done - - # Move performance-analyzer-rca to its final location - local rca_src="${src_path}/plugins/opensearch-performance-analyzer/performance-analyzer-rca" - local rca_dest="${src_path}" - mv "${rca_src}" "${rca_dest}" - - # Set up configuration files - cp $PATH_CONF/security/* $PATH_CONF/opensearch-security/ - cp $PATH_CONF/jvm.prod.options $PATH_CONF/jvm.options - cp $PATH_CONF/opensearch.prod.yml $PATH_CONF/opensearch.yml - - rm -r $PATH_CONF/security - rm $PATH_CONF/jvm.prod.options $PATH_CONF/opensearch.prod.yml - - # Remove symbolic links and bat files - find . -type l -exec rm -rf {} \; - find . -name "*.bat" -exec rm -rf {} \; + # Install plugins + install_plugins + enable_performance_analyzer_rca ${src_path} + # Swap configuration files + add_configuration_files # Generate final package local topdir @@ -253,11 +275,7 @@ function assemble_rpm() { package_name="wazuh-indexer-${version}-1.${SUFFIX}.${EXT}" cp "${TMP_DIR}/RPMS/${SUFFIX}/${package_name}" "${OUTPUT}/dist/" - echo "Cleaning temporary ${TMP_DIR} folder" - rm -r "artifacts/tmp" - echo "After execution, shell path is $(pwd)" - # Store package's name to file. Used by GH Action. - echo "${package_name}" >"${OUTPUT}/artifact_name.txt" + clean } # ==== @@ -269,8 +287,7 @@ function assemble_deb() { cp "distribution/packages/src/deb/debmake_install.sh" "${TMP_DIR}" chmod a+x "${TMP_DIR}/debmake_install.sh" # Copy performance analyzer service file - mkdir -p "${TMP_DIR}"/usr/lib/systemd/system - cp "distribution/packages/src/common/wazuh-indexer-performance-analyzer.service" "${TMP_DIR}"/usr/lib/systemd/system + enable_performance_analyzer cd "${TMP_DIR}" local src_path="./usr/share/wazuh-indexer" @@ -282,29 +299,11 @@ function assemble_deb() { ar xf "${ARTIFACT_BUILD_NAME}" data.tar.gz tar zvxf data.tar.gz - # Install plugins from Maven repository - echo "Install plugins" - for plugin in "${plugins[@]}"; do - plugin_from_maven="org.opensearch.plugin:${plugin}:$VERSION.0" - OPENSEARCH_PATH_CONF=$PATH_CONF "${PATH_BIN}/opensearch-plugin" install --batch --verbose "${plugin_from_maven}" - done - - # Move performance-analyzer-rca to its final location - local rca_src="${src_path}/plugins/opensearch-performance-analyzer/performance-analyzer-rca" - local rca_dest="${src_path}" - mv "${rca_src}" "${rca_dest}" - - # Set up configuration files - cp $PATH_CONF/security/* $PATH_CONF/opensearch-security/ - cp $PATH_CONF/jvm.prod.options $PATH_CONF/jvm.options - cp $PATH_CONF/opensearch.prod.yml $PATH_CONF/opensearch.yml - - rm -r $PATH_CONF/security - rm $PATH_CONF/jvm.prod.options $PATH_CONF/opensearch.prod.yml - - # Remove symbolic links and bat files - find . -type l -exec rm -rf {} \; - find . -name "*.bat" -exec rm -rf {} \; + # Install plugins + install_plugins + enable_performance_analyzer_rca ${src_path} + # Swap configuration files + add_configuration_files # Generate final package local version @@ -324,11 +323,7 @@ function assemble_deb() { # debmake creates the package one level above cp "${TMP_DIR}/../${package_name}" "${OUTPUT}/dist/" - echo "Cleaning temporary ${TMP_DIR} folder" - rm -r "artifacts/tmp" - echo "After execution, shell path is $(pwd)" - # Store package's name to file. Used by GH Action. - echo "${package_name}" >"${OUTPUT}/artifact_name.txt" + clean } # ==== @@ -337,7 +332,7 @@ function assemble_deb() { function main() { parse_args "${@}" - echo "Assembling OpenSearch for $PLATFORM-$DISTRIBUTION-$ARCHITECTURE" + echo "Assembling wazuh-indexer for $PLATFORM-$DISTRIBUTION-$ARCHITECTURE" # wazuh-indexer-min_4.9.0-1-x64_78fcc3db6a5b470294319e48b58c3d715bee39d1.rpm ARTIFACT_BUILD_NAME=$(ls "${OUTPUT}/dist/" | grep "wazuh-indexer-min.*.$EXT") @@ -346,23 +341,14 @@ function main() { mkdir -p "$TMP_DIR" cp "${OUTPUT}/dist/$ARTIFACT_BUILD_NAME" "${TMP_DIR}" - case $SUFFIX.$EXT in - linux-arm64.tar.gz) - assemble_tar - ;; - linux-x64.tar.gz) + case $PACKAGE in + tar) assemble_tar ;; - aarch64.rpm) + rpm) assemble_rpm ;; - x86_64.rpm) - assemble_rpm - ;; - amd64.deb) - assemble_deb - ;; - arm64.deb) + deb) assemble_deb ;; esac