Reworked CI/CD pipeline, added mask_storeu for sse and avx without AV…

…X512VL (#91) * Update Dockerfile * Create target_flags_translate.py * Update entrypoint.sh * Update target_flags_translate.py * Update release-tsl.yml * Update README.md * Update Dockerfile * Update Dockerfile * Update Dockerfile * Update Dockerfile * Update Dockerfile * Update Dockerfile * Update Dockerfile * Update target_flags_translate.py * Update publish-latest.yml * Update Dockerfile * Update tsl.spec * Update entrypoint.sh * added mask_storeu for avx and sse * changed ubuntu:latest to ubuntu:23.04
db-tu-dresden · Apr 26, 2024 · b2dfa63 · b2dfa63
1 parent 8588f92
commit b2dfa63
Show file tree

Hide file tree

Showing 14 changed files with 154 additions and 54 deletions.
diff --git a/.github/actions/tsl-deb-build/Dockerfile b/.github/actions/tsl-deb-build/Dockerfile
@@ -1,4 +1,4 @@
-FROM ubuntu:latest
+FROM ubuntu:23.04
 
 RUN apt-get update
 RUN apt-get -y install bash build-essential binutils lintian debhelper dh-make devscripts

diff --git a/.github/actions/tsl-generate/Dockerfile b/.github/actions/tsl-generate/Dockerfile
@@ -1,22 +1,23 @@
-FROM amd64/ubuntu:latest
+FROM amd64/ubuntu:23.04
 
 RUN apt-get update 
 RUN apt-get -y install software-properties-common \
-  python3 python3-pip python3-venv \
+  python3-full python3-pip python3-venv \
   git wget
 
 RUN ls 
 RUN pwd
 
 COPY requirements.txt /requirements.txt
 
-RUN pip install -r /requirements.txt
+RUN pip3 install --ignore-installed -r /requirements.txt  --break-system-packages
 
 LABEL org.opencontainers.image.source=https://github.com/db-tu-dresden/TSL
 LABEL org.opencontainers.image.description="TSLerator Generation Image"
 LABEL org.opencontainers.image.licenses=Apache-2.0
 
 COPY entrypoint.sh /entrypoint.sh
+COPY target_flags_translate.py /target_flags_translate.py
 
 
 ENTRYPOINT ["/entrypoint.sh"]
diff --git a/.github/actions/tsl-generate/entrypoint.sh b/.github/actions/tsl-generate/entrypoint.sh
@@ -3,7 +3,7 @@
 PARAM_TARGETS=$1
 
 TARGETS=$(echo $PARAM_TARGETS | sed 's/\[//g' | sed 's/\]//g' | sed 's/ //g' | sed 's/,/ /g')
-TARGETS_NAME=$(echo $TARGETS | sed 's/ /-/g' | sed 's/;/-/g' | sed 's/,/-/g')
+TARGETS_NAME=$(python3 /target_flags_translate.py "${PARAM_TARGETS}" --name-only)
 TARGETS_ARRAY_NOTATION=$(echo $PARAM_TARGETS | sed 's/\[//g' | sed 's/\]//g' | sed 's/ //g' | sed 's/,/:/g')
 echo "name=${TARGETS_NAME}" >> $GITHUB_OUTPUT
 
@@ -15,8 +15,8 @@ echo "out=${GENERATION_BASE}" >> $GITHUB_OUTPUT
 
 mkdir -p ${GENERATION_PATH}
 mkdir -p ${LOG_PATH}
-echo "flags: ${TARGETS_ARRAY_NOTATION}" >> ${GENERATION_PATH}/tsl.conf
-echo "path: ${TARGETS_NAME}" >> ${GENERATION_PATH}/tsl.conf
+
+python3 /target_flags_translate.py "${PARAM_TARGETS}" >> ${GENERATION_PATH}/tsl.conf
 
 cd ${REPO_ROOT}
 ls -halt >> ${GENERATION_PATH}/generation.log 2>&1
@@ -29,4 +29,4 @@ if [ $? -ne 0 ]; then
 fi
 
 echo "msg=TSL can be generated (with $TARGETS)." >> $GITHUB_OUTPUT
-echo "success=true" >> $GITHUB_OUTPUT
+echo "success=true" >> $GITHUB_OUTPUT
diff --git a/.github/actions/tsl-generate/target_flags_translate.py b/.github/actions/tsl-generate/target_flags_translate.py
@@ -0,0 +1,42 @@
+#!/usr/bin/env python3
+
+import sys
+
+def get_variants(arr):
+  if len(arr) == 1:
+    for x in arr[0]:
+      yield [x]
+  else:
+    for x in arr[0]:
+      for y in get_variants(arr[1:]):
+        yield [x] + y
+
+def flatten_alternatives(arr):
+  for entry in arr:
+    for x in entry.split("|"):
+      yield x
+
+targets_str = sys.argv[1]
+targets_arr = [x.strip() for x in targets_str.split(",")]
+
+direct_targets = list(filter(lambda x: "|" not in x, targets_arr))
+alternatives_targets = list(filter(lambda x: "|" in x, targets_arr))
+
+targets_all = direct_targets + [x for x in flatten_alternatives(alternatives_targets)]
+targets_name = "-".join(targets_all)
+
+if len(alternatives_targets) > 0:
+  alternatives = [x for x in get_variants([alt.split("|") for alt in alternatives_targets])]
+else:
+  alternatives = []
+
+if len(sys.argv) == 3:
+  print(targets_name)
+else:
+  if len(alternatives) > 0:
+    for alternative in alternatives:
+      print(f"flags: {':'.join(direct_targets + alternative)}")
+      print(f"path: {targets_name}")
+  else:
+    print(f"flags: {':'.join(direct_targets)}")
+    print(f"path: {targets_name}")
diff --git a/.github/actions/tsl-generator-all-in-one/Dockerfile b/.github/actions/tsl-generator-all-in-one/Dockerfile
@@ -1,29 +1,36 @@
-FROM ubuntu:latest
+FROM ubuntu:23.04
+
+ENV TZ=Europe/Berlin
+RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone
+
 RUN apt-get update 
 RUN apt-get -y install software-properties-common \
   build-essential checkinstall \
-  g++ \
-  make cmake lsb-release software-properties-common \
-  python3 python3-pip python3-venv \
+  make cmake lsb-release \
+  python3-full python3-pip python3-venv \
   git wget bash
 
-RUN wget https://apt.llvm.org/llvm.sh && \
-  chmod +x llvm.sh && \
-  ./llvm.sh 17 && \
-  update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-17 100
-
-ENV TZ=Europe/Berlin
-RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone
+# Install gcc 13
+RUN add-apt-repository ppa:ubuntu-toolchain-r/test && \
+    apt-get update && \
+    apt-get install -y g++-13 && \
+    update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-13 100 && \
+    update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-13 100
 
+# Install clang-18
+RUN wget https://apt.llvm.org/llvm.sh && \
+    chmod u+x llvm.sh 
+RUN ./llvm.sh 17 && \
+    update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-17 100
 
 RUN mkdir /tslgen
 COPY root.tar.gz /root.tar.gz
 RUN tar -xvf /root.tar.gz -C /tslgen
 RUN rm /root.tar.gz
 # COPY requirements.txt /requirements.txt
 
-RUN pip install -r /tslgen/requirements.txt
-RUN pip install ruff yamllint
+RUN pip install --ignore-installed -r /tslgen/requirements.txt --break-system-packages
+RUN pip install --ignore-installed ruff yamllint --break-system-packages
 
 
 

diff --git a/.github/actions/tsl-rpm-build/entrypoint.sh b/.github/actions/tsl-rpm-build/entrypoint.sh
@@ -10,18 +10,33 @@ RPM_BASE=/root/rpmbuild
 SPEC_FILE=${RPM_BASE}/SPECS/tsl.spec
 
 
-# sed ${{ VERSION_TAG }} in tsl.spec with $VERSION
-sed -i "s/\${{ VERSION_TAG }}/${VERSION}/g" ${SPEC_FILE}
-sed -i "s/\${{ TSL_TARBALL }}/${TSL_TAR_GZ_NAME}/g" ${SPEC_FILE}
-sed -i "s|\${{ TSL_TARBALL_PREFIX }}|${TSL_TAR_PREFIX}|g" ${SPEC_FILE}
-
 REPO_ROOT=/github/workspace
 TSL_ROOT=${REPO_ROOT}/${TSL_TAR_GZ_NAME}
 OUT_BASE=packages/rpm
 OUT=${REPO_ROOT}/${OUT_BASE}
 mkdir -p ${OUT}
 echo "out=${OUT_BASE}" >> $GITHUB_OUTPUT
 
+RPM_FILE="${OUT}/noarch/libtsl-dev-${VERSION}-1.noarch.rpm"
+
+# sed ${{ VERSION_TAG }} in tsl.spec with $VERSION
+if [[ "$VERSION" == *"-"* ]]; then
+  #release candidate
+  VER=${VERSION%%-*}
+  RC=${VERSION##*-}
+  sed -i "s/\${{ VERSION_TAG }}/${VER}/g" ${SPEC_FILE}
+  sed -i "s/\${{ RELEASE_TAG }}/${RC}/g" ${SPEC_FILE}
+  RMP_FILE="${OUT}/noarch/libtsl-dev-${VER}-${RC}.noarch.rpm"
+else
+  sed -i "s/\${{ VERSION_TAG }}/${VERSION}/g" ${SPEC_FILE}
+  sed -i "s/\${{ RELEASE_TAG }}/1/g" ${SPEC_FILE}
+fi
+
+sed -i "s/\${{ TSL_TARBALL }}/${TSL_TAR_GZ_NAME}/g" ${SPEC_FILE}
+sed -i "s|\${{ TSL_TARBALL_PREFIX }}|${TSL_TAR_PREFIX}|g" ${SPEC_FILE}
+
+
+
 cp ${SPEC_FILE} ${OUT}
 
 cp ${TSL_ROOT} ${RPM_BASE}/SOURCES/${TSL_TAR_GZ_NAME}
@@ -34,7 +49,7 @@ if [ $? -ne 0 ]; then
   exit
 fi
 
-mv ${OUT}/noarch/libtsl-dev-${VERSION}-1.noarch.rpm ${OUT}/libtsl-dev.rpm
+mv ${RPM_FILE} ${OUT}/libtsl-dev.rpm
 # try to install and remove
 dnf install ${OUT}/libtsl-dev.rpm -y
 dnf remove libtsl-dev -y
@@ -50,4 +65,4 @@ echo "name=libtsl-dev.rpm" >> $GITHUB_OUTPUT
 ls -l ${OUT} >> ${OUT}/ls.txt
 
 echo "msg=rpmbuild success" >> $GITHUB_OUTPUT
-echo "success=true" >> $GITHUB_OUTPUT
+echo "success=true" >> $GITHUB_OUTPUT
diff --git a/.github/actions/tsl-rpm-build/tsl.spec b/.github/actions/tsl-rpm-build/tsl.spec
@@ -1,6 +1,6 @@
 Name:           libtsl-dev
 Version:        ${{ VERSION_TAG }}
-Release:        1
+Release:        ${{ RELEASE_TAG }}
 Summary:        Template SIMD Library (TSL) is an open-source C++ library for SIMD programming. It provides a comprehensive collection of SIMD intrinsics and high-level interfaces to exploit the full power of SIMD hardware.
 BuildArch:      noarch
 

diff --git a/.github/actions/tsl-test-arm/Dockerfile b/.github/actions/tsl-test-arm/Dockerfile
@@ -1,4 +1,4 @@
-FROM arm64v8/ubuntu:latest
+FROM arm64v8/ubuntu:23.04
 
 RUN apt update
 RUN apt-get -y install \

diff --git a/.github/actions/tsl-test-x86/Dockerfile b/.github/actions/tsl-test-x86/Dockerfile
@@ -1,4 +1,4 @@
-FROM amd64/ubuntu:latest
+FROM amd64/ubuntu:23.04
 
 RUN apt update
 RUN apt-get -y install \

diff --git a/.github/actions/tsl-validate/Dockerfile b/.github/actions/tsl-validate/Dockerfile
@@ -1,4 +1,4 @@
-FROM amd64/ubuntu:latest
+FROM amd64/ubuntu:23.04
 
 RUN apt update 
 RUN apt-get -y install software-properties-common
@@ -25,28 +25,28 @@ RUN apt-get -y install \
 COPY requirements.txt /requirements.txt
 
 RUN . /py/venvs/py38/bin/activate && \
-  pip install -r /requirements.txt && \
-  pip install ruff && pip install yamllint && \
+  pip install --ignore-installed -r /requirements.txt && \
+  pip install --ignore-installed ruff && pip install --ignore-installed yamllint && \
   deactivate
 
 RUN . /py/venvs/py39/bin/activate && \
-  pip install -r /requirements.txt && \
-  pip install ruff && pip install yamllint && \
+  pip install --ignore-installed -r /requirements.txt && \
+  pip install --ignore-installed ruff && pip install --ignore-installed yamllint && \
   deactivate
 
 RUN . /py/venvs/py310/bin/activate && \
-  pip install -r /requirements.txt && \
-  pip install ruff && pip install yamllint && \
+  pip install --ignore-installed -r /requirements.txt && \
+  pip install --ignore-installed ruff && pip install --ignore-installed yamllint && \
   deactivate
 
 RUN . /py/venvs/py311/bin/activate && \
-  pip install -r /requirements.txt && \
-  pip install ruff && pip install yamllint && \
+  pip install --ignore-installed -r /requirements.txt && \
+  pip install --ignore-installed ruff && pip install --ignore-installed yamllint && \
   deactivate
 
 RUN . /py/venvs/py312/bin/activate && \
-  pip install -r /requirements.txt && \
-  pip install ruff && pip install yamllint && \
+  pip install --ignore-installed -r /requirements.txt && \
+  pip install --ignore-installed ruff && pip install --ignore-installed yamllint && \
   deactivate
 
 LABEL org.opencontainers.image.source=https://github.com/db-tu-dresden/TSL

diff --git a/.github/workflows/publish-latest.yml b/.github/workflows/publish-latest.yml
@@ -51,7 +51,7 @@ jobs:
           LATEST_TAG=$(git tag -l --sort=-v:refname "v*" | head -n 1)
           if [ -z "${LATEST_TAG}" ]; then
             echo "No tag: $LATEST_TAG"
-            LATEST_TAG=v0.0.0
+            LATEST_TAG=v0.0.0-rc1
           else
             echo "Latest tag: $LATEST_TAG"
           fi
@@ -63,10 +63,21 @@ jobs:
           echo "temp=${TEMP}"
           MINOR=${TEMP%%.*}
           echo "minor=${MINOR}"
-          PATCH=${TEMP#*.}
+          PATCHTEMP=${TEMP#*.}
+          echo "patchtemp=$PATCHTEMP"
+          PATCH=${PATCHTEMP%%-*}
           echo "patch=${PATCH}"
-          PATCH=$((PATCH+1))
-          VERSION="v$MAJOR.$MINOR.$PATCH"
+          RC_TMP=${PATCHTEMP#*-}
+          RC=${RC_TMP:2}
+          if [ -z "${RC}" ]; then
+            PATCH=$((PATCH+1))
+            RC=1
+          else
+            RC=$((RC+1))
+          fi
+          echo "rc=${RC}"
+          VERSION="v${MAJOR}.${MINOR}.${PATCH}-rc${RC}"
+          echo "New Version: ${VERSION}"
           echo "tag=${VERSION}"
           echo "tag=${VERSION}" >> $GITHUB_OUTPUT
           git config --global user.email "[email protected]"
@@ -108,4 +119,4 @@ jobs:
       tag-name: ${{ needs.get-tag.outputs.tag }}
     secrets:
       DOCKER_HUB_USER: ${{ secrets.DOCKER_HUB_USER }}
-      DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
+      DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
diff --git a/.github/workflows/release-tsl.yml b/.github/workflows/release-tsl.yml
@@ -58,7 +58,7 @@ jobs:
           - "sse,sse2,ssse3,sse4_1,sse4_2,avx,avx2,avx512f"
           - "sse,sse2,ssse3,sse4_1,sse4_2,avx,avx2,avx512f,avx512cd,avx512er,avx512pf" #avx3.1
           - "sse,sse2,ssse3,sse4_1,sse4_2,avx,avx2,avx512f,avx512cd,avx512bw,avx512dq,avx512vl" #avx3.2
-          - "neon,asimd"
+          - "neon|asimd"
     steps:
       - name: Checkout
         uses: actions/checkout@v4
@@ -160,4 +160,4 @@ jobs:
             ${{ github.workspace }}/.github/workflows/install_tsl.sh
             ${{ github.workspace }}/${{ steps.rpm.outputs.out }}/${{ steps.rpm.outputs.name}}
             ${{ github.workspace }}/${{ steps.deb.outputs.out }}/${{ steps.deb.outputs.name}}
-          tag_name: ${{ inputs.tag-name }}
+          tag_name: ${{ inputs.tag-name }}
diff --git a/README.md b/README.md
@@ -59,18 +59,18 @@ The latest release is available under [Releases](https://github.com/db-tu-dresde
 
 To download the TSL to the current working directory, just run
 ~~~console
-user@host:~ curl -L -s "https://github.com/db-tu-dresden/TSL/releases/latest/download/install_tsl.sh" | /bin/bash
+curl -L -s "https://github.com/db-tu-dresden/TSL/releases/latest/download/install_tsl.sh" | /bin/bash
 ~~~
 
 If you want to "install" the TSL (the header files will be placed at /usr/include/tsl), we prepared an `rpm` and a `deb` package.
 To install the `rpm` package, run
 ~~~console
-user@host:~ sudo dnf install -y https://github.com/db-tu-dresden/TSL/releases/latest/download/libtsl-dev.rpm
+sudo dnf install -y https://github.com/db-tu-dresden/TSL/releases/latest/download/libtsl-dev.rpm
 ~~~
 
 To install the `deb` package, run
 ~~~console
-user@host:~ TSL_DEB_FNAME=$(mktemp -ud --tmpdir libtsl-dev-XXXXX.deb); curl -L -s "https://github.com/db-tu-dresden/TSL/releases/latest/download/libtsl-dev.deb" -o ${TSL_DEB_FNAME} && sudo apt install ${TSL_DEB_FNAME}
+TSL_DEB_FNAME=$(mktemp -ud --tmpdir libtsl-dev-XXXXX.deb); curl -L -s "https://github.com/db-tu-dresden/TSL/releases/latest/download/libtsl-dev.deb" -o ${TSL_DEB_FNAME} && sudo apt install ${TSL_DEB_FNAME}
 ~~~
 
 ### Get the Generator
@@ -79,8 +79,8 @@ user@host:~ TSL_DEB_FNAME=$(mktemp -ud --tmpdir libtsl-dev-XXXXX.deb); curl -L -
 Clone the git repository of the [TSL Generator](https://github.com/db-tu-dresden/TSL) and navigate into your freshly cloned TSLGen directory.
 
 ~~~console
-user@host:~ git clone https://github.com/db-tu-dresden/TSL tslroot
-user@host:~ cd tslroot
+git clone https://github.com/db-tu-dresden/TSL tslroot
+cd tslroot
 ~~~
 
 #### **2. Initialize required environment**