ci: add multi branch build; add initial support for apk

.github/workflows/build.yml

@@ -9,25 +9,41 @@ on:
       - v*
       - '!v2020*'
     paths:
-      - ".github/workflows/*"
+      - ".github/workflows/build.yml"
       - "libremesh.mk"
       - "packages/**"
+  workflow_dispatch:
 
 jobs:
+  # Build all packages for x86_64 using the openwrt sdk
+  # use latest branch of openwrt with opkg to produce feed at https://feed.libremesh.org/master
+  # use main branch of openwrt with apk to produce feed at https://feed.libremesh.org/apk/master
+  # NOTE: this doesn't sign packages for apk
   build:
-    name: Build ${{ github.ref }}
+    name: Build ${{ matrix.version }} ${{ github.ref }}
     runs-on: ubuntu-latest
-
+    strategy:
+      fail-fast: false
+      max-parallel: 1
+      matrix:
+        include:
+          - { version: main, output_path: 'apk/'}
+          - { version: openwrt-23.05, output_path: ''}
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
 
       - name: Build packages
         uses: openwrt/gh-action-sdk@v7
         env:
-          ARCH: "x86_64-openwrt-23.05"
+          ARCH: "x86_64-${{ matrix.version }}"
           FEEDNAME: "libremesh"
           IGNORE_ERRORS: "n m y"
-          KEY_BUILD: "${{ secrets.KEY_BUILD }}"
+          KEY_BUILD: "${{ matrix.version != 'main' && secrets.KEY_BUILD || '' }}"
+          INDEX: 1
+          NO_DEFAULT_FEEDS: 1
+          NO_REFRESH_CHECK: 1
+          NO_SHFMT_CHECK: 1
+          # V: sc
 
       - name: Set package destination
         run: |
@@ -36,12 +52,12 @@ jobs:
           echo "DEST_DIR=$TAG" >> $GITHUB_ENV
 
       - name: Upload packages to GitHub Pages
-        uses: peaceiris/actions-gh-pages@v3
+        uses: peaceiris/actions-gh-pages@v4
         with:
           deploy_key: ${{ secrets.ACTIONS_DEPLOY_KEY }}
           external_repository: libremesh/lime-feed
           publish_dir: bin/packages/x86_64/libremesh/
-          destination_dir: ${{ env.DEST_DIR }}
+          destination_dir: ${{ matrix.output_path }}${{ env.DEST_DIR }}
 
 #     - name: Upload packages to S3
 #       uses: jakejarvis/s3-sync-action@master
@@ -54,4 +70,3 @@ jobs:
 #         AWS_S3_ENDPOINT: ${{ secrets.AWS_S3_ENDPOINT }}
 #         SOURCE_DIR: bin/packages/x86_64/libremesh/
 #         DEST_DIR: ${{ env.DEST_DIR }}
-

.github/workflows/multi-arch-build.yml

@@ -9,79 +9,135 @@ on:
       - v*
       - '!v2020*'
     paths:
+      - ".github/workflows/multi-arch-build.yml"
       - 'packages/shared-state-async/**'
 
   # Allows you to run this workflow manually from the Actions tab
   workflow_dispatch:
 
+# Build packages using the openwrt sdk
+# for each architecture supported
+# in openwrt branches 'main', 'stable', 'old-stable'
+# 
+# NOTE: 
+# This list of branches will be updated manually 
+# - at next openwrt release, ideally adding '25.xx' and removing '23.05'
+# - whether it will be identified an incompatibility of the current libremesh code with an openwrt branch
+env:
+  BUILD_BRANCHES: "main 24.10 23.05"
+
+# workaround: build a package that selects '+shared-state-async'
+# as 'make package/shared-state-async/check' fails with 'Package HASH check failed'
+  PACKAGES: "shared-state-bat_hosts"
+
 jobs:
+  generate_matrix:
+    name: 'Generate matrix'
+    runs-on: ubuntu-latest
+    outputs:
+      matrix_builds: ${{ steps.define_matrix.outputs.matrix_builds }}
+    steps:
+      - name: Set package destination
+        run: |
+          export TAG=$(echo "${{ github.ref }}" | cut -d '/' -f 3- | perl -pe 's/v([0-9])/$1/')
+          echo "$TAG"
+          echo "DEST_DIR=$TAG" >> $GITHUB_ENV
+
+      - name: Define matrix of branches and archs
+        id: define_matrix
+        run: |
+          JSON='['
+          FIRST_BUILD=1
+          echo "${{ env.BUILD_BRANCHES }}"
+          for version in ${{ env.BUILD_BRANCHES }}; do
+
+            VERSION=$([ "$version" == "main" ] \
+              && echo "main" || echo "openwrt-$version")
+            echo $VERSION
+
+            VERSION_PATH=$([ "$version" == "main" ] \
+              && echo "snapshots/packages" || echo "releases/packages-$version")
+            echo $VERSION_PATH
+
+            OPENWRT_BRANCH_PATH=$([ "$version" == "main" ] \
+              && echo "openwrt-main" || echo "openwrt-$version")
+            echo $OPENWRT_BRANCH_PATH
+
+            wget -r -nH --no-parent --level 1 --accept html -P. \
+              --directory-prefix=. https://downloads.openwrt.org/$VERSION_PATH/  
+
+            for arch in $(ls ./$VERSION_PATH/ | grep -v html); do
+              [[ $FIRST_BUILD -ne 1 ]] && JSON="$JSON"','
+              FIRST_BUILD=0
+
+              JSON="$JSON"'{"version": "'"$VERSION"'" ,"arch": "'"$arch"'", "openwrt_branch_path": "'"$OPENWRT_BRANCH_PATH"'" }'
+              echo $JSON
+            done
+          done
+
+          matrix_include='{"include": '"$JSON"']}'
+          echo "matrix_builds=${matrix_include}" >> "$GITHUB_OUTPUT"
+
   build:
-    name: build ${{ matrix.arch }} ${{ github.ref }}
+    name: build ${{ matrix.arch }}-${{ matrix.version }}
     runs-on: ubuntu-latest
+    needs: generate_matrix
     strategy:
       fail-fast: false
-      max-parallel: 1
-      matrix:
-        arch:
-          - aarch64_cortex-a53
-          - aarch64_cortex-a72
-          - aarch64_generic
-          - arm_arm1176jzf-s_vfp
-          - arm_arm926ej-s
-          - arm_cortex-a15_neon-vfpv4
-          - arm_cortex-a5_vfpv4
-          - arm_cortex-a7
-          - arm_cortex-a7_neon-vfpv4
-          - arm_cortex-a7_vfpv4
-          - arm_cortex-a8_vfpv3
-          - arm_cortex-a9
-          - arm_cortex-a9_neon
-          - arm_cortex-a9_vfpv3-d16
-          - arm_fa526
-          - arm_mpcore
-          - arm_xscale
-          - i386_pentium-mmx
-          - i386_pentium4
-          - mips64_octeonplus
-          - mips_24kc
-          - mips_4kec
-          - mips_mips32
-          - mipsel_24kc
-          - mipsel_24kc_24kf
-          - mipsel_74kc
-          - mipsel_mips32
-          - powerpc_464fp
-          - powerpc_8548
-          - riscv64_riscv64
-          - x86_64
-
+      max-parallel: 10
+      matrix: ${{ fromJson(needs.generate_matrix.outputs.matrix_builds) }}
+      # matrix:
+      #   include:
+      #     - { version: main, arch: x86_64, openwrt_branch_path: 'openwrt-main'}
+      #     - { version: main, arch: arc_archs, openwrt_branch_path: 'openwrt-main'}
     steps:
       - uses: actions/checkout@v4
 
-      - name: Build packages ${{ matrix.arch }}
+      - name: Build packages ${{ matrix.arch }}-${{ matrix.version }}
         uses: openwrt/gh-action-sdk@v7
         env:
-          ARCH: "${{ matrix.arch }}"
+          ARCH: "${{ matrix.arch }}-${{ matrix.version }}"
           FEEDNAME: "libremesh"
           IGNORE_ERRORS: "n m y"
-          KEY_BUILD: "${{ secrets.KEY_BUILD }}"
-          PACKAGES: "shared-state-bat_hosts"
+          KEY_BUILD: "${{ matrix.version != 'main' && secrets.KEY_BUILD || '' }}"
+          PACKAGES: "${{ env.PACKAGES }}"
           INDEX: 1
           NO_DEFAULT_FEEDS: 1
           NO_REFRESH_CHECK: 1
           NO_SHFMT_CHECK: 1
 
-      - name: Set package destination
-        run: |
-          export TAG=$(echo "${{ github.ref }}" | cut -d '/' -f 3- | perl -pe 's/v([0-9])/$1/')
-          echo "$TAG"
-          echo "DEST_DIR=$TAG" >> $GITHUB_ENV
+      - name: upload artifact
+        if: ${{ always() }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: ${{ matrix.arch }}-${{ matrix.version }}
+          path: bin/packages/${{ matrix.arch }}/libremesh/
+
+  publish:
+    name: publish ${{ github.ref }}
+    runs-on: ubuntu-latest
+    if: ${{ always() }}
+    needs: [generate_matrix, build]
+    strategy:
+      fail-fast: false
+      max-parallel: 1
+      matrix: ${{ fromJson(needs.generate_matrix.outputs.matrix_builds) }}
+      # matrix:
+      #   include:
+      #     - { version: main, arch: x86_64, openwrt_branch_path: 'openwrt-main'}
+      #     - { version: main, arch: arc_archs, openwrt_branch_path: 'openwrt-main'}
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/download-artifact@v4
+        with:
+          name: "${{ matrix.arch }}-${{ matrix.version }}"
+          path: artifacts
 
       - name: Upload packages to GitHub Pages
-        uses: peaceiris/actions-gh-pages@v3
+        uses: peaceiris/actions-gh-pages@v4
         with:
           deploy_key: ${{ secrets.ACTIONS_DEPLOY_KEY }}
           external_repository: libremesh/lime-feed
-          publish_dir: bin/packages/${{ matrix.arch }}/libremesh/
-          destination_dir: arch_packages/${{ env.DEST_DIR }}/${{ matrix.arch }}
-
+          publish_dir: ./artifacts/
+          destination_dir: arch_packages/${{ env.DEST_DIR }}/${{ matrix.openwrt_branch_path }}/${{ matrix.arch }}/

README.md

@@ -56,22 +56,41 @@ Start an ImageBuilder of your choice, for example ath79-generic if your device i
 
 ```shell
 mkdir ./images/
-docker run -it -v $(pwd)/images:/images/ ghcr.io/openwrt/imagebuilder:ath79-generic-v23.05.3
+docker run -it -v $(pwd)/images:/images/ ghcr.io/openwrt/imagebuilder:ath79-generic-v23.05.5
 ```
 
 If your device is not part of ath79-generic profiles, you can replace it with another <target>-<subtarget> combination. For knowing which target and subtarget is best suited for your router, check out the page about it in the [OpenWrt's Table of Hardware][OpenWrt-ToH].
 
 Within the container, add the `lime-packages` feeds:
 
+**opkg on openwrt 24.10.x and previous**
+
 ```shell
 echo "src/gz libremesh https://feed.libremesh.org/master" >> repositories.conf
 echo "src/gz libremesh_profiles https://feed.libremesh.org/profiles" >> repositories.conf
-echo "src/gz libremesh_arch_packages https://feed.libremesh.org/arch_packages/master/mips_24kc" >> repositories.conf
 echo  "untrusted comment: signed by libremesh.org key a71b3c8285abd28b" > keys/a71b3c8285abd28b
 echo "RWSnGzyChavSiyQ+vLk3x7F0NqcLa4kKyXCdriThMhO78ldHgxGljM/8" >> keys/a71b3c8285abd28b
 ```
+To use the new `shared-state-async` add this repo:
+- replace `openwrt_23.05` with the openwrt branch 
+- replace `mips_24kc` with the architecture of your device based on target/subtarget
+
+```shell 
+echo "src/gz libremesh_arch_packages https://feed.libremesh.org/arch_packages/master/openwrt_23.05/mips_24kc" >> repositories.conf
+```
 
-If your device is not part of ath79-generic replace `mips_24kc` with the architecture of the selected <target>/<subtarget>.
+**apk on openwrt main branch**
+```
+echo "https://feed.libremesh.org/apk/master/packages.adb" >> repositories
+echo "https://feed.libremesh.org/apk/profiles/packages.adb" >> repositories
+```
+To use the new `shared-state-async` add this repo: 
+- replace `openwrt_main` with the openwrt branch 
+- replace `mips_24kc` with the architecture of your device based on target/subtarget
+
+```shell 
+echo "https://feed.libremesh.org/arch_packages/master/openwrt_main/mips_24kc/packages.adb" >> repositories
+```
 
 Ideally add your own `lime-community` files within the container in the folder
 `./files/etc/config/`. To find possible options consult the
@@ -94,14 +113,9 @@ Your images should be available outside of the container in the `./images/` fold
 
 Go to <https://firmware-selector.openwrt.org/>. Find your device. Click on the folder symbol right after "Links: ". Alternatively, find your device in [OpenWrt's Table of Hardware][OpenWrt-ToH], find the image download link, remove the filename from the right side of the link and put the result in your browsers address bar. Scroll down and download openwrt-imagebuilder-*. Unpack the file and open a terminal inside the directory. Add the `lime-packages` feed:
 
-```shell
-echo "src/gz libremesh https://feed.libremesh.org/master" >> repositories.conf
-echo "src/gz libremesh_profiles https://feed.libremesh.org/profiles" >> repositories.conf
-echo "src/gz libremesh_arch_packages https://feed.libremesh.org/arch_packages/master/mips_24kc" >> repositories.conf
-echo  "untrusted comment: signed by libremesh.org key a71b3c8285abd28b" > keys/a71b3c8285abd28b
-echo "RWSnGzyChavSiyQ+vLk3x7F0NqcLa4kKyXCdriThMhO78ldHgxGljM/8" >> keys/a71b3c8285abd28b
-```
-If your device is not part of ath79-generic replace `mips_24kc` with the architecture of the selected &lt;target&gt;/&lt;subtarget&gt;.
+Follow the same steps at With Docker
+- **opkg on openwrt 24.10.x and previous**
+- **apk on openwrt main branch**
 
 Create an image with
 ```shell

libremesh.mk

@@ -3,25 +3,20 @@ include $(TOPDIR)/rules.mk
 PKG_NAME?=$(notdir ${CURDIR})
 
 # from https://github.com/openwrt/luci/blob/master/luci.mk
+# default package version follow this scheme:
+# [year].[day_of_year].[seconds_of_day]~[commit_short_hash] eg. 24.322.80622~a403707
 PKG_VERSION?=$(if $(DUMP),x,$(strip $(shell \
-	if svn info >/dev/null 2>/dev/null; then \
-		revision="svn-r$$(LC_ALL=C svn info | sed -ne 's/^Revision: //p')"; \
-	elif git log -1 >/dev/null 2>/dev/null; then \
-		revision="svn-r$$(LC_ALL=C git log -1 | sed -ne 's/.*git-svn-id: .*@\([0-9]\+\) .*/\1/p')"; \
-		if [ "$$revision" = "svn-r" ]; then \
-			set -- $$(git log -1 --format="%ct %h" --abbrev=7); \
-			secs="$$(($$1 % 86400))"; \
-			yday="$$(date --utc --date="@$$1" "+%y.%j")"; \
-			revision="$$(printf 'git-%s.%05d-%s' "$$yday" "$$secs" "$$2")"; \
-		fi; \
-	else \
-		revision="unknown"; \
-	fi; \
-	echo "$$revision" \
+    if git log -1 >/dev/null 2>/dev/null; then \
+      set -- $$(git log -1 --format="%ct %h" --abbrev=7); \
+        secs="$$(($$1 % 86400))"; \
+        yday="$$(date --utc --date="@$$1" "+%y.%j")"; \
+        printf '%s.%05d~%s' "$$yday" "$$secs" "$$2"; \
+    else \
+      echo "0"; \
+    fi; \
 )))
-PKG_RELEASE?=1
 
-PKG_BUILD_DIR:=$(BUILD_DIR)/$(PKG_NAME)
+PKG_BUILD_DIR?=$(BUILD_DIR)/$(PKG_NAME)
 
 include $(INCLUDE_DIR)/package.mk
 
@@ -31,6 +26,8 @@ define Build/Compile
 	$(CP) ./files ./build
 	$(FIND) ./build -name '*.sh' -exec sed -i '/^\s*#\[Doc\]/d' {} +
 	$(FIND) ./build -name '*.lua' -exec sed -i '/^\s*--!.*/d' {} +
+	$(FIND) ./build -type f -executable -exec sed -i '/^\s*#\[Doc\]/d' {} +
+	$(FIND) ./build -type f -executable -exec sed -i '/^\s*--!.*/d' {} +
 endef
 
 define Package/$(PKG_NAME)/install

packages/altermundi-grafana/Makefile

@@ -4,14 +4,7 @@
 # This is free software, licensed under the GNU General Public License v3.
 #
 
-include $(TOPDIR)/rules.mk
-
-PKG_NAME:=altermundi-grafana
-PKG_VERSION=$(GIT_COMMIT_DATE)-$(GIT_COMMIT_TSTAMP)
-GIT_COMMIT_DATE:=$(shell git log -n 1 --pretty=%ad --date=short . )
-GIT_COMMIT_TSTAMP:=$(shell git log -n 1 --pretty=%at . )
-
-include $(INCLUDE_DIR)/package.mk
+include ../../libremesh.mk
 
 define Package/$(PKG_NAME)
   SECTION:=utils
@@ -36,12 +29,4 @@ define Package/$(PKG_NAME)/description
   that gathers data on the node and pushes to https://grafana.altermundi.net
 endef
 
-define Build/Compile
-endef
-
-define Package/$(PKG_NAME)/install
-	$(INSTALL_DIR) $(1)/
-	$(CP) ./files/* $(1)/
-endef
-
 $(eval $(call BuildPackage,$(PKG_NAME)))