CI: Reduce HTTP requests

Check prebuilt tarballs are existing or not first. If so, skip the downloading of releases tags and tarballs. In the CI, we first call "make artifact" to prepare the tarballs, so the GitHub API for fetching releases tags would not be called and be skipped when "make" is called next time. After this commit, wget in the CI would not complain about error code 403 (rate limit exceeded) anymore. However, the build on Aarch64 still failed because of other reansons, e.g. software bugs of GCC or QEMU. Close #536
sysprog21 · Feb 11, 2025 · c8c8263 · c8c8263
1 parent d1ed003
commit c8c8263
Show file tree

Hide file tree

Showing 3 changed files with 57 additions and 21 deletions.
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
@@ -58,10 +58,20 @@ jobs:
       uses: rlalik/setup-cpp-compiler@master
       with:
         compiler: ${{ matrix.compiler }}
+    - name: fetch artifact first to reduce HTTP requests
+      env:
+        CC: ${{ steps.install_cc.outputs.cc }}
+      run: |
+            make artifact
+            make ENABLE_SYSTEM=1 artifact
+            make ENABLE_ARCH_TEST=1 artifact
+      if: ${{ always() }}
     - name: default build with -g
       env:
         CC: ${{ steps.install_cc.outputs.cc }}
-      run: make OPT_LEVEL=-g -j$(nproc)
+      run: |
+            make distclean
+            make OPT_LEVEL=-g -j$(nproc)
       if: ${{ always() }}
     - name: default build with -Og
       env:
@@ -268,6 +278,7 @@ jobs:
           ./llvm.sh 18
         # Append custom commands here
         run: |
+          make artifact
           make -j$(nproc)
           make check -j$(nproc)
           make ENABLE_JIT=1 clean && make ENABLE_JIT=1 check -j$(nproc)

diff --git a/Makefile b/Makefile
@@ -416,9 +416,6 @@ endif
 clean:
 	$(RM) $(BIN) $(OBJS) $(DEV_OBJS) $(BUILD_DTB) $(BUILD_DTB2C) $(HIST_BIN) $(HIST_OBJS) $(deps) $(WEB_FILES) $(CACHE_OUT) src/rv32_jit.c
 distclean: clean
-	-$(RM) $(DOOM_DATA) $(QUAKE_DATA) $(BUILDROOT_DATA) $(LINUX_DATA)
-	$(RM) -r $(OUT)/linux-image
-	$(RM) -r $(TIMIDITY_DATA)
 	$(RM) -r $(OUT)/id1
 	$(RM) -r $(DEMO_DIR)
 	$(RM) *.zip

diff --git a/mk/artifact.mk b/mk/artifact.mk
@@ -36,15 +36,30 @@ SCIMARK2_SHA1 := de278c5b8cef84ab6dda41855052c7bfef919e36
 
 SHELL_HACK := $(shell mkdir -p $(BIN_DIR)/linux-x86-softfp $(BIN_DIR)/riscv32 $(BIN_DIR)/linux-image)
 
+# $(1): tag of GitHub releases
+# $(2): name of GitHub releases
+# $(3): name showing in terminal
+define fetch-sha1
+    $(if $(wildcard $(BIN_DIR)/$(2)), \
+        $(eval $(info $(3) is found. Skipping downloading.)), \
+        $(eval LATEST_RELEASE := $(shell wget -q https://api.github.com/repos/sysprog21/rv32emu-prebuilt/releases -O- \
+                                    | grep '"tag_name"' \
+                                    | grep "$(1)" \
+                                    | head -n 1 \
+                                    | sed -E 's/.*"tag_name": "([^"]+)".*/\1/')) \
+    )
+endef
+
 ifeq ($(call has, PREBUILT), 1)
-ifeq ($(call has, SYSTEM), 1)
-  LATEST_RELEASE := $(shell wget -q https://api.github.com/repos/sysprog21/rv32emu-prebuilt/releases -O- | grep '"tag_name"' | grep "Linux-Image" | head -n 1 | sed -E 's/.*"tag_name": "([^"]+)".*/\1/')
-else ifeq ($(call has, ARCH_TEST), 1)
-  LATEST_RELEASE := $(shell wget -q https://api.github.com/repos/sysprog21/rv32emu-prebuilt/releases -O- | grep '"tag_name"' | grep "sail" | head -n 1 | sed -E 's/.*"tag_name": "([^"]+)".*/\1/')
-else
-  LATEST_RELEASE := $(shell wget -q https://api.github.com/repos/sysprog21/rv32emu-prebuilt/releases -O- | grep '"tag_name"' | grep "ELF" | head -n 1 | sed -E 's/.*"tag_name": "([^"]+)".*/\1/')
-endif
-  PREBUILT_BLOB_URL = https://github.com/sysprog21/rv32emu-prebuilt/releases/download/$(LATEST_RELEASE)
+    ifeq ($(call has, SYSTEM), 1)
+        $(call fetch-releases-tag,Linux-Image,rv32emu-linux-image-prebuilt.tar.gz,Linux image)
+    else ifeq ($(call has, ARCH_TEST), 1)
+        $(call fetch-releases-tag,sail,rv32emu-prebuilt-sail-$(HOST_PLATFORM),Sail model)
+    else
+        $(call fetch-releases-tag,ELF,rv32emu-prebuilt.tar.gz,Prebuilt benchmark)
+    endif
+
+    PREBUILT_BLOB_URL = https://github.com/sysprog21/rv32emu-prebuilt/releases/download/$(LATEST_RELEASE)
 else
   # Since rv32emu only supports the dynamic binary translation of integer instruction in tiered compilation currently,
   # we disable the hardware floating-point and the related SIMD operation of x86.
@@ -95,14 +110,15 @@ endif
 ifeq ($(call has, ARCH_TEST), 1)
 	$(Q)if [ "$(RES)" = "1" ]; then \
 	    $(PRINTF) "\n$(YELLOW)SHA-1 verification failed! Re-fetching prebuilt binaries from \"rv32emu-prebuilt\" ...\n$(NO_COLOR)"; \
-	    wget -q --show-progress $(PREBUILT_BLOB_URL)/$(RV32EMU_PREBUILT_TARBALL) -O build/$(RV32EMU_PREBUILT_TARBALL);\
+	    wget -q --show-progress $(PREBUILT_BLOB_URL)/$(RV32EMU_PREBUILT_TARBALL) -O build/$(RV32EMU_PREBUILT_TARBALL); \
 	else \
 	    $(call notice, [OK]); \
 	fi
 else
 	$(Q)if [ "$(RES)" = "1" ]; then \
 	    $(PRINTF) "\n$(YELLOW)SHA-1 verification failed! Re-fetching prebuilt binaries from \"rv32emu-prebuilt\" ...\n$(NO_COLOR)"; \
-	    wget -q --show-progress $(PREBUILT_BLOB_URL)/$(RV32EMU_PREBUILT_TARBALL) -O- | tar -C build --strip-components=1 -xz; \
+	    wget -q --show-progress $(PREBUILT_BLOB_URL)/$(RV32EMU_PREBUILT_TARBALL) -O build/$(RV32EMU_PREBUILT_TARBALL); \
+	    tar --strip-components=1 -zxf build/$(RV32EMU_PREBUILT_TARBALL) -C build; \
 	else \
 	    $(call notice, [OK]); \
 	fi
@@ -147,16 +163,28 @@ endif
 
 fetch-checksum:
 ifeq ($(call has, PREBUILT), 1)
-	$(Q)$(PRINTF) "Fetching SHA-1 of prebuilt binaries ...\n"
+	$(Q)$(PRINTF) "Fetching SHA-1 of prebuilt binaries ... "
 ifeq ($(call has, SYSTEM), 1)
-	$(Q)wget -q -O $(BIN_DIR)/sha1sum-linux-image $(PREBUILT_BLOB_URL)/sha1sum-linux-image
-	$(Q)$(call notice, [OK])
+    ifeq ($(wildcard $(BIN_DIR)/rv32emu-linux-image-prebuilt.tar.gz),)
+		$(Q)wget -q -O $(BIN_DIR)/sha1sum-linux-image $(PREBUILT_BLOB_URL)/sha1sum-linux-image
+		$(Q)$(call notice, [OK])
+    else
+		$(Q)$(PRINTF) "Skipped\n"
+    endif
 else ifeq ($(call has, ARCH_TEST), 1)
-	$(Q)wget -q -O $(BIN_DIR)/rv32emu-prebuilt-sail-$(HOST_PLATFORM).sha $(PREBUILT_BLOB_URL)/rv32emu-prebuilt-sail-$(HOST_PLATFORM).sha
+    ifeq ($(wildcard $(BIN_DIR)/rv32emu-prebuilt-sail-$(HOST_PLATFORM)),)
+		wget -q -O $(BIN_DIR)/rv32emu-prebuilt-sail-$(HOST_PLATFORM).sha $(PREBUILT_BLOB_URL)/rv32emu-prebuilt-sail-$(HOST_PLATFORM).sha
+    else
+		$(Q)$(PRINTF) "Skipped\n"
+    endif
 else
-	$(Q)wget -q -O $(BIN_DIR)/sha1sum-linux-x86-softfp $(PREBUILT_BLOB_URL)/sha1sum-linux-x86-softfp
-	$(Q)wget -q -O $(BIN_DIR)/sha1sum-riscv32 $(PREBUILT_BLOB_URL)/sha1sum-riscv32
-	$(Q)$(call notice, [OK])
+    ifeq ($(wildcard $(BIN_DIR)/rv32emu-prebuilt.tar.gz),)
+		$(Q)wget -q -O $(BIN_DIR)/sha1sum-linux-x86-softfp $(PREBUILT_BLOB_URL)/sha1sum-linux-x86-softfp
+		$(Q)wget -q -O $(BIN_DIR)/sha1sum-riscv32 $(PREBUILT_BLOB_URL)/sha1sum-riscv32
+		$(Q)$(call notice, [OK])
+    else
+		$(Q)$(PRINTF) "Skipped\n"
+    endif
 endif
 endif