Taskfile: Replace utility tasks with ones from yscope-dev-utils. (y…

…-scope#456)
Bill-hbrhbr · Jun 24, 2024 · a40a661 · a40a661
1 parent 628ba3d
commit a40a661
Show file tree

Hide file tree

Showing 5 changed files with 35 additions and 109 deletions.
diff --git a/.gitmodules b/.gitmodules
@@ -23,3 +23,6 @@
 [submodule "components/core/submodules/abseil-cpp"]
 	path = components/core/submodules/abseil-cpp
 	url = https://github.com/abseil/abseil-cpp.git
+[submodule "tools/yscope-dev-utils"]
+	path = tools/yscope-dev-utils
+	url = https://github.com/y-scope/yscope-dev-utils.git
diff --git a/Taskfile.yml b/Taskfile.yml
@@ -3,6 +3,7 @@ version: "3"
 includes:
   docs: "docs/tasks.yml"
   lint: "lint-tasks.yml"
+  utils: "tools/yscope-dev-utils/taskfiles/utils.yml"
 
 vars:
   # Paths
@@ -60,7 +61,7 @@ tasks:
       - "init"
       - "job-orchestration"
       - "package-venv"
-      - task: "validate-checksum"
+      - task: "utils:validate-checksum"
         vars:
           CHECKSUM_FILE: "{{.CHECKSUM_FILE}}"
           DATA_DIR: "{{.OUTPUT_DIR}}"
@@ -97,7 +98,7 @@ tasks:
         cd "{{.OUTPUT_DIR}}/var/www/programs/server"
         PATH="{{.G_WEBUI_NODEJS_BIN_DIR}}":$PATH npm install
       # This command must be last
-      - task: "compute-checksum"
+      - task: "utils:compute-checksum"
         vars:
           DATA_DIR: "{{.OUTPUT_DIR}}"
           OUTPUT_FILE: "{{.CHECKSUM_FILE}}"
@@ -163,7 +164,7 @@ tasks:
     deps:
       - "init"
       - "meteor"
-      - task: "validate-checksum"
+      - task: "utils:validate-checksum"
         vars:
           CHECKSUM_FILE: "{{.CHECKSUM_FILE}}"
           DATA_DIR: "{{.OUTPUT_DIR}}"
@@ -187,7 +188,7 @@ tasks:
       # Remove temp files generated by `meteor build` before checksum
       - "find node_modules -type f -name '.meteor-portable-2.json' -exec rm {} +"
       # This command must be last
-      - task: "compute-checksum"
+      - task: "utils:compute-checksum"
         vars:
           DATA_DIR: "{{.OUTPUT_DIR}}"
           OUTPUT_FILE: "{{.CHECKSUM_FILE}}"
@@ -223,14 +224,14 @@ tasks:
       OUTPUT_DIR: "submodules"
     deps:
       - "init"
-      - task: "validate-checksum"
+      - task: "utils:validate-checksum"
         vars:
           CHECKSUM_FILE: "{{.CHECKSUM_FILE}}"
           DATA_DIR: "{{.OUTPUT_DIR}}"
     cmds:
       - "tools/scripts/deps-download/download-all.sh"
       # This command must be last
-      - task: "compute-checksum"
+      - task: "utils:compute-checksum"
         vars:
           DATA_DIR: "{{.OUTPUT_DIR}}"
           OUTPUT_FILE: "{{.CHECKSUM_FILE}}"
@@ -251,7 +252,7 @@ tasks:
       TAR_PATH: "{{.OUTPUT_TMP_DIR}}/{{.TAR_NAME}}"
     deps:
       - "init"
-      - task: "validate-checksum"
+      - task: "utils:validate-checksum"
         vars:
           CHECKSUM_FILE: "{{.CHECKSUM_FILE}}"
           DATA_DIR: "{{.OUTPUT_DIR}}"
@@ -266,7 +267,7 @@ tasks:
       - "mv '{{.EXTRACTED_DIR}}' '{{.OUTPUT_DIR}}'"
       - "rm -rf '{{.OUTPUT_TMP_DIR}}'"
       # This command must be last
-      - task: "compute-checksum"
+      - task: "utils:compute-checksum"
         vars:
           DATA_DIR: "{{.OUTPUT_DIR}}"
           OUTPUT_FILE: "{{.CHECKSUM_FILE}}"
@@ -337,12 +338,12 @@ tasks:
       # `/parents/A` -> `/parents/B` rather than `/parents/A` -> `/parents/B/A`
       - "rsync --archive '{{.G_PACKAGE_BUILD_DIR}}/' '{{.OUTPUT_DIR}}'"
       # Set the storage engine for the package
-      - task: "replace-text"
+      - task: "utils:replace-text"
         vars:
           FILE_PATH: "{{.OUTPUT_DIR}}/lib/python3/site-packages/clp_py_utils/clp_config.py"
           SED_EXP: >-
             s/([[:space:]]*storage_engine: str = ")[^"]+"/\1{{.STORAGE_ENGINE}}"/
-      - task: "replace-text"
+      - task: "utils:replace-text"
         vars:
           FILE_PATH: "{{.OUTPUT_DIR}}/etc/clp-config.yml"
           SED_EXP: >-
@@ -364,18 +365,18 @@ tasks:
       OUTPUT_DIR: "{{.G_PACKAGE_VENV_DIR}}"
     deps:
       - "init"
-      - task: "validate-checksum"
+      - task: "utils:validate-checksum"
         vars:
           CHECKSUM_FILE: "{{.CHECKSUM_FILE}}"
           DATA_DIR: "{{.OUTPUT_DIR}}"
     cmds:
-      - task: "create-venv"
+      - task: "utils:create-venv"
         vars:
           LABEL: "package"
           OUTPUT_DIR: "{{.OUTPUT_DIR}}"
           REQUIREMENTS_FILE: "requirements.txt"
       # This command must be last
-      - task: "compute-checksum"
+      - task: "utils:compute-checksum"
         vars:
           DATA_DIR: "{{.OUTPUT_DIR}}"
           OUTPUT_FILE: "{{.CHECKSUM_FILE}}"
@@ -427,15 +428,15 @@ tasks:
     deps:
       - "init"
       - "meteor"
-      - task: "validate-checksum"
+      - task: "utils:validate-checksum"
         vars:
           CHECKSUM_FILE: "{{.CHECKSUM_FILE}}"
           DATA_DIR: "{{.OUTPUT_DIR}}"
     cmds:
       - "rm -rf '{{.OUTPUT_DIR}}'"
       - "PATH='{{.G_METEOR_BUILD_DIR}}':$PATH meteor npm install --production"
       # This command must be last
-      - task: "compute-checksum"
+      - task: "utils:compute-checksum"
         vars:
           DATA_DIR: "{{.OUTPUT_DIR}}"
           OUTPUT_FILE: "{{.CHECKSUM_FILE}}"
@@ -456,18 +457,18 @@ tasks:
       CHECKSUM_FILE: "{{.G_BUILD_DIR}}/{{.COMPONENT}}-venv.md5"
     deps:
       - "init"
-      - task: "validate-checksum"
+      - task: "utils:validate-checksum"
         vars:
           CHECKSUM_FILE: "{{.CHECKSUM_FILE}}"
           DATA_DIR: "{{.OUTPUT_DIR}}"
     cmds:
-      - task: "create-venv"
+      - task: "utils:create-venv"
         vars:
           LABEL: "{{.COMPONENT}}"
           OUTPUT_DIR: "{{.OUTPUT_DIR}}"
           REQUIREMENTS_FILE: "{{.ROOT_DIR}}/requirements.txt"
       # This command must be last
-      - task: "compute-checksum"
+      - task: "utils:compute-checksum"
         vars:
           DATA_DIR: "{{.OUTPUT_DIR}}"
           OUTPUT_FILE: "{{.CHECKSUM_FILE}}"
@@ -487,87 +488,8 @@ tasks:
     cmds:
       - "rm -rf dist"
 
-  create-venv:
-    internal: true
-    requires:
-      vars: ["LABEL", "OUTPUT_DIR", "REQUIREMENTS_FILE"]
-    label: "create-venv-{{.LABEL}}"
-    cmds:
-      - "rm -rf '{{.OUTPUT_DIR}}'"
-      - "python3 -m venv '{{.OUTPUT_DIR}}'"
-      # Remove calls to `hash` from the venv activation script since Task uses `gosh` rather than
-      # `bash`.
-      # NOTE: Older versions of Python's venv would only call `hash` if they detected the running
-      # shell was one that had the command, but that's not the case in newer versions.
-      - task: "replace-text"
-        vars:
-          FILE_PATH: "{{.OUTPUT_DIR}}/bin/activate"
-          SED_EXP: >-
-            s/^([[:space:]]*)hash[[:space:]]+.*/\1true/g
-      - |-
-        . "{{.OUTPUT_DIR}}/bin/activate"
-        pip3 install --upgrade pip
-        pip3 install --upgrade -r "{{.REQUIREMENTS_FILE}}"
-
   init:
     internal: true
     run: "once"
     silent: true
     cmd: "mkdir -p '{{.G_BUILD_DIR}}'"
-
-  compute-checksum:
-    desc: "Tries to compute a checksum for the given directory and output it to a file."
-    internal: true
-    # Ignore errors so that dependent tasks don't fail
-    ignore_error: true
-    silent: true
-    requires:
-      vars: ["DATA_DIR", "OUTPUT_FILE"]
-    cmds:
-      - >-
-        tar cf -
-        --directory "{{.DATA_DIR}}"
-        --group=0
-        --mtime='UTC 1970-01-01'
-        --numeric-owner
-        --owner=0
-        --sort=name
-        {{.CHECKSUM_TAR_BASE_ARGS}} . 2> /dev/null
-        | md5sum > {{.OUTPUT_FILE}}
-
-  validate-checksum:
-    desc: "Validates the checksum of the given directory matches the checksum in the given file, or
-    deletes the checksum file otherwise."
-    internal: true
-    silent: true
-    requires:
-      vars: ["CHECKSUM_FILE", "DATA_DIR"]
-    vars:
-      TMP_CHECKSUM_FILE: "{{.CHECKSUM_FILE}}.tmp"
-    cmds:
-      - task: "compute-checksum"
-        vars:
-          DATA_DIR: "{{.DATA_DIR}}"
-          OUTPUT_FILE: "{{.TMP_CHECKSUM_FILE}}"
-      - defer: "rm -f '{{.TMP_CHECKSUM_FILE}}'"
-      # Check that the directory exists and the checksum matches; otherwise delete the checksum file
-      - >-
-        (
-        test -d "{{.DATA_DIR}}"
-        && diff -q '{{.TMP_CHECKSUM_FILE}}' '{{.CHECKSUM_FILE}}' 2> /dev/null
-        ) || rm -f '{{.CHECKSUM_FILE}}'
-
-  replace-text:
-    desc: "Task to replace some text in a file using sed."
-    internal: true
-    requires:
-      vars: ["FILE_PATH", "SED_EXP"]
-    cmds:
-      - |-
-        # NOTE:
-        # 1. We can't use `sed -i` since `-i` has different syntax on Linux and macOS
-        # 2. We can't use `--regexp` instead of `-E` since `--regexp` is not supported on macOS
-        src="{{.FILE_PATH}}"
-        dst="{{.FILE_PATH}}.tmp"
-        sed -E '{{.SED_EXP}}' "${src}" > "${dst}"
-        mv "${dst}" "${src}"
diff --git a/docs/tasks.yml b/docs/tasks.yml
@@ -25,7 +25,7 @@ tasks:
     dir: "{{.TASKFILE_DIR}}"
     deps:
       - ":init"
-      - task: ":validate-checksum"
+      - task: ":utils:validate-checksum"
         vars:
           CHECKSUM_FILE: "{{.CHECKSUM_FILE}}"
           DATA_DIR: "{{.OUTPUT_DIR}}"
@@ -43,7 +43,7 @@ tasks:
           --builder html \
           src "{{.OUTPUT_DIR}}"
       # This command must be last
-      - task: ":compute-checksum"
+      - task: ":utils:compute-checksum"
         vars:
           DATA_DIR: "{{.OUTPUT_DIR}}"
           OUTPUT_FILE: "{{.CHECKSUM_FILE}}"
@@ -63,18 +63,18 @@ tasks:
       REQUIREMENTS_FILE: "docs/requirements.txt"
     deps:
       - ":init"
-      - task: ":validate-checksum"
+      - task: ":utils:validate-checksum"
         vars:
           CHECKSUM_FILE: "{{.CHECKSUM_FILE}}"
           DATA_DIR: "{{.OUTPUT_DIR}}"
     cmds:
-      - task: ":create-venv"
+      - task: ":utils:create-venv"
         vars:
           LABEL: "docs"
           OUTPUT_DIR: "{{.OUTPUT_DIR}}"
           REQUIREMENTS_FILE: "{{.REQUIREMENTS_FILE}}"
       # This command must be last
-      - task: ":compute-checksum"
+      - task: ":utils:compute-checksum"
         vars:
           DATA_DIR: "{{.OUTPUT_DIR}}"
           OUTPUT_FILE: "{{.CHECKSUM_FILE}}"
@@ -91,15 +91,15 @@ tasks:
       OUTPUT_DIR: "{{.G_NODE_DEPS_DIR}}"
     deps:
       - ":init"
-      - task: ":validate-checksum"
+      - task: ":utils:validate-checksum"
         vars:
           CHECKSUM_FILE: "{{.CHECKSUM_FILE}}"
           DATA_DIR: "{{.OUTPUT_DIR}}"
     cmds:
       - "rm -rf '{{.OUTPUT_DIR}}'"
       - "npm --prefix '{{.OUTPUT_DIR}}' install http-server"
       # This command must be last
-      - task: ":compute-checksum"
+      - task: ":utils:compute-checksum"
         vars:
           DATA_DIR: "{{.OUTPUT_DIR}}"
           OUTPUT_FILE: "{{.CHECKSUM_FILE}}"

diff --git a/lint-tasks.yml b/lint-tasks.yml
@@ -166,7 +166,7 @@ tasks:
     deps:
       - ":init"
       - ":webui-node-modules"
-      - task: ":validate-checksum"
+      - task: ":utils:validate-checksum"
         vars:
           CHECKSUM_FILE: "{{.CHECKSUM_FILE}}"
           DATA_DIR: "{{.OUTPUT_DIR}}"
@@ -180,7 +180,7 @@ tasks:
       - "rm -rf '{{.OUTPUT_DIR}}'"
       - "PATH='{{.G_LINTER_NODEJS_BIN_DIR}}':$PATH npm update"
       # This command must be last
-      - task: ":compute-checksum"
+      - task: ":utils:compute-checksum"
         vars:
           DATA_DIR: "{{.OUTPUT_DIR}}"
           OUTPUT_FILE: "{{.CHECKSUM_FILE}}"
@@ -199,18 +199,18 @@ tasks:
       OUTPUT_DIR: "{{.G_LINT_VENV_DIR}}"
     deps:
       - ":init"
-      - task: ":validate-checksum"
+      - task: ":utils:validate-checksum"
         vars:
           CHECKSUM_FILE: "{{.CHECKSUM_FILE}}"
           DATA_DIR: "{{.OUTPUT_DIR}}"
     cmds:
-      - task: ":create-venv"
+      - task: ":utils:create-venv"
         vars:
           LABEL: "lint"
           OUTPUT_DIR: "{{.OUTPUT_DIR}}"
           REQUIREMENTS_FILE: "lint-requirements.txt"
       # This command must be last
-      - task: ":compute-checksum"
+      - task: ":utils:compute-checksum"
         vars:
           DATA_DIR: "{{.OUTPUT_DIR}}"
           OUTPUT_FILE: "{{.CHECKSUM_FILE}}"

diff --git a/tools/yscope-dev-utils b/tools/yscope-dev-utils