0.6.0 scripting dev pr flows (#135)

* Updated the Guide builds to include the config-gen * Added config-gen and config-updater to PR flow * Added the config-gen to the release flow * Added config-gen, config-updater to the pr.sh script * Updated config-gen to scope fs and fixed lint errors * Updated dependencies and cleaned up formatting * Removed references to SystemTest * Added optional loglevel parameter to the config-gen like the other webassemblies * Updated config-gen test to also check that the yaml can be parsed back into a config * Added JS unit tests for config-gen * Updated additional dependencies * Updated release flow to release config-gen as a bundler type * Added the config-updater to the release flow * Added tests to validate the exported yaml parses config * Fixed typo * Updated Wasm-pack to 0.12.1
FamilySearch · Aug 15, 2023 · 1aaeedc · 1aaeedc
1 parent 3359f80
commit 1aaeedc
Show file tree

Hide file tree

Showing 27 changed files with 5,321 additions and 1,976 deletions.
diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile
@@ -6,7 +6,7 @@ ARG CARGO_DENY_VER=0.12.1
 # https://github.com/rust-lang/mdBook/releases
 ARG MDBOOK_VER=v0.4.20
 # https://github.com/rustwasm/wasm-pack/releases
-ARG WASM_PACK_VER=v0.11.0
+ARG WASM_PACK_VER=v0.12.1
 # https://nodejs.org/
 ARG NODE_MAJOR_VER=16
 

diff --git a/.github/workflows/pr-js.yml b/.github/workflows/pr-js.yml
@@ -38,7 +38,7 @@ jobs:
           PATH=$PATH:~/bin
           curl -sSL https://github.com/rust-lang/mdBook/releases/download/v0.4.20/mdbook-v0.4.20-x86_64-unknown-linux-gnu.tar.gz \
             | tar -xz -C ~/bin
-          curl -sSL https://github.com/rustwasm/wasm-pack/releases/download/v0.11.0/wasm-pack-v0.11.0-x86_64-unknown-linux-musl.tar.gz \
+          curl -sSL https://github.com/rustwasm/wasm-pack/releases/download/v0.12.1/wasm-pack-v0.12.1-x86_64-unknown-linux-musl.tar.gz \
             | tar -xz --strip-components=1 -C ~/bin --no-anchored wasm-pack
 
           # setup some envs to various paths for convenience
@@ -48,10 +48,18 @@ jobs:
           WASM_LIB_DIR=$(realpath $PROJECT_ROOT/lib/hdr-histogram-wasm)
           WASM_OUTPUT_REACT_DIR=$RESULTS_VIEWER_REACT_DIR/lib/hdr-histogram-wasm
 
+          CFG_GEN_DIR=$(realpath $PROJECT_ROOT/lib/config-gen)
+          mkdir -p "$RESULTS_VIEWER_REACT_DIR/lib/config-gen"
+          CFG_GEN_OUTPUT_REACT_DIR=$RESULTS_VIEWER_REACT_DIR/lib/config-gen
+
           # build the hdr-histogram-wasm for the results viewer
           cd $WASM_LIB_DIR
           wasm-pack build --release -t bundler -d $WASM_OUTPUT_REACT_DIR --scope fs
 
+          # build the config-gen library for the HAR to YAML converter
+          cd $CFG_GEN_DIR
+          wasm-pack build --release -t bundler -d $CFG_GEN_OUTPUT_REACT_DIR --scope fs
+
           cd $RESULTS_VIEWER_REACT_DIR
           npm ci
           npm run linterror

diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml
@@ -120,22 +120,30 @@ jobs:
           use-cross: true
           command: build
           args: -q --release --target ${{ matrix.target }}
-        # uses: ./.github/actions/linux-x86_64-musl/
-        # if: matrix.cross
-        # with:
-        #   args: cross build -q --release --target ${{ matrix.target }}
 
       - name: Build for Linux
         uses: ./.github/actions/linux-x86_64-musl/
         if: matrix.build == 'linux'
         with:
-          args: cargo build -q --release --target x86_64-unknown-linux-musl
+          args: cargo build -q --release --bin pewpew --target x86_64-unknown-linux-musl
+
+      - name: Build Test Server for Linux
+        uses: ./.github/actions/linux-x86_64-musl/
+        if: matrix.build == 'linux'
+        with:
+          args: cargo build -q --release --bin test-server --target x86_64-unknown-linux-musl
+
+      - name: Build Config Updater for Linux
+        uses: ./.github/actions/linux-x86_64-musl/
+        if: matrix.build == 'linux'
+        with:
+          args: cargo build -q --release -p pewpew-config-updater --target x86_64-unknown-linux-musl
 
   wasm-pack:
     name: Wasm Build
     strategy:
       matrix:
-        wasm-dirctory: [config-wasm, hdr-histogram-wasm]
+        wasm-dirctory: [config-wasm, hdr-histogram-wasm, config-gen]
         node-version: [16.x, 18.x, 20.x]
     runs-on: ubuntu-latest
     env:
@@ -157,9 +165,16 @@ jobs:
           # install wasm-pack
           mkdir ~/bin
           PATH=$PATH:~/bin
-          curl -sSL https://github.com/rustwasm/wasm-pack/releases/download/v0.11.0/wasm-pack-v0.11.0-x86_64-unknown-linux-musl.tar.gz \
+          curl -sSL https://github.com/rustwasm/wasm-pack/releases/download/v0.12.1/wasm-pack-v0.12.1-x86_64-unknown-linux-musl.tar.gz \
             | tar -xz --strip-components=1 -C ~/bin --no-anchored wasm-pack
 
+          if [ "${{ matrix.wasm-dirctory }}" = "config-gen" ]; then
+            echo "Running wasm-pack test"
+            wasm-pack test --node
+
+            # The npm tests also need the config-wasm built
+            (cd ../config-wasm; wasm-pack build --release -t nodejs --scope fs)
+          fi
           wasm-pack build --release -t nodejs --scope fs
         working-directory: ${{env.wasm-directory}}
         shell: bash

diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -57,6 +57,11 @@ jobs:
     needs: ['create-release']
     strategy:
       matrix:
+        binary:
+          - build_param: "--bin"
+            binary_name: "pewpew"
+          - build_param: "-p"
+            binary_name: "pewpew-config-updater"
         include:
           - build: linux
             os: ubuntu-latest
@@ -124,7 +129,7 @@ jobs:
         if: matrix.os != 'ubuntu-latest'
         with:
           command: build
-          args: -q --release
+          args: -q --release ${{ matrix.build_param }} ${{ matrix.binary_name }}
 
       # https://github.com/actions-rs/cargo#cross-compilation
       - name: Build with cross # ARM builds
@@ -140,40 +145,36 @@ jobs:
         with:
           use-cross: true
           command: build
-          args: -q --release --target ${{ matrix.target }}
-        # uses: ./.github/actions/linux-x86_64-musl/
-        # if: matrix.cross
-        # with:
-        #   args: cross build -q --release --target ${{ matrix.target }}
+          args: -q --release ${{ matrix.build_param }} ${{ matrix.binary_name }} --target ${{ matrix.target }}
 
       - name: Build for Linux
         uses: ./.github/actions/linux-x86_64-musl/
         if: matrix.build == 'linux'
         with:
-          args: cargo build -q --release --target x86_64-unknown-linux-musl
+          args: cargo build -q --release ${{ matrix.build_param }} ${{ matrix.binary_name }} --target x86_64-unknown-linux-musl
 
       - name: Compress for Linux/Arm
         if: matrix.os == 'ubuntu-latest'
         run: |
           TARGET=$(echo "${{ matrix.target }}" | sed -e "s/-musl.*//" -e "s/-unknown//")
-          asset_name="pewpew-$VERSION-$TARGET.tar.xz"
+          asset_name="${{ matrix.binary_name }}-$VERSION-$TARGET.tar.xz"
           echo "ASSET_NAME=$asset_name" >> $GITHUB_ENV
-          XZ_OPT=-9 tar -C ./target/${{ matrix.target }}/release/ -cJf $asset_name pewpew
+          XZ_OPT=-9 tar -C ./target/${{ matrix.target }}/release/ -cJf $asset_name ${{ matrix.binary_name }}
 
       - name: Compress for Windows
         if: matrix.os == 'windows-latest'
         shell: bash
         run: |
-          asset_name="pewpew-$VERSION-windows-x86_64.zip"
+          asset_name="${{ matrix.binary_name }}-$VERSION-windows-x86_64.zip"
           echo "ASSET_NAME=$asset_name" >> $GITHUB_ENV
-          7z a -mm=Deflate64 -mfb=258 -mpass=15 $asset_name ./target/release/pewpew.exe
+          7z a -mm=Deflate64 -mfb=258 -mpass=15 $asset_name ./target/release/${{ matrix.binary_name }}.exe
 
       - name: Compress for macOS
         if: matrix.os == 'macos-latest'
         run: |
-          asset_name="pewpew-$VERSION-apple-darwin-x86_64.tar.xz"
+          asset_name="${{ matrix.binary_name }}-$VERSION-apple-darwin-x86_64.tar.xz"
           echo "ASSET_NAME=$asset_name" >> $GITHUB_ENV
-          XZ_OPT=-9 tar -C ./target/release/ -cJf $asset_name pewpew
+          XZ_OPT=-9 tar -C ./target/release/ -cJf $asset_name ${{ matrix.binary_name }}
 
       - name: Upload release asset
         uses: actions/[email protected]
@@ -189,9 +190,12 @@ jobs:
   wasm-release-config:
     name: Wasm Pack and Upload Config Parser
     needs: ['create-release']
+    strategy:
+      matrix:
+        wasm-dirctory: [config-wasm]
     runs-on: ubuntu-latest
     env:
-      working-directory: ./lib/config-wasm
+      working-directory: ./lib/${{ matrix.wasm-dirctory }}
     steps:
       - uses: actions/checkout@v3
       - name: Download artifacts
@@ -232,7 +236,7 @@ jobs:
           # install wasm-pack
           mkdir ~/bin
           PATH=$PATH:~/bin
-          curl -sSL https://github.com/rustwasm/wasm-pack/releases/download/v0.11.0/wasm-pack-v0.11.0-x86_64-unknown-linux-musl.tar.gz \
+          curl -sSL https://github.com/rustwasm/wasm-pack/releases/download/v0.12.1/wasm-pack-v0.12.1-x86_64-unknown-linux-musl.tar.gz \
             | tar -xz --strip-components=1 -C ~/bin --no-anchored wasm-pack
 
           wasm-pack build --release -t nodejs --scope fs
@@ -241,7 +245,7 @@ jobs:
 
       - name: Compress for Linux
         run: |
-          asset_name="pewpew-$VERSION-config-wasm.tar.xz"
+          asset_name="pewpew-$VERSION-${{ matrix.wasm-dirctory }}.tar.xz"
           echo "ASSET_NAME=$asset_name" >> $GITHUB_ENV
           XZ_OPT=-9 tar -C ./ -cJf $asset_name package.json config*
         working-directory: ${{env.working-directory}}/pkg/
@@ -257,13 +261,16 @@ jobs:
           asset_content_type: application/octet-stream
 
 
-  # The hdr-histogram-wasm needs to be build with bundler rather than nodejs
+  # The hdr-histogram-wasm and config-gen need to be build with bundler rather than nodejs
   wasm-release-histogram:
     name: Wasm Pack and Upload Histogram
     needs: ['create-release']
+    strategy:
+      matrix:
+        wasm-dirctory: [hdr-histogram-wasm, config-gen]
     runs-on: ubuntu-latest
     env:
-      working-directory: ./lib/hdr-histogram-wasm
+      working-directory: ./lib/${{ matrix.wasm-dirctory }}
     steps:
       - uses: actions/checkout@v3
       - name: Download artifacts
@@ -304,7 +311,7 @@ jobs:
           # install wasm-pack
           mkdir ~/bin
           PATH=$PATH:~/bin
-          curl -sSL https://github.com/rustwasm/wasm-pack/releases/download/v0.11.0/wasm-pack-v0.11.0-x86_64-unknown-linux-musl.tar.gz \
+          curl -sSL https://github.com/rustwasm/wasm-pack/releases/download/v0.12.1/wasm-pack-v0.12.1-x86_64-unknown-linux-musl.tar.gz \
             | tar -xz --strip-components=1 -C ~/bin --no-anchored wasm-pack
 
           wasm-pack build --release -t bundler --scope fs
@@ -313,7 +320,7 @@ jobs:
 
       - name: Compress for Linux
         run: |
-          asset_name="pewpew-$VERSION-hdr-histogram-wasm.tar.xz"
+          asset_name="pewpew-$VERSION-${{ matrix.wasm-dirctory }}.tar.xz"
           echo "ASSET_NAME=$asset_name" >> $GITHUB_ENV
           XZ_OPT=-9 tar -C ./ -cJf $asset_name package.json hdr*
         working-directory: ${{env.working-directory}}/pkg/

diff --git a/.github/workflows/update-guide.yml b/.github/workflows/update-guide.yml
@@ -30,7 +30,7 @@ jobs:
           PATH=$PATH:~/bin
           curl -sSL https://github.com/rust-lang/mdBook/releases/download/v0.4.20/mdbook-v0.4.20-x86_64-unknown-linux-gnu.tar.gz \
             | tar -xz -C ~/bin
-          curl -sSL https://github.com/rustwasm/wasm-pack/releases/download/v0.11.0/wasm-pack-v0.11.0-x86_64-unknown-linux-musl.tar.gz \
+          curl -sSL https://github.com/rustwasm/wasm-pack/releases/download/v0.12.1/wasm-pack-v0.12.1-x86_64-unknown-linux-musl.tar.gz \
             | tar -xz --strip-components=1 -C ~/bin --no-anchored wasm-pack
 
           # setup some envs to various paths for convenience
@@ -40,10 +40,18 @@ jobs:
           WASM_LIB_DIR=$(realpath $PROJECT_ROOT/lib/hdr-histogram-wasm)
           WASM_OUTPUT_REACT_DIR=$RESULTS_VIEWER_REACT_DIR/lib/hdr-histogram-wasm
 
+          CFG_GEN_DIR=$(realpath $PROJECT_ROOT/lib/config-gen)
+          mkdir -p "$RESULTS_VIEWER_REACT_DIR/lib/config-gen"
+          CFG_GEN_OUTPUT_REACT_DIR=$RESULTS_VIEWER_REACT_DIR/lib/config-gen
+
           # build the hdr-histogram-wasm for the results viewer
           cd $WASM_LIB_DIR
           wasm-pack build --release -t bundler -d $WASM_OUTPUT_REACT_DIR --scope fs
 
+          # build the config-gen library for the HAR to YAML converter
+          cd $CFG_GEN_DIR
+          wasm-pack build --release -t bundler -d $CFG_GEN_OUTPUT_REACT_DIR --scope fs
+
           cd $RESULTS_VIEWER_REACT_DIR
           npm ci
           npm run build

diff --git a/Cargo.lock b/Cargo.lock
diff --git a/Cargo.toml b/Cargo.toml
@@ -68,6 +68,9 @@ tempfile = "3"
 panic = "abort"
 lto = "fat"
 
+[profile.release.package.config-gen]
+opt-level = 's'
+
 [profile.release.package.config-wasm]
 opt-level = 's'
 

diff --git a/guide/build-guide.sh b/guide/build-guide.sh
@@ -38,7 +38,7 @@ wasm-pack build --release -t bundler -d $WASM_OUTPUT_REACT_DIR --scope fs
 
 # build the config-gen library for the HAR to YAML converter
 cd $CFG_GEN_DIR
-wasm-pack build --release -t bundler -d $CFG_GEN_OUTPUT_REACT_DIR
+wasm-pack build --release -t bundler -d $CFG_GEN_OUTPUT_REACT_DIR --scope fs
 
 # build the results viewer (which includes putting the output into the book's src)
 cd $RESULTS_VIEWER_REACT_DIR

diff --git a/guide/results-viewer-react/.eslintrc b/guide/results-viewer-react/.eslintrc
@@ -12,7 +12,7 @@
     "plugin:@typescript-eslint/recommended",
     "plugin:@typescript-eslint/stylistic"
   ],
-  "ignorePatterns": ["hdr-histogram-wasm/", "webpack.config.js"],
+  "ignorePatterns": ["hdr-histogram-wasm/", "config-gen/", "webpack.config.js"],
   "rules": {
     "@typescript-eslint/no-explicit-any": 0,
     "@typescript-eslint/no-non-null-assertion": 0,