feat(add): support to count tokens (#1)

* update release script

* use sync io

* support counting tokens

* support choosing tokenizer

* update READMEs

.github/workflows/release.yaml

@@ -8,52 +8,102 @@ on:
 jobs:
   build:
     name: Build Release
-    runs-on: ${{ matrix.os }}
+    permissions:
+      contents: write
     strategy:
+      fail-fast: false
       matrix:
         include:
-          - os: ubuntu-latest
-            rust: stable
-            targets: 
-              - x86_64-unknown-linux-gnu
-              - x86_64-unknown-linux-musl
-              - i686-unknown-linux-musl
-              - aarch64-unknown-linux-musl
-              - arm-unknown-linux-musleabihf
-          - os: macos-latest
-            targets:
-              - x86_64-apple-darwin
-              - aarch64-apple-darwin
-          - os: windows-latest
-            targets:
-              - x86_64-pc-windows-msvc
-              - i686-pc-windows-msvc
-              - aarch64-pc-windows-msvc
-          - os: ubuntu-latest
-            targets:
-              - x86_64-unknown-freebsd
+          - target: x86_64-unknown-linux-gnu
+            os: ubuntu-latest
+            name: tc-x86_64-unknown-linux-gnu.tar.gz
+
+          - target: x86_64-unknown-linux-musl
+            os: ubuntu-latest
+            name: tc-x86_64-unknown-linux-musl.tar.gz
+
+          - target: i686-unknown-linux-musl
+            os: ubuntu-latest
+            name: tc-i686-unknown-linux-musl.tar.gz
+
+          - target: aarch64-unknown-linux-musl
+            os: ubuntu-latest
+            name: tc-aarch64-unknown-linux-musl.tar.gz
+
+          - target: arm-unknown-linux-musleabihf
+            os: ubuntu-latest
+            name: tc-arm-unknown-linux-musleabihf.tar.gz
+
+          - target: x86_64-apple-darwin
+            os: macos-latest
+            name: tc-x86_64-apple-darwin.tar.gz
+
+          - target: aarch64-apple-darwin
+            os: macos-latest
+            name: tc-aarch64-apple-darwin.tar.gz
+
+          - target: x86_64-pc-windows-msvc
+            os: windows-latest
+            name: tc-x86_64-pc-windows-msvc.zip
+            rustflags: -C target-feature=+crt-static
+
+          - target: i686-pc-windows-msvc
+            os: windows-latest
+            name: tc-i686-pc-windows-msvc.zip
+            rustflags: -C target-feature=+crt-static
+
+          - target: aarch64-pc-windows-msvc
+            os: windows-latest
+            name: tc-aarch64-pc-windows-msvc.zip
+            rustflags: -C target-feature=+crt-static
+
+          - target: x86_64-unknown-freebsd
+            os: ubuntu-latest
+            name: tc-x86_64-unknown-freebsd.tar.gz
+
+    runs-on: ${{ matrix.os }}
+    continue-on-error: true
+    env:
+      RUSTFLAGS: ${{ matrix.rustflags || '' }}
 
     steps:
-    - uses: actions/checkout@v4
-
-    - name: Install Rust
-      uses: dtolnay/rust-toolchain@stable
-
-    - name: Add targets
-      run: |
-        for target in ${{ join(matrix.targets, ' ') }}; do
-          rustup target add $target
-        done
-
-    - name: Build
-      run: |
-        for target in ${{ join(matrix.targets, ' ') }}; do
-          cargo build --release --target $target
-          mv target/$target/release/tc target/$target/release/tc-$target
-        done
-
-    - name: Upload artifacts
-      uses: AButler/[email protected]
-      with:
-        files: target/*/release/tc-*
-        repo-token: ${{ secrets.GITHUB_TOKEN }}
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Install Rust Toolchain
+        uses: dtolnay/rust-toolchain@stable
+        with:
+          target: ${{ matrix.target }}
+
+      - name: Setup | Install cross [Linux]
+        if: matrix.os == 'ubuntu-latest'
+        uses: taiki-e/install-action@cross
+
+      - name: Build | Cargo
+        if: matrix.os != 'ubuntu-latest'
+        run: cargo build --release --locked --target ${{ matrix.target }}
+
+      - name: Build | Cross
+        if: matrix.os == 'ubuntu-latest'
+        run: cross build --release --locked --target ${{ matrix.target }}
+
+      - name: Post Build | Prepare artifacts [Windows]
+        if: matrix.os == 'windows-latest'
+        run: |
+          cd target/${{ matrix.target }}/release
+          7z a ../../../${{ matrix.name }} tc.exe
+          cd -
+
+      - name: Post Build | Prepare artifacts [-nix]
+        if: matrix.os != 'windows-latest'
+        run: |
+          cd target/${{ matrix.target }}/release
+          tar czvf ../../../${{ matrix.name }} tc
+          cd -
+
+      - name: Upload artifacts
+        uses: AButler/[email protected]
+        with:
+          files: ${{ matrix.name }}
+          repo-token: ${{ secrets.GITHUB_TOKEN }}
+          release-tag: ${{ github.ref_name }}