Set up a fork with native builds

I've had a number of compilation issues with this gem on different platforms. I don't want us to have to install Rust everywhere as an additional dependency for a single gem. So I created this fork that does cross-compilation in Github Actions. There's one other problem I discovered, which is that Github's package management appears to ignore platforms when uploading gems. When you upload a gem with the same version number but a different platform, Github says you can't upload it since that version has already been uploaded. Rubygems.org doesn't have this problem. HOWEVER, I don't want to push the same gem name to Rubygems.org and get it confused with the upstream project. This is why I renamed the gem to rz_tiktoken_ruby. Please don't confuse it for the original (but do try it out if you need a version pre-built for your platform.)
retailzipline · Aug 24, 2023 · 55f85a0 · 55f85a0
1 parent 0fe5ed3
commit 55f85a0
Show file tree

Hide file tree

Showing 5 changed files with 47 additions and 32 deletions.
diff --git a/.github/workflows/cross_compile.yml → .github/workflows/compile.yml b/.github/workflows/cross_compile.yml → .github/workflows/compile.yml
@@ -1,5 +1,5 @@
 ---
-name: CI
+name: Compile and Upload
 
 on:
   workflow_dispatch:
@@ -9,6 +9,7 @@ on:
 
 jobs:
   ci-data:
+    name: Fetch CI data
     runs-on: ubuntu-latest
     outputs:
       result: ${{ steps.fetch.outputs.result }}
@@ -20,26 +21,40 @@ jobs:
           stable-ruby-versions: |
             exclude: [head]
   cross-gem:
-    name: Compile native gem for ${{ matrix.platform }}
+    name: ${{ matrix.platform }} compilation
     runs-on: ubuntu-latest
     needs: ci-data
     strategy:
       matrix:
         platform: ${{ fromJSON(needs.ci-data.outputs.result).supported-ruby-platforms }}
     steps:
       - uses: actions/checkout@v3
-
+        name: Checkout project
       - uses: ruby/setup-ruby@v1
+        name: Setup ruby
         with:
-          ruby-version: "3.1"
-
+          ruby-version: 3.2
       - uses: oxidize-rb/actions/cross-gem@v1
+        name: Compile ${{ matrix.platform }}
         id: cross-gem
         with:
           platform: ${{ matrix.platform }}
           ruby-versions: ${{ join(fromJSON(needs.ci-data.outputs.result).stable-ruby-versions, ',') }}
-
-      - uses: actions/upload-artifact@v2
+      - name: Display structure of downloaded files
+        run: ls -R
+        working-directory: pkg
+      - uses: fac/ruby-gem-setup-credentials-action@v2
+        name: Set up github creds for pushing
+        with:
+          user: ""
+          key: rubygems
+          token: ${{ secrets.RUBYGEMS_KEY }}
+      - uses: fac/ruby-gem-push-action@v2
+        name: Push gem to Github Packages
+        with:
+          key: rubygems
+      - uses: actions/upload-artifact@v3
+        name: upload push.out
         with:
-          name: cross-gem
-          path: ${{ steps.cross-gem.outputs.gem-path }}
+          name: pushout
+          path: push.out
diff --git a/.github/workflows/build.yml → .github/workflows/test.yml b/.github/workflows/build.yml → .github/workflows/test.yml
@@ -1,26 +1,22 @@
-name: build
+name: Test and Lint
 on: [push, pull_request]
 jobs:
   test:
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-latest]
+        os: ["ubuntu-latest", "macos-latest"]
         ruby: [3.2, 3.0, 2.7]
+        rust: [stable]
     runs-on: ${{ matrix.os }}
     steps:
       - uses: actions/checkout@v3
-      - uses: actions/cache@v3
-        with:
-          path: |
-            ~/.cargo/registry
-            ~/.cargo/git
-            tmp
-          key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}
-      - uses: ruby/setup-ruby@v1
+      - uses: oxidize-rb/actions/setup-ruby-and-rust@v1
         with:
           ruby-version: ${{ matrix.ruby }}
+          rustup-toolchain: ${{ matrix.rust }}
           bundler-cache: true
+          cargo-cache: true
       - run: bundle exec rake build
       - run: bundle exec rake spec
   lint:
@@ -32,6 +28,6 @@ jobs:
       - uses: actions/checkout@v3
       - uses: ruby/setup-ruby@v1
         with:
-          ruby-version: 3.1
+          ruby-version: 3.2
           bundler-cache: true
       - run: bundle exec rake standard
diff --git a/Gemfile.lock b/Gemfile.lock
@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    tiktoken_ruby (0.0.5)
+    rz_tiktoken_ruby (0.0.6)
       rb_sys (~> 0.9.81)
 
 GEM
@@ -85,8 +85,8 @@ DEPENDENCIES
   rake (~> 13.0)
   rake-compiler
   rspec (~> 3.0)
+  rz_tiktoken_ruby!
   standard (~> 1.3)
-  tiktoken_ruby!
   yard-doctest (~> 0.1.17)
 
 BUNDLED WITH

diff --git a/lib/tiktoken_ruby/version.rb b/lib/tiktoken_ruby/version.rb
@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Tiktoken
-  VERSION = "0.0.5"
+  VERSION = "0.0.6"
 end
diff --git a/tiktoken_ruby.gemspec b/tiktoken_ruby.gemspec
@@ -3,25 +3,29 @@
 require_relative "lib/tiktoken_ruby/version"
 
 Gem::Specification.new do |spec|
-  spec.name = "tiktoken_ruby"
+  spec.name = "rz_tiktoken_ruby"
   spec.version = Tiktoken::VERSION
-  spec.authors = ["IAPark"]
-  spec.email = ["[email protected]"]
+  spec.authors = ["IAPark", "judy"]
+  spec.email = ["[email protected]", "[email protected]"]
 
   spec.summary = "Ruby wrapper for Tiktoken"
-  spec.description = "An unofficial Ruby wrapper for Tiktoken, " \
-    "a BPE tokenizer written by and used by OpenAI. It can be used to " \
-    "count the number of tokens in text before sending it to OpenAI APIs."
-
-  spec.homepage = "https://github.com/IAPark/tiktoken_ruby"
+  spec.description = <<~TEXT
+    An unofficial Ruby wrapper for Tiktoken, a BPE tokenizer written by and used by OpenAI. It
+    can be used to count the number of tokens in text before sending it to OpenAI APIs. This
+    is a fork of tiktoken_ruby by IAPark, which has been cross-compiled for multiple platforms.
+    This way compilation with Rust extensions doesn't need to happen wherever you are deploying it.
+  TEXT
+
+  spec.homepage = "https://github.com/retailzipline/tiktoken_ruby"
   spec.license = "MIT"
   spec.required_ruby_version = ">= 2.7.0"
   spec.required_rubygems_version = ">= 3.1.0"
   spec.platform = Gem::Platform::RUBY
 
   spec.metadata["homepage_uri"] = spec.homepage
-  spec.metadata["source_code_uri"] = "https://github.com/IAPark/tiktoken_ruby"
+  spec.metadata["source_code_uri"] = "https://github.com/retailzipline/tiktoken_ruby"
   spec.metadata["documentation_uri"] = "https://rubydoc.info/github/IAPark/tiktoken_ruby/main"
+  spec.metadata['allowed_push_host'] = 'https://rubygems.org'
 
   # spec.metadata["changelog_uri"] = "TODO: Put your gem's CHANGELOG.md URL here."