feat: added automatic fetching of stars

.github/workflows/build.yml

@@ -21,13 +21,16 @@ jobs:
       - name: Install dependencies
         run: pip install -r requirements.txt
 
+      - name: Update stars
+        run: python update_stars.py
+
       - name: Generate README
         run: python generate_readme.py
 
       - name: Commit and push changes
         run: |
           git config --global user.name 'GitHub Actions'
           git config --global user.email '[email protected]'
-          git add README.md
+          git add .
           git commit -m "chore: automatically generated README.md"
           git push origin master

README.md.in

@@ -1,4 +1,4 @@
-<!-- 
+<!--
 
 Hey, you!
 Are you reading this in the generated README.md? Then you're in the wrong place!
@@ -19,7 +19,7 @@ Current trends and state of the art for using open & local LLM models as copilot
 
 ## 📋 Summary
 
-Local Copilots are in an early experimental stage, with most being of MVP-quality. 
+Local Copilots are in an early experimental stage, with most being of MVP-quality.
 
 The reasons for this are:
 
@@ -35,7 +35,7 @@ This document is a curated list of local Copilots, shell assistants, and related
 
 In 2021, GitHub released Copilot which quickly became popular among devs. Since then, with the flurry of AI developments around LLMs, local models that can run on consumer machines have become available, and it has seemed only a matter of time before Copilot will go local.
 
-Many perceived limitations of GitHub's Copilot are related to its closed and cloud-hosted nature. 
+Many perceived limitations of GitHub's Copilot are related to its closed and cloud-hosted nature.
 
 As an alternative, local Copilots enable:
 
@@ -65,7 +65,7 @@ Tools that try to generate projects/features from specification:
 
 ## 🗨️ Chat Interfaces
 
-Chat interfaces with shell/REPL/notebook access. 
+Chat interfaces with shell/REPL/notebook access.
 Similar to/inspired by ChatGPT's "Advanced Data Analysis" feature (previously "Code Interpreter").
 
 | Name                     | :star:  | Notes                                                                                                  |

generate_readme.py

@@ -7,9 +7,18 @@
 with open("data.yaml", "r") as f:
     tables = yaml.safe_load(f)
 
-# print(f"Tables: {tables.keys()}")
+with open("stars.yaml", "r") as f:
+    stars = yaml.safe_load(f)
 
-rendered_template = template.render(tables)
+# combine
+for cat in tables:
+    for row in tables[cat]:
+        print(stars, row["link"])
+        if row["link"] in stars:
+            row["stars"] = stars[row["link"]]["stars"]
+
+
+rendered_template = template.render(tables, stars=stars)
 
 with open("README.md", "w") as f:
     f.write(rendered_template)

stars.yaml

@@ -0,0 +1,19 @@
+# Autogenerated by update_stars.py
+https://huggingface.co/Phind/Phind-CodeLlama-34B-v2:
+  stars: 332
+  updated: '2023-10-03T12:29:52.263572'
+https://huggingface.co/WizardLM/WizardCoder-15B-V1.0:
+  stars: 614
+  updated: '2023-10-03T12:29:52.263572'
+https://huggingface.co/WizardLM/WizardCoder-Python-34B-V1.0:
+  stars: 597
+  updated: '2023-10-03T12:29:52.263572'
+https://huggingface.co/bigcode/santacoder:
+  stars: 298
+  updated: '2023-10-03T12:29:52.263572'
+https://huggingface.co/replit/replit-code-v1-3b:
+  stars: 674
+  updated: '2023-10-03T12:29:52.263572'
+https://huggingface.co/sahil2801/replit-code-instruct-glaive:
+  stars: 83
+  updated: '2023-10-03T12:29:52.263572'

update_stars.py

@@ -0,0 +1,79 @@
+import yaml
+import requests
+
+
+from datetime import datetime
+
+
+from pathlib import Path
+
+
+def _get_stars(url):
+    if url.startswith("https://huggingface.co/"):
+        # examples:
+        #  - datasets/bigcode/the-stack
+        #  - models/Phind/Phind-CodeLlama-34B-v2
+        #  - WizardLM/WizardCoder-Python-34B-V1.0 (assumed to be a model)
+        repo_id = url[23:]
+        if not any([repo_id.startswith("datasets/"), repo_id.startswith("models/")]):
+            repo_id = f"models/{repo_id}"
+        response = requests.get(f"https://huggingface.co/api/{repo_id}")
+        response.raise_for_status()
+        json_data = response.json()
+        stars_count = json_data["likes"]
+    elif url.startswith("https://github.com/"):
+        github_api_url = "https://api.github.com/repos"
+        repo_id = url[19:]
+        response = requests.get(f"{github_api_url}/{repo_id}")
+        response.raise_for_status()
+        json_data = response.json()
+        stars_count = json_data["stargazers_count"]
+    else:
+        raise ValueError(f"Unknown repo url: {url}")
+
+    return stars_count
+
+
+def save_stars(file_path: str, dest: str) -> None:
+    with open(file_path, "r") as f:
+        data = yaml.load(f, Loader=yaml.SafeLoader)
+
+    starmap = {}
+    if Path(dest).exists():
+        with open(dest, "r") as f:
+            starmap = yaml.load(f, Loader=yaml.SafeLoader)
+
+    now = datetime.now()
+
+    def _update_stars(link):
+        try:
+            print(f"Attempting to get stars for {link}")
+            stars_count = _get_stars(link)
+            starmap[link] = {
+                "stars": stars_count,
+                "updated": now.isoformat(),
+            }
+            print(f"  Succeeded: {stars_count}")
+        except Exception as e:
+            print(f"  Error: {e}")
+
+    # Update the existing entries, if it's been more than 3 days:
+    for link in starmap:
+        if (datetime.now() - datetime.fromisoformat(starmap[link]["updated"])).days > 3:
+            _update_stars(link)
+
+    # If any new
+    for category in data:
+        for item in data[category]:
+            if item["link"] not in starmap:
+                print("New entry: ", item["link"])
+                _update_stars(item["link"])
+
+    with open(dest, "w") as f:
+        # add comment to the top of the file, so that it's clear it's autogenerated
+        f.write("# Autogenerated by update_stars.py\n")
+        yaml.dump(starmap, f, default_flow_style=False)
+
+
+if __name__ == "__main__":
+    save_stars("data.yaml", "stars.yaml")