build.yml

name: Scrape Releases

on:
  push:
    branches:
      - main
  schedule:
    - cron: "0 */8 * * *"
  workflow_dispatch:

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

jobs:
  scrape:
    runs-on: ubuntu-latest
    steps:
      - name: 🛒 Checkout the repo
        uses: actions/checkout@v4

      - name: 🐍 Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: '3.x'
          cache: 'pip'
      - run: pip install -r requirements.txt

      - name: 🔖 Get latest Mihomo (stable) version
        id: mihomo-version
        run: |
          echo "stable=$(curl -sL "https://api.github.com/repos/MetaCubeX/mihomo/releases/latest" | jq -r '.tag_name')" >> $GITHUB_OUTPUT
        shell: bash

      - name: 🍱 Restore SQLite Database from Cache
        id: cache-db-restore
        uses: actions/cache/restore@v4
        with:
          path: releases.db
          key: releases_db

      - name: 🍱 Restore Mihomo Installer from Cache
        id: cache-mihomo-restore
        uses: actions/cache/restore@v4
        with:
          path: mihomo-linux-amd64-${{ steps.mihomo-version.outputs.stable }}.deb
          key: mihomo-linux-amd64-${{ steps.mihomo-version.outputs.stable }}

      - name: 🤿 Setup and start Mihomo
        run: |
            # Download Mihomo Installer if needed
            [ ! -f "mihomo-linux-amd64-${{ steps.mihomo-version.outputs.stable }}.deb" ] && curl -fsSL "https://github.com/MetaCubeX/mihomo/releases/download/${{ steps.mihomo-version.outputs.stable }}/mihomo-linux-amd64-${{ steps.mihomo-version.outputs.stable }}.deb" -o "mihomo-linux-amd64-${{ steps.mihomo-version.outputs.stable }}.deb"

            # Install Mihomo
            sudo dpkg -i ./mihomo-linux-amd64-${{ steps.mihomo-version.outputs.stable }}.deb

            # Run Mihomo with our configuration (as daemon)
            ./mihomo_daemon.py
        shell: bash

      - name: 🕸️ Run crawler
        run: |
            scrapy crawl releases -L DEBUG --logfile="scrapy.log"
        shell: bash

      - name: 🥡 Save SQLite Database to Cache
        uses: actions/cache/save@v4
        with:
          path: releases.db
          key: ${{ steps.cache-db-restore.outputs.cache-primary-key }}

      - name: 🥡 Save Mihomo Installer to Cache
        uses: actions/cache/save@v4
        with:
          path: mihomo-linux-amd64-${{ steps.mihomo-version.outputs.stable }}.deb
          key: ${{ steps.cache-mihomo-restore.outputs.cache-primary-key }}

      - name: ✅ Validate stuff
        run: |
            [ -s "${{ github.workspace }}/releases.csv" ] && echo "🎉 CSV file generated successfully!" || exit 1
            [ -s "${{ github.workspace }}/releases.json" ] && echo "🎉 JSON file generated successfully!" || exit 1
            [ -s "${{ github.workspace }}/releases.jsonl" ] && echo "🎉 JSONLines file generated successfully!" || exit 1
            [ -s "${{ github.workspace }}/releases.rss" ] && echo "🎉 RSS Feed generated successfully!" || exit 1
            [ -s "${{ github.workspace }}/releases.db" ] && echo "🎉 SQLite DB updated successfully!" || exit 1
        shell: bash

      - name: 🗒️ Upload Scrapy logs
        if: failure()
        uses: actions/upload-artifact@v4
        with:
          name: scrapy.log
          path: ${{ github.workspace }}/scrapy.log

      - name: 🗒️ Upload Mihomo logs
        if: failure()
        uses: actions/upload-artifact@v4
        with:
          name: mihomo.log
          path: ${{ github.workspace }}/mihomo.log

      - name: ✏️ Sort items
        run: |
            ./sort_items.py && echo "🎉 Items sorted successfully!" || exit 1
        shell: bash

      - name: 🚮 Delete older releases
        uses: dev-drprasad/delete-older-releases@v0.3.4
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        with:
          keep_latest: 3
          delete_tags: true

      - name: 🛠️ Generate stats
        id: gen-stats
        run: |
          # Date
          DATE="$(date -u '+%Y%m%d%H%M%S')"
          echo "date=$DATE" >> $GITHUB_OUTPUT

          # Number of releases
          RELEASES="$(jq length "${{ github.workspace }}/releases.json")"
          echo "releases=$RELEASES" >> $GITHUB_OUTPUT
        shell: bash

      - name: 🚀 Upload new release
        uses: softprops/action-gh-release@v2
        with:
          tag_name: feeds-${{ steps.gen-stats.outputs.date }}
          name: ${{ steps.gen-stats.outputs.date }}
          body: |
            Number of releases - ${{ steps.gen-stats.outputs.releases }}
          files: |
            ${{ github.workspace }}/releases.csv
            ${{ github.workspace }}/releases.json
            ${{ github.workspace }}/releases.jsonl
            ${{ github.workspace }}/releases.rss
            ${{ github.workspace }}/releases.db
          draft: false
          prerelease: false