fetch_all_tools.yaml

name: Fetch all tools

on:
  workflow_dispatch:
  schedule:
    #Every Sunday at 8:00 am
    - cron: "0 8 * * 0" 

# Allow only one concurrent deployment, skipping runs queued between the run in-progress and latest queued.
# However, do NOT cancel in-progress runs as we want to allow these production deployments to complete.
concurrency:
  group: "tools"
  cancel-in-progress: false
  
permissions:
  contents: write

jobs:

  fetch-all-tools-stepwise:
    runs-on: ubuntu-20.04
    environment: fetch-tools
    name: Fetch all tool stepwise
    strategy:
      max-parallel: 1 #need to run one after another, since otherwise there is a chance, that mulitple jobs want to push to the results branch at the same time (which fails due to merge)
      matrix:
        python-version: [3.11]
        subset:
         - repositories01.list
         - repositories02.list
         - repositories03.list
         - repositories04.list
    steps:
      - name: Checkout main
        uses: actions/checkout@v4
        with:
          ref: main #pull latest code produced by job 1, not the revision that started the workflow (https://github.com/actions/checkout/issues/439)
      - name: Checkout results
        uses: actions/checkout@v4
        with:
          ref: results
          path: results
      - uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python-version }}
      - name: Install requirement
        run: python -m pip install -r requirements.txt
      - name: Run script #needs PAT to access other repos
        run: | 
          bash bin/extract_all_tools.sh "${{ matrix.subset }}"
        env: 
          GITHUB_API_KEY: ${{ secrets.GH_API_TOKEN }}
      - name: Commit all tools 
        uses: s0/git-publish-subdir-action@develop
        env:
          BRANCH: results
          FOLDER: results
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
          REPO: self
          SKIP_EMPTY_COMMITS: true
          MESSAGE: "Update results step: fetch all tool stepwise - commmit {sha}"

  fetch-all-tools-merge:
    runs-on: ubuntu-20.04
    needs: fetch-all-tools-stepwise
    name: Fetch all tools merge
    steps:
    - name: Checkout main
      uses: actions/checkout@v4
      with:
        ref: main #pull latest code produced by job 1, not the revision that started the workflow (https://github.com/actions/checkout/issues/439)
    - name: Checkout results
      uses: actions/checkout@v4
      with:
        ref: results
        path: results
    - uses: actions/setup-python@v5
      with:
        python-version: '3.11'
    - name: Install requirement
      run: |
        python -m pip install -r requirements.txt
        sudo apt-get install jq
    - name: Merge all tools
      run: | #merge files with only one header -> https://stackoverflow.com/questions/16890582/unixmerge-multiple-csv-files-with-same-header-by-keeping-the-header-of-the-firs; map(.[]) -> https://stackoverflow.com/questions/42011086/merge-arrays-of-json (get flat array, one tool per entry)
        awk 'FNR==1 && NR!=1{next;}{print}' results/repositories*.list_tools.tsv > results/all_tools.tsv
        jq -s 'map(.[])' results/repositories*.list_tools.json > results/all_tools.json 
    - name: Wordcloud and interactive table
      run: |
        bash bin/format_tools.sh
    - name: Commit all tools
      # add or commit any changes in results if there was a change, merge with main and push as bot
      uses: s0/git-publish-subdir-action@develop
      env:
        BRANCH: results
        FOLDER: results
        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        REPO: self
        SKIP_EMPTY_COMMITS: true
        MESSAGE: "Update results step: fetch all tools merge - commmit {sha}"