crawler.yml

name: crawler

on:
  workflow_dispatch:
  schedule:
    - cron: '0 6 * * *' # Every day at 6:00 AM UTC.

jobs:
  call-build-rippled-workflow:
    uses: runziggurat/xrpl/.github/workflows/build-rippled.yml@main

  crawl-network:
    runs-on: ubuntu-latest
    needs: [ call-build-rippled-workflow ]
    steps:
      - uses: actions/checkout@v3
        with:
          fetch-depth: 0
      - run: rustup toolchain install stable --profile minimal
      - name: Configure rippled
        run: |
          git clone https://github.com/XRPLF/rippled
          mkdir -p ~/.config/ripple
          cp rippled/cfg/rippled-example.cfg ~/.config/ripple/rippled.cfg
          cp rippled/cfg/validators-example.txt ~/.config/ripple/validators.txt
          sed -i 's#^path=/var/lib/rippled/db/nudb#path=/home/runner/work/rippled/rippled/db/nudb#' ~/.config/ripple/rippled.cfg
          sed -i 's#^/var/lib/rippled/db#/home/runner/work/rippled/rippled/db#' ~/.config/ripple/rippled.cfg
          sed -i 's#^/var/log/rippled/debug.log#/home/runner/work/rippled/rippled/debug.log#' ~/.config/ripple/rippled.cfg
      - uses: actions/download-artifact@v3
        with:
          name: node-executable
          path: ./rippled
      - name: Begin crawling
        run: |
          chmod +x rippled/rippled
          ./rippled/rippled &
          cargo run --features crawler --bin crawler -- --seed-addrs 127.0.0.1:51235 --rpc-addr 127.0.0.1:54321 &
          # After 30 min, query rpc and send SIGTERM.
          sleep 30m
          curl --data-binary '{"jsonrpc": "2.0", "id":0, "method": "dumpmetrics", "params": { "file": "latest.json" } }' -H 'content-type: application/json' http://127.0.0.1:54321/
          kill $(pidof crawler) $(pidof rippled)
      - name: Check for error
        run: |
          # If the result contains any error, fail workflow
          if grep "error" latest.json; then
            echo "Aborting. Crawler results contained an error"
            exit 1
          fi
          cat latest.json
      - uses: actions/upload-artifact@v3
        with:
          name: latest-result
          path: latest.json

  call-build-viz-state-workflow:
    needs: [ crawl-network ]
    uses: runziggurat/ziggurat-core/.github/workflows/build-viz-state.yml@main
    with:
      filter: Ripple

  call-process-results-workflow:
    needs: [ call-build-viz-state-workflow ]
    uses: runziggurat/ziggurat-core/.github/workflows/process-results.yml@main
    with:
      name: crawler
      extension: json
      repository: xrpl
    secrets:
      gcp_credentials: ${{ secrets.GCP_CREDENTIALS }}