crawler #158
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: crawler | |
| on: | |
| workflow_dispatch: | |
| schedule: | |
| - cron: '0 6 * * *' # Every day at 6:00 AM UTC. | |
| jobs: | |
| call-build-rippled-workflow: | |
| uses: runziggurat/xrpl/.github/workflows/build-rippled.yml@main | |
| crawl-network: | |
| runs-on: ubuntu-latest | |
| needs: [ call-build-rippled-workflow ] | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| fetch-depth: 0 | |
| - run: rustup toolchain install stable --profile minimal | |
| - name: Configure rippled | |
| run: | | |
| git clone https://github.com/XRPLF/rippled | |
| mkdir -p ~/.config/ripple | |
| cp rippled/cfg/rippled-example.cfg ~/.config/ripple/rippled.cfg | |
| cp rippled/cfg/validators-example.txt ~/.config/ripple/validators.txt | |
| sed -i 's#^path=/var/lib/rippled/db/nudb#path=/home/runner/work/rippled/rippled/db/nudb#' ~/.config/ripple/rippled.cfg | |
| sed -i 's#^/var/lib/rippled/db#/home/runner/work/rippled/rippled/db#' ~/.config/ripple/rippled.cfg | |
| sed -i 's#^/var/log/rippled/debug.log#/home/runner/work/rippled/rippled/debug.log#' ~/.config/ripple/rippled.cfg | |
| - uses: actions/download-artifact@v3 | |
| with: | |
| name: node-executable | |
| path: ./rippled | |
| - name: Begin crawling | |
| run: | | |
| chmod +x rippled/rippled | |
| ./rippled/rippled & | |
| cargo run --features crawler --bin crawler -- --seed-addrs 127.0.0.1:51235 --rpc-addr 127.0.0.1:54321 & | |
| # After 30 min, query rpc and send SIGTERM. | |
| sleep 30m | |
| curl --data-binary '{"jsonrpc": "2.0", "id":0, "method": "dumpmetrics", "params": { "file": "latest.json" } }' -H 'content-type: application/json' http://127.0.0.1:54321/ | |
| kill $(pidof crawler) $(pidof rippled) | |
| - name: Check for error | |
| run: | | |
| # If the result contains any error, fail workflow | |
| if grep "error" latest.json; then | |
| echo "Aborting. Crawler results contained an error" | |
| exit 1 | |
| fi | |
| cat latest.json | |
| - uses: actions/upload-artifact@v3 | |
| with: | |
| name: latest-result | |
| path: latest.json | |
| call-build-viz-state-workflow: | |
| needs: [ crawl-network ] | |
| uses: runziggurat/ziggurat-core/.github/workflows/build-viz-state.yml@main | |
| with: | |
| filter: Ripple | |
| call-process-results-workflow: | |
| needs: [ call-build-viz-state-workflow ] | |
| uses: runziggurat/ziggurat-core/.github/workflows/process-results.yml@main | |
| with: | |
| name: crawler | |
| extension: json | |
| repository: xrpl | |
| secrets: | |
| gcp_credentials: ${{ secrets.GCP_CREDENTIALS }} |