Merge all data sources #2197
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # SPDX-FileCopyrightText: © 2022–2023 Kevin Lu | |
| # SPDX-Licence-Identifier: AGPL-3.0-or-later | |
| name: Merge all data sources | |
| on: | |
| workflow_dispatch: | |
| repository_dispatch: | |
| types: | |
| - update | |
| jobs: | |
| merge: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| path: yaml-yugi | |
| - uses: actions/checkout@v4 | |
| with: | |
| repository: DawnbrandBots/yaml-yugipedia | |
| path: yaml-yugipedia | |
| - uses: actions/checkout@v4 | |
| with: | |
| path: yaml-yugi-ko | |
| repository: DawnbrandBots/yaml-yugi-ko | |
| sparse-checkout: | | |
| /*.csv | |
| /overrides.tsv | |
| sparse-checkout-cone-mode: false | |
| - uses: actions/checkout@v4 | |
| with: | |
| repository: DawnbrandBots/yaml-yugi-zh | |
| path: yaml-yugi-zh | |
| - uses: actions/checkout@v4 | |
| with: | |
| path: aggregate | |
| ref: aggregate | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.10" | |
| cache: pip | |
| cache-dependency-path: yaml-yugi/src/requirements.txt | |
| - name: Download yaml-yugi-ko exports and TCG/OCG limit regulations | |
| run: | | |
| curl -fsSLO https://dawnbrandbots.github.io/yaml-yugi-ko/ocg.csv | |
| curl -fsSLo tcg.vector.json https://dawnbrandbots.github.io/yaml-yugi-limit-regulation/tcg/current.vector.json | |
| curl -fsSLo ocg.vector.json https://dawnbrandbots.github.io/yaml-yugi-limit-regulation/ocg/current.vector.json | |
| - run: pip install -r yaml-yugi/src/requirements.txt | |
| - name: Transform (series) | |
| working-directory: yaml-yugi/data/series | |
| run: python3 ../../src/main_archetypes.py ../../../yaml-yugipedia/wikitext/TCG_and_OCG_archetypes | |
| - name: Transform (OCG+TCG) | |
| working-directory: yaml-yugi/data/cards | |
| run: | | |
| git rm --ignore-unmatch kdb*.json kdb*.yaml yugipedia*.json yugipedia*.yaml | |
| python3 ../../src/main_ocgtcg.py \ | |
| ../../../yaml-yugipedia/wikitext/Duel_Monsters_cards \ | |
| --zh-CN ../../../yaml-yugi-zh/zh-CN \ | |
| --assignments ../../src/assignments/assignments.yaml \ | |
| --tcg ../../../tcg.vector.json \ | |
| --ocg ../../../ocg.vector.json \ | |
| --ko ../../../yaml-yugi-ko/overrides.tsv \ | |
| --ko-official ../../../ocg.csv \ | |
| --ko-override ../../../yaml-yugi-ko/ocg-override.csv \ | |
| --ko-prerelease ../../../yaml-yugi-ko/ocg-prerelease.csv \ | |
| --master-duel ../../../aggregate/master-duel-raw.json \ | |
| --aggregate ../../../aggregate/cards.json | |
| - name: Transform (Rush Duel) | |
| working-directory: yaml-yugi/data/rush | |
| run: | | |
| git rm --ignore-unmatch yugipedia*.json yugipedia*.yaml | |
| python3 ../../src/main_rush.py \ | |
| ../../../yaml-yugipedia/wikitext/Rush_Duel_cards \ | |
| --ko-override ../../../yaml-yugi-ko/rush-override.csv \ | |
| --ko-prerelease ../../../yaml-yugi-ko/rush-prerelease.csv \ | |
| --aggregate ../../../aggregate/rush.json | |
| - name: Transform (TCG Speed Duel Skills) | |
| working-directory: yaml-yugi/data/tcg-speed-skill | |
| run: | | |
| git rm --ignore-unmatch *.json *.yaml | |
| python3 ../../src/main_speed.py \ | |
| ../../../yaml-yugipedia/wikitext/Skill_Cards \ | |
| --aggregate ../../../aggregate/skill.json | |
| - id: commit | |
| uses: DawnbrandBots/.github/actions/commit-push@main | |
| with: | |
| message: "Transform: ${{ github.run_number }} (${{ github.run_id }})" | |
| working-directory: yaml-yugi | |
| - if: steps.commit.outputs.status > 0 | |
| working-directory: yaml-yugi | |
| run: gh workflow run validate-data.yaml | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| - name: Transform (Master Duel) | |
| working-directory: yaml-yugi | |
| run: python src/main_masterduel.py '../yaml-yugipedia/wikitext/Yu-Gi-Oh!_Master_Duel_cards' > ../aggregate/master-duel-raw.json | |
| - if: steps.commit.outputs.status > 0 | |
| working-directory: yaml-yugi | |
| name: Merge | |
| run: | | |
| sed -s '1i---' data/cards/*.yaml > ../aggregate/cards.yaml | |
| sed -s '1i---' data/rush/*.yaml > ../aggregate/rush.yaml | |
| - if: steps.commit.outputs.status > 0 | |
| working-directory: aggregate | |
| name: Commit aggregate | |
| run: | | |
| git config user.name GitHub Actions | |
| git config user.email [email protected] | |
| git add . | |
| git commit -m "Merge: ${{ github.run_number }} (${{ github.run_id }})" -m "$(git -C ../yaml-yugi rev-parse @)" | |
| git pull --rebase origin aggregate | |
| git push | |
| - if: steps.commit.outputs.status > 0 | |
| uses: actions/setup-node@v4 | |
| with: | |
| node-version: 20 | |
| cache: yarn | |
| cache-dependency-path: yaml-yugi/yarn.lock | |
| - if: steps.commit.outputs.status > 0 | |
| working-directory: yaml-yugi | |
| run: yarn | |
| - if: steps.commit.outputs.status > 0 | |
| working-directory: yaml-yugi | |
| name: Load (push) | |
| env: | |
| OPENSEARCH_URL: ${{ secrets.OS_URL }} | |
| run: | | |
| yarn ts-node src/load.ts ../aggregate/cards.yaml yu-gi-oh_ocg-tcg_20230501 | |
| yarn ts-node src/load.ts ../aggregate/rush.yaml yu-gi-oh_rush_20230501 |