Merge all data sources #4169
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # SPDX-FileCopyrightText: © 2022–2024 Kevin Lu | |
| # SPDX-Licence-Identifier: AGPL-3.0-or-later | |
| name: Merge all data sources | |
| on: | |
| workflow_dispatch: | |
| repository_dispatch: | |
| types: | |
| - update | |
| jobs: | |
| merge: | |
| runs-on: ubuntu-latest | |
| outputs: | |
| status: ${{ steps.commit.outputs.status }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| path: yaml-yugi | |
| - uses: actions/checkout@v4 | |
| with: | |
| repository: DawnbrandBots/yaml-yugipedia | |
| path: yaml-yugipedia | |
| - uses: actions/checkout@v4 | |
| with: | |
| path: yaml-yugi-ko | |
| repository: DawnbrandBots/yaml-yugi-ko | |
| sparse-checkout: | | |
| *.csv | |
| sparse-checkout-cone-mode: false | |
| - uses: actions/checkout@v4 | |
| with: | |
| repository: DawnbrandBots/yaml-yugi-zh | |
| path: yaml-yugi-zh | |
| - uses: actions/checkout@v4 | |
| with: | |
| repository: db-ygoresources-com/yugioh-card-history | |
| path: yugioh-card-history | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.12" | |
| cache: pip | |
| cache-dependency-path: yaml-yugi/src/requirements.txt | |
| - name: Setup dependencies | |
| run: | | |
| mkdir aggregate | |
| curl -fsSLo aggregate/master-duel-raw.json https://dawnbrandbots.github.io/yaml-yugi/master-duel-raw.json | |
| curl -fsSLo tcg.vector.json https://dawnbrandbots.github.io/yaml-yugi-limit-regulation/tcg/current.vector.json | |
| curl -fsSLo ocg.vector.json https://dawnbrandbots.github.io/yaml-yugi-limit-regulation/ocg/current.vector.json | |
| pip install -r yaml-yugi/src/requirements.txt | |
| - name: Transform (series) | |
| working-directory: yaml-yugi/data/series | |
| run: python3 ../../src/main_archetypes.py ../../../yaml-yugipedia/wikitext/TCG_and_OCG_archetypes | |
| - name: Transform (OCG+TCG) | |
| working-directory: yaml-yugi/data/cards | |
| run: | | |
| git rm --ignore-unmatch *.json *.yaml | |
| python3 ../../src/main_ocgtcg.py \ | |
| ../../../yaml-yugipedia/wikitext/Duel_Monsters_cards \ | |
| --zh-CN ../../../yaml-yugi-zh/zh-CN \ | |
| --assignments ../../src/assignments/assignments.yaml \ | |
| --tcg ../../../tcg.vector.json \ | |
| --ocg ../../../ocg.vector.json \ | |
| --unreleased ../../../yaml-yugipedia/semantic-mediawiki/unreleased.csv \ | |
| --ko-official ../../../yaml-yugi-ko/_site/ocg.csv \ | |
| --ko-override ../../../yaml-yugi-ko/ocg-override.csv \ | |
| --ko-prerelease ../../../yaml-yugi-ko/ocg-prerelease.csv \ | |
| --master-duel ../../../aggregate/master-duel-raw.json \ | |
| --aggregate ../../../aggregate/cards.json | |
| - name: Transform (Rush Duel) | |
| working-directory: yaml-yugi/data/rush | |
| run: | | |
| git rm --ignore-unmatch *.json *.yaml | |
| python3 ../../src/main_rush.py \ | |
| ../../../yaml-yugipedia/wikitext/Rush_Duel_cards \ | |
| --ko-override ../../../yaml-yugi-ko/rush-override.csv \ | |
| --ko-prerelease ../../../yaml-yugi-ko/rush-prerelease.csv \ | |
| --ocg-aggregate ../../../aggregate/cards.json \ | |
| --aggregate ../../../aggregate/rush.json | |
| - name: Transform (TCG Speed Duel Skills) | |
| working-directory: yaml-yugi/data/tcg-speed-skill | |
| run: | | |
| git rm --ignore-unmatch *.json *.yaml | |
| python3 ../../src/main_speed.py \ | |
| ../../../yaml-yugipedia/wikitext/Skill_Cards \ | |
| --aggregate ../../../aggregate/skill.json | |
| - id: commit | |
| uses: DawnbrandBots/.github/actions/commit-push@main | |
| with: | |
| message: "Transform: ${{ github.run_number }} (${{ github.run_id }})" | |
| working-directory: yaml-yugi | |
| - if: steps.commit.outputs.status > 0 | |
| working-directory: yaml-yugi | |
| run: gh workflow run validate-data.yaml | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| - name: Transform (Master Duel) | |
| working-directory: yaml-yugi | |
| run: python src/main_masterduel.py '../yaml-yugipedia/wikitext/Yu-Gi-Oh!_Master_Duel_cards' > ../aggregate/master-duel-raw.json | |
| - if: steps.commit.outputs.status > 0 | |
| working-directory: yaml-yugi | |
| name: Merge | |
| run: | | |
| sed -s '1i---' data/cards/*.yaml > ../aggregate/cards.yaml | |
| sed -s '1i---' data/rush/*.yaml > ../aggregate/rush.yaml | |
| - if: steps.commit.outputs.status > 0 | |
| uses: actions/setup-node@v4 | |
| with: | |
| node-version-file: yaml-yugi/.node-version | |
| cache: yarn | |
| cache-dependency-path: yaml-yugi/yarn.lock | |
| - if: steps.commit.outputs.status > 0 | |
| working-directory: yaml-yugi | |
| run: yarn | |
| - if: steps.commit.outputs.status > 0 | |
| working-directory: yaml-yugi | |
| run: yarn --silent check-for-ygoresources-discrepancies ../yugioh-card-history ja > ../aggregate/discrepancy.ja.json || true | |
| - if: steps.commit.outputs.status > 0 | |
| working-directory: yaml-yugi | |
| run: yarn --silent check-for-ygoresources-discrepancies ../yugioh-card-history en > ../aggregate/discrepancy.en.json || true | |
| - if: steps.commit.outputs.status > 0 | |
| working-directory: yaml-yugi | |
| run: yarn --silent check-for-ygoresources-discrepancies ../yugioh-card-history de > ../aggregate/discrepancy.de.json || true | |
| - if: steps.commit.outputs.status > 0 | |
| working-directory: yaml-yugi | |
| run: yarn --silent check-for-ygoresources-discrepancies ../yugioh-card-history es > ../aggregate/discrepancy.es.json || true | |
| - if: steps.commit.outputs.status > 0 | |
| working-directory: yaml-yugi | |
| run: yarn --silent check-for-ygoresources-discrepancies ../yugioh-card-history fr > ../aggregate/discrepancy.fr.json || true | |
| - if: steps.commit.outputs.status > 0 | |
| working-directory: yaml-yugi | |
| run: yarn --silent check-for-ygoresources-discrepancies ../yugioh-card-history it > ../aggregate/discrepancy.it.json || true | |
| - if: steps.commit.outputs.status > 0 | |
| working-directory: yaml-yugi | |
| run: yarn --silent check-for-ygoresources-discrepancies ../yugioh-card-history pt > ../aggregate/discrepancy.pt.json || true | |
| - if: steps.commit.outputs.status > 0 | |
| run: cp aggregate/* yaml-yugi/src/web | |
| - if: steps.commit.outputs.status > 0 | |
| uses: actions/upload-pages-artifact@v3 | |
| with: | |
| path: yaml-yugi/src/web | |
| - if: steps.commit.outputs.status > 0 | |
| working-directory: yaml-yugi | |
| name: Load (push) | |
| env: | |
| OPENSEARCH_URL: ${{ secrets.OS_URL }} | |
| run: | | |
| yarn load ../aggregate/cards.yaml yu-gi-oh_ocg-tcg | |
| yarn load ../aggregate/rush.yaml yu-gi-oh_rush | |
| deploy-pages: | |
| needs: merge | |
| if: needs.merge.outputs.status > 0 && !cancelled() | |
| # Grant GITHUB_TOKEN the permissions required to make a Pages deployment | |
| permissions: | |
| pages: write # to deploy to Pages | |
| id-token: write # to verify the deployment originates from an appropriate source | |
| # Deploy to the github-pages environment | |
| environment: | |
| name: github-pages | |
| url: ${{ steps.deployment.outputs.page_url }} | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/deploy-pages@v4 | |
| id: deployment |