Merge all data sources #3273
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # SPDX-FileCopyrightText: © 2022–2024 Kevin Lu | |
| # SPDX-Licence-Identifier: AGPL-3.0-or-later | |
| name: Merge all data sources | |
| on: | |
| workflow_dispatch: | |
| repository_dispatch: | |
| types: | |
| - update | |
| jobs: | |
| merge: | |
| runs-on: ubuntu-latest | |
| outputs: | |
| status: ${{ steps.commit.outputs.status }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| path: yaml-yugi | |
| - uses: actions/checkout@v4 | |
| with: | |
| repository: DawnbrandBots/yaml-yugipedia | |
| path: yaml-yugipedia | |
| - uses: actions/checkout@v4 | |
| with: | |
| path: yaml-yugi-ko | |
| repository: DawnbrandBots/yaml-yugi-ko | |
| sparse-checkout: | | |
| *.csv | |
| sparse-checkout-cone-mode: false | |
| - uses: actions/checkout@v4 | |
| with: | |
| repository: DawnbrandBots/yaml-yugi-zh | |
| path: yaml-yugi-zh | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.10" | |
| cache: pip | |
| cache-dependency-path: yaml-yugi/src/requirements.txt | |
| - name: Setup dependencies | |
| run: | | |
| mkdir aggregate | |
| curl -fsSLo aggregate/master-duel-raw.json https://dawnbrandbots.github.io/yaml-yugi/master-duel-raw.json | |
| curl -fsSLo tcg.vector.json https://dawnbrandbots.github.io/yaml-yugi-limit-regulation/tcg/current.vector.json | |
| curl -fsSLo ocg.vector.json https://dawnbrandbots.github.io/yaml-yugi-limit-regulation/ocg/current.vector.json | |
| pip install -r yaml-yugi/src/requirements.txt | |
| - name: Transform (series) | |
| working-directory: yaml-yugi/data/series | |
| run: python3 ../../src/main_archetypes.py ../../../yaml-yugipedia/wikitext/TCG_and_OCG_archetypes | |
| - name: Transform (OCG+TCG) | |
| working-directory: yaml-yugi/data/cards | |
| run: | | |
| git rm --ignore-unmatch *.json *.yaml | |
| python3 ../../src/main_ocgtcg.py \ | |
| ../../../yaml-yugipedia/wikitext/Duel_Monsters_cards \ | |
| --zh-CN ../../../yaml-yugi-zh/zh-CN \ | |
| --assignments ../../src/assignments/assignments.yaml \ | |
| --tcg ../../../tcg.vector.json \ | |
| --ocg ../../../ocg.vector.json \ | |
| --ko-official ../../../yaml-yugi-ko/_site/ocg.csv \ | |
| --ko-override ../../../yaml-yugi-ko/ocg-override.csv \ | |
| --ko-prerelease ../../../yaml-yugi-ko/ocg-prerelease.csv \ | |
| --master-duel ../../../aggregate/master-duel-raw.json \ | |
| --aggregate ../../../aggregate/cards.json | |
| - name: Transform (Rush Duel) | |
| working-directory: yaml-yugi/data/rush | |
| run: | | |
| git rm --ignore-unmatch *.json *.yaml | |
| python3 ../../src/main_rush.py \ | |
| ../../../yaml-yugipedia/wikitext/Rush_Duel_cards \ | |
| --ko-override ../../../yaml-yugi-ko/rush-override.csv \ | |
| --ko-prerelease ../../../yaml-yugi-ko/rush-prerelease.csv \ | |
| --ocg-aggregate ../../../aggregate/cards.json \ | |
| --aggregate ../../../aggregate/rush.json | |
| - name: Transform (TCG Speed Duel Skills) | |
| working-directory: yaml-yugi/data/tcg-speed-skill | |
| run: | | |
| git rm --ignore-unmatch *.json *.yaml | |
| python3 ../../src/main_speed.py \ | |
| ../../../yaml-yugipedia/wikitext/Skill_Cards \ | |
| --aggregate ../../../aggregate/skill.json | |
| - id: commit | |
| uses: DawnbrandBots/.github/actions/commit-push@main | |
| with: | |
| message: "Transform: ${{ github.run_number }} (${{ github.run_id }})" | |
| working-directory: yaml-yugi | |
| - if: steps.commit.outputs.status > 0 | |
| working-directory: yaml-yugi | |
| run: gh workflow run validate-data.yaml | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| - name: Transform (Master Duel) | |
| working-directory: yaml-yugi | |
| run: python src/main_masterduel.py '../yaml-yugipedia/wikitext/Yu-Gi-Oh!_Master_Duel_cards' > ../aggregate/master-duel-raw.json | |
| - if: steps.commit.outputs.status > 0 | |
| working-directory: yaml-yugi | |
| name: Merge | |
| run: | | |
| sed -s '1i---' data/cards/*.yaml > ../aggregate/cards.yaml | |
| sed -s '1i---' data/rush/*.yaml > ../aggregate/rush.yaml | |
| - if: steps.commit.outputs.status > 0 | |
| run: cp aggregate/* yaml-yugi/src/web | |
| - if: steps.commit.outputs.status > 0 | |
| uses: actions/upload-pages-artifact@v3 | |
| with: | |
| path: yaml-yugi/src/web | |
| - if: steps.commit.outputs.status > 0 | |
| uses: actions/setup-node@v4 | |
| with: | |
| node-version: 20 | |
| cache: yarn | |
| cache-dependency-path: yaml-yugi/yarn.lock | |
| - if: steps.commit.outputs.status > 0 | |
| working-directory: yaml-yugi | |
| run: yarn | |
| - if: steps.commit.outputs.status > 0 | |
| working-directory: yaml-yugi | |
| name: Load (push) | |
| env: | |
| OPENSEARCH_URL: ${{ secrets.OS_URL }} | |
| run: | | |
| yarn ts-node src/load.ts ../aggregate/cards.yaml yu-gi-oh_ocg-tcg | |
| yarn ts-node src/load.ts ../aggregate/rush.yaml yu-gi-oh_rush | |
| deploy-pages: | |
| needs: merge | |
| if: needs.merge.outputs.status > 0 && !cancelled() | |
| # Grant GITHUB_TOKEN the permissions required to make a Pages deployment | |
| permissions: | |
| pages: write # to deploy to Pages | |
| id-token: write # to verify the deployment originates from an appropriate source | |
| # Deploy to the github-pages environment | |
| environment: | |
| name: github-pages | |
| url: ${{ steps.deployment.outputs.page_url }} | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/deploy-pages@v4 | |
| id: deployment |