From b566ec1fd69669c52e5baeb8d0437ece0c4c998a Mon Sep 17 00:00:00 2001 From: aquacluck Date: Sat, 8 Jul 2023 15:34:59 -0700 Subject: [PATCH] Add Dockerfile, read raw romfs with zstd+sarc+byml --- Dockerfile | 7 ++++++ README.md | 8 +++++-- beco.ts | 3 +-- build.ts | 64 ++++++++++++++++++++++++++++++++++++++---------------- 4 files changed, 59 insertions(+), 23 deletions(-) create mode 100644 Dockerfile diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..a820f0d --- /dev/null +++ b/Dockerfile @@ -0,0 +1,7 @@ +FROM node:18 +WORKDIR /radar +COPY . . +RUN npm install && npm install typescript -g +RUN apt-get update && apt-get install -y zstd python3-pip && pip3 install sarc byml --break-system-packages +CMD ./node_modules/.bin/ts-node ./build.ts -r /romfs -e ./tools && \ + npm run dev diff --git a/README.md b/README.md index fe23790..1a47c43 100644 --- a/README.md +++ b/README.md @@ -3,6 +3,10 @@ A server for querying placement objects in *The Legend of Zelda: Tears of the Ki Run build.ts to generate a map database before starting the server for the first time. - ts-node build.ts -d ../totk/Banc + ts-node build.ts -r ../totk -e tools -This assumes the `totk/Banc` directory contains the YAML data object map files +This assumes the `totk` directory contains the unaltered romfs contents. + +For docker usage: `docker build -t radar .; docker run -it --rm --name radar -v /path/to/your/romfs:/romfs radar` + +It's possible to build the db within docker and copy it out for the server to use, if you'd rather not install the extraction tools used in build.ts on your local machine. diff --git a/beco.ts b/beco.ts index 044b26a..f4373d9 100644 --- a/beco.ts +++ b/beco.ts @@ -28,9 +28,8 @@ export class Beco { // Offsets to row data, divided by 2 and relative to the start of the row section offsets: number[]; // u32, size num_rows segments: BecoSegment[][]; // Rows x Segments - constructor(file: string) { + constructor(buf: Buffer) { let little = true; - let buf = fs.readFileSync(file); let arr = new Uint8Array(buf.byteLength); buf.copy(arr, 0, 0, buf.byteLength); let dv = new DataView(arr.buffer); diff --git a/build.ts b/build.ts index ca45d46..b6eb060 100644 --- a/build.ts +++ b/build.ts @@ -1,3 +1,4 @@ +import { execSync } from 'child_process'; import sqlite3 from 'better-sqlite3'; import fs from 'fs'; import yaml from 'js-yaml'; @@ -6,22 +7,26 @@ import { Beco } from './beco'; let parseArgs = require('minimist'); let argv = parseArgs(process.argv); -if (!argv.d || !argv.b || !argv.e) { +if (!argv.e || !argv.r) { console.log("Error: Must specify paths to directories with "); - console.log(" -d Banc extracted YAML files"); - console.log(" -b field map area beco files"); console.log(" -e Ecosystem json files"); - console.log(" e.g. % ts-node build.ts -d path/to/Banc -b path/to/beco -e path/to/Ecosystem") + console.log(" -r Bare game romfs"); + console.log(" e.g. % ts-node build.ts -r path/to/romfs -e tools") process.exit(1); } -const totkData = argv.d -const becoPath = argv.b; const ecoPath = argv.e; +const romfsPath = argv.r; +const totkData = path.join(romfsPath, 'Banc'); +const becoPath = path.join(romfsPath, 'Ecosystem', 'FieldMapArea'); fs.rmSync('map.db.tmp', { force: true }); const db = sqlite3('map.db.tmp'); db.pragma('journal_mode = WAL'); +const zsDicPath = fs.mkdtempSync('zsdicpack'); +execSync(`zstd -d "${romfsPath}/Pack/ZsDic.pack.zs" -o "${zsDicPath}/ZsDic.pack"`); +execSync(`sarc x --directory "${zsDicPath}" "${zsDicPath}/ZsDic.pack"`); + class GenGroupIdGenerator { private nextId = 0; @@ -85,14 +90,16 @@ const LOCATIONS = JSON.parse(fs.readFileSync('LocationMarker.json', 'utf8')) const KOROKS = JSON.parse(fs.readFileSync('koroks_id.json', 'utf8')) const DROP_TABLES = JSON.parse(fs.readFileSync('drop_tables.json', 'utf8')) +const BCETT_YAML_SUFFIXES = /\.bcett\.b?yml(\.zs)?$/; + const DropTableDefault = "Default"; const DROP_TYPE_ACTOR = "Actor"; const DROP_TYPE_TABLE = "Table"; -const BecoGround = new Beco(path.join(becoPath, 'Ground.beco')); -const BecoMinus = new Beco(path.join(becoPath, 'MinusField.beco')); -const BecoSky = new Beco(path.join(becoPath, 'Sky.beco')); -const BecoCave = new Beco(path.join(becoPath, 'Cave.beco')); +const BecoGround = new Beco(readRawBeco('Ground')); +const BecoMinus = new Beco(readRawBeco('MinusField')); +const BecoSky = new Beco(readRawBeco('Sky')); +const BecoCave = new Beco(readRawBeco('Cave')); // Should probably be yaml not json for consistency const Ecosystem = Object.fromEntries(['Cave', 'Ground', 'MinusField', 'Sky'].map(name => { @@ -192,6 +199,28 @@ function parseHash(hash: string) { return '0x' + BigInt(hash).toString(16).padStart(16, '0'); } +function readRawBeco(name: string): Buffer { + let filePath = path.join(becoPath, name + '.beco'); + if (fs.existsSync(filePath)) { + return fs.readFileSync(filePath); + } else if (fs.existsSync(filePath + '.zs')) { + return execSync(`zstd -D "${zsDicPath}/zs.zsdic" -d ${filePath}.zs -c`, {maxBuffer: 1073741824}); + } + throw Error(`No beco file found for ${name}`); +} + +function readRawYaml(filePath: string): string { + let rawYaml: string = ""; + if (filePath.endsWith('.byml.zs')) { + rawYaml = execSync(`zstd -D "${zsDicPath}/bcett.byml.zsdic" -d ${filePath} -c | byml_to_yml - -`, {maxBuffer: 1073741824}).toString(); + } else if (filePath.endsWith('.byml')) { + rawYaml = execSync(`byml_to_yml ${filePath} -`, {maxBuffer: 1073741824}).toString(); + } else { + rawYaml = fs.readFileSync(filePath, 'utf-8').toString(); + } + return rawYaml; +} + function getKorokType(hideType: number | undefined, name: string) { if (name == 'KorokCarryProgressKeeper') { return 'Korok Friends'; @@ -215,9 +244,7 @@ function getKorokType(hideType: number | undefined, name: string) { function processBanc(filePath: string, mapType: string, mapName: string) { let doc: any = null; try { - doc = yaml.load(fs.readFileSync(filePath, 'utf-8'), - { schema: schema } - ); + doc = yaml.load(readRawYaml(filePath), { schema: schema }); } catch (e: any) { console.log("Error: ", e); process.exit(1); @@ -443,13 +470,13 @@ function processBancs() { const dirPath = path.join(totkData, field); let files = fs.readdirSync(dirPath); for (const file of files) { - if (!file.endsWith('.bcett.yml')) + if (!file.match(BCETT_YAML_SUFFIXES)) continue; let filePath = path.join(dirPath, file); const fieldParts = field.split("/"); let mapName = file - .replace(".bcett.yml", "") + .replace(BCETT_YAML_SUFFIXES, "") .replace("_Static", "") .replace("_Dynamic", ""); const mapType = fieldParts[0]; @@ -463,12 +490,12 @@ function processBancs() { for (const mapType of ["SmallDungeon", "LargeDungeon", "NormalStage"]) { const dirPath = path.join(totkData, mapType); for (const file of fs.readdirSync(dirPath)) { - if (!file.endsWith('.bcett.yml')) + if (!file.match(BCETT_YAML_SUFFIXES)) continue; const filePath = path.join(dirPath, file); const mapName = file - .replace(".bcett.yml", "") + .replace(BCETT_YAML_SUFFIXES, "") .replace("_Static", "") .replace("_Dynamic", ""); processBanc(filePath, mapType, mapName); @@ -496,8 +523,7 @@ function processRecycleBox() { console.log("process recyclebox: ", filePath) let doc: any = null; try { - doc = yaml.load(fs.readFileSync(filePath, 'utf-8'), - { schema: schema }); + doc = yaml.load(readRawYaml(filePath), { schema: schema }); } catch (e: any) { console.log("Error: ", e); process.exit(1);