diff --git a/package.json b/package.json index de49d52..9e46ff6 100644 --- a/package.json +++ b/package.json @@ -26,6 +26,7 @@ "pre-commit": "pnpm nano-staged" }, "devDependencies": { + "@types/node": "20", "eslint": "^8.57.0", "eslint-config-foxkit": "4.1.0", "eslint-config-prettier": "^9.1.0", @@ -35,6 +36,8 @@ "typescript": "5.5.4" }, "dependencies": { + "@foxkit/list": "^1.2.0", + "@foxkit/node-util": "^0.6.0", "tsx": "^4.19.0" } } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index a4ddecf..330d334 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -8,10 +8,19 @@ importers: .: dependencies: + '@foxkit/list': + specifier: ^1.2.0 + version: 1.2.0 + '@foxkit/node-util': + specifier: ^0.6.0 + version: 0.6.0(picocolors@1.0.1) tsx: specifier: ^4.19.0 version: 4.19.0 devDependencies: + '@types/node': + specifier: '20' + version: 20.16.5 eslint: specifier: ^8.57.0 version: 8.57.0 @@ -198,6 +207,17 @@ packages: resolution: {integrity: sha512-Ys+3g2TaW7gADOJzPt83SJtCDhMjndcDMFVQ/Tj9iA1BfJzFKD9mAUXT3OenpuPHbI6P/myECxRJrofUsDx/5g==} engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0} + '@foxkit/list@1.2.0': + resolution: {integrity: sha512-dXlD0JRnQNwnEj4NStYTomoM2bDVcVsKjb0P/6jWY0Ud6StMxjxlADSqv5k601Tg2Tmg1Uk5b5NULBK3cuMMxQ==} + + '@foxkit/node-util@0.6.0': + resolution: {integrity: sha512-e60lhhUgW+MVHRJCtis9HfZ2+2mNGI7ZNQkVwgTe6P4WQ6dqQn/pMKjkltHPaXCRk3XWMzkLD7Iy0KMM0ZpRHw==} + peerDependencies: + picocolors: '>=1.0.0' + peerDependenciesMeta: + picocolors: + optional: true + '@humanwhocodes/config-array@0.11.14': resolution: {integrity: sha512-3T8LkOmg45BV5FICb15QQMsyUSWrQ8AygVfC7ZG32zOalnqrilm018ZVCw0eapXux8FtA33q8PSRSstjee3jSg==} engines: {node: '>=10.10.0'} @@ -223,6 +243,9 @@ packages: resolution: {integrity: sha512-oGB+UxlgWcgQkgwo8GcEGwemoTFt3FIO9ababBmaGwXIoBKZ+GTy0pP185beGg7Llih/NSHSV2XAs1lnznocSg==} engines: {node: '>= 8'} + '@types/node@20.16.5': + resolution: {integrity: sha512-VwYCweNo3ERajwy0IUlqqcyZ8/A7Zwa9ZP3MnENWcB11AejO+tLy3pu850goUW2FC/IJMdZUfKpX/yxL1gymCA==} + '@typescript-eslint/eslint-plugin@8.3.0': resolution: {integrity: sha512-FLAIn63G5KH+adZosDYiutqkOkYEx0nvcwNNfJAf+c7Ae/H35qWwTYvPZUKFj5AS+WfHG/WJJfWnDnyNUlp8UA==} engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} @@ -715,6 +738,9 @@ packages: engines: {node: '>=14.17'} hasBin: true + undici-types@6.19.8: + resolution: {integrity: sha512-ve2KP6f/JnbPBFyobGHuerC9g1FYGn/F8n1LWTwNxCEzd6IfqTwUQcNXgEtmmQ6DlRrC1hrSrBnCZPokRrDHjw==} + uri-js@4.4.1: resolution: {integrity: sha512-7rKUyy33Q1yc98pQ1DAmLtwX109F7TIfWlW1Ydo8Wl1ii1SeHieeh0HHfPeL2fMXK6z0s8ecKs9frCuLJvndBg==} @@ -831,6 +857,12 @@ snapshots: '@eslint/js@8.57.0': {} + '@foxkit/list@1.2.0': {} + + '@foxkit/node-util@0.6.0(picocolors@1.0.1)': + optionalDependencies: + picocolors: 1.0.1 + '@humanwhocodes/config-array@0.11.14': dependencies: '@humanwhocodes/object-schema': 2.0.3 @@ -855,6 +887,10 @@ snapshots: '@nodelib/fs.scandir': 2.1.5 fastq: 1.17.1 + '@types/node@20.16.5': + dependencies: + undici-types: 6.19.8 + '@typescript-eslint/eslint-plugin@8.3.0(@typescript-eslint/parser@8.3.0(eslint@8.57.0)(typescript@5.5.4))(eslint@8.57.0)(typescript@5.5.4)': dependencies: '@eslint-community/regexpp': 4.11.0 @@ -1375,6 +1411,8 @@ snapshots: typescript@5.5.4: {} + undici-types@6.19.8: {} + uri-js@4.4.1: dependencies: punycode: 2.3.1 diff --git a/src/download-monster-data.ts b/src/download-monster-data.ts new file mode 100644 index 0000000..5519ab8 --- /dev/null +++ b/src/download-monster-data.ts @@ -0,0 +1,77 @@ +import fs from "fs/promises"; +import { isDirectory, isFile, writeFile } from "@foxkit/node-util/fs"; +import { sleep } from "./utils/sleep"; +import { List } from "@foxkit/list"; + +const [, , ...args] = process.argv; +const isForced = args.includes("--force"); + +async function setupDirs() { + if (!(await isDirectory("data"))) { + await fs.mkdir("data"); + } + + if (!(await isDirectory("data/monsters"))) { + await fs.mkdir("data/monsters"); + } +} + +async function fetchMonsterList(): Promise> { + const res = await fetch("https://api.flyff.com/monster"); + if (!res.ok) { + throw new Error(`Failed to fetch Monster list: ${res.statusText}`); + } + + await sleep(200); + + return res.json(); +} + +async function fetchMonsterData(id: number) { + const res = await fetch(`https://api.flyff.com/monster/${id}`); + if (!res.ok) { + throw new Error( + `Failed to fetch Monster data for id ${id}: ${res.statusText}` + ); + } + + await sleep(220); + + return res.text(); +} + +async function main() { + if (isForced) { + console.log("Forceing redownloading of all data"); + } + + await setupDirs(); + + console.log("Downloading Monster List"); + const list = await fetchMonsterList(); + console.log(`Found ${list.length} Monster IDs`); + + const queue = new List(list); + let id: number | undefined; + while ((id = queue.shift())) { + if (!isForced && (await isFile(`data/monsters/${id}.json`))) { + console.log(`[SKIP] Already have data for id ${id}`); + continue; + } + + console.log(`[LOG] Fetching data for id ${id}`); + const data = await fetchMonsterData(id); + await writeFile(`data/monsters/${id}.json`, data); + console.log(`[DONE] completed download of data for id ${id}`); + } + + // TODO: fetching skills + // TODO: global constants for paths such as data/monsters/*.json +} + +main() + .then(() => console.log("Completed")) + .catch(e => { + console.error(e); + process.exit(1); + }); diff --git a/src/utils/sleep.ts b/src/utils/sleep.ts new file mode 100644 index 0000000..1e5174e --- /dev/null +++ b/src/utils/sleep.ts @@ -0,0 +1,3 @@ +export function sleep(length: number) { + return new Promise(resolve => setTimeout(() => resolve(), length)); +} diff --git a/tsconfig.json b/tsconfig.json index d724a3e..fe48a0e 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -7,8 +7,9 @@ "strict": true, "noEmit": true, "isolatedModules": true, - "skipLibCheck": true + "skipLibCheck": true, + "allowSyntheticDefaultImports": true }, - "include": ["**/*.ts"], + "include": ["**/*.ts", "src/types/**/*.d.ts"], "exclude": ["node_modules", "dist"] }