feat: All versions now are available offline! (#174)

cypress/e2e/index.spec.ts

@@ -14,7 +14,7 @@ const compareRenderedFlatWorld = () => {
 }
 
 const testWorldLoad = () => {
-  return cy.document().then({ timeout: 25_000 }, doc => {
+  return cy.document().then({ timeout: 35_000 }, doc => {
     return new Cypress.Promise(resolve => {
       doc.addEventListener('cypress-world-ready', resolve)
     })

experiments/decode.html

@@ -0,0 +1 @@
+<script src="decode.ts" type="module"></script>

experiments/decode.ts

@@ -0,0 +1,26 @@
+// Include the pako library
+import pako from 'pako';
+import compressedJsRaw from './compressed.js?raw'
+
+function decompressFromBase64(input) {
+    // Decode the Base64 string
+    const binaryString = atob(input);
+    const len = binaryString.length;
+    const bytes = new Uint8Array(len);
+
+    // Convert the binary string to a byte array
+    for (let i = 0; i < len; i++) {
+        bytes[i] = binaryString.charCodeAt(i);
+    }
+
+    // Decompress the byte array
+    const decompressedData = pako.inflate(bytes, { to: 'string' });
+
+    return decompressedData;
+}
+
+// Use the function
+console.time('decompress');
+const decompressedData = decompressFromBase64(compressedJsRaw);
+console.timeEnd('decompress')
+console.log(decompressedData)

prismarine-viewer/esbuild.mjs

@@ -6,20 +6,19 @@ import * as esbuild from 'esbuild'
 import { polyfillNode } from 'esbuild-plugin-polyfill-node'
 import path, { dirname, join } from 'path'
 import { fileURLToPath } from 'url'
+import childProcess from 'child_process'
+import supportedVersions from '../src/supportedVersions.mjs'
 
 const dev = process.argv.includes('-w')
 
 const __dirname = path.dirname(fileURLToPath(new URL(import.meta.url)))
 
-const mcDataPath = join(__dirname, '../dist/mc-data')
+const mcDataPath = join(__dirname, '../generated/minecraft-data-optimized.json')
 if (!fs.existsSync(mcDataPath)) {
-  // shouldn't it be in the viewer instead?
-  await import('../scripts/prepareData.mjs')
+  childProcess.execSync('tsx ../scripts/makeOptimizedMcData.mjs', { stdio: 'inherit', cwd: __dirname })
 }
 
 fs.copyFileSync(join(__dirname, 'playground.html'), join(__dirname, 'public/index.html'))
-fsExtra.copySync(mcDataPath, join(__dirname, 'public/mc-data'))
-const availableVersions = fs.readdirSync(mcDataPath).map(ver => ver.replace('.js', ''))
 
 /** @type {import('esbuild').BuildOptions} */
 const buildOptions = {
@@ -37,7 +36,7 @@ const buildOptions = {
   ],
   keepNames: true,
   banner: {
-    js: `globalThis.global = globalThis;globalThis.includedVersions = ${JSON.stringify(availableVersions)};`,
+    js: `globalThis.global = globalThis;globalThis.includedVersions = ${JSON.stringify(supportedVersions)};`,
   },
   alias: {
     events: 'events',
@@ -63,13 +62,14 @@ const buildOptions = {
         }, () => {
           const defaultVersionsObj = {}
           return {
-            contents: `window.mcData ??= ${JSON.stringify(defaultVersionsObj)};module.exports = { pc: window.mcData }`,
-            loader: 'js',
+            contents: fs.readFileSync(join(__dirname, '../src/shims/minecraftData.ts'), 'utf8'),
+            loader: 'ts',
+            resolveDir: join(__dirname, '../src/shims'),
           }
         })
         build.onEnd((e) => {
           if (e.errors.length) return
-          // fs.writeFileSync(join(__dirname, 'dist/metafile.json'), JSON.stringify(e.metafile), 'utf8')
+          fs.writeFileSync(join(__dirname, './public/metafile.json'), JSON.stringify(e.metafile), 'utf8')
         })
       }
     },

prismarine-viewer/examples/playground.ts

@@ -15,6 +15,7 @@ import { loadScript } from '../viewer/lib/utils'
 import { TWEEN_DURATION } from '../viewer/lib/entities'
 import { EntityMesh } from '../viewer/lib/entity/EntityMesh'
 import { WorldDataEmitter, Viewer } from '../viewer'
+import '../../src/getCollisionShapes'
 import { toMajorVersion } from '../../src/utils'
 
 window.THREE = THREE
@@ -65,21 +66,21 @@ async function main () {
   let continuousRender = false
 
   const { version } = params
+  await window._LOAD_MC_DATA()
   // temporary solution until web worker is here, cache data for faster reloads
-  const globalMcData = window['mcData']
-  if (!globalMcData['version']) {
-    const major = toMajorVersion(version)
-    const sessionKey = `mcData-${major}`
-    if (sessionStorage[sessionKey]) {
-      Object.assign(globalMcData, JSON.parse(sessionStorage[sessionKey]))
-    } else {
-      if (sessionStorage.length > 1) sessionStorage.clear()
-      await loadScript(`./mc-data/${major}.js`)
-      try {
-        sessionStorage[sessionKey] = JSON.stringify(Object.fromEntries(Object.entries(globalMcData).filter(([ver]) => ver.startsWith(major))))
-      } catch { }
-    }
-  }
+  // const globalMcData = window['mcData']
+  // if (!globalMcData['version']) {
+  //   const major = toMajorVersion(version)
+  //   const sessionKey = `mcData-${major}`
+  //   if (sessionStorage[sessionKey]) {
+  //     Object.assign(globalMcData, JSON.parse(sessionStorage[sessionKey]))
+  //   } else {
+  //     if (sessionStorage.length > 1) sessionStorage.clear()
+  //     try {
+  //       sessionStorage[sessionKey] = JSON.stringify(Object.fromEntries(Object.entries(globalMcData).filter(([ver]) => ver.startsWith(major))))
+  //     } catch { }
+  //   }
+  // }
 
   const mcData: IndexedData = require('minecraft-data')(version)
   window['loadedData'] = mcData

prismarine-viewer/package.json

@@ -38,5 +38,8 @@
   "optionalDependencies": {
     "canvas": "^2.11.2",
     "node-canvas-webgl": "^0.3.0"
+  },
+  "devDependencies": {
+    "live-server": "^1.2.2"
   }
 }

prismarine-viewer/viewer/lib/worldrendererCommon.ts

@@ -2,7 +2,7 @@
 import { EventEmitter } from 'events'
 import { Vec3 } from 'vec3'
 import * as THREE from 'three'
-import mcDataRaw from 'minecraft-data/data.js' // handled correctly in esbuild plugin
+import mcDataRaw from 'minecraft-data/data.js' // note: using alias
 import blocksAtlases from 'mc-assets/dist/blocksAtlases.json'
 import blocksAtlasLatest from 'mc-assets/dist/blocksAtlasLatest.png'
 import blocksAtlasLegacy from 'mc-assets/dist/blocksAtlasLegacy.png'
@@ -223,8 +223,12 @@ export abstract class WorldRendererCommon<WorkerSend = any, WorkerReceive = any>
 
   sendMesherMcData () {
     const allMcData = mcDataRaw.pc[this.version] ?? mcDataRaw.pc[toMajorVersion(this.version)]
-    const mcData = Object.fromEntries(Object.entries(allMcData).filter(([key]) => dynamicMcDataFiles.includes(key)))
-    mcData.version = JSON.parse(JSON.stringify(mcData.version))
+    const mcData = {
+      version: JSON.parse(JSON.stringify(allMcData.version))
+    }
+    for (const key of dynamicMcDataFiles) {
+      mcData[key] = allMcData[key]
+    }
 
     for (const worker of this.workers) {
       worker.postMessage({ type: 'mcData', mcData, config: this.mesherConfig })

rsbuild.config.ts

@@ -101,9 +101,10 @@ export default defineConfig({
                 const prep = async () => {
                     console.time('total-prep')
                     fs.mkdirSync('./generated', { recursive: true })
-                    if (!fs.existsSync('./generated/minecraft-data-data.js')) {
-                        childProcess.execSync('tsx ./scripts/genShims.ts', { stdio: 'inherit' })
+                    if (!fs.existsSync('./generated/minecraft-data-optimized.json') || require('./generated/minecraft-data-optimized.json').versionKey !== require('minecraft-data/package.json').version) {
+                        childProcess.execSync('tsx ./scripts/makeOptimizedMcData.mjs', { stdio: 'inherit' })
                     }
+                    childProcess.execSync('tsx ./scripts/genShims.ts', { stdio: 'inherit' })
                     if (!fs.existsSync('./generated/latestBlockCollisionsShapes.json')) {
                         childProcess.execSync('tsx ./scripts/optimizeBlockCollisions.ts', { stdio: 'inherit' })
                     }
@@ -117,7 +118,6 @@ export default defineConfig({
                         configJson.defaultProxy = ':8080'
                     }
                     fs.writeFileSync('./dist/config.json', JSON.stringify(configJson), 'utf8')
-                    childProcess.execSync('node ./scripts/prepareData.mjs', { stdio: 'inherit' })
                     // childProcess.execSync('./scripts/prepareSounds.mjs', { stdio: 'inherit' })
                     // childProcess.execSync('tsx ./scripts/genMcDataTypes.ts', { stdio: 'inherit' })
                     // childProcess.execSync('tsx ./scripts/genPixelartTypes.ts', { stdio: 'inherit' })
@@ -164,7 +164,7 @@ export default defineConfig({
                     // throw new Error(`${resource.request} was requested by ${resource.contextInfo.issuer}`)
                 }
                 if (absolute.endsWith('/minecraft-data/data.js')) {
-                    resource.request = path.join(__dirname, './generated/minecraft-data-data.js')
+                    resource.request = path.join(__dirname, './src/shims/minecraftData.ts')
                 }
             }))
             addRules([

scripts/build.js

@@ -45,7 +45,6 @@ exports.getSwAdditionalEntries = () => {
     // need to be careful with this
     const filesToCachePatterns = [
         'index.html',
-        `mc-data/${defaultLocalServerOptions.versionMajor}.js`,
         'background/**',
         // todo-low copy from assets
         '*.mp3',

scripts/genShims.ts

@@ -1,24 +1,7 @@
 import fs from 'fs'
-import MinecraftData from 'minecraft-data'
-import MCProtocol from 'minecraft-protocol'
 import { appReplacableResources } from '../src/resourcesSource'
 
-const { supportedVersions, defaultVersion } = MCProtocol
-
-// gen generated/minecraft-data-data.js
-
-const data = MinecraftData(defaultVersion)
-const defaultVersionObj = {
-  [defaultVersion]: {
-    version: data.version,
-    protocol: data.protocol,
-  }
-}
-
-const mcDataContents = `window.mcData ??= ${JSON.stringify(defaultVersionObj)};module.exports = { pc: window.mcData }`
-
 fs.mkdirSync('./generated', { recursive: true })
-fs.writeFileSync('./generated/minecraft-data-data.js', mcDataContents, 'utf8')
 
 // app resources