Merge pull request #13 from kion-dgl/11-add-compression-test

11 add compression test

package.json

@@ -11,7 +11,6 @@
     "typescript": "^5.0.0"
   },
   "dependencies": {
-    "bytereader": "^0.1.0",
     "pngjs": "^7.0.0",
     "three": "^0.165.0"
   }

src/EncodeRom.ts

@@ -1,4 +1,4 @@
-import { readFileSync, writeFileSync, readdirSync } from "fs";
+import { readFileSync, writeFileSync } from "fs";
 import { RGBA_ASTC_10x5_Format } from "three";
 const CHUNK_SIZE = 0x800;
 const STRIDE_SIZE = 0x930;
@@ -70,15 +70,17 @@ const encodeRom = () => {
   if (!romDst) {
     throw new Error("Need to set DST_ROM value in .env");
   }
+
   const rom = readFileSync(sourceRom);
 
-  const replaceFiles = readdirSync("./mod");
-  replaceFiles.forEach((filename) => {
-    const sourceFile = readFileSync(`./bin/${filename}`);
-    const moddedFile = readFileSync(`./mod/${filename}`);
-    replaceInRom(rom, sourceFile, moddedFile);
-  });
+  // Replace Textures
+  const mikuTexture = readFileSync("mod/PL00T.BIN");
+  const pl00t = readFileSync("bin/PL00T.BIN");
+  const pl00t2 = readFileSync("bin/PL00T2.BIN");
+  replaceInRom(rom, pl00t, mikuTexture);
+  replaceInRom(rom, pl00t, mikuTexture);
 
+  // Write the result
   writeFileSync(romDst, rom);
 };
 

src/EncodeTexture.ts

@@ -126,11 +126,11 @@ const replaceTexture = (
 
 const encodeTexture = (bodyTexture: string, faceTexture: string) => {
   // Encode the body and face texture to write to ROM
-  const srcTexture = readFileSync("bin/PL01T.BIN");
+  const srcTexture = readFileSync("bin/PL00T.BIN");
   const bodyBuffer = readFileSync(bodyTexture);
   const faceBuffer = readFileSync(faceTexture);
   const modTexture = replaceTexture(srcTexture, bodyBuffer, faceBuffer);
-  writeFileSync("mod/PL01T.BIN", modTexture);
+  writeFileSync("mod/PL00T.BIN", modTexture);
 };
 
 export { encodeTexture };

test/body.test.ts

@@ -17,11 +17,12 @@ const ROT = new Matrix4();
 ROT.makeRotationX(Math.PI);
 
 test("Reading the strip offsets for the body", () => {
-  const buffer = readFileSync(`./bin/PL00P000.BIN`);
-  const dat = buffer.subarray(0x30, 0x30 + 0x2b40);
-  const reader = new ByteReader(dat.buffer as ArrayBuffer);
+  const file = readFileSync(`./bin/PL00P000.BIN`);
+  const dat = file.subarray(0x30, 0x30 + 0x2b40);
+  const { buffer } = Buffer.from(dat);
+  const reader = new ByteReader(buffer as ArrayBuffer);
 
-  const BODY_OFS = 0xb0;
+  const BODY_OFS = 0x80;
   const names = [
     "00_BODY",
     "01_HIP",
@@ -35,9 +36,10 @@ test("Reading the strip offsets for the body", () => {
 });
 
 test.skip("Reading the vertex and face data for the body", () => {
-  const buffer = readFileSync(`./bin/PL00P000.BIN`);
-  const dat = buffer.subarray(0x30, 0x30 + 0x2b40);
-  const reader = new ByteReader(dat.buffer as ArrayBuffer);
+  const file = readFileSync(`./bin/PL00P000.BIN`);
+  const dat = file.subarray(0x30, 0x30 + 0x2b40);
+  const { buffer } = Buffer.from(dat);
+  const reader = new ByteReader(buffer as ArrayBuffer);
 
   const geometry = body.map((mesh) => {
     const { name, vertOfs, vertCount, triOfs, triCount, quadOfs, quadCount } =
@@ -57,9 +59,10 @@ test.skip("Reading the vertex and face data for the body", () => {
 });
 
 test("Re-encoding the vertices read from the body", () => {
-  const buffer = readFileSync(`./bin/PL00P000.BIN`);
-  const dat = buffer.subarray(0x30, 0x30 + 0x2b40);
-  const reader = new ByteReader(dat.buffer as ArrayBuffer);
+  const file = readFileSync(`./bin/PL00P000.BIN`);
+  const dat = file.subarray(0x30, 0x30 + 0x2b40);
+  const { buffer } = Buffer.from(dat);
+  const reader = new ByteReader(buffer as ArrayBuffer);
 
   body.map((mesh) => {
     const { vertOfs, vertCount } = mesh;
@@ -124,9 +127,10 @@ test("Re-encoding the vertices read from the body", () => {
 });
 
 test("Re-encoding the tris read from the body", () => {
-  const buffer = readFileSync(`./bin/PL00P000.BIN`);
-  const dat = buffer.subarray(0x30, 0x30 + 0x2b40);
-  const reader = new ByteReader(dat.buffer as ArrayBuffer);
+  const file = readFileSync(`./bin/PL00P000.BIN`);
+  const dat = file.subarray(0x30, 0x30 + 0x2b40);
+  const { buffer } = Buffer.from(dat);
+  const reader = new ByteReader(buffer as ArrayBuffer);
 
   const FACE_MASK = 0x7f;
   const PIXEL_TO_FLOAT_RATIO = 0.00390625;
@@ -197,9 +201,10 @@ test("Re-encoding the tris read from the body", () => {
 });
 
 test("Re-encoding the quad read from the body", () => {
-  const buffer = readFileSync(`./bin/PL00P000.BIN`);
-  const dat = buffer.subarray(0x30, 0x30 + 0x2b40);
-  const reader = new ByteReader(dat.buffer as ArrayBuffer);
+  const file = readFileSync(`./bin/PL00P000.BIN`);
+  const dat = file.subarray(0x30, 0x30 + 0x2b40);
+  const { buffer } = Buffer.from(dat);
+  const reader = new ByteReader(buffer as ArrayBuffer);
 
   const FACE_MASK = 0x7f;
   const PIXEL_TO_FLOAT_RATIO = 0.00390625;

test/compress.test.ts

@@ -0,0 +1,187 @@
+import { readFileSync } from "fs";
+import { test, expect } from "bun:test";
+
+const encodeBitfield = (bits: boolean[]): Buffer => {
+  const length = Math.ceil(bits.length / 32) * 4;
+  let ofs = 0;
+  const buffer = Buffer.alloc(length);
+  const dword = new Uint32Array(1);
+
+  for (let i = 0; i < bits.length; i += 32) {
+    dword[0] = 0;
+    bits[i + 0] && (dword[0] |= 0x80000000);
+    bits[i + 1] && (dword[0] |= 0x40000000);
+    bits[i + 2] && (dword[0] |= 0x20000000);
+    bits[i + 3] && (dword[0] |= 0x10000000);
+
+    bits[i + 4] && (dword[0] |= 0x8000000);
+    bits[i + 5] && (dword[0] |= 0x4000000);
+    bits[i + 6] && (dword[0] |= 0x2000000);
+    bits[i + 7] && (dword[0] |= 0x1000000);
+
+    bits[i + 8] && (dword[0] |= 0x800000);
+    bits[i + 9] && (dword[0] |= 0x400000);
+    bits[i + 10] && (dword[0] |= 0x200000);
+    bits[i + 11] && (dword[0] |= 0x100000);
+
+    bits[i + 12] && (dword[0] |= 0x80000);
+    bits[i + 13] && (dword[0] |= 0x40000);
+    bits[i + 14] && (dword[0] |= 0x20000);
+    bits[i + 15] && (dword[0] |= 0x10000);
+
+    bits[i + 16] && (dword[0] |= 0x8000);
+    bits[i + 17] && (dword[0] |= 0x4000);
+    bits[i + 18] && (dword[0] |= 0x2000);
+    bits[i + 19] && (dword[0] |= 0x1000);
+
+    bits[i + 20] && (dword[0] |= 0x800);
+    bits[i + 21] && (dword[0] |= 0x400);
+    bits[i + 22] && (dword[0] |= 0x200);
+    bits[i + 23] && (dword[0] |= 0x100);
+
+    bits[i + 24] && (dword[0] |= 0x80);
+    bits[i + 25] && (dword[0] |= 0x40);
+    bits[i + 26] && (dword[0] |= 0x20);
+    bits[i + 27] && (dword[0] |= 0x10);
+
+    bits[i + 28] && (dword[0] |= 0x8);
+    bits[i + 29] && (dword[0] |= 0x4);
+    bits[i + 30] && (dword[0] |= 0x2);
+    bits[i + 31] && (dword[0] |= 0x1);
+
+    buffer.writeUInt32LE(dword[0], ofs);
+    ofs += 4;
+  }
+
+  return buffer;
+};
+
+const decompressPayload = (
+  payload: Buffer,
+  bitfield: boolean[],
+  fullsize: number,
+): Buffer => {
+  const target = Buffer.alloc(fullsize);
+
+  let outOfs = 0;
+  let windowOfs = 0;
+  let ofs = 0;
+
+  for (let i = 0; i < bitfield.length; i++) {
+    const bit = bitfield[i];
+    if (outOfs === fullsize) {
+      break;
+    }
+
+    const word = payload.readUInt16LE(ofs);
+    ofs += 2;
+
+    if (!bit) {
+      target.writeUInt16LE(word, outOfs);
+      outOfs += 2;
+      continue;
+    } else {
+      // Bit is true
+      if (word === 0xffff) {
+        windowOfs += 0x2000;
+      } else {
+        const whence = (word >> 3) & 0x1fff;
+        const copyFrom = windowOfs + whence;
+        const copyLen = ((word & 0x07) + 2) * 2;
+        for (let n = 0; n < copyLen; n++) {
+          target[outOfs++] = target[copyFrom + n];
+        }
+      }
+    }
+  }
+
+  return target;
+};
+
+const compressSegment = (segment: Buffer): [boolean[], Buffer] => {
+  // Create a boolean array and out buffer
+  const bucket: boolean[] = [];
+  const compressed = Buffer.alloc(segment.length);
+
+  // Number of min and max number of words to match
+  const MAX_CAP = 9;
+  const MIN_CAP = 2;
+
+  let inOfs = 0;
+  let outOfs = 0;
+
+  do {
+    // Check ahead
+    let found = false;
+    const wordsLeft = (segment.length - inOfs) / 2;
+    const maxCount = wordsLeft < MAX_CAP ? wordsLeft : MAX_CAP;
+
+    // Check ahead
+
+    for (let count = maxCount; count >= MIN_CAP; count--) {
+      const len = count * 2;
+      const needle = segment.subarray(inOfs, inOfs + len);
+      const window = segment.subarray(0, inOfs);
+      const needleOfs = window.indexOf(needle);
+
+      if (needleOfs === -1) {
+        continue;
+      }
+
+      found = true;
+      const lowBits = count - 2;
+      const highBits = needleOfs << 3;
+      const word = highBits | lowBits;
+      compressed.writeUInt16LE(word, outOfs);
+      bucket.push(true);
+      outOfs += 2;
+      inOfs += len;
+      break;
+    }
+
+    if (!found) {
+      const word = segment.readUInt16LE(inOfs);
+      inOfs += 2;
+      bucket.push(false);
+      compressed.writeUInt16LE(word, outOfs);
+      outOfs += 2;
+    }
+  } while (inOfs < segment.length);
+
+  // Write a true bit and 0xffff to finish the segment
+  bucket.push(true);
+  compressed.writeUInt16LE(0xffff, outOfs);
+  outOfs += 2;
+
+  const payload = compressed.subarray(0, outOfs);
+  return [bucket, payload];
+};
+
+test("Can I compress and decompress some payload", () => {
+  const decompressed = readFileSync("fixtures/face-texture.bin");
+
+  // First we need to split the decompressed file into segments
+  const segments = [
+    decompressed.subarray(0x0000, 0x2000),
+    decompressed.subarray(0x2000, 0x4000),
+    decompressed.subarray(0x4000, 0x6000),
+    decompressed.subarray(0x6000, 0x8000),
+    decompressed.subarray(0x8000),
+  ];
+
+  const bits: boolean[] = [];
+  const loads: Buffer[] = [];
+  segments.forEach((segment, index) => {
+    const [b, p] = compressSegment(segment);
+    b.forEach((bit) => bits.push(bit));
+    loads.push(p);
+    const dsx = decompressPayload(p, b, segment.length);
+    expect(dsx).toEqual(segment);
+  });
+
+  const bitfied = encodeBitfield(bits);
+  const payload = Buffer.concat(loads);
+
+  const dsx = decompressPayload(payload, bits, 0x8080);
+  expect(dsx).toEqual(decompressed);
+});