Add @mcap/support, @mcap/nodejs, and @mcap/browser libraries

.gitattributes

@@ -11,3 +11,4 @@ python/mcap-ros1-support/mcap_ros1/vendor/** linguist-vendored=true
 tests/conformance/data/** linguist-generated=true
 typescript/examples/flatbuffer/output/** linguist-generated=true
 go/ros/testdata/markers.bz2.bag filter=lfs diff=lfs merge=lfs -text
+typescript/support/src/fixtures/byte-vector.ts linguist-generated=true

.github/workflows/ci.yml

@@ -34,7 +34,7 @@ jobs:
           cache: yarn
       - run: corepack enable && yarn install --immutable
       - run: yarn workspace @foxglove/mcap-conformance lint:ci
-      - run: yarn workspace @foxglove/mcap-conformance typecheck
+      - run: yarn workspace @foxglove/mcap-conformance build
 
   conformance-cpp:
     runs-on: ubuntu-latest
@@ -199,6 +199,8 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
+        with:
+          lfs: true
       - uses: actions/setup-node@v3
         with:
           node-version: 16.x
@@ -208,15 +210,40 @@ jobs:
       - run: yarn dedupe --check
       - run: yarn prettier:check
       - run: yarn workspace @mcap/core lint:ci
-      - run: yarn workspace @mcap/core typecheck
-      - run: yarn workspace @mcap/core test
-
-      - name: Publish to NPM
+      - run: yarn workspace @mcap/core build
+      - run: yarn workspace @mcap/support lint:ci
+      - run: yarn workspace @mcap/support build
+      - run: yarn workspace @mcap/nodejs lint:ci
+      - run: yarn workspace @mcap/nodejs build
+      - run: yarn workspace @mcap/browser lint:ci
+      - run: yarn workspace @mcap/browser build
+      - run: yarn typescript:test
+
+      - name: Publish @mcap/core to NPM
         if: ${{ startsWith(github.ref, 'refs/tags/releases/typescript/core/v') }}
         run: yarn workspace @mcap/core npm publish --access public
         env:
           YARN_NPM_AUTH_TOKEN: ${{ secrets.NPM_PUBLISH_TOKEN }}
 
+      # To be enabled once API has settled
+      # - name: Publish @mcap/support to NPM
+      #   if: ${{ startsWith(github.ref, 'refs/tags/releases/typescript/support/v') }}
+      #   run: yarn workspace @mcap/support publish --access public
+      #   env:
+      #     NODE_AUTH_TOKEN: ${{ secrets.NPM_PUBLISH_TOKEN }}
+
+      # - name: Publish @mcap/nodejs to NPM
+      #   if: ${{ startsWith(github.ref, 'refs/tags/releases/typescript/nodejs/v') }}
+      #   run: yarn workspace @mcap/nodejs publish --access public
+      #   env:
+      #     NODE_AUTH_TOKEN: ${{ secrets.NPM_PUBLISH_TOKEN }}
+
+      # - name: Publish @mcap/browser to NPM
+      #   if: ${{ startsWith(github.ref, 'refs/tags/releases/typescript/browser/v') }}
+      #   run: yarn workspace @mcap/browser publish --access public
+      #   env:
+      #     NODE_AUTH_TOKEN: ${{ secrets.NPM_PUBLISH_TOKEN }}
+
   typescript-examples:
     runs-on: ubuntu-latest
     steps:

.vscode/settings.json

@@ -14,15 +14,18 @@
   "eslint.options": {
     "reportUnusedDisableDirectives": "error"
   },
-  "jest.jestCommandLine": "yarn workspace @mcap/core test",
+  "jest.jestCommandLine": "yarn typescript:test",
 
   "cSpell.enabled": true,
 
   "search.exclude": {
     "**/node_modules": true,
     "tests/conformance/data": true,
     "python/**/build": true,
-    "python/docs/*-apidoc": true
+    "python/docs/*-apidoc": true,
+    ".yarn/**": true,
+    "yarn.lock": true,
+    "**/dist": true
   },
 
   "python.formatting.provider": "black",

cspell.config.yaml

@@ -14,6 +14,7 @@ ignorePaths:
   - bin
   - dist
   - yarn-error.log
+  - "*.bfbs"
   - "*.csv"
   - "*.Dockerfile"
   - Dockerfile
@@ -88,6 +89,7 @@ words:
   - rostime
   - schemaless
   - serde
+  - sfixed
   - srgb
   - stoull
   - struct
@@ -101,6 +103,7 @@ words:
   - velodyne
   - waabi
   - webp
+  - xcdr
   - zstandard
   - zstd
   - zustand

package.json

@@ -6,23 +6,32 @@
       "typescript/benchmarks",
       "typescript/core",
       "typescript/examples/*",
+      "typescript/support",
+      "typescript/nodejs",
+      "typescript/browser",
       "website"
     ]
   },
   "scripts": {
     "prettier": "prettier --write .",
     "prettier:check": "prettier --check .",
     "docs:swift:start": "swift package --disable-sandbox preview-documentation --target MCAP",
-    "typedoc": "typedoc --out __docs__/typescript typescript/core/src/index.ts --tsconfig typescript/core/tsconfig.json",
+    "typedoc": "yarn typescript:build && typedoc --out __docs__/typescript --options typescript/typedoc.json",
     "start": "yarn workspace website start",
     "spellcheck": "cspell --relative '**'",
+    "typescript:test": "yarn jest --config typescript/jest.config.json",
+    "typescript:build": "yarn workspace @mcap/core build && yarn workspace @mcap/support build && yarn workspace @mcap/nodejs build && yarn workspace @mcap/browser build",
+    "typescript:clean": "yarn workspace @mcap/core build --clean && yarn workspace @mcap/support build --clean && yarn workspace @mcap/nodejs build --clean && yarn workspace @mcap/browser build --clean",
     "test:conformance:generate-inputs": "yarn workspace @foxglove/mcap-conformance generate-inputs --data-dir \"$(pwd)/tests/conformance/data\"",
     "test:conformance": "yarn workspace @foxglove/mcap-conformance run-tests --data-dir \"$(pwd)/tests/conformance/data\""
   },
   "packageManager": "[email protected]",
   "devDependencies": {
     "cspell": "^6.26.3",
+    "jest": "29.4.3",
     "prettier": "^2.8.4",
+    "ts-jest": "29.0.5",
+    "ts-node": "10.9.1",
     "typedoc": "^0.23.25"
   }
 }

python/mcap-protobuf-support/README.md

@@ -35,4 +35,4 @@ pipenv run python point_cloud_example.py output.mcap
 
 ## Stay in touch
 
-Join our [Slack channel](https://foxglove.dev/join-slack) to ask questions, share feedback, and stay up to date on what our team is working on.
+Join our [Slack channel](https://foxglove.dev/slack) to ask questions, share feedback, and stay up to date on what our team is working on.

python/mcap-ros1-support/README.md

@@ -36,5 +36,5 @@ ros_writer.finish()
 
 ## Stay in touch
 
-Join our [Slack channel](https://foxglove.dev/join-slack) to ask questions,
+Join our [Slack channel](https://foxglove.dev/slack) to ask questions,
 share feedback, and stay up to date on what our team is working on.

python/mcap-ros2-support/README.md

@@ -22,5 +22,5 @@ for msg in read_ros2_messages("my_data.mcap"):
 
 ## Stay in touch
 
-Join our [Slack channel](https://foxglove.dev/join-slack) to ask questions,
+Join our [Slack channel](https://foxglove.dev/slack) to ask questions,
 share feedback, and stay up to date on what our team is working on.

tests/conformance/.eslintrc.js

@@ -8,8 +8,11 @@ module.exports = {
       files: ["*.ts", "*.tsx"],
       extends: ["plugin:@foxglove/typescript"],
       parserOptions: {
-        project: "tsconfig.json",
+        project: ["tsconfig.json", "../../typescript/*/tsconfig.json"],
         tsconfigRootDir: __dirname,
+        // Enable typescript-eslint to use `src` files for type information across project references
+        // <https://github.com/typescript-eslint/typescript-eslint/issues/2094>
+        EXPERIMENTAL_useSourceOfProjectReferenceRedirect: true,
       },
     },
   ],

tests/conformance/package.json

@@ -3,17 +3,18 @@
   "version": "0.0.0",
   "private": true,
   "scripts": {
-    "typecheck": "tsc -p tsconfig.json --noEmit",
+    "build": "tsc -b tsconfig.json tsconfig.cjs.json",
     "lint:ci": "eslint --report-unused-disable-directives .",
     "lint": "eslint --report-unused-disable-directives --fix .",
     "generate-inputs": "ts-node --files --project tsconfig.cjs.json scripts/generate-inputs",
-    "run-tests": "ts-node --files --project tsconfig.cjs.json scripts/run-tests"
+    "run-tests": "tsc -b tsconfig.json tsconfig.cjs.json && ts-node --files --project tsconfig.cjs.json scripts/run-tests"
   },
   "devDependencies": {
     "@foxglove/crc": "^0.0.3",
     "@foxglove/eslint-plugin": "0.21.0",
     "@foxglove/tsconfig": "1.1.0",
-    "@mcap/core": "*",
+    "@mcap/core": "workspace:*",
+    "@mcap/support": "workspace:*",
     "@types/diff": "^5.0.2",
     "@types/js-yaml": "^4.0.5",
     "@types/node": "18.13.0",

tests/conformance/scripts/run-tests/runners/TypescriptIndexedReaderTestRunner.ts

@@ -1,4 +1,5 @@
 import { McapIndexedReader } from "@mcap/core";
+import { FileHandleReadable } from "@mcap/nodejs";
 import fs from "fs/promises";
 import { TestFeatures, TestVariant } from "variants/types";
 
@@ -41,28 +42,7 @@ export default class TypescriptIndexedReaderTestRunner extends IndexedReadTestRu
   }
 
   private async _run(fileHandle: fs.FileHandle): Promise<IndexedReadTestResult> {
-    let buffer = new ArrayBuffer(4096);
-    const readable = {
-      size: async () => BigInt((await fileHandle.stat()).size),
-      read: async (offset: bigint, length: bigint) => {
-        if (offset > Number.MAX_SAFE_INTEGER || length > Number.MAX_SAFE_INTEGER) {
-          throw new Error(`Read too large: offset ${offset}, length ${length}`);
-        }
-        if (length > buffer.byteLength) {
-          buffer = new ArrayBuffer(Number(length * 2n));
-        }
-        const result = await fileHandle.read({
-          buffer: new DataView(buffer, 0, Number(length)),
-          position: Number(offset),
-        });
-        if (result.bytesRead !== Number(length)) {
-          throw new Error(
-            `Read only ${result.bytesRead} bytes from offset ${offset}, expected ${length}`,
-          );
-        }
-        return new Uint8Array(result.buffer.buffer, result.buffer.byteOffset, result.bytesRead);
-      },
-    };
+    const readable = new FileHandleReadable(fileHandle);
 
     const reader = await McapIndexedReader.Initialize({ readable });
     if (reader.chunkIndexes.length === 0) {

tests/conformance/tsconfig.cjs.json

@@ -3,5 +3,9 @@
   "compilerOptions": {
     "outDir": "./dist/cjs",
     "module": "commonjs"
-  }
+  },
+  "references": [
+    { "path": "../../typescript/core/tsconfig.cjs.json" },
+    { "path": "../../typescript/support/tsconfig.cjs.json" }
+  ]
 }

tests/conformance/tsconfig.json

@@ -6,12 +6,19 @@
     "noEmit": true,
     "lib": ["es2020", "dom"],
     "paths": {
-      "@mcap/core": ["../../typescript/core/src"]
+      "@mcap/core": ["../../typescript/core/src"],
+      "@mcap/nodejs": ["../../typescript/nodejs/src"],
+      "@mcap/support": ["../../typescript/support/src"]
     },
 
     // required for tsconfig-paths https://github.com/dividab/tsconfig-paths/issues/143
     "baseUrl": "."
   },
+  "references": [
+    { "path": "../../typescript/core" },
+    { "path": "../../typescript/nodejs" },
+    { "path": "../../typescript/support" }
+  ],
   "ts-node": {
     "require": ["tsconfig-paths/register"]
   }

typescript/CONTRIBUTING.md

@@ -0,0 +1,42 @@
+# Development guide
+
+Install dependencies:
+
+```
+corepack enable
+yarn install
+```
+
+Run lint/tests:
+
+```
+yarn workspace @mcap/core lint
+yarn workspace @mcap/core test
+```
+
+Read and validate an MCAP file:
+
+```
+yarn workspace @foxglove/mcap-example-validate validate file.mcap
+```
+
+Run benchmarks:
+
+```
+yarn workspace @foxglove/mcap-benchmarks bench
+```
+
+Run benchmarks with Chrome debugger attached to use profiling tools:
+
+```
+yarn workspace @foxglove/mcap-benchmarks bench:debug
+```
+
+## Releasing to NPM
+
+- Check out the version of the code you want to release
+- Update package.json in `typescript/{pkg}/package.json` with the new version.
+- Make a PR with your changes to package.json
+- Wait for the PR to pass CI and merge
+- Checkout main and tag the merged commit with `releases/typescript/{pkg}/v#.#.#` (replace #.#.# with the version you used in package.json)
+- Push the new tag to the repo with `git push origin releases/typescript/{pkg}/v#.#.#`

typescript/README.md

@@ -1,33 +1,10 @@
 # TypeScript libraries for MCAP
 
-Install dependencies:
+[MCAP](https://mcap.dev/) is a modular container format and logging library for pub/sub messages with arbitrary message serialization. It is primarily intended for use in robotics applications, and works well under various workloads, resource constraints, and durability requirements.
 
-```
-corepack enable
-yarn install
-```
+The following NPM packages are provided for use with JavaScript and TypeScript:
 
-Run lint/tests:
-
-```
-yarn workspace @mcap/core lint
-yarn workspace @mcap/core test
-```
-
-Read and validate an MCAP file:
-
-```
-yarn workspace @foxglove/mcap-example-validate validate file.mcap
-```
-
-Run benchmarks:
-
-```
-yarn workspace @foxglove/mcap-benchmarks bench
-```
-
-Run benchmarks with Chrome debugger attached to use profiling tools:
-
-```
-yarn workspace @foxglove/mcap-benchmarks bench:debug
-```
+- **@mcap/core** – low-level readers and writers
+- **@mcap/support** – support for well-known compression formats and encodings
+- **@mcap/nodejs** – support for Node.js environment
+- **@mcap/browser** – support for browser environment

typescript/browser/.eslintrc.js

@@ -0,0 +1,28 @@
+/* eslint-env node */
+module.exports = {
+  env: { es2020: true },
+  ignorePatterns: ["dist"],
+  extends: ["plugin:@foxglove/base", "plugin:@foxglove/jest", "plugin:import/recommended"],
+  overrides: [
+    {
+      files: ["*.ts", "*.tsx"],
+      extends: ["plugin:@foxglove/typescript"],
+      parserOptions: {
+        project: "../*/tsconfig.json",
+        tsconfigRootDir: __dirname,
+        // Enable typescript-eslint to use `src` files for type information across project references
+        // <https://github.com/typescript-eslint/typescript-eslint/issues/2094>
+        EXPERIMENTAL_useSourceOfProjectReferenceRedirect: true,
+      },
+    },
+  ],
+  rules: {
+    "no-warning-comments": ["error", { terms: ["fixme"], location: "anywhere" }],
+  },
+  settings: {
+    "import/resolver": {
+      typescript: true,
+      node: true,
+    },
+  },
+};