Multi-threaded runtime for WASM

cmake/arrow/CMakeLists.txt

@@ -261,6 +261,9 @@ add_library(arrow STATIC ${ARROW_SRCS})
 target_compile_definitions(arrow PUBLIC ARROW_NO_DEPRECATED_API)
 target_compile_definitions(arrow PUBLIC ARROW_STATIC)
 
+
+set(CMAKE_CXX_FLAGS_RELEASE " -pthread ")
+
 # will need built boost filesystem and system .lib to work, even though
 # perspective itself does not use those dependencies
 target_link_libraries(arrow

cmake/re2/CMakeLists.txt

@@ -99,9 +99,7 @@ target_include_directories(re2 PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_D
 set_target_properties(re2 PROPERTIES SOVERSION ${SONAME} VERSION ${SONAME}.0.0)
 add_library(re2::re2 ALIAS re2)
 
-if(UNIX)
-  target_link_libraries(re2 PUBLIC Threads::Threads)
-endif()
+target_link_libraries(re2 PUBLIC Threads::Threads)
 
 if(RE2_BUILD_TESTING)
   set(TESTING_SOURCES
@@ -165,6 +163,8 @@ set(RE2_HEADERS
     re2/stringpiece.h
     )
 
+set(CMAKE_CXX_FLAGS_RELEASE " -pthread ")
+
 install(FILES ${RE2_HEADERS}
         DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/re2)
 install(TARGETS re2 EXPORT re2Config

cpp/perspective/CMakeLists.txt

@@ -287,9 +287,8 @@ if (PSP_WASM_BUILD)
             -O3 \
             -g0 \
             -flto \
+            -pthread \
             ")
-
-        # TODO: -flto
     endif()
 
     set(ASYNC_MODE_FLAGS "")
@@ -610,6 +609,8 @@ if (PSP_WASM_BUILD)
         -s NO_FILESYSTEM=1 \
         -s ALLOW_MEMORY_GROWTH=1 \
         -s MODULARIZE=1 \
+        -s USE_PTHREADS=1 \
+        -s PROXY_TO_PTHREAD=1 \
         -s EXPORT_NAME=\"load_perspective\" \
         -s MAXIMUM_MEMORY=4gb \
         -s USE_ES6_IMPORT_META=0 \

cpp/perspective/src/cpp/compat_impl_linux.cpp

@@ -145,7 +145,7 @@ void
 set_thread_name(std::thread& thr, const std::string& name) {
 #ifdef PSP_PARALLEL_FOR
     auto handle = thr.native_handle();
-    pthread_setname_np(handle, name.c_str());
+    // pthread_setname_np(handle, name.c_str());
 #endif
 }
 

cpp/perspective/src/cpp/emscripten.cpp

@@ -1843,31 +1843,38 @@ namespace binding {
 
 using namespace perspective::binding;
 
+void
+_main_completed() {
+    // clang-format off
+    EM_ASM({
+        if (typeof self !== "undefined") {
+            try {
+                if (self.dispatchEvent && !self._perspective_initialized
+                    && !!self.document) {
+                    self._perspective_initialized = true;
+                    var event = self.document.createEvent("Event");
+                    event.initEvent("perspective-ready", false, true);
+                    self.dispatchEvent(event);
+                } else if (!self.document && self.postMessage) {
+                    self.__on_wasm_init__("done");
+                    self.postMessage({});
+                }
+            } catch (e) {
+                console.error(e);
+            }
+        }
+    });
+    // clang-format on
+}
+
 /**
  * Main
  */
 int
 main(int argc, char** argv) {
     t_computed_expression_parser::init();
-
-    // clang-format off
-EM_ASM({
-
-    if (typeof self !== "undefined") {
-        try {
-            if (self.dispatchEvent && !self._perspective_initialized && self.document !== null) {
-                self._perspective_initialized = true;
-                var event = self.document.createEvent("Event");
-                event.initEvent("perspective-ready", false, true);
-                self.dispatchEvent(event);
-            } else if (!self.document && self.postMessage) {                
-                self.postMessage({});
-            }
-        } catch (e) {}
-    }
-
-});
-    // clang-format on
+    parallel_for(8, [](int cidx) {});
+    emscripten_sync_run_in_main_runtime_thread(EM_FUNC_SIG_V, _main_completed);
 }
 
 /******************************************************************************

cpp/perspective/src/include/perspective/emscripten.h

@@ -12,6 +12,7 @@
 #include <boost/optional.hpp>
 #include <emscripten.h>
 #include <emscripten/bind.h>
+#include <emscripten/threading.h>
 
 #ifdef PSP_ENABLE_WASM
 #include <emscripten/val.h>

cpp/perspective/src/include/perspective/first.h

@@ -7,11 +7,11 @@
  *
  */
 
-#ifndef PSP_ENABLE_WASM
+// #ifndef PSP_ENABLE_WASM
 #ifndef PSP_PARALLEL_FOR
 #define PSP_PARALLEL_FOR
 #endif
-#endif
+// #endif
 
 #if !defined(__linux__) && !defined(__APPLE__) && !defined(WIN32)
 // default to linux

cpp/perspective/src/include/perspective/parallel_for.h

@@ -9,10 +9,8 @@
 
 #pragma once
 
-#ifdef PSP_ENABLE_PYTHON
 #include <arrow/util/parallel.h>
 #include <arrow/status.h>
-#endif
 
 namespace perspective {
 

packages/perspective/build.js

@@ -6,6 +6,7 @@ const {WorkerPlugin} = require("@finos/perspective-build/worker");
 const {NodeModulesExternal} = require("@finos/perspective-build/external");
 const {UMDLoader} = require("@finos/perspective-build/umd");
 const {build} = require("@finos/perspective-build/build");
+const {BlobPlugin} = require("@finos/perspective-build/blob");
 
 const BUILD = [
     {
@@ -30,7 +31,7 @@ const BUILD = [
         },
         format: "esm",
         entryPoints: ["src/js/perspective.browser.js"],
-        plugins: [WasmPlugin(false), WorkerPlugin(false)],
+        plugins: [WasmPlugin(false), WorkerPlugin(false), BlobPlugin()],
         outfile: "dist/cdn/perspective.js",
     },
     {

packages/perspective/src/js/perspective.js

@@ -14,6 +14,7 @@ import {bindall, get_column_type} from "./utils.js";
 import {Server} from "./api/server.js";
 
 import formatters from "./view_formatters";
+import {promise} from "sinon";
 
 // IE fix - chrono::steady_clock depends on performance.now() which does not
 // exist in IE workers
@@ -35,7 +36,7 @@ const WARNED_KEYS = new Set();
  *
  * @module perspective
  */
-export default function (Module) {
+export default function (Module, module_url, worker_url) {
     let __MODULE__ = Module;
     let accessor = new DataAccessor();
     const SIDES = ["zero", "one", "two"];
@@ -2258,17 +2259,39 @@ export default function (Module) {
          * @param {ArrayBuffer} buffer an ArrayBuffer or Buffer containing the
          * Perspective WASM code
          */
-        init(msg) {
+        async init(msg) {
             if (typeof WebAssembly === "undefined") {
                 throw new Error("WebAssembly not supported");
             } else {
-                __MODULE__({
+                const promise = new Promise((resolve) => {
+                    // The easiest way to deal with emscripten's main thread
+                    // completion, which has an awkward API.
+                    self.__on_wasm_init__ = resolve;
+                });
+
+                // `__MODULE__` is async because we want both the source text
+                // (to pass to the worker threads) and the module (for this
+                // thread).
+                const {default: mod_constructor} = await __MODULE__;
+                const wasm_mod = await mod_constructor({
                     wasmBinary: msg.buffer,
                     wasmJSMethod: "native-wasm",
-                }).then((mod) => {
-                    __MODULE__ = mod;
-                    super.init(msg);
+                    mainScriptUrlOrBlob: module_url,
+                    locateFile(file) {
+                        // The `perspective.cpp.worker.js` file can't be
+                        // correctly linked by esbuild/webpack due to because
+                        // emscripten generated code outputs a dynamic string
+                        // passed to `import()`, so we can only provide this
+                        // file via `locateFile()`.
+                        if (file.endsWith("worker.js")) {
+                            return worker_url;
+                        }
+                    },
                 });
+
+                __MODULE__ = wasm_mod;
+                await promise;
+                super.init(msg);
             }
         }
     }

packages/perspective/src/js/perspective.node.js

@@ -100,6 +100,8 @@ function perspective_assets(assets, host_psp) {
         response.setHeader("Access-Control-Request-Method", "*");
         response.setHeader("Access-Control-Allow-Methods", "OPTIONS,GET");
         response.setHeader("Access-Control-Allow-Headers", "*");
+        response.setHeader(`Cross-Origin-Embedder-Policy`, `require-corp`);
+        response.setHeader(`Cross-Origin-Opener-Policy`, `same-origin`);
 
         let url = request.url.split(/[\?\#]/)[0];
 

packages/perspective/src/js/perspective.worker.js

@@ -7,21 +7,43 @@
  *
  */
 
-import load_perspective from "@finos/perspective/pkg/esm/perspective.cpp.js";
+import load_perspective_txt from "@finos/perspective/pkg/esm/perspective.cpp.js";
+import perspective_worker_txt from "@finos/perspective/pkg/esm/perspective.cpp.worker.js";
 import perspective from "./perspective.js";
 
-let _perspective_instance;
+const psp_url = (() => {
+    const blob = new Blob([load_perspective_txt], {type: "text/javascript"});
+    return URL.createObjectURL(blob);
+})();
+
+const load_perspective = (async () => {
+    return await import(psp_url);
+})();
+
+const worker_url = (() => {
+    const worker_blob = new Blob([perspective_worker_txt], {
+        type: "text/javascript",
+    });
+    return URL.createObjectURL(worker_blob);
+})();
 
+let _perspective_instance;
 if (global.document !== undefined && typeof WebAssembly !== "undefined") {
     _perspective_instance = global.perspective = perspective(
         load_perspective({
             wasmJSMethod: "native-wasm",
             printErr: (x) => console.error(x),
             print: (x) => console.log(x),
-        })
+        }),
+        psp_url,
+        worker_url
     );
 } else {
-    _perspective_instance = global.perspective = perspective(load_perspective);
+    _perspective_instance = global.perspective = perspective(
+        load_perspective,
+        psp_url,
+        worker_url
+    );
 }
 
 export default _perspective_instance;

rust/perspective-viewer/src/rust/components/filter_item.rs

@@ -68,7 +68,9 @@ impl DragDropListItemProps for FilterItemProps {
 impl FilterItemProps {
     /// Does this filter item get a "suggestions" auto-complete modal?
     fn is_suggestable(&self) -> bool {
-        (self.filter.1 == FilterOp::EQ || self.filter.1 == FilterOp::NE || self.filter.1 == FilterOp::In)
+        (self.filter.1 == FilterOp::EQ
+            || self.filter.1 == FilterOp::NE
+            || self.filter.1 == FilterOp::In)
             && self.get_filter_type() == Some(Type::String)
     }
 
@@ -259,7 +261,7 @@ impl Component for FilterItem {
                     ctx.props().filter_dropdown.autocomplete(
                         column,
                         if ctx.props().filter.1 == FilterOp::In {
-                            input.split(",").last().unwrap().to_owned()
+                            input.split(',').last().unwrap().to_owned()
                         } else {
                             input.clone()
                         },

rust/perspective-viewer/src/rust/model/structural.rs

@@ -35,28 +35,28 @@ pub trait HasDragDrop {
 #[macro_export]
 macro_rules! derive_model {
     (DragDrop for $key:ty) => {
-        impl crate::model::HasDragDrop for $key {
+        impl $crate::model::HasDragDrop for $key {
             fn dragdrop(&self) -> &'_ DragDrop {
                 &self.dragdrop
             }
         }
     };
     (Renderer for $key:ty) => {
-        impl crate::model::HasRenderer for $key {
+        impl $crate::model::HasRenderer for $key {
             fn renderer(&self) -> &'_ Renderer {
                 &self.renderer
             }
         }
     };
     (Session for $key:ty) => {
-        impl crate::model::HasSession for $key {
+        impl $crate::model::HasSession for $key {
             fn session(&self) -> &'_ Session {
                 &self.session
             }
         }
     };
     (Theme for $key:ty) => {
-        impl crate::model::HasTheme for $key {
+        impl $crate::model::HasTheme for $key {
             fn theme(&self) -> &'_ Theme {
                 &self.theme
             }