Copy changes from Phabricator

.dockerignore

@@ -14,3 +14,4 @@
 !test
 !ext
 **/_build
+!mocha-webdriver-0.3.2.tgz

.github/workflows/docker_latest.yml

@@ -18,7 +18,7 @@ jobs:
     strategy:
       matrix:
         python-version: [3.9]
-        node-version: [14.x]
+        node-version: [18.x]
     steps:
       - name: Check out the repo
         uses: actions/checkout@v2

.github/workflows/main.yml

@@ -18,7 +18,7 @@ jobs:
       fail-fast: false
       matrix:
         python-version: [3.9]
-        node-version: [14.x]
+        node-version: [18.x]
         tests:
           - ':lint:python:client:common:smoke:'
           - ':server-1-of-2:'
@@ -30,10 +30,10 @@ jobs:
           - ':nbrowser-^[^A-S]:'
         include:
           - tests: ':lint:python:client:common:smoke:'
-            node-version: 14.x
+            node-version: 18.x
             python-version: '3.10'
           - tests: ':lint:python:client:common:smoke:'
-            node-version: 14.x
+            node-version: 18.x
             python-version: '3.11'
     steps:
       - uses: actions/checkout@v3
@@ -72,6 +72,10 @@ jobs:
       - name: Build Node.js code
         run: yarn run build:prod
 
+      - name: Install chromedriver
+        if: contains(matrix.tests, ':nbrowser-') || contains(matrix.tests, ':smoke:')
+        run: ./node_modules/selenium-webdriver/bin/linux/selenium-manager --driver chromedriver
+
       - name: Run smoke test
         if: contains(matrix.tests, ':smoke:')
         run: VERBOSE=1 DEBUG=1 MOCHA_WEBDRIVER_HEADLESS=1 yarn run test:smoke

.gitignore

@@ -79,3 +79,4 @@ timings.txt
 xunit.xml
 
 **/_build
+!mocha-webdriver-0.3.2.tgz

.nvmrc

@@ -1 +1 @@
-14.20.1
+18.17.1

Dockerfile

@@ -10,7 +10,7 @@ FROM scratch as ext
 ## Javascript build stage
 ################################################################################
 
-FROM node:14-buster as builder
+FROM node:18-buster as builder
 
 # Install all node dependencies.
 WORKDIR /grist
@@ -67,7 +67,7 @@ FROM gristlabs/gvisor-unprivileged:buster as sandbox
 ################################################################################
 
 # Now, start preparing final image.
-FROM node:14-buster-slim
+FROM node:18-buster-slim
 
 # Install libexpat1, libsqlite3-0 for python3 library binary dependencies.
 # Install pgrep for managing gvisor processes.

app/server/lib/FlexServer.ts

@@ -67,7 +67,6 @@ import {addUploadRoute} from 'app/server/lib/uploads';
 import {buildWidgetRepository, IWidgetRepository} from 'app/server/lib/WidgetRepository';
 import {setupLocale} from 'app/server/localization';
 import axios from 'axios';
-import * as bodyParser from 'body-parser';
 import * as cookie from 'cookie';
 import express from 'express';
 import * as fse from 'fs-extra';
@@ -752,7 +751,7 @@ export class FlexServer implements GristServer {
 
   public addJsonSupport() {
     if (this._check('json')) { return; }
-    this.app.use(bodyParser.json({limit: '1mb'}));  // Increase from the default 100kb
+    this.app.use(express.json({limit: '1mb'}));  // Increase from the default 100kb
   }
 
   public addSessions() {

app/server/lib/NSandbox.ts

@@ -737,6 +737,10 @@ function gvisor(options: ISandboxOptions): SandboxProcess {
     wrapperArgs.push(process.env.GRIST_CHECKPOINT!);
   }
   const child = spawn(command, [...wrapperArgs.get(), `python${pythonVersion}`, '--', ...pythonArgs]);
+  if (!child.pid) {
+    throw new Error(`failed to spawn python${pythonVersion}`);
+  }
+
   // For gvisor under ptrace, main work is done by a traced process identifiable as
   // being labeled "exe" and having a parent also labeled "exe".
   const recognizeTracedProcess = (p: ProcessInfo) => {

app/server/lib/SamlConfig.ts

@@ -52,7 +52,6 @@
  *
  */
 
-import * as bodyParser from 'body-parser';
 import * as express from 'express';
 import * as fse from 'fs-extra';
 import * as saml2 from 'saml2-js';
@@ -161,7 +160,7 @@ export class SamlConfig {
     }));
 
     // Assert endpoint for when the login completes as POST.
-    app.post("/saml/assert", bodyParser.urlencoded({extended: true}), expressWrap(async (req, res, next) => {
+    app.post("/saml/assert", express.urlencoded({extended: true}), expressWrap(async (req, res, next) => {
       const relayState: string = req.body.RelayState;
       if (!relayState) { throw new Error('Login or logout failed to complete'); }
       const permitStore = this._gristServer.getExternalPermitStore();

app/server/lib/SandboxControl.ts

@@ -30,12 +30,16 @@ export interface ISandboxControl {
  * Control a single process directly. A thin wrapper around the Throttle class.
  */
 export class DirectProcessControl implements ISandboxControl {
+  private _pid: number;
   private _throttle?: Throttle;
 
   constructor(private _process: childProcess.ChildProcess, logMeta?: log.ILogMeta) {
+    if (!_process.pid) { throw new Error(`process identifier (PID) is undefined`); }
+
+    this._pid = _process.pid;
     if (process.env.GRIST_THROTTLE_CPU) {
       this._throttle = new Throttle({
-        pid: _process.pid,
+        pid: this._pid,
         logMeta: {...logMeta, pid: _process.pid},
       });
     }
@@ -55,7 +59,7 @@ export class DirectProcessControl implements ISandboxControl {
   }
 
   public async getUsage() {
-    const memory = (await pidusage(this._process.pid)).memory;
+    const memory = (await pidusage(this._pid)).memory;
     return { memory };
   }
 }

app/server/lib/expressWrap.ts

@@ -2,10 +2,14 @@ import {RequestWithLogin} from 'app/server/lib/Authorizer';
 import log from 'app/server/lib/log';
 import * as express from 'express';
 
+export interface AsyncRequestHandler {
+  (req: express.Request, res: express.Response, next: express.NextFunction): any | Promise<any>;
+}
+
 /**
  * Wrapper for async express endpoints to catch errors and forward them to the error handler.
  */
-export function expressWrap(callback: express.RequestHandler): express.RequestHandler {
+export function expressWrap(callback: AsyncRequestHandler): express.RequestHandler {
   return async (req, res, next) => {
     try {
       await callback(req, res, next);

package.json

@@ -49,7 +49,7 @@
     "@types/diff-match-patch": "1.0.32",
     "@types/dompurify": "2.4.0",
     "@types/double-ended-queue": "2.1.0",
-    "@types/express": "4.16.0",
+    "@types/express": "4.17.17",
     "@types/form-data": "2.2.1",
     "@types/fs-extra": "5.0.4",
     "@types/http-proxy": "1.17.9",
@@ -64,16 +64,17 @@
     "@types/marked": "4.0.8",
     "@types/mime-types": "2.1.0",
     "@types/minio": "7.0.15",
-    "@types/mocha": "5.2.5",
+    "@types/mocha": "10.0.1",
     "@types/moment-timezone": "0.5.9",
-    "@types/node": "^14",
+    "@types/node": "18.11.9",
     "@types/node-fetch": "2.6.2",
     "@types/pidusage": "2.0.1",
     "@types/plotly.js": "2.12.1",
+    "@types/proper-lockfile": "4.1.2",
     "@types/qrcode": "1.4.2",
     "@types/redlock": "3.0.2",
     "@types/saml2-js": "2.0.1",
-    "@types/selenium-webdriver": "4.0.0",
+    "@types/selenium-webdriver": "4.1.16",
     "@types/sinon": "5.0.5",
     "@types/sqlite3": "3.1.6",
     "@types/tmp": "0.0.33",
@@ -93,10 +94,10 @@
     "i18next-scanner": "4.1.0",
     "jsdom": "16.5.0",
     "mocha": "10.2.0",
-    "mocha-webdriver": "0.2.13",
     "moment-locales-webpack-plugin": "^1.2.0",
     "nodemon": "^2.0.4",
     "otplib": "12.0.1",
+    "proper-lockfile": "4.1.2",
     "sinon": "7.1.1",
     "source-map-loader": "^0.2.4",
     "tmp-promise": "1.0.5",
@@ -136,7 +137,7 @@
     "dompurify": "3.0.0",
     "double-ended-queue": "2.1.0-0",
     "exceljs": "4.2.1",
-    "express": "4.16.4",
+    "express": "4.18.2",
     "file-type": "16.5.4",
     "fs-extra": "7.0.0",
     "grain-rpc": "0.1.7",
@@ -147,7 +148,7 @@
     "https-proxy-agent": "5.0.1",
     "i18n-iso-countries": "6.1.0",
     "i18next": "21.9.1",
-    "i18next-http-middleware": "3.2.1",
+    "i18next-http-middleware": "3.3.2",
     "image-size": "0.6.3",
     "jquery": "3.5.0",
     "js-yaml": "3.14.1",
@@ -158,6 +159,7 @@
     "lodash": "4.17.21",
     "marked": "4.2.12",
     "minio": "7.0.32",
+    "mocha-webdriver": "./mocha-webdriver-0.3.2.tgz",
     "moment": "2.29.4",
     "moment-timezone": "0.5.35",
     "morgan": "1.9.1",

static/test.html

@@ -24,8 +24,8 @@
             }
 
             mocha.checkLeaks();
-            // fxdriver_id is set by selenium, execWebdriverJQuery by webdriverjq.js.
-            mocha.globals(['cmd', 'fxdriver_id', 'execWebdriverJQuery']);
+            // fxdriver_id and ret_nodes are set by selenium, execWebdriverJQuery by webdriverjq.js.
+            mocha.globals(['cmd', 'fxdriver_id', 'execWebdriverJQuery', 'ret_nodes']);
             var runner = mocha.run();
             mocha.failedTests = [];
             runner.on('fail', function(test, err) {

test/declarations.d.ts

@@ -15,31 +15,3 @@ declare namespace Chai {
     notIncludeMembers<T>(superset: T[], subset: T[], message?: string): void;
   }
 }
-
-declare module "selenium-webdriver" {
-  interface WebDriver {
-    withActions(cb: (actions: WebActions) => void): Promise<void>;
-  }
-
-  // This is not a complete definition of available methods, but only those that we use for now.
-  // TODO: find documentation for this interface or update selenium-webdriver.
-  interface WebActions {
-    contextClick(el?: WebElement): WebActions;
-    click(el?: WebElement): WebActions;
-    press(): WebActions;
-    move(params: {origin?: WebElement|string, x?: number, y?: number}): WebActions;
-    keyDown(key: string): WebActions;
-    keyUp(key: string): WebActions;
-    dragAndDrop(element: WebElement, target: WebElement): WebActions;
-    release(): WebActions;
-    doubleClick(element: WebElement): WebActions;
-    pause(ms: number): WebActions;
-  }
-}
-
-import "mocha-webdriver";
-declare module "mocha-webdriver" {
-  // It looks like this hack makes tsc see our definition as primary, adding
-  // the typed version override (of the withActions method) as the default one.
-  export declare let driver: import("selenium-webdriver").WebDriver;
-}

test/gen-server/seed.ts

@@ -23,7 +23,7 @@
  */
 
 import {addPath} from 'app-module-path';
-import {IHookCallbackContext} from 'mocha';
+import {Context} from 'mocha';
 import * as path from 'path';
 import {Connection, getConnectionManager, Repository} from 'typeorm';
 
@@ -643,7 +643,7 @@ function _generateData(numOrgs: number, numWorkspaces: number, numDocs: number)
  * To set up TYPEORM_* environment variables for testing, call this in a before() call of a test
  * suite, using setUpDB(this);
  */
-export function setUpDB(context?: IHookCallbackContext) {
+export function setUpDB(context?: Context) {
   if (!process.env.TYPEORM_DATABASE) {
     process.env.TYPEORM_DATABASE = ":memory:";
   } else {