[Tests] Basic System Tests (#1176)

* test(systest): implement structure for basic system tests (#1164)

* lint-fix(systest): fix any value (#1164)

* test(systests): run test for flowr server (#1164)

* lint(systests): lin fix (#1164)

* refactor(systests): use childprocess instead of zx (#1164)

* test-fix(systests): newline at end of file (#1164)

* ci(systets): add systests to ci (#1164)

* ci-fix(systest): extend test timeout (#1164)

* lint(systest): format config (#1164)

* refactor(test): defer math ast parsing in normalized ast fold

* refactor(system-tests): concat buffers in checking server output

* refactor(system-tests): sequentialize

* wip(debug): print stdout for server test

* wip(server-tests): only run server test for output

* refactor(server-test): kill the server thread with a sigkill

---------

Co-authored-by: Florian Sihler <[email protected]>

.github/workflows/qa.yaml

@@ -118,6 +118,9 @@ jobs:
       - name: "🧪 Run the Tests"
         run: bash .github/workflows/scripts/run-flowr-command.sh "test-full -- --allowOnly=false"
 
+      - name: "⚙️ Run System Tests"
+        run: bash .github/workflows/scripts/run-flowr-command.sh "test:system -- --no-watch"
+
       - name: "⬆️ Upload Coverage Reports to Codecov"
         if: github.ref == 'refs/heads/main'
         uses: codecov/codecov-action@v4

.github/workflows/release.yaml

@@ -159,7 +159,7 @@ jobs:
         run: npm ci
 
       - name: "⬆️ Publish on NPM"
-        # we use publish library because flowr itself is a runnable, and we do not want to publish something with extra
+        # we use publish-library because flowr itself is a runnable, and we do not want to publish something with extra
         # dist paths etc.
         # besides, we make dead-sure we have a clean directory to work on!
         run: |

package.json

@@ -36,12 +36,13 @@
     "lint": "npm run license-compat -- --summary && npx eslint --version && npx eslint src/ test/",
     "license-compat": "license-checker --onlyAllow 'MIT;MIT OR X11;GPLv2;LGPL;GNUGPL;ISC;Apache-2.0;FreeBSD;BSD-2-Clause;clearbsd;ModifiedBSD;BSD-3-Clause;Python-2.0;Unlicense;WTFPL;BlueOak-1.0.0;CC-BY-4.0;CC-BY-3.0;CC0-1.0;0BSD'",
     "doc": "typedoc",
-    "test": "vitest --config test/vitest.config.mts",
+    "test": "vitest --exclude \"test/system-tests/**\" --config test/vitest.config.mts",
+    "test:system": "vitest --dir test/system-tests --config test/system-tests/vitest.config.mts",
     "test:coverage": "npm run test -- --coverage",
     "performance-test": "func() { cd test/performance/ && bash run-all-suites.sh $1 $2 $3; cd ../../; }; func",
     "test-full": "npm run test:coverage -- --no-watch -- --make-summary --test-installation",
     "detect-circular-deps": "npx madge  --extensions ts,tsx --circular src/",
-    "checkup": "npm run flowr -- --execute \":version\" && npm run lint && npm run test-full -- --allowOnly=false && docker build -t test-flowr -f scripts/Dockerfile . && npm run doc && npm-run-all wiki:*"
+    "checkup": "npm run flowr -- --execute \":version\" && npm run lint && npm run test-full -- --allowOnly=false && npm run test:system -- --no-watch && docker build -t test-flowr -f scripts/Dockerfile . && npm run doc && npm-run-all wiki:*"
   },
   "keywords": [
     "static code analysis",

test/functionality/r-bridge/normalize-ast-fold.test.ts

@@ -9,8 +9,11 @@ import type { RExpressionList } from '../../../src/r-bridge/lang-4.x/ast/model/n
 
 describe('normalize-visitor', withShell(shell => {
 	let normalized: NormalizedAst | undefined;
+	let mathAst: NormalizedAst | undefined;
 	beforeAll(async() => {
 		normalized = await retrieveNormalizedAst(shell, 'x <- 42\ny <- "hello world"\nprint("foo")');
+		mathAst = await retrieveNormalizedAst(shell, '1 + 3 * 2');
+
 	});
 	test('find the number', () => {
 		let marker = false;
@@ -42,7 +45,7 @@ describe('normalize-visitor', withShell(shell => {
 		const result = astFold.fold(normalized?.ast);
 		expect(result).toBe(2);
 	});
-	test('do basic math (monoid)', async() => {
+	test('do basic math (monoid)', () => {
 		class MyMathFold<Info> extends DefaultNormalizedAstFold<number, Info> {
 			constructor() {
 				super(0);
@@ -67,11 +70,10 @@ describe('normalize-visitor', withShell(shell => {
 			}
 		}
 		const astFold = new MyMathFold();
-		const math = await retrieveNormalizedAst(shell, '1 + 3 * 2');
-		const result = astFold.fold(math?.ast);
+		const result = astFold.fold(mathAst?.ast);
 		expect(result).toBe(7);
 	});
-	test('fold should stop if overwritten and no continue', async() => {
+	test('fold should stop if overwritten and no continue', () => {
 		let foundNumber = false;
 		class MyMathFold<Info> extends DefaultNormalizedAstFold<void, Info> {
 			override foldRNumber(_node: RNumber<Info>) {
@@ -83,8 +85,7 @@ describe('normalize-visitor', withShell(shell => {
 			}
 		}
 		const astFold = new MyMathFold();
-		const math = await retrieveNormalizedAst(shell, '1 + 3 * 2');
-		astFold.fold(math?.ast);
+		astFold.fold(mathAst?.ast);
 		expect(foundNumber).toBe(false);
 	});
 }));

test/system-tests/commands.test.ts

@@ -0,0 +1,16 @@
+import { assert, describe, test } from 'vitest';
+import { run } from './utility/utility';
+
+
+describe('commands', () => {
+	test('flowr as server', async() => {
+		const expected = 'Server listening on port';
+		const output = await run('npm run flowr -- --server', expected);
+		assert.include(output, expected);
+	});
+
+	test('slicer', async() => {
+		const output = await run('npm run slicer -- -c "3@a" -r "a <- 3\\nb <- 4\\nprint(a)"');
+		assert.include(output, 'a <- 3\na');
+	});
+});

test/system-tests/repl.test.ts

@@ -0,0 +1,24 @@
+import { assert, describe, test } from 'vitest';
+import { flowrRepl } from './utility/utility';
+
+describe('repl', () => {
+	test(':df', async() => {
+		const output = await flowrRepl([':df test', ':quit']);
+		assert.include(output, 'flowchart');
+	});
+
+	test(':df x <- 3', async() => {
+		const output = await flowrRepl([':df x <- 3 ', ':quit']);
+		assert.include(output, 'flowchart');
+	});
+
+	test(':df "x <- 3\nprint(x)"', async() => {
+		const output = await flowrRepl([':df "x <- 3\\nprint(x)"', ':quit']);
+		assert.include(output, 'flowchart');
+	});
+
+	test(':slicer', async() => {
+		const output = await flowrRepl([':slicer -c "3@a" -r "a <- 3\\nb <- 4\\nprint(a)"', ':quit']);
+		assert.include(output, 'a <- 3\na');
+	});
+});

test/system-tests/utility/utility.ts

@@ -0,0 +1,70 @@
+import { exec } from 'child_process';
+
+/**
+ * Runs the flowr repl and feeds input to the repl
+ * @param input - input to feed
+ * @returns Repl Output
+ */
+
+export async function flowrRepl(input: string[]): Promise<string> {
+	const process = new Promise<string>((resolve, reject) => {
+		const child = exec('npm run flowr', { timeout: 60 * 1000 }, (error, stdout, _) => {
+			if(error) {
+				reject(new Error(`${error.name}: ${error.message}\n${stdout}`));
+			}
+
+			resolve(stdout);
+		});
+
+		// Send new data when flowr sends us the 'R>' prompt to avoid
+		// sending data too fast
+		let i = 0;
+		child.stdout?.on('data', (d) => {
+			const data = d as Buffer;
+
+			if(data.toString().includes('R>')) {
+				if(i < input.length) {
+					child.stdin?.write(`${input[i++]}\n`);
+				}
+			}
+		});
+	});
+
+	return await process;
+}
+
+/**
+ * Runs a command and terminates it automatically if it outputs a certain string
+ * This is useful, so we don't have to set timeouts and hope the output will be produced in time.
+ *
+ * @param command - Command to run
+ * @param terminateOn - (optional) string to kill the process on
+ * @param timeout - (optional) timeout in milliseconds
+ * @returns output of command
+ */
+export async function run(command: string, terminateOn?: string, timeout = 60 * 1000): Promise<string> {
+	const process = new Promise<string>((resolve, reject) => {
+		const child = exec(command, { timeout }, (error, stdout, _) => {
+			if(error) {
+				reject(new Error(`${error.name}: ${error.message}\n${stdout}`));
+			}
+
+			resolve(stdout);
+		});
+
+		if(terminateOn) {
+			let buffer = '';
+			child.stdout?.on('data', (d: Buffer) => {
+				buffer += d.toString();
+
+				if(buffer.includes(terminateOn)) {
+					child.kill('SIGKILL');
+					resolve(buffer);
+				}
+			});
+		}
+	});
+
+	return await process;
+}
+

test/system-tests/vitest.config.mts

@@ -0,0 +1,22 @@
+import { defineConfig } from 'vitest/config';
+
+export default defineConfig({
+	test: {
+		testTimeout: 60 * 2000,
+		sequence:    {
+			/* each test file that does not support parallel execution will be executed in sequence by stating this explicitly */
+			concurrent: true,
+			setupFiles: 'parallel'
+		},
+		reporters: process.env.GITHUB_ACTIONS ? ['basic', 'github-actions'] : ['dot'],
+		isolate:   false,
+		pool:      'threads',
+		deps:      {
+			optimizer: {
+				ssr: {
+					enabled: true
+				}
+			}
+		}
+	},
+});