feat(variables): handle iterations

src/use-lunatic/commons/calculated-variables.ts

@@ -1,5 +1,5 @@
 import { executeVtlExpression } from './execute-expression/execute-expression';
-import getVtlCompatibleValue from '../../utils/vtl';
+import { getVTLCompatibleValue } from '../../utils/vtl';
 import { CALCULATED } from '../../constants';
 import type { LunaticState, LunaticValues } from '../type';
 
@@ -86,6 +86,6 @@ const buildScopedBindings = ({
 
 const buildVectorialBindings = ({ bindings }: { bindings: LunaticValues }) =>
 	Object.entries(bindings).reduce((acc, [k, v]) => {
-		if (Array.isArray(v)) return { ...acc, [k]: getVtlCompatibleValue(v) };
+		if (Array.isArray(v)) return { ...acc, [k]: getVTLCompatibleValue(v) };
 		return { ...acc, [k]: v };
 	}, {});

src/use-lunatic/commons/execute-expression/create-execute-expression.ts

@@ -3,7 +3,7 @@ import getSafetyExpression from './get-safety-expression';
 import getExpressionVariables from './get-expressions-variables';
 import createMemoizer from './create-memoizer';
 import createRefreshCalculated from './create-refresh-calculated';
-import getVtlCompatibleValue from '../../../utils/vtl';
+import { getVTLCompatibleValue } from '../../../utils/vtl';
 import { VTL, VTL_MD, X_AXIS, Y_AXIS } from '../../../utils/constants';
 import type { LunaticExpression, LunaticState } from '../../type';
 
@@ -171,7 +171,7 @@ function createExecuteExpression(
 
 			return null;
 		}
-		return getVtlCompatibleValue(value);
+		return getVTLCompatibleValue(value);
 	}
 
 	function fillVariablesValues(

src/use-lunatic/commons/execute-expression/execute-expression.tsx

@@ -1,7 +1,7 @@
 import { type ReactNode } from 'react';
 import { interpret } from '@inseefr/trevas';
 import MdLabel from '../../../components/commons/components/md-label';
-import getVtlCompatibleValue from '../../../utils/vtl';
+import { getVTLCompatibleValue } from '../../../utils/vtl';
 import { MD, VTL } from '../../../utils/constants';
 import type { VTLBindings } from '../../type';
 
@@ -36,7 +36,7 @@ export function executeVtlExpression(
 ) {
 	// Make sur all values can be handled by VTL
 	const legalVtlBindings = Object.entries(vtlBindings).reduce(
-		(acc, [k, v]) => ({ ...acc, [k]: getVtlCompatibleValue(v) }),
+		(acc, [k, v]) => ({ ...acc, [k]: getVTLCompatibleValue(v) }),
 		{}
 	);
 	const result = interpret(expression, legalVtlBindings);

src/use-lunatic/commons/lunatic-variables.spec.ts

@@ -72,4 +72,53 @@ describe('lunatic-variables', () => {
 		variables.set('FIRSTNAME', 'Jane');
 		expect(variables.run('FIRSTNAME || " " || LASTNAME')).toEqual('Jane Doe');
 	});
+
+	describe('with iteration', () => {
+		it('should handle arrays', () => {
+			variables.set('FIRSTNAME', ['John', 'Jane']);
+			expect(variables.get('FIRSTNAME')).toEqual(['John', 'Jane']);
+			expect(variables.get('FIRSTNAME', 0)).toEqual('John');
+			expect(variables.get('FIRSTNAME', 1)).toEqual('Jane');
+		});
+		it('should ignore non array values', () => {
+			variables.set('FIRSTNAME', 'John');
+			expect(variables.get('FIRSTNAME', 0)).toEqual('John');
+		});
+		it('should handle iteration in calculation', () => {
+			variables.set('FIRSTNAME', ['John', 'Jane']);
+			variables.set('LASTNAME', ['Doe', 'Dae']);
+			variables.setCalculated('FULLNAME', 'FIRSTNAME || " " || LASTNAME', [
+				'FIRSTNAME',
+				'LASTNAME',
+			]);
+			expect(variables.get('FULLNAME', 0)).toEqual('John Doe');
+			expect(variables.get('FULLNAME', 1)).toEqual('Jane Dae');
+			expect(variables.interpretCount).toBe(2);
+			expect(variables.get('FULLNAME', 0)).toEqual('John Doe');
+			expect(variables.get('FULLNAME', 1)).toEqual('Jane Dae');
+			expect(variables.interpretCount).toBe(2);
+			expect(variables.get('FULLNAME', 0)).toEqual('John Doe');
+			variables.set('FIRSTNAME', ['John', 'Marc']);
+			expect(variables.get('FULLNAME', 0)).toEqual('John Doe');
+			expect(variables.get('FULLNAME', 1)).toEqual('Marc Dae');
+			// Only the second iteration should be calculated
+			expect(variables.interpretCount).toBe(3);
+		});
+		it('should handle aggregation expression', () => {
+			variables.set('FIRSTNAME', ['John', 'Jane']);
+			expect(variables.run('count(FIRSTNAME)')).toEqual(2);
+			variables.set('FIRSTNAME', ['John', 'Jane', 'Marc']);
+			expect(variables.run('count(FIRSTNAME)')).toEqual(3);
+		});
+		it('should handle non array values', () => {
+			variables.set('FIRSTNAME', ['John', 'Jane']);
+			variables.set('LASTNAME', 'Doe');
+			expect(variables.run('FIRSTNAME || " " || LASTNAME', 0)).toEqual(
+				'John Doe'
+			);
+			expect(variables.run('FIRSTNAME || " " || LASTNAME', 1)).toEqual(
+				'Jane Doe'
+			);
+		});
+	});
 });

src/use-lunatic/commons/lunatic-variables.ts

@@ -1,10 +1,10 @@
-import { VtlLexer } from '@inseefr/vtl-2.0-antlr-tools';
-import antlr4 from 'antlr4';
-import { interpret } from '@inseefr/trevas';
+import { interpretVTL, parseVTLVariables } from '../../utils/vtl';
 
 // Interpret counter, used for testing purpose
 let interpretCount = 0;
 
+type IterationLevel = number;
+
 export class LunaticVariables {
 	private dictionary = new Map<string, LunaticVariable>();
 
@@ -15,11 +15,11 @@ export class LunaticVariables {
 	/**
 	 * Retrieve variable value
 	 */
-	public get<T>(name: string) {
+	public get<T>(name: string, iteration?: IterationLevel) {
 		if (!this.dictionary.has(name)) {
 			return undefined;
 		}
-		return this.dictionary.get(name)!.getValue() as T;
+		return this.dictionary.get(name)!.getValue(iteration) as T;
 	}
 
 	/**
@@ -54,8 +54,8 @@ export class LunaticVariables {
 		return variable;
 	}
 
-	public run(expression: string): unknown {
-		return this.setCalculated(expression, expression).getValue();
+	public run(expression: string, iteration?: IterationLevel): unknown {
+		return this.setCalculated(expression, expression).getValue(iteration);
 	}
 
 	// Retrieve the number of interpret() run, used in testing
@@ -66,16 +66,16 @@ export class LunaticVariables {
 
 class LunaticVariable {
 	// Last time the value was updated (changed)
-	public updatedAt = 0;
+	public updatedAt = new Map<undefined | IterationLevel, number>();
 	// Last time "calculation" was run (for calculated variable)
-	private calculatedAt = 0;
+	private calculatedAt = new Map<undefined | IterationLevel, number>();
 	// Internal value for the variable
 	private value: unknown;
 	// List of dependencies, ex: ['FIRSTNAME', 'LASTNAME']
 	private dependencies?: string[];
 	// Expression for calculated variable
 	private readonly expression?: string;
-	// Dictionnary holding all the available variables
+	// Dictionary holding all the available variables
 	private readonly dictionary?: Map<string, LunaticVariable>;
 
 	constructor(
@@ -96,77 +96,89 @@ class LunaticVariable {
 		this.dependencies = dependencies;
 	}
 
-	getValue(): unknown {
+	getValue(iteration?: IterationLevel): unknown {
 		// The variable is not calculated
 		if (!this.expression) {
-			return this.value;
+			return this.getSavedValue(iteration);
 		}
 		// Calculate bindings first to refresh "updatedAt" on calculated dependencies
-		const bindings = this.getDependenciesValues();
-		if (!this.isOutdated()) {
-			return this.value;
+		const bindings = this.getDependenciesValues(iteration);
+		if (!this.isOutdated(iteration)) {
+			return this.getSavedValue(iteration);
 		}
 		if ((import.meta as any).env.MODE === 'test') {
 			interpretCount++;
 		}
 		// Remember the value
-		this.setValue(interpret(this.expression, bindings));
-		this.calculatedAt = performance.now();
-		return this.value;
+		try {
+			this.setValue(interpretVTL(this.expression, bindings), iteration);
+		} catch (e) {
+			throw new Error(
+				`Cannot interpret expression "${
+					this.expression
+				}" with bindings ${JSON.stringify(bindings)}, error : ${(
+					e as Error
+				).toString()}`
+			);
+		}
+		this.calculatedAt.set(iteration, performance.now());
+		this.calculatedAt.set(undefined, performance.now());
+		return this.getSavedValue(iteration);
 	}
 
-	setValue(v: unknown) {
-		if (v === this.value) {
+	setValue(value: unknown, iteration?: IterationLevel): void {
+		if (value === this.getSavedValue(iteration)) {
+			return;
+		}
+		// Decompose arrays, to only update items that changed
+		if (Array.isArray(value) && iteration === undefined) {
+			value.forEach((v, k) => this.setValue(v, k));
 			return;
 		}
-		this.value = v;
-		this.updatedAt = performance.now();
+		// We want to save a value at a specific iteration, but the value is not an array yet
+		if (iteration !== undefined && !Array.isArray(this.value)) {
+			this.value = [];
+		}
+		if (iteration === undefined) {
+			this.value = value;
+		} else {
+			(this.value as unknown[])[iteration] = value;
+		}
+		this.updatedAt.set(iteration, performance.now());
+		this.updatedAt.set(undefined, performance.now());
+	}
+
+	private getSavedValue(iteration?: IterationLevel): unknown {
+		if (iteration !== undefined && Array.isArray(this.value)) {
+			return this.value[iteration];
+		}
+		return this.value;
 	}
 
 	private getDependencies(): string[] {
 		// Calculate dependencies from expression on the fly if necessary
 		if (this.dependencies === undefined) {
-			this.dependencies = getExpressionVariables(this.expression!);
+			this.dependencies = parseVTLVariables(this.expression!);
 		}
 		return this.dependencies;
 	}
 
-	private getDependenciesValues(): Record<string, unknown> {
+	private getDependenciesValues(
+		iteration?: IterationLevel
+	): Record<string, unknown> {
 		return Object.fromEntries(
-			this.getDependencies().map((dep) => [
-				dep,
-				this.dictionary?.get(dep)?.getValue(),
-			])
+			this.getDependencies().map((dep) => {
+				return [dep, this.dictionary?.get(dep)?.getValue(iteration)];
+			})
 		);
 	}
 
-	private isOutdated(): boolean {
+	private isOutdated(iteration?: IterationLevel): boolean {
 		const dependenciesUpdatedAt = Math.max(
 			...this.getDependencies().map(
-				(dep) => this.dictionary?.get(dep)?.updatedAt ?? 0
+				(dep) => this.dictionary?.get(dep)?.updatedAt.get(iteration) ?? 0
 			)
 		);
-		return dependenciesUpdatedAt > this.calculatedAt;
-	}
-}
-
-/**
- * Find variables used in an expression
- */
-function getExpressionVariables(expression: string): string[] {
-	try {
-		const chars = new antlr4.InputStream(expression);
-		const lexer = new VtlLexer(chars);
-		const dependencySet = lexer
-			.getAllTokens()
-			.reduce(function (acc, { start, stop, type }) {
-				if (type === 234) {
-					acc.add(expression.substring(start, stop + 1));
-				}
-				return acc;
-			}, new Set<string>());
-		return Array.from(dependencySet);
-	} catch (e) {
-		return [];
+		return dependenciesUpdatedAt > (this.calculatedAt.get(iteration) ?? -1);
 	}
 }

src/utils/vtl.ts

@@ -0,0 +1,77 @@
+import { objectMap } from './object';
+import { interpret } from '@inseefr/trevas';
+import antlr4 from 'antlr4';
+import { VtlLexer } from '@inseefr/vtl-2.0-antlr-tools';
+
+type DataSet = { dataPoints: { result: unknown } };
+
+/**
+ * Simplified version of interpret (that converts binding and value)
+ */
+export function interpretVTL<T>(
+	expression: string,
+	bindings: Record<string, unknown>
+): T {
+	const result = interpret(
+		expression,
+		objectMap(bindings, (k, v) => [k, getVTLCompatibleValue(v)])
+	);
+	if (isDataSet(result)) {
+		return extractDataSetResult(result) as T;
+	}
+	return result as T;
+}
+
+export function parseVTLVariables(expression: string): string[] {
+	try {
+		const chars = new antlr4.InputStream(expression);
+		const lexer = new VtlLexer(chars);
+		const dependencySet = lexer
+			.getAllTokens()
+			.reduce(function (acc, { start, stop, type }) {
+				if (type === 234) {
+					acc.add(expression.substring(start, stop + 1));
+				}
+				return acc;
+			}, new Set<string>());
+		return Array.from(dependencySet);
+	} catch (e) {
+		return [];
+	}
+}
+
+export function getVTLCompatibleValue(value: unknown) {
+	if (value === undefined) {
+		return null;
+	}
+	if (Array.isArray(value)) {
+		return {
+			dataStructure: { result: {} },
+			dataPoints: {
+				result: value,
+			},
+		};
+	}
+
+	return value;
+}
+
+function isDataSet(result: unknown): result is DataSet {
+	return (
+		typeof result === 'object' &&
+		result !== null &&
+		'dataPoints' in result &&
+		result.dataPoints !== null &&
+		typeof result.dataPoints === 'object' &&
+		'result' in result.dataPoints
+	);
+}
+
+function extractDataSetResult(dataSet: DataSet) {
+	const { dataPoints } = dataSet;
+	if (dataPoints) {
+		const { result } = dataPoints;
+		return result;
+	}
+	return undefined;
+}