Merge pull request #86 from complexdatacollective/rollup

Rollup

apps/documentation/docs/desktop/project-information/faq.en.md

@@ -23,15 +23,15 @@ Throughout the remaining duration of our funding we will be working through a ba
 
 ## How much does the software cost? What is your funding model?
 
-The Network Canvas suite is free! It is licensed under the GPLv3 open source license. The project team has a strong ideological commitment to producing high quality free academic software for the benefit of researchers, students, and any other interested parties. We were funded to produce this software via [a grant from the National Institutes for Health](https://projectreporter.nih.gov/project_info_description.cfm?aid=9306043&icde=35540823&ddparam=&ddvalue=&ddsub=&cr=1&csb=default&cs=ASC&pball=), for which we are extremely grateful.
+The Network Canvas suite is free! It is licensed under the GPLv3 open source license. The project team has a strong ideological commitment to producing high quality free academic software for the benefit of researchers, students, and any other interested parties. We were funded to produce this software via [a grant from the National Institutes for Health](https://reporter.nih.gov/search/My1lXoKhnEyzXX5kIyl7Mw/project-details/9306043), for which we are extremely grateful.
 
 Our license means that you are free to use, modify, and extend the software however you wish. It is our intention to foster a community to support the ongoing development of these tools, and we welcome collaboration. If you do extend or improve the software, we welcome contributions back into our main [GitHub repositories](https://github.com/complexdatacollective) in the form of pull requests.
 
 We can also provide development and consultancy services to support ideas you have for developing the software to support specific features or research projects. Please create a post on our [user community](https://community.networkcanvas.com) to discuss your ideas further.
 
 ## Who is developing the Network Canvas suite?
 
-The software is being developed by a team of researchers and developers based at Northwestern University and the University of Oxford, as well as several external contracted developers. We are grateful to be funded through a grant from the [National Institutes of Health](https://projectreporter.nih.gov/project_info_description.cfm?aid=9306043&icde=35540823&ddparam=&ddvalue=&ddsub=&cr=1&csb=default&cs=ASC&pball=).
+The software is being developed by a team of researchers and developers based at Northwestern University and the University of Oxford, as well as several external contracted developers. We are grateful to be funded through a grant from the [National Institutes of Health](https://reporter.nih.gov/search/My1lXoKhnEyzXX5kIyl7Mw/project-details/9306043).
 
 The intellectual property and copyright associated with the software is controlled by a registered not-for-profit, the Complex Data Collective, comprising the core project staff.
 

biome.json

@@ -24,7 +24,12 @@
 		"useIgnoreFile": true
 	},
 	"files": {
-		"ignore": ["public/**/*.*", "packages/development-protocol/assets", "packages/development-protocol/protocol.json"]
+		"ignore": [
+			"public/**/*.*",
+			"packages/development-protocol/assets",
+			"packages/development-protocol/protocol.json",
+			"packages/protocol-validation/src/schemas/compiled"
+		]
 	},
 	"javascript": { "formatter": { "quoteStyle": "double" } }
 }

packages/protocol-validation/.gitignore

@@ -1,2 +1,3 @@
 /src/__tests__/data
-/src/__tests__/.cache
+/src/__tests__/.cache
+/src/schemas/compiled

packages/protocol-validation/package.json

@@ -13,37 +13,40 @@
 	"description": "Utilities for validating and migrating Network Canvas protocol files.",
 	"scripts": {
 		"prebuild": "pnpm run compile-schemas",
-		"build": "tsup src/index.ts",
+		"build": "tsc --noEmit && vite build",
 		"dev": "npm run build -- --watch",
 		"pretest": "pnpm run build",
 		"test": "vitest",
-		"zod-to-json": "tsx ./src/scripts/zodToJson.ts",
-		"compile-schemas": "tsx ./src/scripts/compileSchemas.ts"
+		"zod-to-json": "tsx ./scripts/zodToJson.ts",
+		"compile-schemas": "tsx ./scripts/compileSchemas.ts"
 	},
 	"bin": {
-		"@codaco/protocol-validation": "./cli.js"
+		"@codaco/protocol-validation": "./scripts/cli.js"
 	},
 	"files": ["dist"],
+	"dependencies": {
+		"ajv": "^8.17.1",
+		"zod": "^3.24.1"
+	},
 	"devDependencies": {
 		"@codaco/shared-consts": "workspace:*",
 		"@codaco/tsconfig": "workspace:*",
 		"@types/gunzip-maybe": "^1.4.2",
 		"@types/lodash-es": "^4.17.12",
 		"@types/tar": "^6.1.13",
 		"@types/tar-stream": "^3.1.3",
-		"ajv": "^8.17.1",
 		"chalk": "^5.3.0",
 		"dotenv": "^16.4.5",
-		"es-toolkit": "^1.31.0",
+		"es-toolkit": "^1.32.0",
 		"gunzip-maybe": "^1.4.2",
 		"jszip": "^3.10.1",
 		"lodash-es": "^4.17.21",
 		"tar-stream": "^3.1.7",
-		"tsup": "^8.3.6",
 		"tsx": "^4.19.2",
 		"typescript": "^5.6.3",
+		"vite": "^6.2.0",
+		"vite-plugin-dts": "^4.5.3",
 		"vitest": "^2.1.8",
-		"zod": "^3.24.1",
 		"zod-to-json-schema": "^3.24.1"
 	}
 }

packages/protocol-validation/cli.js → packages/protocol-validation/scripts/cli.js

@@ -4,7 +4,7 @@
 import chalk from "chalk";
 import fs from "node:fs";
 import path from "node:path";
-import { errToString, validateProtocol } from "./dist/index.js";
+import { errToString, validateProtocol } from "../dist/index.js";
 
 async function main() {
 	const [, , filePath, forceSchema = undefined] = process.argv;

packages/protocol-validation/src/scripts/compileSchemas.ts → packages/protocol-validation/scripts/compileSchemas.ts

@@ -4,7 +4,7 @@ import { mkdir, readdir, readFile, writeFile } from "node:fs/promises";
 import { basename, extname, join, resolve } from "node:path";
 
 const SCHEMA_SRC_PATH = "./src/schemas/";
-const SCHEMA_OUTPUT_PATH = "./dist/schemas";
+const SCHEMA_OUTPUT_PATH = "./src/schemas/compiled";
 
 const ajv = new Ajv({
 	code: { source: true, esm: true, lines: true },
@@ -18,42 +18,11 @@ ajv.addFormat("date-time", /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}.\d{3}Z$/);
 const isJsonFile = (fileName: string) => extname(fileName) === ".json";
 const getBaseName = (schemaFileName: string) => basename(schemaFileName, ".json");
 
-const asVariableName = (schemaName: string) => `version_${schemaName.replace(/\./g, "_")}`;
-
-const asIntName = (schemaVersion: string | number) => {
-	if (Number.isNaN(Number.parseInt(schemaVersion as string, 10))) {
-		throw Error("Schema version could not be converted to integer");
-	}
-
-	return Number.parseInt(schemaVersion as string, 10);
-};
-
 const getSchemas = async (directory: string) => {
 	const files = await readdir(directory);
 	return files.filter(isJsonFile).map(getBaseName);
 };
 
-const generateModuleIndex = (schemas: string[]) => {
-	const formatRequire = (baseSchemaName: string) => {
-		const relativeModulePath = join(`./${baseSchemaName}.js`);
-		return `import ${asVariableName(baseSchemaName)} from './${relativeModulePath}';`;
-	};
-
-	const formatVersions = (baseSchemaName: string) =>
-		`  { version: ${asIntName(baseSchemaName)}, validator: ${asVariableName(baseSchemaName)} },`;
-
-	const schemaRequires = schemas.map(formatRequire).join("\n");
-	const schemaVersions = `${schemas.map(formatVersions).join("\n")}`;
-
-	return `${schemaRequires}
-
-const versions = [
-${schemaVersions}
-];
-export default versions;
-\r\n`;
-};
-
 const compileSchemas = async () => {
 	const schemaSrcDirectory = resolve(SCHEMA_SRC_PATH);
 	const schemaOutputDirectory = resolve(SCHEMA_OUTPUT_PATH);
@@ -77,10 +46,6 @@ const compileSchemas = async () => {
 
 		console.log(`${baseSchemaName} done.`);
 	}
-
-	const moduleIndexPath = join(schemaOutputDirectory, "index.js");
-	const moduleIndex = generateModuleIndex(schemas);
-	await writeFile(moduleIndexPath, moduleIndex);
 };
 
 compileSchemas().catch((err) => {

packages/protocol-validation/src/scripts/zodToJson.ts → packages/protocol-validation/scripts/zodToJson.ts

@@ -1,4 +1,5 @@
 import chalk from "chalk";
+import { execSync } from "node:child_process";
 import { writeFile } from "node:fs/promises";
 import { basename } from "node:path";
 import { ensureError } from "src/utils/ensureError";
@@ -20,15 +21,21 @@ const logError = (msg: string) => console.log(chalk.red(msg));
 const logSuccess = (msg: string) => console.log(chalk.green(msg));
 
 export const convertZodToJson = async (zodSchema: ZodType<unknown>, schemaName: string) => {
-	console.log("zodSchema", zodSchema);
-	const convertedSchema = zodToJsonSchema(zodSchema, schemaName);
+	const convertedSchema = zodToJsonSchema(zodSchema, {
+		name: schemaName,
+		// $refStrategy: "seen", // this reduces the size of the schema but is possibly too permissive
+	});
 
 	const outputPath = `${SCHEMA_DIR}/${schemaFileName}.json`;
 
 	const jsonSchemaString = JSON.stringify(convertedSchema, null, 2);
 	try {
 		await writeFile(outputPath, jsonSchemaString);
 		logSuccess(`Successfully converted zod schema to json schema: ${outputPath}`);
+
+		// format the json schema with biome
+		execSync(`npx biome check ${outputPath} --write `);
+		logSuccess(`Successfully linted json schema: ${outputPath}`);
 	} catch (e) {
 		const error = ensureError(e);
 		logError(`Error saving json schema to file: ${error.message}`);

packages/protocol-validation/src/migrations/migrations/index.ts

@@ -1,8 +1,13 @@
 import type { ProtocolMigration } from "../migrateProtocol";
+// @ts-ignore
 import version4 from "./4";
+// @ts-ignore
 import version5 from "./5";
+// @ts-ignore
 import version6 from "./6";
+// @ts-ignore
 import version7 from "./7";
+// @ts-ignore
 import version8 from "./8";
 
 /**

packages/protocol-validation/src/validation/helpers.ts

@@ -1,7 +1,7 @@
 import type { EntityTypeDefinition, FilterRule, NcNode, StageSubject } from "@codaco/shared-consts";
 import { get } from "es-toolkit/compat";
-import type { ValidationError } from "src";
-import type { Codebook } from "src/schemas/8.zod";
+import type { ValidationError } from "../";
+import type { Codebook } from "../schemas/8.zod";
 
 // For some error types, AJV returns info separate from message
 const additionalErrorInfo = (errorObj: ValidationError) => {

packages/protocol-validation/src/validation/validateSchema.ts

@@ -1,25 +1,6 @@
 import type { DefinedError, ValidateFunction } from "ajv";
-import validator1 from "../../dist/schemas/1.js";
-import validator2 from "../../dist/schemas/2.js";
-import validator3 from "../../dist/schemas/3.js";
-import validator4 from "../../dist/schemas/4.js";
-import validator5 from "../../dist/schemas/5.js";
-import validator6 from "../../dist/schemas/6.js";
-import validator7 from "../../dist/schemas/7.js";
-import validator8 from "../../dist/schemas/8.js";
 import type { Protocol } from "../schemas/8.zod";
 
-const validators: Record<number, ValidateFunction> = {
-	1: validator1 as ValidateFunction,
-	2: validator2 as ValidateFunction,
-	3: validator3 as ValidateFunction,
-	4: validator4 as ValidateFunction,
-	5: validator5 as ValidateFunction,
-	6: validator6 as ValidateFunction,
-	7: validator7 as ValidateFunction,
-	8: validator8 as ValidateFunction,
-};
-
 export const validateSchema = async (protocol: Protocol, forceVersion?: number) => {
 	if (!protocol) {
 		throw new Error("Protocol is undefined");
@@ -35,7 +16,9 @@ export const validateSchema = async (protocol: Protocol, forceVersion?: number)
 		console.log(`Forcing validation against schema version ${version}...`);
 	}
 
-	const validator = validators[version];
+	const validator = await import(`../schemas/compiled/${version}.js`).then(
+		(module) => module.default as ValidateFunction,
+	);
 
 	if (!validator) {
 		throw new Error(`Couldn't find validator for schema version ${version}.`);

packages/protocol-validation/tsconfig.json

@@ -2,8 +2,10 @@
 	"extends": "@codaco/tsconfig/base.json",
 	"compilerOptions": {
 		"baseUrl": ".",
-		"tsBuildInfoFile": "node_modules/.cache/tsbuildinfo.json"
+		"tsBuildInfoFile": "node_modules/.cache/tsbuildinfo.json",
+		"allowJs": false,
+		"incremental": false
 	},
 	"include": ["."],
-	"exclude": ["dist", "build", "node_modules"]
+	"exclude": ["dist", "node_modules"]
 }

packages/protocol-validation/vite.config.js

@@ -0,0 +1,48 @@
+import { execSync } from "node:child_process";
+import path, { dirname, resolve } from "node:path";
+import { fileURLToPath } from "node:url";
+import { defineConfig } from "vite";
+import dts from "vite-plugin-dts";
+
+const schemaPlugin = () => {
+	return {
+		name: "schema",
+
+		// watches the schema files for changes
+		buildStart() {
+			this.addWatchFile(path.resolve("src/schemas/"));
+		},
+		// runs when a file changes
+		watchChange(file) {
+			if (file.endsWith("zod.ts")) {
+				console.log("🔄 Converting zod schema to json...", file);
+				execSync("pnpm run zod-to-json src/schemas/8.zod.ts");
+			}
+
+			if (file.endsWith(".json")) {
+				console.log("🔄 Recompiling all json schemas...", file);
+				execSync("pnpm run compile-schemas");
+			}
+		},
+	};
+};
+
+const __dirname = dirname(fileURLToPath(import.meta.url));
+
+export default defineConfig({
+	build: {
+		lib: {
+			entry: resolve(__dirname, "src/index.ts"),
+			name: "ProtocolValidation",
+			// the proper extensions will be added
+			fileName: "index",
+			formats: ["es"],
+		},
+	},
+	plugins: [
+		schemaPlugin(),
+		dts({
+			rollupTypes: true,
+		}),
+	],
+});