fix(dev): use ts-morph to parse template configs (#387)

Previous to this change, we used esbuild to compile the templates into
modules in order to pull off the config const. This led to various
issues for people. It works fine with Vite since Vite does a lot of
CJS/polyfill magic, but we weren't handling that with esbuild. This was
overkill anyways as we only need to grab the config off of the
templates, so compilation is irrelevant at this step.

---------

Co-authored-by: github-actions <41898282+github-actions[bot]@users.noreply.github.com>

packages/pages/THIRD-PARTY-NOTICES

@@ -1283,6 +1283,36 @@ Repository: https://github.com/yargs/yargs-parser.git
 
 -----------
 
+The following npm package may be included in this product:
+
+ - [email protected]
+
+This package contains the following license and notice below:
+
+The MIT License (MIT)
+
+Copyright (c) 2017-2023 David Sherret
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+
+-----------
+
 The following npm package may be included in this product:
 
  - [email protected]

packages/pages/package.json

@@ -74,6 +74,7 @@
     "pretty-ms": "^8.0.0",
     "prompts": "^2.4.2",
     "rollup": "^3.28.0",
+    "ts-morph": "^19.0.0",
     "typescript": "^5.1.6",
     "vite": "^4.4.9",
     "vite-plugin-node-polyfills": "^0.11.0"

packages/pages/src/common/src/parsers/sourceFileParser.ts

@@ -0,0 +1,113 @@
+import {
+  Project,
+  SourceFile,
+  SyntaxKind,
+  FunctionDeclaration,
+  ObjectLiteralExpression,
+  ArrowFunction,
+} from "ts-morph";
+import vm from "node:vm";
+import typescript from "typescript";
+import { processEnvVariables } from "../../../util/processEnvVariables.js";
+
+export function createTsMorphProject() {
+  return new Project({
+    compilerOptions: {
+      jsx: typescript.JsxEmit.ReactJSX,
+    },
+  });
+}
+
+export default class SourceFileParser {
+  private sourceFile: SourceFile;
+
+  constructor(
+    private filepath: string,
+    project: Project
+  ) {
+    if (!project.getSourceFile(filepath)) {
+      project.addSourceFileAtPath(filepath);
+    }
+    this.sourceFile = project.getSourceFileOrThrow(filepath);
+  }
+
+  getExportedObjectExpression(
+    variableName: string
+  ): ObjectLiteralExpression | undefined {
+    const variableStatement = this.sourceFile
+      .getVariableStatements()
+      .find((variableStatement) => {
+        return (
+          variableStatement.isExported() &&
+          variableStatement
+            .getFirstDescendantByKindOrThrow(SyntaxKind.VariableDeclaration)
+            .getName() === variableName
+        );
+      });
+    if (!variableStatement) {
+      return;
+    }
+    const objectLiteralExp = variableStatement.getFirstDescendantByKind(
+      SyntaxKind.ObjectLiteralExpression
+    );
+    if (!objectLiteralExp) {
+      throw new Error(
+        `Could not find ObjectLiteralExpression within \`${variableStatement.getFullText()}\`.`
+      );
+    }
+    return objectLiteralExp;
+  }
+
+  getExportFunctionExpression(
+    functionName?: string
+  ): ObjectLiteralExpression[] | ObjectLiteralExpression | undefined {
+    if (functionName) {
+      const functionDeclaration = this.sourceFile.getFunction(
+        (f) => f.isExported() && f.getName() === functionName
+      );
+      const objectLiteralExp = functionDeclaration?.getFirstDescendantByKind(
+        SyntaxKind.ObjectLiteralExpression
+      );
+      return objectLiteralExp;
+    }
+    const functionDeclarations = this.sourceFile.getFunctions();
+    const objectLiteralExpressions: ObjectLiteralExpression[] = [];
+    functionDeclarations.forEach((fd) => {
+      if (fd.isExported()) {
+        objectLiteralExpressions.push(
+          fd.getFirstAncestorByKindOrThrow(SyntaxKind.ObjectLiteralExpression)
+        );
+      }
+    });
+    return objectLiteralExpressions;
+  }
+
+  getFunctions() {
+    return this.sourceFile.getFunctions();
+  }
+
+  getFunctionNode(
+    funcName: string
+  ): FunctionDeclaration | ArrowFunction | undefined {
+    return (
+      this.sourceFile.getFunction(funcName) ??
+      this.sourceFile
+        .getVariableDeclaration(funcName)
+        ?.getFirstChildByKind(SyntaxKind.ArrowFunction)
+    );
+  }
+
+  /**
+   * This function takes in an {@link ObjectLiteralExpression} and returns it's data.
+   *
+   * It converts a js object string and converts it into an object using vm.runInNewContext,
+   * which can be thought of as a safe version of `eval`. Note that we cannot use JSON.parse here,
+   * because we are working with a js object not a JSON.
+   */
+  getCompiledObjectLiteral<T>(objectLiteralExp: ObjectLiteralExpression): T {
+    return vm.runInNewContext(
+      "(" + objectLiteralExp.getText() + ")",
+      processEnvVariables("YEXT_PUBLIC")
+    );
+  }
+}

packages/pages/src/common/src/parsers/templateConfigParser.ts

@@ -0,0 +1,25 @@
+import { TemplateConfig } from "../template/types.js";
+import SourceFileParser from "./sourceFileParser.js";
+
+export const TEMPLATE_CONFIG_VARIABLE_NAME = "config";
+
+/**
+ * TemplateConfigParser is a class for parsing the template config.
+ */
+export default class TemplateConfigParser {
+  constructor(private sourceFileParser: SourceFileParser) {}
+
+  /** Returns the template config exported from the page if one is present. */
+  getTemplateConfig(): TemplateConfig | undefined {
+    const configObjectLiteralExp =
+      this.sourceFileParser.getExportedObjectExpression(
+        TEMPLATE_CONFIG_VARIABLE_NAME
+      );
+    return (
+      configObjectLiteralExp &&
+      this.sourceFileParser.getCompiledObjectLiteral<TemplateConfig>(
+        configObjectLiteralExp
+      )
+    );
+  }
+}

packages/pages/src/common/src/template/internal/types.ts

@@ -146,7 +146,7 @@ export const convertTemplateModuleToTemplateModuleInternal = (
   return templateModuleInternal;
 };
 
-const convertTemplateConfigToTemplateConfigInternal = (
+export const convertTemplateConfigToTemplateConfigInternal = (
   templateName: string,
   templateConfig: TemplateConfig | undefined
 ): TemplateConfigInternal => {

packages/pages/src/dev/server/ssr/generateTestData.ts

@@ -10,7 +10,6 @@ import {
 import path from "path";
 import fs from "fs";
 import { ProjectStructure } from "../../../common/src/project/structure.js";
-import { getFeaturesConfig } from "../../../generate/features/createFeaturesJson.js";
 import { getTemplateFilepathsFromProjectStructure } from "../../../common/src/template/internal/getTemplateFilepaths.js";
 import {
   convertTemplateModuleToTemplateModuleInternal,
@@ -20,6 +19,7 @@ import { ViteDevServer } from "vite";
 import { loadViteModule } from "./loadViteModule.js";
 import { TemplateModuleCollection } from "../../../common/src/template/internal/loader.js";
 import { TemplateModule } from "../../../common/src/template/types.js";
+import { getFeaturesConfig } from "../../../generate/templates/createTemplatesJsonFromModule.js";
 
 /**
  * generateTestData will run yext pages generate-test-data and return true in

packages/pages/src/generate/features/features.ts

@@ -1,21 +1,15 @@
 import { ProjectStructure } from "../../common/src/project/structure.js";
 import { getTemplateFilepaths } from "../../common/src/template/internal/getTemplateFilepaths.js";
-import { loadTemplateModules } from "../../common/src/template/internal/loader.js";
-import { createFeaturesJson } from "./createFeaturesJson.js";
 import { Command } from "commander";
+import { createTemplatesJson } from "../templates/createTemplatesJson.js";
 
 const handler = async ({ scope }: { scope: string }): Promise<void> => {
   const projectStructure = await ProjectStructure.init({ scope });
   const templateFilepaths = getTemplateFilepaths(
     projectStructure.getTemplatePaths()
   );
-  const templateModules = await loadTemplateModules(
-    templateFilepaths,
-    true,
-    false
-  );
 
-  createFeaturesJson(templateModules, projectStructure);
+  createTemplatesJson(templateFilepaths, projectStructure, "FEATURES");
 };
 
 export const featureCommand = (program: Command) => {

packages/pages/src/generate/templates/createTemplatesJson.ts

@@ -5,53 +5,127 @@ import {
   FeatureConfig,
   convertTemplateConfigToFeatureConfig,
 } from "../../common/src/feature/features.js";
-import { TemplateModuleCollection } from "../../common/src/template/internal/loader.js";
 import { ProjectStructure } from "../../common/src/project/structure.js";
 import {
   StreamConfig,
   convertTemplateConfigToStreamConfig,
 } from "../../common/src/feature/stream.js";
-
-export const getTemplatesConfig = (
-  templateModules: TemplateModuleCollection
-): FeaturesConfig => {
-  const features: FeatureConfig[] = [];
-  const streams: StreamConfig[] = [];
-  for (const module of templateModules.values()) {
-    const featureConfig = convertTemplateConfigToFeatureConfig(module.config);
-    features.push(featureConfig);
-    const streamConfig = convertTemplateConfigToStreamConfig(module.config);
-    if (streamConfig) {
-      streams.push(streamConfig);
-    }
-  }
-
-  return { features, streams };
-};
+import SourceFileParser, {
+  createTsMorphProject,
+} from "../../common/src/parsers/sourceFileParser.js";
+import TemplateConfigParser from "../../common/src/parsers/templateConfigParser.js";
+import {
+  convertTemplateConfigToTemplateConfigInternal,
+  parse,
+} from "../../common/src/template/internal/types.js";
 
 /**
- * Generates a templates.json from the templates.
+ * Generates a templates.json or features.json from the templates.
  */
 export const createTemplatesJson = (
-  templateModules: TemplateModuleCollection,
-  projectStructure: ProjectStructure
+  templateFilepaths: string[],
+  projectStructure: ProjectStructure,
+  type: "FEATURES" | "TEMPLATES"
 ): void => {
   // Note: the object used to be known as "features" but it's been changed to "templates".
   // We're allowing the generation of a features.json file still for backwards compatibility,
   // which is why all of the types have not yet been renamed.
-  const { features, streams } = getTemplatesConfig(templateModules);
-  const templatesAbsolutePath = path.resolve(
-    projectStructure.config.rootFolders.dist,
-    projectStructure.config.distConfigFiles.templates
-  );
+  const { features, streams } = getTemplatesConfig(templateFilepaths);
+  let templatesAbsolutePath;
+
+  switch (type) {
+    case "FEATURES":
+      templatesAbsolutePath = path.resolve(
+        projectStructure.getSitesConfigPath().path,
+        projectStructure.config.sitesConfigFiles.features
+      );
+      break;
+    case "TEMPLATES":
+      templatesAbsolutePath = path.resolve(
+        projectStructure.config.rootFolders.dist,
+        projectStructure.config.distConfigFiles.templates
+      );
+      break;
+  }
 
   const templatesDir = path.dirname(templatesAbsolutePath);
   if (!fs.existsSync(templatesDir)) {
     fs.mkdirSync(templatesDir);
   }
 
+  // features.json is merged with the existing data a user can set. For templates.json the extra
+  // data is defined in config.yaml, so the merging is unncessary there.
+  let templatesJson;
+  switch (type) {
+    case "FEATURES":
+      templatesJson = mergeFeatureJson(
+        templatesAbsolutePath,
+        features,
+        streams
+      );
+      break;
+    case "TEMPLATES":
+      templatesJson = { features, streams };
+      break;
+  }
+
   fs.writeFileSync(
     templatesAbsolutePath,
-    JSON.stringify({ features, streams }, null, "  ")
+    JSON.stringify(templatesJson, null, "  ")
   );
 };
+
+/**
+ * Uses ts-morph to parse out the config function of each template.
+ */
+const getTemplatesConfig = (templateFilepaths: string[]): FeaturesConfig => {
+  const features: FeatureConfig[] = [];
+  const streams: StreamConfig[] = [];
+  for (const templateFilepath of templateFilepaths) {
+    const sfp = new SourceFileParser(templateFilepath, createTsMorphProject());
+    const tcp = new TemplateConfigParser(sfp);
+
+    const templateConfig = tcp.getTemplateConfig();
+    const templatePath = parse(templateFilepath, false);
+
+    const templateConfigInternal =
+      convertTemplateConfigToTemplateConfigInternal(
+        templatePath.name,
+        templateConfig
+      );
+
+    const featureConfig = convertTemplateConfigToFeatureConfig(
+      templateConfigInternal
+    );
+    features.push(featureConfig);
+    const streamConfig = convertTemplateConfigToStreamConfig(
+      templateConfigInternal
+    );
+    if (streamConfig) {
+      streams.push(streamConfig);
+    }
+  }
+
+  return { features, streams };
+};
+
+/**
+ * Overwrites the "features" and "streams" fields in the feature.json while keeping other fields
+ * if the feature.json already exists.
+ */
+export const mergeFeatureJson = (
+  featurePath: string,
+  features: FeatureConfig[],
+  streams: any
+): FeaturesConfig => {
+  let originalFeaturesJson = {} as any;
+  if (fs.existsSync(featurePath)) {
+    originalFeaturesJson = JSON.parse(fs.readFileSync(featurePath).toString());
+  }
+
+  return {
+    ...originalFeaturesJson,
+    features,
+    streams,
+  };
+};