NEOS-1272 Add neosync transformers auto complete to javascript transf…

…ormer (#2388)

frontend/apps/web/app/(mgmt)/[account]/transformers/Sheetforms/GenerateJavascriptForm.tsx

@@ -85,7 +85,7 @@ export default function GenerateJavascriptForm(props: Props): ReactElement {
                   <FormLabel>Transformer Code</FormLabel>
                   <div className="text-[0.8rem] text-muted-foreground">
                     Define your own Transformation below using Javascript.{' '}
-                    <LearnMoreTag href="https://docs.neosync.dev/transformers/user-defined#custom-code-transformers" />
+                    <LearnMoreTag href="https://docs.neosync.dev/transformers/javascript" />
                   </div>
                 </div>
                 <div className="flex flex-row gap-2">

frontend/apps/web/app/(mgmt)/[account]/transformers/Sheetforms/TransformJavascriptForm.tsx

@@ -12,14 +12,15 @@ import {
   FormItem,
   FormLabel,
 } from '@/components/ui/form';
+import { useReadNeosyncTransformerDeclarationFile } from '@/libs/hooks/useReadNeosyncTransfomerDeclarationFile';
 import { useMutation } from '@connectrpc/connect-query';
 import { yupResolver } from '@hookform/resolvers/yup';
-import { Editor } from '@monaco-editor/react';
+import { Editor, useMonaco } from '@monaco-editor/react';
 import { TransformJavascript } from '@neosync/sdk';
 import { validateUserJavascriptCode } from '@neosync/sdk/connectquery';
 import { CheckCircledIcon, CrossCircledIcon } from '@radix-ui/react-icons';
 import { useTheme } from 'next-themes';
-import { ReactElement, useState } from 'react';
+import { ReactElement, useEffect, useState } from 'react';
 import { useForm } from 'react-hook-form';
 import { ValidCode } from '../../new/transformer/UserDefinedTransformerForms/UserDefinedTransformJavascriptForm';
 import { TRANSFORMER_SCHEMA_CONFIGS } from '../../new/transformer/schema';
@@ -49,6 +50,18 @@ export default function TransformJavascriptForm(props: Props): ReactElement {
   const [codeStatus, setCodeStatus] = useState<ValidCode>('null');
   const { resolvedTheme } = useTheme();
 
+  const monaco = useMonaco();
+  const { data: fileContent } = useReadNeosyncTransformerDeclarationFile();
+
+  useEffect(() => {
+    if (monaco && fileContent) {
+      monaco.languages.typescript.javascriptDefaults.addExtraLib(
+        fileContent,
+        'neosync-transformer.d.ts'
+      );
+    }
+  }, [monaco, fileContent]);
+
   async function handleValidateCode(): Promise<void> {
     if (!account) {
       return;
@@ -95,7 +108,7 @@ export default function TransformJavascriptForm(props: Props): ReactElement {
                       input.{'{'}column_name{'}'}
                     </code>
                     .{' '}
-                    <LearnMoreTag href="https://docs.neosync.dev/transformers/user-defined#custom-code-transformers" />
+                    <LearnMoreTag href="https://docs.neosync.dev/transformers/javascript" />
                   </div>
                 </div>
                 <div className="flex flex-row gap-2 w-[80px]">
@@ -170,3 +183,22 @@ export default function TransformJavascriptForm(props: Props): ReactElement {
     </div>
   );
 }
+
+// function shouldTriggerAutocomplete(text: string): boolean {
+//   const trimmedText = text.trim();
+//   const textSplit = trimmedText.split(/\s+/);
+//   const lastSignificantWord = trimmedText.split(/\s+/).pop()?.toUpperCase();
+//   const triggerKeywords = ['neo'];
+
+//   if (textSplit.length == 2 && textSplit[0].toUpperCase() == 'WHERE') {
+//     /* since we pre-pend the 'WHERE', we want the autocomplete to show up for the first letter typed
+//      which would come through as 'WHERE a' if the user just typed the letter 'a'
+//      so the when we split that text, we check if the length is 2 (as a way of checking if the user has only typed one letter or is still on the first word) and if it is and the first word is 'WHERE' which it should be since we pre-pend it, then show the auto-complete */
+//     return true;
+//   } else {
+//     return (
+//       triggerKeywords.includes(lastSignificantWord || '') ||
+//       triggerKeywords.some((keyword) => trimmedText.endsWith(keyword + ' '))
+//     );
+//   }
+// }

frontend/apps/web/app/api/files/neosync-transformer-declarations/route.ts

@@ -0,0 +1,13 @@
+import { withNeosyncContext } from '@/api-only/neosync-context';
+import * as fs from 'fs';
+import { NextRequest, NextResponse } from 'next/server';
+
+export async function GET(req: NextRequest): Promise<NextResponse> {
+  return withNeosyncContext(async () => {
+    const data = await fs.promises.readFile(
+      '/app/apps/web/@types/neosync-transformers.d.ts',
+      'utf8'
+    );
+    return data;
+  })(req);
+}

frontend/apps/web/libs/hooks/useReadNeosyncTransfomerDeclarationFile.ts

@@ -0,0 +1,9 @@
+import { useQuery, UseQueryResult } from '@tanstack/react-query';
+import { fetcher } from '../fetcher';
+
+export function useReadNeosyncTransformerDeclarationFile(): UseQueryResult<string> {
+  return useQuery({
+    queryKey: [`/api/files/neosync-transformer-declarations`],
+    queryFn: (ctx) => fetcher(ctx.queryKey.join('/')),
+  });
+}

frontend/apps/web/tsconfig.json

@@ -38,6 +38,7 @@
     "**/*.ts",
     "**/*.tsx",
     ".next/types/**/*.ts",
+    "@types/*.d.ts"
   ],
   "exclude": [
     "node_modules"

worker/pkg/benthos/transformers/generator_utils.go

@@ -2,7 +2,12 @@ package transformers
 
 import (
 	"bufio"
+	"fmt"
+	"go/parser"
+	"go/token"
+	"io/fs"
 	"os"
+	"path/filepath"
 	"regexp"
 	"strings"
 )
@@ -52,6 +57,39 @@ type ParsedBenthosSpec struct {
 	SpecDescription string
 }
 
+func ExtractBenthosSpec(fileSet *token.FileSet) ([]*BenthosSpec, error) {
+	transformerSpecs := []*BenthosSpec{}
+
+	err := filepath.WalkDir(".", func(path string, d fs.DirEntry, err error) error {
+		if !d.IsDir() && filepath.Ext(path) == ".go" {
+			node, err := parser.ParseFile(fileSet, path, nil, parser.ParseComments)
+			if err != nil {
+				return fmt.Errorf("Failed to parse file %s: %v", path, err)
+			}
+			for _, cgroup := range node.Comments {
+				for _, comment := range cgroup.List {
+					if strings.HasPrefix(comment.Text, "// +neosyncTransformerBuilder:") {
+						parts := strings.Split(comment.Text, ":")
+						if len(parts) < 3 {
+							continue
+						}
+						transformerSpecs = append(transformerSpecs, &BenthosSpec{
+							SourceFile: path,
+							Name:       parts[2],
+							Type:       parts[1],
+						})
+					}
+				}
+			}
+		}
+		return nil
+	})
+	if err != nil {
+		return nil, fmt.Errorf("impossible to walk directories: %s", err)
+	}
+	return transformerSpecs, nil
+}
+
 func ParseBloblangSpec(benthosSpec *BenthosSpec) (*ParsedBenthosSpec, error) {
 	paramRegex := regexp.MustCompile(`bloblang\.New(\w+)Param\("(\w+)"\)(?:\.Optional\(\))?(?:\.Default\(([^()]*(?:\([^()]*\))?[^()]*)\))?(?:\.Description\("([^"]*)"\))?`)
 	specDescriptionRegex := regexp.MustCompile(`\.Description\("([^"]*)"\)`)
@@ -125,3 +163,10 @@ func lowercaseFirst(s string) string {
 	}
 	return strings.ToLower(string(s[0])) + s[1:]
 }
+
+func CapitalizeFirst(s string) string {
+	if s == "" {
+		return s
+	}
+	return strings.ToUpper(string(s[0])) + s[1:]
+}

worker/pkg/benthos/transformers/generators.go

@@ -3,3 +3,4 @@ package transformers
 //go:generate go run neosync_transformer_generator.go $GOPACKAGE
 //go:generate go run neosync_transformer_list_generator.go $GOPACKAGE
 //go:generate go run neosync_js_transformer_docs_generator.go ../../../../docs/docs/transformers/gen-javascript-transformer.md
+//go:generate go run neosync_transformer_typescript_declaration_generator.go ../../../../frontend/apps/web/@types/neosync-transformers.d.ts

worker/pkg/benthos/transformers/neosync_js_transformer_docs_generator.go

@@ -62,9 +62,6 @@ func main() {
 		if err != nil {
 			fmt.Println("Error parsing bloblang params:", err)
 		}
-		// fmt.Println(tf.Name)
-		// jsonF, _ := json.MarshalIndent(parsedSpec.Params, "", " ")
-		// fmt.Printf("%s \n", string(jsonF))
 		tf.Params = sanitizeParamDefaults(parsedSpec.Params)
 		tf.Description = parsedSpec.SpecDescription
 		exampleStr, err := generateExample(tf)

worker/pkg/benthos/transformers/neosync_transformer_list_generator.go

@@ -13,6 +13,8 @@ import (
 	"path/filepath"
 	"strings"
 	"text/template"
+
+	transformers_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers"
 )
 
 type TransformerInfo struct {
@@ -46,9 +48,9 @@ func main() {
 							continue
 						}
 						if parts[1] == "transform" {
-							transformers = append(transformers, fmt.Sprintf("New%s()", capitalizeFirst(parts[2])))
+							transformers = append(transformers, fmt.Sprintf("New%s()", transformers_utils.CapitalizeFirst(parts[2])))
 						} else if parts[1] == "generate" {
-							generators = append(generators, fmt.Sprintf("New%s()", capitalizeFirst(parts[2])))
+							generators = append(generators, fmt.Sprintf("New%s()", transformers_utils.CapitalizeFirst(parts[2])))
 						}
 					}
 				}
@@ -122,10 +124,3 @@ func generateCode(transformers, generators []string, pkgName string) (string, er
 	}
 	return out.String(), nil
 }
-
-func capitalizeFirst(s string) string {
-	if len(s) == 0 {
-		return s
-	}
-	return strings.ToUpper(string(s[0])) + s[1:]
-}