Adjustments to parser API (#112)

* Adjustments to parser API

The ParserInput source is now a `Buffer`, which I think better reflects what we need.

Also added some initial setup for the `JavaScriptParser`.

* Set `allowJs` compiler option

* Add `.gitignore` file

* Add some more exception handling

* Declare visitor as top-level function instead

* Some more fixes to parser

* Remove logging

* Add Parser.Builder

* Some more initial work on parser

* Implement `Space.format()`

* Implement AST-based alternative to `Space.format()`

* Add `isTree()`

* Split up tests

* Merge main

openrewrite/.gitignore

@@ -15,4 +15,6 @@ yarn-error.log*
 .env
 .env.test
 .env.production
-.env.*.local
+.env.*.local
+
+tsconfig.build.tsbuildinfo

openrewrite/jest.config.js

@@ -5,14 +5,9 @@ module.exports = {
   moduleFileExtensions: ['ts', 'tsx', 'js', 'jsx', 'json', 'node'],
   transform: {
     '^.+\\.tsx?$': ['ts-jest', {
-      tsconfig: 'tsconfig.json', // Adjust if your tsconfig file is named or located differently
+      tsconfig: 'tsconfig.test.json', // Adjust if your tsconfig file is named or located differently
     }],
   },
   testMatch: ['**/__tests__/**/*.+(ts|tsx|js)', '**/?(*.)+(spec|test).+(ts|tsx|js)'],
   collectCoverageFrom: ['src/**/*.{ts,tsx}', '!src/**/*.d.ts'],
-  globals: {
-    'ts-jest': {
-      tsconfig: 'tsconfig.test.json'
-    }
-  }
 };

openrewrite/src/core/parser.ts

@@ -7,13 +7,13 @@ export class ParserInput {
     private readonly _path: string;
     private readonly _fileAttributes: FileAttributes | null;
     private readonly _synthetic: boolean;
-    private readonly _source: () => fs.ReadStream;
+    private readonly _source: () => Buffer;
 
     constructor(
         path: string,
         fileAttributes: FileAttributes | null,
         synthetic: boolean,
-        source: () => fs.ReadStream
+        source: () => Buffer
     ) {
         this._path = path;
         this._fileAttributes = fileAttributes;
@@ -33,7 +33,7 @@ export class ParserInput {
         return this._synthetic;
     }
 
-    get source(): () => fs.ReadStream {
+    get source(): () => Buffer {
         return this._source;
     }
 }
@@ -50,18 +50,18 @@ export abstract class Parser {
     abstract sourcePathFromSourceText(prefix: string, sourceCode: string): string;
 
     parse(
-        sourceFiles: Iterable<string>,
+        sourceFilesPaths: Iterable<string>,
         relativeTo: string | null,
         ctx: ExecutionContext
     ): Iterable<SourceFile> {
         const inputs: ParserInput[] = [];
-        for (const path of sourceFiles) {
+        for (const path of sourceFilesPaths) {
             inputs.push(
                 new ParserInput(
                     path,
                     null,
                     false,
-                    () => fs.createReadStream(path)
+                    () => fs.readFileSync(path)
                 )
             );
         }
@@ -79,13 +79,7 @@ export abstract class Parser {
                 path,
                 null,
                 true,
-                () => {
-                    // FIXME handling of streams
-                    const stream = new fs.ReadStream(null!);
-                    stream.push(source);
-                    stream.push(null);
-                    return stream;
-                }
+                () => Buffer.from(source)
             );
         });
         return this.parseInputs(inputs, null, ctx);
@@ -115,7 +109,7 @@ export abstract class Parser {
 }
 
 export namespace Parser {
-    abstract class Builder {
+    export abstract class Builder {
         protected _sourceFileType: any;
 
         get sourceFileType(): any {
@@ -126,7 +120,7 @@ export namespace Parser {
     }
 }
 
-function requirePrintEqualsInput(
+export function requirePrintEqualsInput(
     parser: Parser,
     sourceFile: SourceFile,
     parserInput: ParserInput,
@@ -136,7 +130,7 @@ function requirePrintEqualsInput(
     const required = ctx.getMessage(ExecutionContext.REQUIRE_PRINT_EQUALS_INPUT, true);
     if (required && !sourceFile.printEqualsInput(parserInput, ctx)) {
         const diff = Result.diff(
-            parserInput.source().read().toString(),
+            parserInput.source().toString(),
             sourceFile.printAll(),
             parserInput.path
         );

openrewrite/src/core/tree.ts

@@ -18,6 +18,10 @@ export interface Tree {
     accept<R extends Tree, P>(v: TreeVisitor<R, P>, p: P): R | null;
 }
 
+export function isTree(tree: any): tree is Tree {
+    return !!tree.constructor.isTree || !!tree.isTree;
+}
+
 export abstract class TreeVisitor<T extends Tree, P> {
     private _cursor: Cursor;
     private _visitCount: number = 0;
@@ -352,6 +356,7 @@ type AbstractConstructor<T = {}> = abstract new (...args: any[]) => T;
 
 export function SourceFileMixin<TBase extends AbstractConstructor<Tree>>(Base: TBase) {
     abstract class SourceFileMixed extends Base implements SourceFile {
+        static isTree = true;
         static isSourceFile = true;
 
         abstract get sourcePath(): string;
@@ -479,6 +484,7 @@ export abstract class PrinterFactory {
 
 @LstType("org.openrewrite.tree.ParseError")
 export class ParseError implements SourceFile {
+    static isTree = true;
     static isParseError = true;
     static isSourceFile = true;
 
@@ -530,7 +536,7 @@ export class ParseError implements SourceFile {
             parser.getCharset(ctx),
             false,
             null,
-            input.source().read().toString(),
+            input.source().toString(),
             erroneous
         );
     }

openrewrite/src/java/tree/support_types.ts

@@ -32,6 +32,7 @@ export function isJava(tree: any): tree is J {
 
 export function JMixin<TBase extends Constructor<Object>>(Base: TBase) {
     abstract class JMixed extends Base implements J {
+        static isTree = true;
         static isJava = true;
 
         abstract get prefix(): Space;
@@ -107,10 +108,80 @@ export class Space {
             } else if (whitespace == ' ') {
                 return Space.SINGLE_SPACE;
             }
+            // FIXME add flyweights
         }
         return new Space(comments, whitespace);
     }
 
+    static format(formatting: string, beginIndex: number, toIndex: number): Space {
+        if (beginIndex == toIndex) {
+            return Space.EMPTY;
+        } else if (toIndex == beginIndex + 1 && formatting[beginIndex] === ' ') {
+            return Space.SINGLE_SPACE;
+        }
+
+        let comments: Comment[] = [];
+        let whitespaceStart = beginIndex;
+        let commentStart = -1;
+        let commentEnd = -1;
+        let suffixStart = -1;
+        let i = beginIndex;
+
+        // Step 1: Process leading whitespace
+        while (i < toIndex && (formatting[i] === ' ' || formatting[i] === '\t' || formatting[i] === '\n' || formatting[i] === '\r')) {
+            i++;
+        }
+        let whitespaceEnd = i;  // Capture end of leading whitespace
+
+        // Step 2: Parse comments
+        while (i < toIndex) {
+            const char = formatting[i];
+
+            // Handle single-line comment (//)
+            if (char === '/' && i + 1 < toIndex && formatting[i + 1] === '/') {
+                commentStart = i + 2;  // Skip the "//"
+                i += 2;
+                while (i < toIndex && formatting[i] !== '\n' && formatting[i] !== '\r') {
+                    i++;  // Continue until end of line or end of input
+                }
+                commentEnd = i;
+                suffixStart = i;  // Capture newline as suffix
+                while (i < toIndex && (formatting[i] === '\n' || formatting[i] === '\r')) {
+                    i++;
+                }
+                const commentText = formatting.slice(commentStart, commentEnd);
+                const suffix = formatting.slice(suffixStart, i);
+                comments.push(new TextComment(false, commentText, suffix, Markers.EMPTY));
+
+                // Handle multi-line comment (/* ... */)
+            } else if (char === '/' && i + 1 < toIndex && formatting[i + 1] === '*') {
+                commentStart = i + 2;  // Skip the "/*"
+                i += 2;
+                while (i + 1 < toIndex && !(formatting[i] === '*' && formatting[i + 1] === '/')) {
+                    i++;  // Continue until "*/" or end of input
+                }
+                commentEnd = i;  // Position before */
+                i += 2;  // Skip the closing "*/"
+
+                suffixStart = i;
+                while (i < toIndex && (formatting[i] === ' ' || formatting[i] === '\t' || formatting[i] === '\n' || formatting[i] === '\r')) {
+                    i++;  // Capture any trailing whitespace after comment
+                }
+                const commentText = formatting.slice(commentStart, commentEnd);
+                const suffix = formatting.slice(suffixStart, i);
+                comments.push(new TextComment(true, commentText, suffix, Markers.EMPTY));
+            } else {
+                i++;  // Skip non-comment characters
+            }
+        }
+
+        // Step 3: Extract leading whitespace
+        const whitespace = whitespaceEnd > whitespaceStart ? formatting.slice(whitespaceStart, whitespaceEnd) : null;
+
+        // Step 4: Return a Space object with accumulated whitespace and comments
+        return Space.build(comments, whitespace);
+    }
+
     public constructor(comments: Comment[], whitespace: string | null) {
         this._comments = comments;
         this._whitespace = whitespace;