Build macros, run them, serve a (test) service, host, connect to it (#20

)

* Build macros, run them, serve a (test) service, host.

* Address review comments.

README.md

@@ -8,7 +8,18 @@ This repository is home to various macro related Dart packages.
 
 | Package | Description | Version |
 |---|---|---|
-| `dart_model` | Serializable data model for Dart code. | 0.0.1-wip |
+| [_analyzer_macros](pkgs/_analyzer_macros/) | Macro support for the analyzer. |  |
+| [_cfe_macros](pkgs/_cfe_macros/) | Macro support for the CFE. |  |
+| [_macro_builder](pkgs/_macro_builder/) | Builds macros. |  |
+| [_macro_client](pkgs/_macro_client/) | Connects user macro code to a macro host. |  |
+| [_macro_host](pkgs/_macro_host/) | Hosts macros. |  |
+| [_macro_runner](pkgs/_macro_runner/) | Runs macros. |  |
+| [_macro_server](pkgs/_macro_server/) | Serves a `macro_service`. |  |
+| [_test_macros](pkgs/_test_macros/) | Some test macros. |  |
+| [dart_model](pkgs/dart_model/) | Data model for information about Dart code, queries about Dart code and augmentations to Dart code. Serializable with a versioned JSON schema for use by macros, generators and other tools. | [![pub package](https://img.shields.io/pub/v/dart_model.svg)](https://pub.dev/packages/dart_model) |
+| [macro](pkgs/macro/) | For implementing a macro. | [![pub package](https://img.shields.io/pub/v/macro.svg)](https://pub.dev/packages/macro) |
+| [macro_service](pkgs/macro_service/) | Macro communication with the macro host. | [![pub package](https://img.shields.io/pub/v/macro_service.svg)](https://pub.dev/packages/macro_service) |
+| [generate_dart_model](tool/dart_model_generator/) |  |  |
 
 ## Publishing automation
 

pkgs/_cfe_macros/test/cfe_test.dart

@@ -80,7 +80,7 @@ class TestMacroPackageConfigs implements injected.MacroPackageConfigs {
     if (result == true) {
       macroWasFound = true;
     }
-    return macroWasFound;
+    return result;
   }
 }
 

pkgs/_macro_builder/lib/macro_builder.dart

@@ -2,22 +2,100 @@
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
+import 'dart:io';
+
 import 'package:dart_model/dart_model.dart';
 
+import 'src/bootstrap.dart';
+
+/// Builds macros.
+///
+/// TODO(davidmorgan): add a way to clean up generated files and built output.
 class MacroBuilder {
   /// Builds an executable from user-written macro code.
   ///
   /// Each `QualifiedName` in [macroImplementations] must point to a class that
   /// implements `Macro` from `package:macro`.
+  ///
+  /// The [packageConfig] must include the macros and all their deps.
+  ///
+  /// TODO(davidmorgan): figure out builder lifecycle: is it one builder per
+  /// host, one per workspace, one per build?
+  /// TODO(davidmorgan): replace `File` packageConfig with a concept of version
+  /// solve and workspace.
+  /// TODO(davidmorgan): support for multi-root workspaces.
+  /// TODO(davidmorgan): support (or decide not to support) in-memory overlay
+  /// filesystems.
   Future<BuiltMacroBundle> build(
-      Iterable<QualifiedName> macroImplementations) async {
-    // TODO(davidmorgan): implement.
-    // Generated entrypoint will instantiate all the `Macro` instances pointed
-    // to by `macroImplementations` then pass them to `MacroClient.run` in
-    // `package:_macro_client`.
-    return BuiltMacroBundle();
+      Uri packageConfig, Iterable<QualifiedName> macroImplementations) async {
+    final script = createBootstrap(macroImplementations.toList());
+
+    return await MacroBuild(packageConfig, script).build();
   }
 }
 
 /// A bundle of one or more macros that's ready to execute.
-class BuiltMacroBundle {}
+class BuiltMacroBundle {
+  // TODO(davidmorgan): other formats besides executable.
+  final String executablePath;
+
+  BuiltMacroBundle(this.executablePath);
+}
+
+/// A single build.
+///
+/// TODO(davidmorgan): split to interface+implementations as we add different
+/// ways to build.
+class MacroBuild {
+  final Uri packageConfig;
+  final String script;
+  final Directory workspace =
+      Directory.systemTemp.createTempSync('macro_builder');
+
+  /// Creates a build for [script] with [packageConfig], which must have all
+  /// the needed deps.
+  MacroBuild(this.packageConfig, this.script);
+
+  /// Runs the build.
+  ///
+  /// Throws on failure to build.
+  Future<BuiltMacroBundle> build() async {
+    final scriptFile = File.fromUri(workspace.uri.resolve('bin/main.dart'));
+    await scriptFile.create(recursive: true);
+    await scriptFile.writeAsString(script.toString());
+
+    final targetPackageConfig =
+        File.fromUri(workspace.uri.resolve('.dart_tool/package_config.json'));
+    targetPackageConfig.parent.createSync(recursive: true);
+    targetPackageConfig
+        .writeAsStringSync(_makePackageConfigAbsolute(packageConfig));
+
+    // See package:analyzer/src/summary2/kernel_compilation_service.dart for an
+    // example of compiling macros using the frontend server.
+    //
+    // For now just use the command line.
+
+    final result = Process.runSync(
+        // TODO(davidmorgan): this is wrong if run from an AOT-compiled
+        // executable.
+        Platform.resolvedExecutable,
+        ['compile', 'exe', 'bin/main.dart', '--output=bin/main.exe'],
+        workingDirectory: workspace.path);
+    if (result.exitCode != 0) {
+      throw StateError('Compile failed: ${result.stderr}');
+    }
+
+    return BuiltMacroBundle(
+        File.fromUri(scriptFile.parent.uri.resolve('main.exe')).path);
+  }
+
+  /// Returns the contents of [packageConfig] with relative paths replaced to
+  /// absolute paths, so the pubspec will work from any location.
+  String _makePackageConfigAbsolute(Uri packageConfig) {
+    final file = File.fromUri(packageConfig);
+    final root = file.parent.parent.absolute.uri;
+    return file
+        .readAsStringSync()
+        .replaceAll('"rootUri": "../', '"rootUri": "$root');
+  }
+}

pkgs/_macro_builder/lib/src/bootstrap.dart

@@ -0,0 +1,34 @@
+// Copyright (c) 2024, the Dart project authors. Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+import 'package:dart_model/dart_model.dart';
+
+/// Creates the entrypoint script for [macros].
+String createBootstrap(List<QualifiedName> macros) {
+  final script = StringBuffer();
+  for (var i = 0; i != macros.length; ++i) {
+    final macro = macros[i];
+    // TODO(davidmorgan): pick non-clashing prefixes.
+    script.writeln("import '${macro.uri}' as m$i;");
+  }
+  script.write('''
+import 'dart:convert' as convert;
+
+import 'package:_macro_client/macro_client.dart' as macro_client;
+import 'package:macro_service/macro_service.dart' as macro_service;
+
+void main(List<String> arguments) {
+   macro_client.MacroClient.run(
+      endpoint: macro_service.HostEndpoint.fromJson(
+        convert.json.decode(arguments[0])),
+      macros: [''');
+  for (var i = 0; i != macros.length; ++i) {
+    final macro = macros[i];
+    script.write('m$i.${macro.name}()');
+    if (i != macros.length - 1) script.write(', ');
+  }
+  script.writeln(']);');
+  script.writeln('}');
+  return script.toString();
+}

pkgs/_macro_builder/mono_pkg.yaml

@@ -8,3 +8,8 @@ stages:
   - format:
     sdk:
     - dev
+- unit_test:
+  - test: --test-randomize-ordering-seed=random
+    os:
+    - linux
+    - windows

pkgs/_macro_builder/pubspec.yaml

@@ -13,3 +13,4 @@ dependencies:
 
 dev_dependencies:
   dart_flutter_team_lints: ^3.0.0
+  test: ^1.25.0

pkgs/_macro_builder/test/macro_builder_test.dart

@@ -0,0 +1,52 @@
+// Copyright (c) 2024, the Dart project authors. Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+import 'dart:io';
+import 'dart:isolate';
+
+import 'package:_macro_builder/macro_builder.dart';
+import 'package:_macro_builder/src/bootstrap.dart';
+import 'package:dart_model/dart_model.dart';
+import 'package:test/test.dart';
+
+void main() {
+  group(MacroBuilder, () {
+    test('bootstrap matches golden', () async {
+      final script = createBootstrap([
+        QualifiedName('package:_test_macros/declare_x_macro.dart#DeclareX'),
+        QualifiedName('package:_test_macros/declare_y_macro.dart#DeclareY'),
+        QualifiedName(
+            'package:_more_macros/other_macro.dart#OtherMacroImplementation')
+      ]);
+
+      expect(script, '''
+import 'package:_test_macros/declare_x_macro.dart' as m0;
+import 'package:_test_macros/declare_y_macro.dart' as m1;
+import 'package:_more_macros/other_macro.dart' as m2;
+import 'dart:convert' as convert;
+
+import 'package:_macro_client/macro_client.dart' as macro_client;
+import 'package:macro_service/macro_service.dart' as macro_service;
+
+void main(List<String> arguments) {
+   macro_client.MacroClient.run(
+      endpoint: macro_service.HostEndpoint.fromJson(
+        convert.json.decode(arguments[0])),
+      macros: [m0.DeclareX(), m1.DeclareY(), m2.OtherMacroImplementation()]);
+}
+''');
+    });
+
+    test('builds macros', () async {
+      final builder = MacroBuilder();
+
+      final bundle = await builder.build(Isolate.packageConfigSync!, [
+        QualifiedName(
+            'package:_test_macros/declare_x_macro.dart#DeclareXImplementation')
+      ]);
+
+      expect(File(bundle.executablePath).existsSync(), true);
+    });
+  });
+}

pkgs/_macro_client/lib/macro_client.dart

@@ -2,12 +2,34 @@
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
+import 'dart:convert';
+import 'dart:io';
+
 import 'package:macro/macro.dart';
 import 'package:macro_service/macro_service.dart';
 
+/// Runs macros, connecting them to a macro host.
+///
+/// TODO(davidmorgan): handle shutdown and dispose.
+/// TODO(davidmorgan): split to multpile implementations depending on
+/// transport used to connect to host.
 class MacroClient {
+  MacroClient._(Iterable<Macro> macros, Socket socket) {
+    // TODO(davidmorgan): negotiation about protocol version goes here.
+
+    // Tell the host which macros are in this bundle.
+    for (final macro in macros) {
+      final request = MacroStartedRequest(macroDescription: macro.description);
+      // TODO(davidmorgan): currently this is JSON with one request per line,
+      // switch to binary.
+      socket.writeln(json.encode(request.node));
+    }
+  }
+
   /// Runs [macros] for the host at [endpoint].
-  void run(HostEndpoint endpoint, Iterable<Macro> macros) {
-    // TODO(davidmorgan): implement.
+  static Future<MacroClient> run(
+      {required HostEndpoint endpoint, required Iterable<Macro> macros}) async {
+    final socket = await Socket.connect('localhost', endpoint.port);
+    return MacroClient._(macros, socket);
   }
 }

pkgs/_macro_client/mono_pkg.yaml

@@ -8,3 +8,8 @@ stages:
   - format:
     sdk:
     - dev
+- unit_test:
+  - test: --test-randomize-ordering-seed=random
+    os:
+    - linux
+    - windows

pkgs/_macro_client/pubspec.yaml

@@ -13,4 +13,6 @@ dependencies:
   macro_service: any
 
 dev_dependencies:
+  _test_macros: any
   dart_flutter_team_lints: ^3.0.0
+  test: ^1.25.0

pkgs/_macro_client/test/macro_client_test.dart

@@ -0,0 +1,27 @@
+// Copyright (c) 2024, the Dart project authors. Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+import 'dart:async';
+import 'dart:io';
+
+import 'package:_macro_client/macro_client.dart';
+import 'package:_test_macros/declare_x_macro.dart';
+import 'package:macro_service/macro_service.dart';
+import 'package:test/test.dart';
+
+void main() {
+  group(MacroClient, () {
+    test('connects to service', () async {
+      final serverSocket = await ServerSocket.bind('localhost', 0);
+      addTearDown(serverSocket.close);
+
+      unawaited(MacroClient.run(
+          endpoint: HostEndpoint(port: serverSocket.port),
+          macros: [DeclareXImplementation()]));
+
+      expect(
+          serverSocket.first.timeout(const Duration(seconds: 10)), completes);
+    });
+  });
+}