Build macros, run them, serve a (test) service, host.

README.md

@@ -8,7 +8,18 @@ This repository is home to various macro related Dart packages.
 
 | Package | Description | Version |
 |---|---|---|
-| `dart_model` | Serializable data model for Dart code. | 0.0.1-wip |
+| [_analyzer_macros](pkgs/_analyzer_macros/) | Macro support for the analyzer. |  |
+| [_cfe_macros](pkgs/_cfe_macros/) | Macro support for the CFE. |  |
+| [_macro_builder](pkgs/_macro_builder/) | Builds macros. |  |
+| [_macro_client](pkgs/_macro_client/) | Connects user macro code to a macro host. |  |
+| [_macro_host](pkgs/_macro_host/) | Hosts macros. |  |
+| [_macro_runner](pkgs/_macro_runner/) | Runs macros. |  |
+| [_macro_server](pkgs/_macro_server/) | Serves a `macro_service`. |  |
+| [_test_macros](pkgs/_test_macros/) | Some test macros. |  |
+| [dart_model](pkgs/dart_model/) | Data model for information about Dart code, queries about Dart code and augmentations to Dart code. Serializable with a versioned JSON schema for use by macros, generators and other tools. | [![pub package](https://img.shields.io/pub/v/dart_model.svg)](https://pub.dev/packages/dart_model) |
+| [macro](pkgs/macro/) | For implementing a macro. | [![pub package](https://img.shields.io/pub/v/macro.svg)](https://pub.dev/packages/macro) |
+| [macro_service](pkgs/macro_service/) | Macro communication with the macro host. | [![pub package](https://img.shields.io/pub/v/macro_service.svg)](https://pub.dev/packages/macro_service) |
+| [generate_dart_model](tool/dart_model_generator/) |  |  |
 
 ## Publishing automation
 

pkgs/_macro_builder/lib/macro_builder.dart

@@ -2,22 +2,110 @@
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
+import 'dart:io';
+
 import 'package:dart_model/dart_model.dart';
 
+/// Builds macros.
+///
+/// TODO(davidmorgan): add a way to clean up generated files and built output.
 class MacroBuilder {
   /// Builds an executable from user-written macro code.
   ///
   /// Each `QualifiedName` in [macroImplementations] must point to a class that
   /// implements `Macro` from `package:macro`.
+  ///
+  /// The [packageConfig] must include the macros and all their deps.
   Future<BuiltMacroBundle> build(
-      Iterable<QualifiedName> macroImplementations) async {
-    // TODO(davidmorgan): implement.
-    // Generated entrypoint will instantiate all the `Macro` instances pointed
-    // to by `macroImplementations` then pass them to `MacroClient.run` in
-    // `package:_macro_client`.
-    return BuiltMacroBundle();
+      File packageConfig, Iterable<QualifiedName> macroImplementations) async {
+    final script = createBootstrap(macroImplementations.toList());
+
+    return await MacroBuild(packageConfig, script).build();
+  }
+
+  /// Creates the entrypoint script for [macros].
+  static String createBootstrap(List<QualifiedName> macros) {
+    final script = StringBuffer();
+    for (var i = 0; i != macros.length; ++i) {
+      final macro = macros[i];
+      // TODO(davidmorgan): pick non-clashing prefixes.
+      script.writeln("import '${macro.uri}' as m$i;");
+    }
+    script.write('''
+import 'dart:convert';
+
+import 'package:_macro_client/macro_client.dart';
+import 'package:macro_service/macro_service.dart';
+
+void main(List<String> arguments) {
+   MacroClient.run(
+      endpoint: HostEndpoint.fromJson(json.decode(arguments[0])),
+      macros: [''');
+    for (var i = 0; i != macros.length; ++i) {
+      final macro = macros[i];
+      script.write('m$i.${macro.name}()');
+      if (i != macros.length - 1) script.write(', ');
+    }
+    script.writeln(']);');
+    script.writeln('}');
+    return script.toString();
   }
 }
 
 /// A bundle of one or more macros that's ready to execute.
-class BuiltMacroBundle {}
+class BuiltMacroBundle {
+  // TODO(davidmorgan): other formats besides executable.
+  final String executablePath;
+
+  BuiltMacroBundle(this.executablePath);
+}
+
+/// A single build.
+class MacroBuild {
+  final File packageConfig;
+  final String script;
+  final Directory workspace =
+      Directory.systemTemp.createTempSync('macro_builder');
+
+  /// Creates a build for [script] with [packageConfig], which must have all
+  /// the needed deps.
+  MacroBuild(this.packageConfig, this.script);
+
+  /// Runs the build.
+  ///
+  /// Throws on failure to build.
+  Future<BuiltMacroBundle> build() async {
+    final scriptFile = File.fromUri(workspace.uri.resolve('bin/main.dart'));
+    scriptFile.parent.createSync(recursive: true);
+    scriptFile.writeAsStringSync(script.toString());
+
+    final targetPackageConfig =
+        File.fromUri(workspace.uri.resolve('.dart_tool/package_config.json'));
+    targetPackageConfig.parent.createSync(recursive: true);
+    targetPackageConfig
+        .writeAsStringSync(_makePackageConfigAbsolute(packageConfig));
+
+    // See package:analyzer/src/summary2/kernel_compilation_service.dart for an
+    // example of compiling macros using the frontend server.
+    //
+    // For now just use the command line.
+
+    final result = Process.runSync('dart', ['compile', 'exe', 'bin/main.dart'],
+        workingDirectory: workspace.path);
+    if (result.exitCode != 0) {
+      throw StateError('Compile failed: ${result.stderr}');
+    }
+
+    return BuiltMacroBundle(
+        File.fromUri(scriptFile.parent.uri.resolve('main.exe')).path);
+  }
+
+  /// Returns the contents of [pubspec] with relative paths replaced to
+  /// absolute paths, so the pubspec will work from any location.
+  String _makePackageConfigAbsolute(File pubspec) {
+    final root = pubspec.parent.parent.absolute.uri;
+    return pubspec
+        .readAsStringSync()
+        .replaceAll('"rootUri": "../', '"rootUri": "$root');
+  }
+}

pkgs/_macro_builder/mono_pkg.yaml

@@ -8,3 +8,8 @@ stages:
   - format:
     sdk:
     - dev
+- unit_test:
+  - test: --test-randomize-ordering-seed=random
+    os:
+    - linux
+    - windows

pkgs/_macro_builder/pubspec.yaml

@@ -13,3 +13,4 @@ dependencies:
 
 dev_dependencies:
   dart_flutter_team_lints: ^3.0.0
+  test: ^1.25.0

pkgs/_macro_builder/test/macro_builder_test.dart

@@ -0,0 +1,51 @@
+// Copyright (c) 2024, the Dart project authors. Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+import 'dart:io';
+import 'dart:isolate';
+
+import 'package:_macro_builder/macro_builder.dart';
+import 'package:dart_model/dart_model.dart';
+import 'package:test/test.dart';
+
+void main() {
+  group(MacroBuilder, () {
+    test('bootstrap matches golden', () async {
+      final script = MacroBuilder.createBootstrap([
+        QualifiedName('package:_test_macros/declare_x_macro.dart#DeclareX'),
+        QualifiedName('package:_test_macros/declare_y_macro.dart#DeclareY'),
+        QualifiedName(
+            'package:_more_macros/other_macro.dart#OtherMacroImplementation')
+      ]);
+
+      expect(script, '''
+import 'package:_test_macros/declare_x_macro.dart' as m0;
+import 'package:_test_macros/declare_y_macro.dart' as m1;
+import 'package:_more_macros/other_macro.dart' as m2;
+import 'dart:convert';
+
+import 'package:_macro_client/macro_client.dart';
+import 'package:macro_service/macro_service.dart';
+
+void main(List<String> arguments) {
+   MacroClient.run(
+      endpoint: HostEndpoint.fromJson(json.decode(arguments[0])),
+      macros: [m0.DeclareX(), m1.DeclareY(), m2.OtherMacroImplementation()]);
+}
+''');
+    });
+
+    test('builds macros', () async {
+      final builder = MacroBuilder();
+
+      final bundle =
+          await builder.build(File.fromUri(Isolate.packageConfigSync!), [
+        QualifiedName(
+            'package:_test_macros/declare_x_macro.dart#DeclareXImplementation')
+      ]);
+
+      expect(File(bundle.executablePath).existsSync(), true);
+    });
+  });
+}

pkgs/_macro_client/lib/macro_client.dart

@@ -2,12 +2,36 @@
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
+import 'dart:convert';
+import 'dart:io';
+
 import 'package:macro/macro.dart';
 import 'package:macro_service/macro_service.dart';
 
+/// Runs macros, connecting them to a macro host.
 class MacroClient {
+  final Iterable<Macro> macros;
+  final Socket socket;
+
+  MacroClient._(this.macros, this.socket) {
+    // TODO(davidmorgan): negotiation about protocol version goes here.
+
+    // Tell the host which macros are in this bundle.
+    for (final macro in macros) {
+      _send(MacroStartedRequest(macroDescription: macro.description).node);
+    }
+  }
+
   /// Runs [macros] for the host at [endpoint].
-  void run(HostEndpoint endpoint, Iterable<Macro> macros) {
-    // TODO(davidmorgan): implement.
+  static Future<MacroClient> run(
+      {required HostEndpoint endpoint, required Iterable<Macro> macros}) async {
+    final socket = await Socket.connect('localhost', endpoint.port);
+    return MacroClient._(macros, socket);
+  }
+
+  void _send(Map<String, Object?> node) {
+    // TODO(davidmorgan): currently this is JSON with one request per line,
+    // switch to binary.
+    socket.writeln(json.encode(node));
   }
 }

pkgs/_macro_client/mono_pkg.yaml

@@ -8,3 +8,8 @@ stages:
   - format:
     sdk:
     - dev
+- unit_test:
+  - test: --test-randomize-ordering-seed=random
+    os:
+    - linux
+    - windows

pkgs/_macro_client/pubspec.yaml

@@ -13,4 +13,6 @@ dependencies:
   macro_service: any
 
 dev_dependencies:
+  _test_macros: any
   dart_flutter_team_lints: ^3.0.0
+  test: ^1.25.0

pkgs/_macro_client/test/macro_client_test.dart

@@ -0,0 +1,26 @@
+// Copyright (c) 2024, the Dart project authors. Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+import 'dart:async';
+import 'dart:io';
+
+import 'package:_macro_client/macro_client.dart';
+import 'package:_test_macros/declare_x_macro.dart';
+import 'package:macro_service/macro_service.dart';
+import 'package:test/test.dart';
+
+void main() {
+  group(MacroClient, () {
+    test('connects to service', () async {
+      final serverSocket = await ServerSocket.bind('localhost', 0);
+
+      unawaited(MacroClient.run(
+          endpoint: HostEndpoint(port: serverSocket.port),
+          macros: [DeclareXImplementation()]));
+
+      expect(
+          serverSocket.first.timeout(const Duration(seconds: 10)), completes);
+    });
+  });
+}

pkgs/_macro_host/lib/macro_host.dart

@@ -2,26 +2,91 @@
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
+import 'dart:io';
+
 import 'package:_macro_builder/macro_builder.dart';
 import 'package:_macro_runner/macro_runner.dart';
 import 'package:_macro_server/macro_server.dart';
+import 'package:dart_model/dart_model.dart';
 import 'package:macro_service/macro_service.dart';
 
-class MacroHost {
+/// Hosts macros: builds them, runs them, serves the macro service.
+///
+/// Tools that want to support macros, such as the Analyzer and the CFE, can
+/// do so by running a `MacroHost` and providing their own `MacroService`.
+class MacroHost implements MacroService {
   final MacroServer macroServer;
+  final ListOfServices services;
   final MacroBuilder macroBuilder = MacroBuilder();
   final MacroRunner macroRunner = MacroRunner();
-  late final HostEndpoint hostEndpoint;
 
-  MacroHost._(this.macroServer, this.hostEndpoint);
+  // TODO(davidmorgan): this should be per macro, as part of tracking per-macro
+  // lifecycle state.
+  Set<int>? _macroPhases;
+
+  MacroHost._(this.macroServer, this.services) {
+    services.services.insert(0, this);
+  }
+
+  /// Starts a macro host serving the provided [service].
+  ///
+  /// The service passed in should handle introspection RPCs, it does not need
+  /// to handle others.
+  ///
+  /// TODO(davidmorgan): make this split clearer, it should be in the protocol
+  /// definition somewhere which requests the host handles.
+  static Future<MacroHost> serve({required MacroService service}) async {
+    final listOfServices = ListOfServices();
+    listOfServices.services.add(service);
+    final server = await MacroServer.serve(service: listOfServices);
+    return MacroHost._(server, listOfServices);
+  }
+
+  /// Whether [name] is a macro according to that package's `pubspec.yaml`.
+  bool isMacro(File packageConfig, QualifiedName name) {
+    // TODO(language/3728): this is a placeholder, use package config when
+    // available.
+    return true;
+  }
 
-  static Future<MacroHost> serve({required MacroHostService service}) async {
-    final server = MacroServer(service: service);
-    final endpoint = await server.serve();
-    return MacroHost._(server, endpoint);
+  /// Determines which phases the macro implemented at [name] runs in.
+  Future<Set<int>> queryMacroPhases(
+      File packageConfig, QualifiedName name) async {
+    if (_macroPhases != null) return _macroPhases!;
+    final macroBundle = await macroBuilder.build(packageConfig, [name]);
+    macroRunner.run(macroBundle: macroBundle, endpoint: macroServer.endpoint);
+    // TODO(davidmorgan): wait explicitly for the MacroStartedRequest to
+    // arrive, remove this hard-coded wait.
+    await Future<void>.delayed(const Duration(seconds: 2));
+    return _macroPhases!;
   }
 
-  // TODO(davidmorgan): methods for integration with analyzer+CFE go here:
-  // check if an annotation is linked to a macro, run a macro and ask it to
-  // produce augmentations.
+  /// Handle requests that are for the host.
+  @override
+  Future<Object?> handle(Object request) async {
+    // TODO(davidmorgan): don't assume the type. Return `null` for types
+    // that should be passed through to the service that was passed in.
+    final macroStartedRequest =
+        MacroStartedRequest.fromJson(request as Map<String, Object?>);
+    _macroPhases = macroStartedRequest.macroDescription.runsInPhases.toSet();
+    return MacroStartedResponse();
+  }
+
+  // TODO(davidmorgan): add method here for running macro phases.
+}
+
+// TODO(davidmorgan): this is used to handle some requests in the host while
+// letting some fall through to the passed in service. Differentiate in a
+// better way.
+class ListOfServices implements MacroService {
+  List<MacroService> services = [];
+
+  @override
+  Future<Object> handle(Object request) async {
+    for (final service in services) {
+      final result = await service.handle(request);
+      if (result != null) return result;
+    }
+    throw StateError('No service handled: $request');
+  }
 }

pkgs/_macro_host/mono_pkg.yaml

@@ -8,3 +8,8 @@ stages:
   - format:
     sdk:
     - dev
+- unit_test:
+  - test: --test-randomize-ordering-seed=random
+    os:
+    - linux
+    - windows