Fix dstr unparsing

* This is an entirely new approach.
* Instead to find the "correct" dstr segments we simply try all and unparse the first one
  that round trips.
* This so far guarantees we always get good concrete syntax, but it can be time intensive as
  the combinatoric space of possible dynamic string sequence is quadratic with the dstr children size.
* For this reason we try above (currently) dstr children to unparse as heredoc first.
* Passes the entire corpus and fixes bugs.

[fix #249]

Author: mbj

.rubocop.yml

@@ -5,7 +5,7 @@ AllCops:
     - '**/*.rake'
     - 'Gemfile'
     - 'Gemfile.triage'
-  TargetRubyVersion: 3.0
+  TargetRubyVersion: 3.1
   Exclude:
     - tmp/**/*
     - vendor/**/*
@@ -106,3 +106,7 @@ Naming/RescuedExceptionsVariableName:
 
 Layout/MultilineMethodCallIndentation:
   Enabled: false
+
+# Useful for code structure
+Lint/UselessConstantScoping:
+  Enabled: false

Changelog.md

@@ -1,3 +1,10 @@
+# v0.7.0 2024-09-16
+
+[#366](https://github.com/mbj/unparser/pull/366)
+
+* Fix all known dstring issues.
+* Interface changes.
+
 # v0.6.15 2024-06-10
 
 [#373](https://github.com/mbj/unparser/pull/373)

Gemfile

@@ -2,4 +2,6 @@
 
 source 'https://rubygems.org'
 
+gem 'mutant', github: 'mbj/mutant', ref: '152465f02b2c4a18b80f28fee120f58d71a41391'
+
 gemspec

Gemfile.lock

@@ -1,7 +1,19 @@
+GIT
+  remote: https://github.com/mbj/mutant.git
+  revision: 152465f02b2c4a18b80f28fee120f58d71a41391
+  ref: 152465f02b2c4a18b80f28fee120f58d71a41391
+  specs:
+    mutant (0.12.4)
+      diff-lcs (~> 1.3)
+      parser (~> 3.3.0)
+      regexp_parser (~> 2.9.0)
+      sorbet-runtime (~> 0.5.0)
+      unparser (~> 0.7.0)
+
 PATH
   remote: .
   specs:
-    unparser (0.6.15)
+    unparser (0.7.0)
       diff-lcs (~> 1.6)
       parser (>= 3.3.0)
 
@@ -12,12 +24,7 @@ GEM
     diff-lcs (1.6.0)
     json (2.10.2)
     language_server-protocol (3.17.0.4)
-    mutant (0.12.4)
-      diff-lcs (~> 1.3)
-      parser (~> 3.3.0)
-      regexp_parser (~> 2.9.0)
-      sorbet-runtime (~> 0.5.0)
-      unparser (~> 0.6.14)
+    lint_roller (1.1.0)
     mutant-rspec (0.12.4)
       mutant (= 0.12.4)
       rspec-core (>= 3.8.0, < 4.0.0)
@@ -44,31 +51,33 @@ GEM
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.13.0)
     rspec-support (3.13.2)
-    rubocop (1.71.2)
+    rubocop (1.74.0)
       json (~> 2.3)
-      language_server-protocol (>= 3.17.0)
+      language_server-protocol (~> 3.17.0.2)
+      lint_roller (~> 1.1.0)
       parallel (~> 1.10)
       parser (>= 3.3.0.2)
       rainbow (>= 2.2.2, < 4.0)
       regexp_parser (>= 2.9.3, < 3.0)
       rubocop-ast (>= 1.38.0, < 2.0)
       ruby-progressbar (~> 1.7)
       unicode-display_width (>= 2.4.0, < 4.0)
-    rubocop-ast (1.38.0)
+    rubocop-ast (1.38.1)
       parser (>= 3.3.1.0)
     rubocop-packaging (0.5.2)
       rubocop (>= 1.33, < 2.0)
     ruby-progressbar (1.13.0)
-    sorbet-runtime (0.5.11826)
+    sorbet-runtime (0.5.11934)
     unicode-display_width (3.1.4)
       unicode-emoji (~> 4.0, >= 4.0.4)
     unicode-emoji (4.0.4)
 
 PLATFORMS
   ruby
+  x86_64-linux
 
 DEPENDENCIES
-  mutant (~> 0.12.4)
+  mutant!
   mutant-rspec (~> 0.12.4)
   rspec (~> 3.13)
   rspec-core (~> 3.13)
@@ -78,4 +87,4 @@ DEPENDENCIES
   unparser!
 
 BUNDLED WITH
-   2.5.10
+   2.5.22

bin/corpus

@@ -1,10 +1,14 @@
 #!/usr/bin/env ruby
 # frozen_string_literal: true
 
+require 'etc'
 require 'mutant'
 require 'optparse'
+require 'pathname'
 require 'unparser'
 
+Thread.abort_on_exception = true
+
 module Unparser
   module Corpus
     ROOT = Pathname.new(__dir__).parent
@@ -16,16 +20,92 @@ module Unparser
       # Perform verification via unparser cli
       #
       # @return [Boolean]
+      #
+      # rubocop:disable Metrics/AbcSize
+      # rubocop:disable Metrics/MethodLength
       def verify
+        puts("Verifiying: #{name}")
         checkout
-        command = %W[unparser #{repo_path}]
-        exclude.each do |name|
-          command.push('--ignore', repo_path.join(name).to_s)
+
+        paths = Pathname.glob(Pathname.new(repo_path).join('**/*.rb'))
+
+        driver = Mutant::Parallel.async(
+          config: Mutant::Parallel::Config.new(
+            block:            method(:verify_path),
+            jobs:             Etc.nprocessors,
+            on_process_start: ->(*) {},
+            process_name:     'unparser-corpus-test',
+            sink:             Sink.new,
+            source:           Mutant::Parallel::Source::Array.new(jobs: paths),
+            thread_name:      'unparser-corpus-test',
+            timeout:          nil
+          ),
+          world:  Mutant::WORLD
+        )
+
+        loop do
+          status = driver.wait_timeout(1)
+
+          puts("Processed: #{status.payload.total}")
+
+          # rubocop:disable Lint/UnreachableLoop
+          status.payload.errors.each do |report|
+            puts report
+            fail
+          end
+          # rubocop:enable Lint/UnreachableLoop
+
+          break if status.done?
+        end
+
+        true
+      end
+      # rubocop:enable Metrics/AbcSize
+      # rubocop:enable Metrics/MethodLength
+
+      private
+
+      class Sink
+        include Mutant::Parallel::Sink
+
+        attr_reader :errors, :total
+
+        def initialize
+          @errors = []
+          @total  = 0
+        end
+
+        def stop?
+          !@errors.empty?
+        end
+
+        def status
+          self
+        end
+
+        def response(response)
+          if response.error
+            Mutant::WORLD.stderr.puts(response.log)
+            fail response.error
+          end
+
+          @total += 1
+
+          if response.result
+            @errors << response.result
+          end
         end
-        Kernel.system(*command)
       end
 
-    private
+      def verify_path(path)
+        validation = Validation.from_path(path)
+
+        if original_syntax_error?(validation) || generated_encoding_error?(validation) || validation.success?
+          return
+        end
+
+        validation.report
+      end
 
       def checkout
         TMP.mkdir unless TMP.directory?
@@ -50,6 +130,20 @@ module Unparser
         TMP.join(name)
       end
 
+      # This happens if the original source contained a non UTF charset meta comment.
+      # These are not exposed to the AST in a way unparser could know about to generate a non UTF-8
+      # target and emit that meta comment itself.
+      # For the purpose of corpus testing these cases are ignored.
+      def generated_encoding_error?(validation)
+        exception = validation.generated_node.from_left { return false }
+        exception.instance_of?(Parser::SyntaxError) &&
+          exception.message.eql?('literal contains escape sequences incompatible with UTF-8')
+      end
+
+      def original_syntax_error?(validation)
+        validation.original_node.from_left { return false }.instance_of?(Parser::SyntaxError)
+      end
+
       def system(arguments)
         return if Kernel.system(*arguments)
 

bin/parser-round-trip-test

@@ -40,7 +40,14 @@ class Test
     :rubies
   )
 
-  EXPECT_FAILURE = {}.freeze
+  EXPECT_FAILURE         = {}.freeze
+  STATIC_LOCAL_VARIABLES = %w[foo bar baz].to_set.freeze
+
+  NO_ROUND_TRIP = %i[
+    test_int___LINE__
+    test_pattern_matching__FILE__LINE_literals
+    test_string___FILE__
+  ].freeze
 
   def legacy_attributes
     default_builder_attributes.reject do |attribute_name, value|
@@ -56,6 +63,8 @@ class Test
       "Non targeted rubies: #{rubies.join(',')}"
     elsif validation.original_node.left?
       'Test specifies a syntax error'
+    elsif NO_ROUND_TRIP.include?(name)
+      'Test not round trippable'
     end
   end
 
@@ -77,20 +86,25 @@ class Test
 
   # rubocop:disable Metrics/AbcSize
   def validation
-    identification   = name.to_s
+    identification = name.to_s
+
+    ast = Unparser::AST.new(
+      comments:               [],
+      explicit_encoding:      nil,
+      node:                   node,
+      static_local_variables: STATIC_LOCAL_VARIABLES
+    )
 
-    generated_source = Unparser.unparse_either(node)
+    generated_source = Unparser.unparse_ast_either(ast)
       .fmap { |string| string.dup.force_encoding(parser_source.encoding).freeze }
 
-    generated_node = generated_source.bind do |source|
-      parse_either(source, identification)
-    end
+    generated_node = generated_source.bind { |source| parse_either(source, identification) }
 
     Unparser::Validation.new(
       generated_node:   generated_node,
       generated_source: generated_source,
       identification:   identification,
-      original_node:    parse_either(parser_source, identification).fmap { node },
+      original_ast:     parse_either_ast(parser_source, identification),
       original_source:  right(parser_source)
     )
   end
@@ -99,7 +113,7 @@ class Test
 
   def parser
     Unparser.parser.tap do |parser|
-      %w[foo bar baz].each(&parser.static_env.method(:declare))
+      STATIC_LOCAL_VARIABLES.each(&parser.static_env.method(:declare))
     end
   end
 
@@ -108,6 +122,17 @@ class Test
       parser.parse(Unparser.buffer(source, identification))
     end
   end
+
+  def parse_either_ast(source, identification)
+    parse_either(source, identification).fmap do |node|
+      Unparser::AST.new(
+        comments:               [],
+        explicit_encoding:      nil,
+        node:                   node,
+        static_local_variables: Set.new
+      )
+    end
+  end
 end
 
 class Execution

config/mutant.yml

@@ -12,38 +12,3 @@ coverage_criteria:
 matcher:
   subjects:
   - 'Unparser*'
-  ignore:
-  - 'Unparser::Builder#initialize'
-  - 'Unparser::CLI*'
-  - 'Unparser::Concord#define_readers'
-  - 'Unparser::Emitter#emit_comments'
-  - 'Unparser::Emitter#emit_comments_before'
-  - 'Unparser::Emitter#emit_eol_comments'
-  - 'Unparser::Emitter.handle'
-  - 'Unparser::Emitter::Args#normal_arguments'
-  - 'Unparser::Emitter::Args#shadowargs'
-  - 'Unparser::Emitter::Array#emitters'
-  - 'Unparser::Emitter::Binary#writer'
-  - 'Unparser::Emitter::Block#target_writer'
-  - 'Unparser::Emitter::BlockPass#dispatch' # 3.1+ specific
-  - 'Unparser::Emitter::Class#dispatch'
-  - 'Unparser::Emitter::Class#local_variable_scope'
-  - 'Unparser::Emitter::Def#local_variable_scope'
-  - 'Unparser::Emitter::FindPattern#dispatch' # 3.0+ specific
-  - 'Unparser::Emitter::Hash#emit_heredoc_reminder_member' # 3.2+ specific
-  - 'Unparser::Emitter::HashPattern#write_symbol_body'
-  - 'Unparser::Emitter::LocalVariableRoot*'
-  - 'Unparser::Emitter::LocalVariableRoot.included'
-  - 'Unparser::Emitter::MatchPatternP#dispatch' # 3.0+ specific
-  - 'Unparser::Emitter::MatchRest#dispatch' # 3.0+ specific
-  - 'Unparser::Emitter::Module#local_variable_scope'
-  - 'Unparser::Emitter::Root#local_variable_scope'
-  - 'Unparser::Emitter::Send#writer'
-  - 'Unparser::NodeDetails.included'
-  - 'Unparser::Validation.from_string'
-  - 'Unparser::Validation::Literal*'
-  - 'Unparser::Writer.included'
-  - 'Unparser::Writer::Binary#left_emitter'
-  - 'Unparser::Writer::Binary#right_emitter'
-  - 'Unparser::Writer::Send#details'
-  - 'Unparser::Writer::Send#effective_writer'