Skip to content

Commit

Permalink
[release] fixes to the IWYU parser
Browse files Browse the repository at this point in the history 
Fixed IWYU deadly parser bugs (no newline on the end of the file)
  • Loading branch information
@Agrael1
Agrael1 authored Aug 26, 2022
2 parents d11a58b + d27cdbd commit a43bb6e
Show file tree
Hide file tree
Showing 8 changed files with 355 additions and 356 deletions.
132 changes: 57 additions & 75 deletions IncludeToolboxShared/IWYU/IWYUApply.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -64,108 +64,90 @@ public static async Task ApplyAsync(IWYUOptions settings, string output)
{
if (output == "") return;

while (true)
{
int pos = output.IndexOf(match);
if (pos == -1) return;
int pos = output.IndexOf(match);
if (pos == -1) return;

pos += match.Length;
string part = output.Substring(pos);
pos += match.Length;
string part = output.Substring(pos);

int endp = part.IndexOf("---");
string path = part.Substring(0, part.IndexOf(':', 3));
var doc = await VS.Documents.OpenAsync(path);
using var edit = doc.TextBuffer.CreateEdit();
int endp = part.IndexOf("---");
string path = part.Substring(0, part.IndexOf(':', 3));
var doc = await VS.Documents.OpenAsync(path);
using var edit = doc.TextBuffer.CreateEdit();

int endl = part.IndexOf("\n");
string result = part.Substring(endl, endp - endl);
ApplyCheap(edit,
result,
settings.Comms != Comment.No);
int endl = part.IndexOf("\n");
string result = part.Substring(endl, endp - endl);
ApplyCheap(edit,
result,
settings.Comms != Comment.No);

edit.Apply();
output = part.Substring(endp);
}
edit.Apply();
}

public static async Task ApplyPreciseAsync(IWYUOptions settings, Parser.Output parsed, string output, Standard std)
{
if (output == "") return;

while (true)
{
int pos = output.IndexOf(match);
if (pos == -1) return;

var retasks = Parser.Parse(output.AsSpan().Slice(0, pos), true, true);
int sep_index = output.IndexOf(" should remove these lines:"); //find middle ground
int pos = output.IndexOf(match);
if (pos == -1) return;

var retasks = Parser.Parse(output.AsSpan().Slice(0, pos), true, true);
int sep_index = output.IndexOf(" should remove these lines:"); //find middle ground

pos += match.Length;
string part = output.Substring(pos);


int endp = part.IndexOf("---");
string path = part.Substring(0, part.IndexOf(':', 3));
var doc = await VS.Documents.OpenAsync(path);
using var edit = doc.TextBuffer.CreateEdit();
var lb = Utils.GetLineBreak(edit);
pos += match.Length;
string part = output.Substring(pos);

string path = part.Substring(0, part.IndexOf(':', 3));
var doc = await VS.Documents.OpenAsync(path);
using var edit = doc.TextBuffer.CreateEdit();
var lb = Utils.GetLineBreak(doc.TextView);

var add_f = retasks.Declarations.Where(s => s.span.begin < sep_index);
var rem_f = retasks.Declarations.Where(s => s.span.begin > sep_index);

var add_i = retasks.Includes.Where(s => s.span.begin < sep_index);
var rem_i = retasks.Includes.Where(s => s.span.begin > sep_index);
var add_f = retasks.Declarations.Where(s => s.span.Start < sep_index);
var rem_f = retasks.Declarations.Where(s => s.span.Start > sep_index);

var add_i = retasks.Includes.Where(s => s.span.Start < sep_index);
var rem_i = retasks.Includes.Where(s => s.span.Start > sep_index);

foreach (var item in add_i)
{
edit.Insert(parsed.LastInclude, lb + item.span.str(output));
}

DeclNode tree = new(Lexer.TType.Namespace)
{
LineBreak = lb
};

if (settings.MoveDecls)
{
tree.AddChildren(parsed.Declarations.Where(s => !rem_f.Contains(s)));
foreach (var task in parsed.Declarations)
edit.Delete(task.AsSpan());
}

tree.AddChildren(add_f);
string result = tree.ToString(std >= Standard.cpp17);
edit.Insert(parsed.LastInclude, lb + result);

foreach (var item in add_i)
{
edit.Insert(parsed.LastInclude, item.Project(output) + lb);
}

DeclNode tree = new(Lexer.TType.Namespace)
{
LineBreak = lb
};

if (!settings.MoveDecls)
foreach (var task in rem_f)
{
var found = parsed.Declarations.FindLast(s => s == task);
if (settings.MoveDecls)
{
tree.AddChildren(parsed.Declarations.Where(s => !rem_f.Contains(s)));
foreach (var task in parsed.Declarations)
edit.Delete(task.span);
}

if (!found.Valid()) continue;
edit.Delete(found.AsSpan());
parsed.Declarations.Remove(found);
tree.AddChildren(add_f);
string result = tree.ToString(std >= Standard.cpp17);
edit.Insert(parsed.LastInclude, result + lb);

}

foreach (var task in rem_i)
if (!settings.MoveDecls)
foreach (var task in rem_f)
{
var found = parsed.Includes.FindLast(s => s == task);

if (!found.Valid()) continue;
edit.Delete(found.AsSpan());
parsed.Includes.Remove(found);
var found = parsed.Declarations.FindLast(s => s == task);
edit.Delete(found.span);
parsed.Declarations.Remove(found);
}


edit.Apply();
output = part.Substring(endp);
foreach (var task in rem_i)
{
var found = parsed.Includes.FindLast(s => s == task);
edit.Delete(found.span);
parsed.Includes.Remove(found);
}

edit.Apply();
}
}
}
1 change: 1 addition & 0 deletions IncludeToolboxShared/IncludeToolboxShared.projitems
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@
<Compile Include="$(MSBuildThisFileDirectory)Util\Output.cs" />
<Compile Include="$(MSBuildThisFileDirectory)Util\ParseIncludes.cs" />
<Compile Include="$(MSBuildThisFileDirectory)Util\Parser.cs" />
<Compile Include="$(MSBuildThisFileDirectory)Util\ParserStructures.cs" />
<Compile Include="$(MSBuildThisFileDirectory)Util\RegexUtils.cs" />
<Compile Include="$(MSBuildThisFileDirectory)Util\Standard.cs" />
<Compile Include="$(MSBuildThisFileDirectory)Implementation\TrialAndErrorRemoval.cs" />
Expand Down
24 changes: 9 additions & 15 deletions IncludeToolboxShared/Util/EmptyNS.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,36 +7,30 @@ namespace IncludeToolbox
{
internal struct NSTracker
{
Stack<KeyValuePair<string_view, bool>> nsscan = new();
string_view ns = new();
bool empty = true;

public int Start { get => ns.begin; set => ns.begin = value; }
public bool Empty { get => empty; set => empty = value; }
Stack<KeyValuePair<int, bool>> nsscan = new();
public int Start { get; set ; } = 0;
public bool Empty { get; set; } = true;

public NSTracker()
{
}

public void Push()
{
nsscan.Push(new(ns, empty));
empty = true;
nsscan.Push(new(Start, Empty));
Empty = true;
}
public Span Pop(int end)
{
var v = nsscan.Pop();
empty = v.Value && empty;
var s = ns;
s.end = end + 1;
ns = v.Key;
return s.AsSpan();
Empty = v.Value && Empty;
return new(v.Key, end - v.Key);
}
public void Drop()
{
if (nsscan.Count > 0)
nsscan.Pop();
empty = false;
Empty = false;
}
}

Expand Down Expand Up @@ -140,7 +134,7 @@ public static Span[] ParseEmptyNamespaces(string text)
case TType.CloseBr:
if (tracker.Empty)
{
var c = tracker.Pop(tok.Position);
var c = tracker.Pop(tok.End);
namespaces.RemoveAll(x => c.Contains(x));
namespaces.Add(c);
}
Expand Down
13 changes: 7 additions & 6 deletions IncludeToolboxShared/Util/Lexer.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -201,10 +201,11 @@ private bool IsDelim(char c)
internal Token TryAssociateWith(ReadOnlySpan<char> tk, TType type)
{
int pos = FindDelim();
var sl = code.Slice(0, pos + 1);
var sl = code.Slice(0, pos);
bool a = sl.SequenceEqual(tk);

Token t = sl.StartsWith(tk) && IsDelim(sl[tk.Length]) ? (new(type, current_pos - 1)) : (new());
if (!t.valid()) return t;
if (!a) return new();
Token t = new(type, current_pos - 1);
RemovePrefix(pos);
return t;
}
Expand Down Expand Up @@ -332,18 +333,18 @@ private Token GetToken(bool expect_id, Desc desc = default)
}
break;
case ';':
tk = new Token(TType.Semicolon, Position);
tk = MakeValueToken(TType.Semicolon, 1);
break;
case '<':
case '"':
if (expect_id)
tk = GetHeader(c);
break;
case '{':
tk = new Token(TType.OpenBr, Position);
tk = MakeValueToken(TType.OpenBr, 1);
break;
case '}':
tk = new Token(TType.CloseBr, Position);
tk = MakeValueToken(TType.CloseBr, 1);
break;
default:
break;
Expand Down
85 changes: 1 addition & 84 deletions IncludeToolboxShared/Util/ParseIncludes.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,88 +9,6 @@

namespace IncludeToolbox
{
public enum NewlineChar
{
N,
CR,
LF,
CRLF
}

public struct IncludeLine
{
private string file = "";
public DelimiterMode delimiter = DelimiterMode.Unchanged;
public Span span = new();
public Span file_subspan = new();
public int line = 0;
public bool keep = false;
public NewlineChar newlineChar = NewlineChar.N;

public IncludeLine()
{}

public string Content => Valid ? file.Substring(1, file.Length - 2) : "";
public string FullFile { get => file; set => file = value; }
public bool Keep => keep;
public bool Valid => !string.IsNullOrEmpty(file);
public int NewlineLength => newlineChar switch { NewlineChar.N => 0, NewlineChar.CR => 2, _ => 1 };


public Span ReplaceSpan(int relative_pos) => new(relative_pos + span.Start, span.Length);
public Span ReplaceSpan(int relative_pos, int offset_end) =>
offset_end >= span.Length ? new() : new(relative_pos + span.Start, span.Length - offset_end);
public Span ReplaceSpanWithoutNewline(int relative_pos) =>
ReplaceSpan(relative_pos, NewlineLength);

public string Project(string over)
{
if (!Valid) return "";
var x = over.Substring(span.Start, span.Length);
return x.Remove(file_subspan.Start, file_subspan.Length).Insert(file_subspan.Start, FullFile);
}
public void SetFullContent(string content) { FullFile = content; }

public void SetFile(string val)
{
switch (delimiter)
{
case DelimiterMode.AngleBrackets:
FullFile = '<' + val + '>';
break;
case DelimiterMode.Quotes:
FullFile = '"' + val + '"';
break;
}
}
public void SetDelimiter(DelimiterMode delimiter)
{
if (this.delimiter == delimiter) return;
this.delimiter = delimiter;
SetFile(Content);
}
public void ToForward()
{
FullFile.Replace('\\', '/');
}
public void ToBackward()
{
FullFile.Replace('/', '\\');
}

public string Resolve(IEnumerable<string> includeDirectories)
{
foreach (string dir in includeDirectories)
{
string candidate = Path.Combine(dir, Content);
if (System.IO.File.Exists(candidate))
return Utils.GetExactPathName(candidate);
}

Output.WriteLine($"Unable to resolve include: '{Content}'");
return "";
}
}
public static partial class Parser
{
static readonly Regex pragma = new("(?:\\/\\*|\\/\\/)(?:\\s*IWYU\\s+pragma:\\s+keep)");// IWYU pragma: keep
Expand Down Expand Up @@ -130,8 +48,7 @@ public static IncludeLine[] ParseInclues(ReadOnlySpan<char> text, bool ignore_if
"\r\n" => NewlineChar.CRLF,
_ => NewlineChar.N
};
end_pos = tok.End;
xline.span = new(start_pos, end_pos - start_pos);
xline.span = new(start_pos, tok.End - start_pos);
lines.Add(xline);
xline = new IncludeLine();
}
Expand Down
Loading

0 comments on commit a43bb6e

Please sign in to comment.