Skip to content

Commit

Permalink
Add: Makes CreateToken public
Browse files Browse the repository at this point in the history
  • Loading branch information
@neolithos
neolithos committed Dec 28, 2018
1 parent f668c19 commit 62f986c
Showing 1 changed file with 88 additions and 20 deletions.
108 changes: 88 additions & 20 deletions NeoLua/LuaLexer.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -345,6 +345,7 @@ public interface ILuaLexer : IDisposable
{
/// <summary>Read next token.</summary>
void Next();

/// <summary>Look a head token, aka next token.</summary>
Token LookAhead { get; }
/// <summary>Look a head token, aka next, next token.</summary>
Expand Down Expand Up @@ -592,12 +593,32 @@ public void Eat()
public Token CreateToken(LuaToken kind)
{
var endPosition = CurrentPosition;
var tok = new Token(kind, CurValue, startPosition, endPosition);
var tok = LuaLexer.CreateToken(kind, CurValue, startPosition, endPosition);
startPosition = endPosition;
currentStringBuilder.Clear();
return tok;
} // func CreateToken

/// <summary>Create a new token.</summary>
/// <param name="kind">Token type</param>
/// <param name="value"></param>
/// <returns>Token</returns>
public Token CreateTokenAtStart(LuaToken kind, string value = null)
{
var pos = StartPosition;
return LuaLexer.CreateToken(kind, value, pos, pos);
} // func CreateTokenAtStart

/// <summary>Create a new token.</summary>
/// <param name="kind">Token type</param>
/// <param name="value"></param>
/// <returns>Token</returns>
public Token CreateTokenAtEnd(LuaToken kind, string value = null)
{
var pos = CurrentPosition;
return LuaLexer.CreateToken(kind, value, pos, pos);
} // func CreateTokenAtEnd

/// <summary></summary>
/// <param name="c"></param>
/// <param name="idx"></param>
Expand Down Expand Up @@ -1465,23 +1486,19 @@ private static IEnumerable<Token> CreateHtmlTokenStream(LuaCharLexer chars, bool
foreach (var preamble in scriptPreamble)
yield return preamble;
}
var pos = chars.StartPosition;
yield return new Token(LuaToken.Identifier, "print", pos, pos);
yield return chars.CreateTokenAtStart(LuaToken.Identifier, "print");
yield return chars.CreateToken(LuaToken.String);
pos = chars.CurrentPosition;
yield return new Token(LuaToken.Semicolon, String.Empty, pos, pos);
yield return chars.CreateTokenAtStart(LuaToken.Semicolon);
}

codeEmitted = true;
chars.Next(2);
if (chars.Cur == '=')
{
var pos = chars.CurrentPosition;

chars.Next();

yield return new Token(LuaToken.Identifier, "printValue", pos, pos);
yield return new Token(LuaToken.BracketOpen, String.Empty, pos, pos);
yield return chars.CreateTokenAtEnd(LuaToken.Identifier, "printValue");
yield return chars.CreateTokenAtEnd(LuaToken.BracketOpen);

state = 2;
}
Expand Down Expand Up @@ -1539,8 +1556,7 @@ private static IEnumerable<Token> CreateHtmlTokenStream(LuaCharLexer chars, bool
{
if (chars.HasCurValue)
{
var pos = chars.StartPosition;
yield return new Token(LuaToken.Comma, String.Empty, pos, pos);
yield return chars.CreateTokenAtStart(LuaToken.Comma);
yield return chars.CreateToken(LuaToken.String);
}
goto case 4;
Expand All @@ -1553,8 +1569,8 @@ private static IEnumerable<Token> CreateHtmlTokenStream(LuaCharLexer chars, bool
var startAt = chars.CurrentPosition;
chars.Next(2);
var endAt = chars.CurrentPosition;
yield return new Token(LuaToken.BracketClose, String.Empty, startAt, endAt);
yield return new Token(LuaToken.Semicolon, String.Empty, startAt, endAt);
yield return CreateToken(LuaToken.BracketClose, String.Empty, startAt, endAt);
yield return CreateToken(LuaToken.Semicolon, String.Empty, startAt, endAt);
state = 0;
}
break;
Expand All @@ -1579,19 +1595,16 @@ private static IEnumerable<Token> CreateHtmlTokenStream(LuaCharLexer chars, bool
{
if (chars.HasCurValue) // we have something in the buffer
{
var pos = chars.StartPosition;
yield return new Token(LuaToken.Identifier, "print", pos, pos);
yield return chars.CreateTokenAtStart(LuaToken.Identifier, "print");
yield return chars.CreateToken(LuaToken.String);
pos = chars.CurrentPosition;
yield return new Token(LuaToken.Semicolon, String.Empty, pos, pos);
yield return chars.CreateTokenAtStart(LuaToken.Semicolon);
}
}
else // no code emitted --> create a return statement
{
var pos = chars.StartPosition;
yield return new Token(LuaToken.KwReturn, "return", pos, pos);
yield return chars.CreateTokenAtStart(LuaToken.KwReturn);
yield return chars.CreateToken(LuaToken.String);
yield return new Token(LuaToken.Semicolon, String.Empty, pos, pos);
yield return chars.CreateTokenAtStart(LuaToken.Semicolon);
}
yield return chars.CreateToken(LuaToken.Eof);
} // func CreateHtmlTokenStream
Expand Down Expand Up @@ -1621,12 +1634,58 @@ public static ILuaLexer CreateHtml(LuaCharLexer charStream, bool enforceCode)
public static bool IsIdentifierChar(char c)
=> Char.IsLetterOrDigit(c) || c == '_';

/// <summary>Is the given identifier a keyword.</summary>
/// <param name="token"></param>
/// <returns></returns>
public static bool IsKeyWord(LuaToken token)
{
switch(token)
{
case LuaToken.KwAnd:
case LuaToken.KwBreak:
case LuaToken.KwCast:
case LuaToken.KwConst:
case LuaToken.KwDo:
case LuaToken.KwElse:
case LuaToken.KwElseif:
case LuaToken.KwEnd:
case LuaToken.KwFalse:
case LuaToken.KwFor:
case LuaToken.KwForEach:
case LuaToken.KwFunction:
case LuaToken.KwGoto:
case LuaToken.KwIf:
case LuaToken.KwIn:
case LuaToken.KwLocal:
case LuaToken.KwNil:
case LuaToken.KwNot:
case LuaToken.KwOr:
case LuaToken.KwRepeat:
case LuaToken.KwReturn:
case LuaToken.KwThen:
case LuaToken.KwTrue:
case LuaToken.KwUntil:
case LuaToken.KwWhile:
return true;
default:
return false;
}
} // func IsKeyWord

/// <summary>Is the given identifier a keyword.</summary>
/// <param name="member"></param>
/// <returns></returns>
public static bool IsKeyWord(string member)
=> Array.BinarySearch(keywords.Value, member) >= 0;

/// <summary></summary>
/// <param name="token"></param>
/// <returns></returns>
public static string GetDefaultTokenValue(LuaToken token)
=> IsKeyWord(token)
? GetTokenName(token)
: String.Empty;

/// <summary>Resolves the name of the token.</summary>
/// <param name="type"></param>
/// <returns></returns>
Expand All @@ -1646,6 +1705,15 @@ public static string GetTokenName(LuaToken type)

return name;
} // func GetTokenName

/// <summary>Create a new token.</summary>
/// <param name="kind"></param>
/// <param name="value"></param>
/// <param name="startAt"></param>
/// <param name="endAt"></param>
/// <returns></returns>
public static Token CreateToken(LuaToken kind, string value, Position startAt, Position endAt)
=> new Token(kind, value ?? GetDefaultTokenValue(kind), startAt, endAt);
} // class LuaLexer

#endregion
Expand Down

0 comments on commit 62f986c

Please sign in to comment.