Skip to content

Cleanup whitespace stripping and blockwise Placeables #25

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
wants to merge 4 commits into
base: master
Choose a base branch
from
Draft

Cleanup whitespace stripping and blockwise Placeables #25

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
53eec7c
Squish bugs in whitespace removal sequences
alerque Oct 10, 2019
a9868cb
Handle bad EOF senarios so Junk doesn't end up with our PEG grammar hack
alerque Oct 11, 2019
17fb72c
Handle inline vs. start of block placables differently on append()
alerque Oct 11, 2019
2a2df8c
Try another approach to blockwise Placeables
alerque Oct 12, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions fluent/parser.lua
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@ local ftl_grammar = epnf.define(function (_ENV)
VariantKey = P"[" * blank^-1 * (V"NumberLiteral" + V"Identifier") * blank^-1 * P"]"
NumberLiteral = Cg(C(P"-"^-1 * digits * (P"." * digits)^-1), "value")
local inline_placeable = P"{" * blank^-1 * (V"SelectExpression" + V"InlineExpression") * blank^-1 * P"}"
local block_placeable = blank_block * blank_inline^-1 * inline_placeable
local block_placeable = Cg(blank_block, "_blockwise") * blank_inline^-1 * Cg(inline_placeable, "expression")
local inline_text = text_char^1
local block_text = blank_block * blank_inline * indented_char * inline_text^-1
StringLiteral = P'"' * Cg(C(quoted_char^0), "value") * P'"'
Expand All @@ -66,7 +66,7 @@ local ftl_grammar = epnf.define(function (_ENV)
InlineExpression = V"StringLiteral" + V"NumberLiteral" + V"FunctionReference" + V"MessageReference" + V"TermReference" + V"VariableReference" + inline_placeable
_InlineExpression = V"StringLiteral" + V"NumberLiteral" + V"FunctionReference" + V"_TermReference" + V"VariableReference"
SelectExpression = V"_InlineExpression" * blank^-1 * P"->" * blank_inline^-1 * V"variant_list"
PatternElement = Cg(C(inline_text + block_text), "value") + Cg(inline_placeable + block_placeable, "expression")
PatternElement = Cg(C(inline_text + block_text), "value") + Cg(inline_placeable, "expression") + block_placeable
Pattern = V"PatternElement"^1
Attribute = line_end * blank^-1 * P"." * V"Identifier" * blank_inline^-1 * "=" * blank_inline^-1 * V"Pattern"
local junk_line = (1-P"\n"-P(nulleof))^0 * (P"\n" + P(nulleof))
Expand Down
52 changes: 39 additions & 13 deletions fluent/resource.lua
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,8 @@
local class = require("pl.class")
local tablex = require("pl.tablex")

local nulleof = "NULL\000"

local FTL = {}
local node_to_type

Expand All @@ -14,8 +16,8 @@ local FluentNode = class({
if key == "id" then
self.type = value
elseif key == "value" then
self[key] = string.gsub(string.gsub(value, "\r\n?","\n"), "^\n+ +", "")
elseif key ~= "pos" and key ~= "sigil" then
self[key] = string.gsub(value, "\r\n?","\n")
elseif key ~= "pos" and key ~= "sigil" and key ~= "_blockwise" then
self[key] = value
end
end
Expand Down Expand Up @@ -80,6 +82,10 @@ end

FTL.Junk = class({
_base = FluentNode,
_init = function (self, node, resource)
self:super(node, resource)
self.content = string.gsub(self.content, nulleof.."$", "")
end
})

FTL.Message = class({
Expand Down Expand Up @@ -136,27 +142,33 @@ FTL.Pattern = class({
return tablex.reduce(math.min, indents) or 0
end
local striplen = tablex.reduce(math.min, tablex.imap(mindent, self.elements)) or 0
local i, strippref = 1, "\n"
local i, common = 1, ""
while i <= striplen do
strippref = strippref .. " "
common = common .. " "
i = i + 1
end
local strip = function (node, key, len)
local strip = function (node, key, common)
if type(node.value) == "string" then
local value = node.value
if len >= 1 then
value = string.gsub(value, strippref, "\n\n")
end
value = key == 1 and string.gsub(value, "^[\n ]+", "") or value
-- Strip spaces from otherwise empty lines
local value = string.gsub(node.value, "\n +\n", "\n\n")
-- Strip leading whitespace from first element
value = key == 1 and string.gsub(value, "^\n+", "") or value
-- Strip trailing whitespace from last element
value = key == #self.elements and string.gsub(value, "[\n ]+$", "") or value
-- Strip lowest common donominator indent from all elements
if string.len(common) >= 1 then
value = string.gsub(value, "\n"..common, "\n")
end
if string.len(value) == 0 then
-- Delete from elements if result is completely empty
self.elements[key] = nil
else
-- Otherwise update value
self.elements[key].value = value
end
end
end
tablex.foreachi(self.elements, strip, striplen)
tablex.foreachi(self.elements, strip, common)
end,
__mul = function (self, node)
if node:is_a(FTL.Message) or node:is_a(FTL.Attribute) or node:is_a(FTL.Variant) then
Expand All @@ -179,7 +191,7 @@ FTL.TextElement = class({
end,
__add = function (self, node)
if self:is_a(node:is_a()) and self.appendable and node.appendable then
node.value = (node.value or "") .. "\n" .. (self.value or "")
node.value = (node.value or "") .. (self.value or "")
return node
end
end,
Expand All @@ -189,13 +201,27 @@ FTL.TextElement = class({
})

FTL.Placeable = class({
appendable = true,
_base = FluentNode,
_init = function (self, node, resource)
node.id = "Placeable"
D{nobe._blockwise}
if node._blockwise then getmetatable(self).blockwise = true end
-- if node.block_expression then
-- getmetatable(self).blockwise = true
-- node.expression = node.block_expression
-- node.block_expression = nil
-- end
node.expression = node_to_type(node.expression, resource)
self:super(node, resource)
end,
__add = function (self, node)
-- D{self.type, self.blockwise, self.expression, node.value}
if node:is_a(FTL.TextElement) and self.blockwise then
node.value = node.value .. "\n"
-- don't acknowledge this even happened so we go on to attach self as well as modify node
return nil
end
end,
__mod = function (self, node)
if node:is_a(FTL.Pattern) then
table.insert(node.elements, self)
Expand Down
10 changes: 5 additions & 5 deletions spec/fixtures_spec.lua
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,9 +29,9 @@ describe('upstream reference fixture', function ()
or fname:match("/crlf.ftl$")
or fname:match("/eof_comment.ftl$")
or fname:match("/eof_empty.ftl$")
-- or fname:match("/eof_id_equals.ftl$")
-- or fname:match("/eof_id.ftl$")
-- or fname:match("/eof_junk.ftl$")
or fname:match("/eof_id_equals.ftl$")
or fname:match("/eof_id.ftl$")
or fname:match("/eof_junk.ftl$")
or fname:match("/eof_value.ftl$")
-- or fname:match("/escaped_characters.ftl$")
or fname:match("/junk.ftl$")
Expand All @@ -40,14 +40,14 @@ describe('upstream reference fixture', function ()
-- or fname:match("/member_expressions.ftl$")
or fname:match("/messages.ftl$")
or fname:match("/mixed_entries.ftl$")
-- or fname:match("/multiline_values.ftl$")
or fname:match("/multiline_values.ftl$")
or fname:match("/numbers.ftl$")
or fname:match("/obsolete.ftl$")
or fname:match("/placeables.ftl$")
or fname:match("/reference_expressions.ftl$")
or fname:match("/select_expressions.ftl$")
or fname:match("/select_indent.ftl$")
-- or fname:match("/sparse_entries.ftl$")
or fname:match("/sparse_entries.ftl$")
or fname:match("/tab.ftl$")
-- or fname:match("/term_parameters.ftl$")
or fname:match("/terms.ftl$")
Expand Down