From 8e122aec5aa6b064b5722307fa23014697aed38e Mon Sep 17 00:00:00 2001 From: brigadier Date: Sun, 9 Oct 2016 00:59:10 +0300 Subject: [PATCH 1/3] allow unicode and everything in values --- src/eini_lexer.xrl | 5 +++-- test/eini_tests.erl | 37 ++++++++++++++++++++++++++++++------- test/test.txt | 2 ++ 3 files changed, 35 insertions(+), 9 deletions(-) create mode 100644 test/test.txt diff --git a/src/eini_lexer.xrl b/src/eini_lexer.xrl index d0b5e50..d66a249 100644 --- a/src/eini_lexer.xrl +++ b/src/eini_lexer.xrl @@ -22,12 +22,13 @@ Definitions. %% Characters for keys K = [a-zA-Z0-9_\-\.]+ -%% Characters for values, printable except =, [ and ] + %% \x3b : $; %% \x3d : $= %% \x5b : $[ %% \x5d : $] -V = [\x21-\x3a\x3c\x3e-\x5a\x5c\x5e-\x7e]+ +V = [^\x3b\x3d\x5b\x5d\x0d\x0a\s\t]+ + %% spaces and breaks S = [\s\t] diff --git a/test/eini_tests.erl b/test/eini_tests.erl index d1df935..524abeb 100644 --- a/test/eini_tests.erl +++ b/test/eini_tests.erl @@ -205,7 +205,7 @@ one_section_title_and_one_prop_test_() -> )), %% Multi blank lines after a prop ?_assertEqual({ok, [ - {title, + {title, [{key1, <<"value1">>}]} ]}, parse( @@ -407,9 +407,9 @@ lex_error_title_test_() -> fun teardown/ 1, [ %% vertical tab in section title - ?_assertMatch({error, {illegal_character, 1, _Reason}}, + ?_assertMatch({error, {syntax_error, 1, _Reason}}, parse("[ti\vtle]")), - ?_assertMatch({error, {illegal_character, 3, _Reason}}, + ?_assertMatch({error, {syntax_error, 3, _Reason}}, parse( "[titleA]\n" "keyA1=valueA1\n" @@ -417,7 +417,7 @@ lex_error_title_test_() -> "keyB1=valueB1\n" )) ]}. - + syntax_error_title_test_() -> %% TODO: Erlang 17 lost the ability to correctly report line numbers from errors. %% Put the numbers back in some day when the fix is released @@ -464,7 +464,7 @@ syntax_error_title_test_() -> parse("[title]\n" " ;comment\n")) ]}. - + syntax_error_property_test_() -> {setup, fun setup/0, @@ -481,7 +481,7 @@ syntax_error_property_test_() -> parse("[title]\n" "key;comment=value\n")) ]}. - + dup_title_test_() -> {setup, fun setup/0, @@ -495,7 +495,7 @@ dup_title_test_() -> "keyB1=valueB1\n" )) ]}. - + dup_key_test_() -> {setup, fun setup/0, @@ -575,3 +575,26 @@ is_section_test_() -> end} ] }. + +unicode_test_() -> + {setup, + fun setup/0, + fun teardown/1, + [ + ?_assertEqual({ok, [ + {title, [{key1, <<"юникод"/utf8>>}]} + ]}, + parse( + "[title]\n" + "key1= \xD1\x8E\xD0\xBD\xD0\xB8\xD0\xBA\xD0\xBE\xD0\xB4 \n" + )), + ?_assertEqual({ok, [ + {title, [{key1, <<"юникод"/utf8>>}]} + ]}, + parse( + begin + {ok, Data} = file:read_file("test/test.txt"), + Data + end + )) + ]}. diff --git a/test/test.txt b/test/test.txt new file mode 100644 index 0000000..74131a4 --- /dev/null +++ b/test/test.txt @@ -0,0 +1,2 @@ +[title] +key1=юникод From 362c27003fe802efcced7a3f850a0df51fc05241 Mon Sep 17 00:00:00 2001 From: Nathaniel Waisbrot Date: Mon, 10 Oct 2016 09:45:46 -0400 Subject: [PATCH 2/3] paste the results of file:read_file/1 directly into the code --- test/eini_tests.erl | 7 ++----- test/test.txt | 2 -- 2 files changed, 2 insertions(+), 7 deletions(-) delete mode 100644 test/test.txt diff --git a/test/eini_tests.erl b/test/eini_tests.erl index 524abeb..d5e69e3 100644 --- a/test/eini_tests.erl +++ b/test/eini_tests.erl @@ -591,10 +591,7 @@ unicode_test_() -> ?_assertEqual({ok, [ {title, [{key1, <<"юникод"/utf8>>}]} ]}, - parse( - begin - {ok, Data} = file:read_file("test/test.txt"), - Data - end + parse( + <<91,116,105,116,108,101,93,10,107,101,121,49,61,209,142,208,189,208,184,208,186,208,190,208,180,10>> )) ]}. diff --git a/test/test.txt b/test/test.txt deleted file mode 100644 index 74131a4..0000000 --- a/test/test.txt +++ /dev/null @@ -1,2 +0,0 @@ -[title] -key1=юникод From aa578ae55faa1c2784d756bd440343d85179fb03 Mon Sep 17 00:00:00 2001 From: Evgeny M Date: Mon, 10 Oct 2016 18:17:38 +0400 Subject: [PATCH 3/3] make the tests compatible with R16B Encoding of source files in R17+ is utf8 by default, in R16 ascii, so utf8 strings removed from the tests altogether for compatiblity. --- test/eini_tests.erl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/eini_tests.erl b/test/eini_tests.erl index d5e69e3..b6e5fa5 100644 --- a/test/eini_tests.erl +++ b/test/eini_tests.erl @@ -582,14 +582,14 @@ unicode_test_() -> fun teardown/1, [ ?_assertEqual({ok, [ - {title, [{key1, <<"юникод"/utf8>>}]} + {title, [{key1, <<"\xD1\x8E\xD0\xBD\xD0\xB8\xD0\xBA\xD0\xBE\xD0\xB4">>}]} ]}, parse( "[title]\n" "key1= \xD1\x8E\xD0\xBD\xD0\xB8\xD0\xBA\xD0\xBE\xD0\xB4 \n" )), ?_assertEqual({ok, [ - {title, [{key1, <<"юникод"/utf8>>}]} + {title, [{key1, <<"\xD1\x8E\xD0\xBD\xD0\xB8\xD0\xBA\xD0\xBE\xD0\xB4">>}]} ]}, parse( <<91,116,105,116,108,101,93,10,107,101,121,49,61,209,142,208,189,208,184,208,186,208,190,208,180,10>>