Skip to content

Commit

Permalink
handle "1*{2}"; make bitwise handling more consistent
Browse files Browse the repository at this point in the history
  • Loading branch information
@moregan @wchristian
moregan authored and wchristian committed May 11, 2017
1 parent bb03bcd commit c511660
Show file tree
Hide file tree
Showing 2 changed files with 25 additions and 32 deletions.
40 changes: 24 additions & 16 deletions lib/PPI/Token/Unknown.pm
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,11 +52,17 @@ sub __TOKENIZER__on_char {

# Now, we split on the different values of the current content
if ( $c eq '*' ) {
if ( $char =~ /(?:(?!\d)\w|\:)/ ) {
# Is it a number?
if ( $char =~ /\d/ ) {
# bitwise operator
$t->{class} = $t->{token}->set_class( 'Operator' );
return $t->_finalize_token->__TOKENIZER__on_char( $t );
}

if ( $char =~ /[\w:]/ ) {
# Symbol (unless the thing before it is a number
my $tokens = $t->_previous_significant_tokens(1);
my $p0 = $tokens->[0];
if ( $p0 and ! $p0->isa('PPI::Token::Number') ) {
my ( $prev ) = @{ $t->_previous_significant_tokens(1) };
if ( $prev and ! $prev->isa('PPI::Token::Number') ) {
$t->{class} = $t->{token}->set_class( 'Symbol' );
return 1;
}
Expand All @@ -69,10 +75,6 @@ sub __TOKENIZER__on_char {
# control-character symbol (e.g. *{^_Foo})
$t->{class} = $t->{token}->set_class( 'Magic' );
return 1;
} else {
# Obvious GLOB cast
$t->{class} = $t->{token}->set_class( 'Cast' );
return $t->_finalize_token->__TOKENIZER__on_char( $t );
}
}

Expand Down Expand Up @@ -150,7 +152,7 @@ sub __TOKENIZER__on_char {
} elsif ( $c eq '%' ) {
# Is it a number?
if ( $char =~ /\d/ ) {
# This is %2 (modulus number)
# bitwise operator
$t->{class} = $t->{token}->set_class( 'Operator' );
return $t->_finalize_token->__TOKENIZER__on_char( $t );
}
Expand All @@ -161,10 +163,13 @@ sub __TOKENIZER__on_char {
return 1;
}

# Is it a symbol?
if ( $char =~ /[\w:]/ ) {
$t->{class} = $t->{token}->set_class( 'Symbol' );
return 1;
# Symbol (unless the thing before it is a number
my ( $prev ) = @{ $t->_previous_significant_tokens(1) };
if ( $prev and ! $prev->isa('PPI::Token::Number') ) {
$t->{class} = $t->{token}->set_class( 'Symbol' );
return 1;
}
}

if ( $char eq '{' ) {
Expand All @@ -188,15 +193,18 @@ sub __TOKENIZER__on_char {
} elsif ( $c eq '&' ) {
# Is it a number?
if ( $char =~ /\d/ ) {
# This is &2 (bitwise-and number)
# bitwise operator
$t->{class} = $t->{token}->set_class( 'Operator' );
return $t->_finalize_token->__TOKENIZER__on_char( $t );
}

# Is it a symbol
if ( $char =~ /[\w:]/ ) {
$t->{class} = $t->{token}->set_class( 'Symbol' );
return 1;
# Symbol (unless the thing before it is a number
my ( $prev ) = @{ $t->_previous_significant_tokens(1) };
if ( $prev and ! $prev->isa('PPI::Token::Number') ) {
$t->{class} = $t->{token}->set_class( 'Symbol' );
return 1;
}
}

return $self->_as_cast_or_op($t) if $self->_is_cast_or_op($char);
Expand Down
17 changes: 1 addition & 16 deletions t/ppi_token_unknown.t
View file
Original file line number Diff line number Diff line change
Expand Up @@ -85,12 +85,9 @@ OPERATOR_CAST: {
test_varying_whitespace( @number, @asterisk_op, @scalar );
test_varying_whitespace( @number, @asterisk_op, @list );
test_varying_whitespace( @number, @asterisk_op, @hash );
{
local %known_bad_seps = map { $_ => 1 } qw( null );
test_varying_whitespace( @number, @asterisk_op, @hashctor1 );
test_varying_whitespace( @number, @asterisk_op, @hashctor2 );
test_varying_whitespace( @number, @asterisk_op, @hashctor3 );
}
test_varying_whitespace( @number, @exp_op, @bareword );
test_varying_whitespace( @number, @exp_op, @hashctor3 ); # doesn't compile, but make sure ** is operator
test_varying_whitespace( @number, @asteriskeq_op, @bareword );
Expand Down Expand Up @@ -149,21 +146,15 @@ OPERATOR_CAST: {

my @single = ( "'3'", [ 'PPI::Token::Quote::Single' => "'3'", ] );
test_varying_whitespace( @single, @asterisk_op, @scalar );
{
local %known_bad_seps = map { $_ => 1 } qw( null );
test_varying_whitespace( @single, @asterisk_op, @hashctor3 );
}
test_varying_whitespace( @single, @percent_op, @scalar );
test_varying_whitespace( @single, @percent_op, @hashctor3 );
test_varying_whitespace( @single, @ampersand_op, @scalar );
test_varying_whitespace( @single, @ampersand_op, @hashctor3 );

my @double = ( '"3"', [ 'PPI::Token::Quote::Double' => '"3"', ] );
test_varying_whitespace( @double, @asterisk_op, @scalar );
{
local %known_bad_seps = map { $_ => 1 } qw( null );
test_varying_whitespace( @double, @asterisk_op, @hashctor3 );
}
test_varying_whitespace( @double, @percent_op, @scalar );
test_varying_whitespace( @double, @percent_op, @hashctor3 );
test_varying_whitespace( @double, @ampersand_op, @scalar );
Expand Down Expand Up @@ -293,10 +284,7 @@ OPERATOR_CAST: {
]
);
test_varying_whitespace( @evalblock, @asterisk_op, @scalar );
{
local %known_bad_seps = map { $_ => 1 } qw( null );
test_varying_whitespace( @evalblock, @asterisk_op, @hashctor3 );
}
test_varying_whitespace( @double, @asterisk_op, @hashctor3 );
test_varying_whitespace( @evalblock, @percent_op, @scalar );
test_varying_whitespace( @evalblock, @percent_op, @hashctor3 );
test_varying_whitespace( @evalblock, @ampersand_op, @scalar );
Expand All @@ -310,10 +298,7 @@ OPERATOR_CAST: {
]
);
test_varying_whitespace( @evalstring, @asterisk_op, @scalar );
{
local %known_bad_seps = map { $_ => 1 } qw( null );
test_varying_whitespace( @evalstring, @asterisk_op, @hashctor3 );
}
test_varying_whitespace( @evalstring, @percent_op, @scalar );
test_varying_whitespace( @evalstring, @percent_op, @hashctor3 );
test_varying_whitespace( @evalstring, @ampersand_op, @scalar );
Expand Down

0 comments on commit c511660

Please sign in to comment.