From c511660d3c7531ef7bace6f1ffdffa6815f1de75 Mon Sep 17 00:00:00 2001 From: Mike O'Regan Date: Fri, 26 Dec 2014 08:12:42 -0600 Subject: [PATCH] handle "1*{2}"; make bitwise handling more consistent --- lib/PPI/Token/Unknown.pm | 40 ++++++++++++++++++++++++---------------- t/ppi_token_unknown.t | 17 +---------------- 2 files changed, 25 insertions(+), 32 deletions(-) diff --git a/lib/PPI/Token/Unknown.pm b/lib/PPI/Token/Unknown.pm index c628a7a2..142b10ea 100644 --- a/lib/PPI/Token/Unknown.pm +++ b/lib/PPI/Token/Unknown.pm @@ -52,11 +52,17 @@ sub __TOKENIZER__on_char { # Now, we split on the different values of the current content if ( $c eq '*' ) { - if ( $char =~ /(?:(?!\d)\w|\:)/ ) { + # Is it a number? + if ( $char =~ /\d/ ) { + # bitwise operator + $t->{class} = $t->{token}->set_class( 'Operator' ); + return $t->_finalize_token->__TOKENIZER__on_char( $t ); + } + + if ( $char =~ /[\w:]/ ) { # Symbol (unless the thing before it is a number - my $tokens = $t->_previous_significant_tokens(1); - my $p0 = $tokens->[0]; - if ( $p0 and ! $p0->isa('PPI::Token::Number') ) { + my ( $prev ) = @{ $t->_previous_significant_tokens(1) }; + if ( $prev and ! $prev->isa('PPI::Token::Number') ) { $t->{class} = $t->{token}->set_class( 'Symbol' ); return 1; } @@ -69,10 +75,6 @@ sub __TOKENIZER__on_char { # control-character symbol (e.g. *{^_Foo}) $t->{class} = $t->{token}->set_class( 'Magic' ); return 1; - } else { - # Obvious GLOB cast - $t->{class} = $t->{token}->set_class( 'Cast' ); - return $t->_finalize_token->__TOKENIZER__on_char( $t ); } } @@ -150,7 +152,7 @@ sub __TOKENIZER__on_char { } elsif ( $c eq '%' ) { # Is it a number? if ( $char =~ /\d/ ) { - # This is %2 (modulus number) + # bitwise operator $t->{class} = $t->{token}->set_class( 'Operator' ); return $t->_finalize_token->__TOKENIZER__on_char( $t ); } @@ -161,10 +163,13 @@ sub __TOKENIZER__on_char { return 1; } - # Is it a symbol? if ( $char =~ /[\w:]/ ) { - $t->{class} = $t->{token}->set_class( 'Symbol' ); - return 1; + # Symbol (unless the thing before it is a number + my ( $prev ) = @{ $t->_previous_significant_tokens(1) }; + if ( $prev and ! $prev->isa('PPI::Token::Number') ) { + $t->{class} = $t->{token}->set_class( 'Symbol' ); + return 1; + } } if ( $char eq '{' ) { @@ -188,15 +193,18 @@ sub __TOKENIZER__on_char { } elsif ( $c eq '&' ) { # Is it a number? if ( $char =~ /\d/ ) { - # This is &2 (bitwise-and number) + # bitwise operator $t->{class} = $t->{token}->set_class( 'Operator' ); return $t->_finalize_token->__TOKENIZER__on_char( $t ); } - # Is it a symbol if ( $char =~ /[\w:]/ ) { - $t->{class} = $t->{token}->set_class( 'Symbol' ); - return 1; + # Symbol (unless the thing before it is a number + my ( $prev ) = @{ $t->_previous_significant_tokens(1) }; + if ( $prev and ! $prev->isa('PPI::Token::Number') ) { + $t->{class} = $t->{token}->set_class( 'Symbol' ); + return 1; + } } return $self->_as_cast_or_op($t) if $self->_is_cast_or_op($char); diff --git a/t/ppi_token_unknown.t b/t/ppi_token_unknown.t index 2367b9ee..1f81ed16 100644 --- a/t/ppi_token_unknown.t +++ b/t/ppi_token_unknown.t @@ -85,12 +85,9 @@ OPERATOR_CAST: { test_varying_whitespace( @number, @asterisk_op, @scalar ); test_varying_whitespace( @number, @asterisk_op, @list ); test_varying_whitespace( @number, @asterisk_op, @hash ); -{ - local %known_bad_seps = map { $_ => 1 } qw( null ); test_varying_whitespace( @number, @asterisk_op, @hashctor1 ); test_varying_whitespace( @number, @asterisk_op, @hashctor2 ); test_varying_whitespace( @number, @asterisk_op, @hashctor3 ); -} test_varying_whitespace( @number, @exp_op, @bareword ); test_varying_whitespace( @number, @exp_op, @hashctor3 ); # doesn't compile, but make sure ** is operator test_varying_whitespace( @number, @asteriskeq_op, @bareword ); @@ -149,10 +146,7 @@ OPERATOR_CAST: { my @single = ( "'3'", [ 'PPI::Token::Quote::Single' => "'3'", ] ); test_varying_whitespace( @single, @asterisk_op, @scalar ); -{ - local %known_bad_seps = map { $_ => 1 } qw( null ); test_varying_whitespace( @single, @asterisk_op, @hashctor3 ); -} test_varying_whitespace( @single, @percent_op, @scalar ); test_varying_whitespace( @single, @percent_op, @hashctor3 ); test_varying_whitespace( @single, @ampersand_op, @scalar ); @@ -160,10 +154,7 @@ OPERATOR_CAST: { my @double = ( '"3"', [ 'PPI::Token::Quote::Double' => '"3"', ] ); test_varying_whitespace( @double, @asterisk_op, @scalar ); -{ - local %known_bad_seps = map { $_ => 1 } qw( null ); test_varying_whitespace( @double, @asterisk_op, @hashctor3 ); -} test_varying_whitespace( @double, @percent_op, @scalar ); test_varying_whitespace( @double, @percent_op, @hashctor3 ); test_varying_whitespace( @double, @ampersand_op, @scalar ); @@ -293,10 +284,7 @@ OPERATOR_CAST: { ] ); test_varying_whitespace( @evalblock, @asterisk_op, @scalar ); -{ - local %known_bad_seps = map { $_ => 1 } qw( null ); - test_varying_whitespace( @evalblock, @asterisk_op, @hashctor3 ); -} + test_varying_whitespace( @double, @asterisk_op, @hashctor3 ); test_varying_whitespace( @evalblock, @percent_op, @scalar ); test_varying_whitespace( @evalblock, @percent_op, @hashctor3 ); test_varying_whitespace( @evalblock, @ampersand_op, @scalar ); @@ -310,10 +298,7 @@ OPERATOR_CAST: { ] ); test_varying_whitespace( @evalstring, @asterisk_op, @scalar ); -{ - local %known_bad_seps = map { $_ => 1 } qw( null ); test_varying_whitespace( @evalstring, @asterisk_op, @hashctor3 ); -} test_varying_whitespace( @evalstring, @percent_op, @scalar ); test_varying_whitespace( @evalstring, @percent_op, @hashctor3 ); test_varying_whitespace( @evalstring, @ampersand_op, @scalar );