handle "1*{2}"; make bitwise handling more consistent

Author: moregan

lib/PPI/Token/Unknown.pm

@@ -52,11 +52,17 @@ sub __TOKENIZER__on_char {
 
 	# Now, we split on the different values of the current content
 	if ( $c eq '*' ) {
-		if ( $char =~ /(?:(?!\d)\w|\:)/ ) {
+		# Is it a number?
+		if ( $char =~ /\d/ ) {
+			# bitwise operator
+			$t->{class} = $t->{token}->set_class( 'Operator' );
+			return $t->_finalize_token->__TOKENIZER__on_char( $t );
+		}
+
+		if ( $char =~ /[\w:]/ ) {
 			# Symbol (unless the thing before it is a number
-			my $tokens = $t->_previous_significant_tokens(1);
-			my $p0     = $tokens->[0];
-			if ( $p0 and ! $p0->isa('PPI::Token::Number') ) {
+			my ( $prev ) = @{ $t->_previous_significant_tokens(1) };
+			if ( $prev and ! $prev->isa('PPI::Token::Number') ) {
 				$t->{class} = $t->{token}->set_class( 'Symbol' );
 				return 1;
 			}
@@ -69,10 +75,6 @@ sub __TOKENIZER__on_char {
 				# control-character symbol (e.g. *{^_Foo})
 				$t->{class} = $t->{token}->set_class( 'Magic' );
 				return 1;
-			} else {
-				# Obvious GLOB cast
-				$t->{class} = $t->{token}->set_class( 'Cast' );
-				return $t->_finalize_token->__TOKENIZER__on_char( $t );
 			}
 		}
 
@@ -150,7 +152,7 @@ sub __TOKENIZER__on_char {
 	} elsif ( $c eq '%' ) {
 		# Is it a number?
 		if ( $char =~ /\d/ ) {
-			# This is %2 (modulus number)
+			# bitwise operator
 			$t->{class} = $t->{token}->set_class( 'Operator' );
 			return $t->_finalize_token->__TOKENIZER__on_char( $t );
 		}
@@ -161,10 +163,13 @@ sub __TOKENIZER__on_char {
 			return 1;
 		}
 
-		# Is it a symbol?
 		if ( $char =~ /[\w:]/ ) {
-			$t->{class} = $t->{token}->set_class( 'Symbol' );
-			return 1;
+			# Symbol (unless the thing before it is a number
+			my ( $prev ) = @{ $t->_previous_significant_tokens(1) };
+			if ( $prev and ! $prev->isa('PPI::Token::Number') ) {
+				$t->{class} = $t->{token}->set_class( 'Symbol' );
+				return 1;
+			}
 		}
 
 		if ( $char eq '{' ) {
@@ -188,15 +193,18 @@ sub __TOKENIZER__on_char {
 	} elsif ( $c eq '&' ) {
 		# Is it a number?
 		if ( $char =~ /\d/ ) {
-			# This is &2 (bitwise-and number)
+			# bitwise operator
 			$t->{class} = $t->{token}->set_class( 'Operator' );
 			return $t->_finalize_token->__TOKENIZER__on_char( $t );
 		}
 
-		# Is it a symbol
 		if ( $char =~ /[\w:]/ ) {
-			$t->{class} = $t->{token}->set_class( 'Symbol' );
-			return 1;
+			# Symbol (unless the thing before it is a number
+			my ( $prev ) = @{ $t->_previous_significant_tokens(1) };
+			if ( $prev and ! $prev->isa('PPI::Token::Number') ) {
+				$t->{class} = $t->{token}->set_class( 'Symbol' );
+				return 1;
+			}
 		}
 
 		return $self->_as_cast_or_op($t) if $self->_is_cast_or_op($char);

t/ppi_token_unknown.t

@@ -85,12 +85,9 @@ OPERATOR_CAST: {
 	test_varying_whitespace( @number, @asterisk_op, @scalar );
 	test_varying_whitespace( @number, @asterisk_op, @list );
 	test_varying_whitespace( @number, @asterisk_op, @hash );
-{
-	local %known_bad_seps = map { $_ => 1 } qw( null );
 	test_varying_whitespace( @number, @asterisk_op, @hashctor1 );
 	test_varying_whitespace( @number, @asterisk_op, @hashctor2 );
 	test_varying_whitespace( @number, @asterisk_op, @hashctor3 );
-}
 	test_varying_whitespace( @number, @exp_op, @bareword );
 	test_varying_whitespace( @number, @exp_op, @hashctor3 );  # doesn't compile, but make sure ** is operator
 	test_varying_whitespace( @number, @asteriskeq_op, @bareword );
@@ -149,21 +146,15 @@ OPERATOR_CAST: {
 
 	my @single = ( "'3'", [ 'PPI::Token::Quote::Single' => "'3'", ] );
 	test_varying_whitespace( @single, @asterisk_op, @scalar );
-{
-	local %known_bad_seps = map { $_ => 1 } qw( null );
 	test_varying_whitespace( @single, @asterisk_op, @hashctor3 );
-}
 	test_varying_whitespace( @single, @percent_op, @scalar );
 	test_varying_whitespace( @single, @percent_op, @hashctor3 );
 	test_varying_whitespace( @single, @ampersand_op, @scalar );
 	test_varying_whitespace( @single, @ampersand_op, @hashctor3 );
 
 	my @double = ( '"3"', [ 'PPI::Token::Quote::Double' => '"3"', ] );
 	test_varying_whitespace( @double, @asterisk_op, @scalar );
-{
-	local %known_bad_seps = map { $_ => 1 } qw( null );
 	test_varying_whitespace( @double, @asterisk_op, @hashctor3 );
-}
 	test_varying_whitespace( @double, @percent_op, @scalar );
 	test_varying_whitespace( @double, @percent_op, @hashctor3 );
 	test_varying_whitespace( @double, @ampersand_op, @scalar );
@@ -293,10 +284,7 @@ OPERATOR_CAST: {
 		]
 	);
 	test_varying_whitespace( @evalblock, @asterisk_op, @scalar );
-{
-	local %known_bad_seps = map { $_ => 1 } qw( null );
-	test_varying_whitespace( @evalblock, @asterisk_op, @hashctor3 );
-}
+	test_varying_whitespace( @double, @asterisk_op, @hashctor3 );
 	test_varying_whitespace( @evalblock, @percent_op, @scalar );
 	test_varying_whitespace( @evalblock, @percent_op, @hashctor3 );
 	test_varying_whitespace( @evalblock, @ampersand_op, @scalar );
@@ -310,10 +298,7 @@ OPERATOR_CAST: {
 		]
 	);
 	test_varying_whitespace( @evalstring, @asterisk_op, @scalar );
-{
-	local %known_bad_seps = map { $_ => 1 } qw( null );
 	test_varying_whitespace( @evalstring, @asterisk_op, @hashctor3 );
-}
 	test_varying_whitespace( @evalstring, @percent_op, @scalar );
 	test_varying_whitespace( @evalstring, @percent_op, @hashctor3 );
 	test_varying_whitespace( @evalstring, @ampersand_op, @scalar );