Lexer.php

<?php

namespace Phug;

use Phug\Lexer\Event\EndLexEvent;
use Phug\Lexer\Event\LexEvent;
use Phug\Lexer\Event\TokenEvent;
use Phug\Lexer\HandleTokenInterface;
use Phug\Lexer\Partial\DumpTokenTrait;
use Phug\Lexer\Partial\StateTrait;
use Phug\Lexer\Scanner\TextScanner;
use Phug\Lexer\ScannerInterface;
use Phug\Lexer\State;
use Phug\Lexer\TokenInterface;
use Phug\Util\Collection;
use Phug\Util\ModuleContainerInterface;
use Phug\Util\Partial\ModuleContainerTrait;

/**
 * Performs lexical analysis and provides a token generator.
 *
 * Tokens are defined as single units of code
 * (e.g. tag, class, id, attributeStart, attribute, attributeEnd)
 *
 * These will run through the parser and be converted to an AST
 *
 * The lexer works sequentially, `lex()` will return a generator and
 * you can read that generator in any manner you like.
 * The generator will produce valid tokens until the end of the passed
 * input.
 *
 * Usage example:
 * <code>
 *
 *     use Phug\Lexer;
 *
 *     $lexer = new Lexer();
 *
 *     foreach ($lexer->lex($pugInput) as $token)
 *          var_dump($token);
 *
 * </code>
 */
class Lexer implements LexerInterface, ModuleContainerInterface
{
    use ModuleContainerTrait;
    use StateTrait;
    use DumpTokenTrait;

    const INDENT_SPACE = ' ';
    const INDENT_TAB = "\t";
    const DEFAULT_TAB_WIDTH = 4;

    /**
     * @var TokenInterface
     */
    private $lastToken;

    /**
     * @var TokenInterface|null
     */
    private $previousToken;

    /**
     * Creates a new lexer instance.
     *
     * The options should be an associative array
     *
     * Valid options are:
     *
     * lexer_state_class_name:  The class of the lexer state to use
     * level:           The internal indentation level to start on
     * indent_style:     The indentation character (auto-detected)
     * indent_width:     How often to repeat indentStyle (auto-detected)
     * encoding:        The encoding when working with mb_*-functions (auto-detected)
     * scanners:        An array of scans that will be performed
     *
     * Add a new scan to 'scans' to extend the lexer.
     * Notice that the parser needs to be able to handle newly introduced tokens provided by scanners.
     *
     * @param array|null $options the options passed to the lexer instance
     *
     * @throws \Exception
     */
    public function __construct($options = null)
    {
        $this->setOptionsDefaults($options ?: [], [
            'lexer_state_class_name'   => State::class,
            'level'                    => 0,
            'indent_style'             => null,
            'indent_width'             => null,
            'allow_mixed_indent'       => true,
            'multiline_interpolation'  => true,
            'multiline_markup_enabled' => true,
            'encoding'                 => null,
            'lexer_modules'            => [],
            'keywords'                 => [],
            'scanners'                 => Scanners::getList(),
            'mixin_keyword'            => 'mixin',
            'mixin_call_keyword'       => '\\+',

            //Events
            'on_lex'                   => null,
            'on_lex_end'               => null,
            'on_token'                 => null,
        ]);

        $this->state = null;

        $this->updateOptions();
    }

    /**
     * @return TokenInterface|null
     */
    public function getPreviousToken()
    {
        return $this->previousToken;
    }

    /**
     * Synchronize the lexer to the new options values.
     */
    public function updateOptions()
    {
        if ($onLex = $this->getOption('on_lex')) {
            $this->attach(LexerEvent::LEX, $onLex);
        }

        if ($onLex = $this->getOption('on_lex_end')) {
            $this->attach(LexerEvent::END_LEX, $onLex);
        }

        if ($onToken = $this->getOption('on_token')) {
            $this->attach(LexerEvent::TOKEN, $onToken);
        }

        $this->addModules($this->getOption('lexer_modules'));
    }

    /**
     * Returns the current scanners registered for the lexing process.
     *
     * @return ScannerInterface[]
     */
    public function getScanners()
    {
        return $this->getOption('scanners');
    }

    /**
     * Adds a new scanner class to use in the lexing process at the top of scanning order.
     *
     * The scanner class needs to extend Phug\Lexer\ScannerInterface. It can be the class name itself
     * or an instance of it.
     *
     * @param string                  $name
     * @param ScannerInterface|string $scanner
     *
     * @return $this
     */
    public function prependScanner($name, $scanner)
    {
        $this->filterScanner($scanner);

        $scanners = [
            $name => $scanner,
        ];

        foreach ($this->getScanners() as $scannerName => $classNameOrInstance) {
            if ($scannerName !== $name) {
                $scanners[$scannerName] = $classNameOrInstance;
            }
        }

        return $this->setOption('scanners', $scanners);
    }

    /**
     * Adds a new scanner class to use in the lexing process.
     *
     * The scanner class needs to extend Phug\Lexer\ScannerInterface. It can be the class name itself
     * or an instance of it.
     *
     * @param string                  $name
     * @param ScannerInterface|string $scanner
     *
     * @return $this
     */
    public function addScanner($name, $scanner)
    {
        $this->filterScanner($scanner);

        return $this->setOption(['scanners', $name], $scanner);
    }

    /**
     * Returns a generator that will lex the passed input sequentially.
     *
     * If you don't move the generator, the lexer does nothing.
     * Only as soon as you iterate the generator or call `next()`/`current()` on it
     * the lexer will start its work and spit out tokens sequentially.
     * This approach requires less memory during the lexing process.
     *
     * The returned tokens are required to be `Phug\Lexer\TokenInterface` instances.
     *
     * @param string $input the pug-string to lex into tokens.
     * @param null   $path
     *
     * @return iterable a generator that can be iterated sequentially
     */
    public function lex($input, $path = null)
    {
        $stateClassName = $this->getOption('lexer_state_class_name');
        $lexEvent = new LexEvent($input, $path, $stateClassName, [
            'encoding'                 => $this->getOption('encoding'),
            'indent_style'             => $this->getOption('indent_style'),
            'indent_width'             => $this->getOption('indent_width'),
            'allow_mixed_indent'       => $this->getOption('allow_mixed_indent'),
            'multiline_interpolation'  => $this->getOption('multiline_interpolation'),
            'multiline_markup_enabled' => $this->getOption('multiline_markup_enabled'),
            'level'                    => $this->getOption('level'),
            'mixin_keyword'            => $this->getRegExpOption('mixin_keyword'),
            'mixin_call_keyword'       => $this->getRegExpOption('mixin_call_keyword'),
        ]);

        $this->trigger($lexEvent);

        $input = $lexEvent->getInput();
        $path = $lexEvent->getPath();
        $stateClassName = $lexEvent->getStateClassName();
        $stateOptions = $lexEvent->getStateOptions();

        $stateOptions['path'] = $path;
        $stateOptions['keyword_names'] = array_keys($this->getOption('keywords') ?: []);

        if (!is_a($stateClassName, State::class, true)) {
            throw new \InvalidArgumentException(
                'lexer_state_class_name needs to be a valid '.State::class.' sub class, '.
                $stateClassName.' given'
            );
        }

        //Put together our initial state
        $this->state = new $stateClassName($this, $input, $stateOptions);

        $scanners = $this->getOption('scanners');

        //We always scan for text at the very end.
        $scanners['final_plain_text'] = TextScanner::class;

        //Scan for tokens
        //N> yield from $this->handleTokens($this->state->loopScan($scanners));
        $tokens = $this->state->loopScan($scanners);

        foreach ($this->handleTokens($tokens) as $token) {
            yield $token;
        }

        $this->trigger(new EndLexEvent($lexEvent));

        //Free state
        $this->state = null;
        $this->previousToken = null;
        $this->lastToken = null;
    }

    /**
     * @param TokenInterface $lastToken
     */
    private function setLastToken($lastToken)
    {
        $this->previousToken = $this->lastToken;
        $this->lastToken = $lastToken;
    }

    private function getRegExpOption($name)
    {
        $value = $this->getOption($name);

        return is_array($value) ? '(?:'.implode('|', $value).')' : $value;
    }

    private function proceedTokenEvent($token)
    {
        $event = new TokenEvent($token);

        if (!($token instanceof HandleTokenInterface) || !$token->isHandled()) {
            $this->trigger($event);

            if ($token instanceof HandleTokenInterface) {
                $token->markAsHandled();
            }
        }

        return $event;
    }

    private function handleToken($token)
    {
        $event = $this->proceedTokenEvent($token);
        $tokens = $event->getTokenGenerator();

        if ($tokens) {
            //N> yield from $this->handleTokens($tokens)
            foreach ($this->handleTokens($tokens) as $tok) {
                $this->setLastToken($tok);

                yield $tok;
            }

            return;
        }

        $token = $event->getToken();
        $this->setLastToken($token);

        yield $token;
    }

    /**
     * @param iterable $tokens
     *
     * @return iterable
     */
    private function handleTokens($tokens)
    {
        foreach ($tokens as $rawToken) {
            foreach ($this->handleToken($rawToken) as $token) {
                yield $token;
            }
        }
    }

    /**
     * @return TokenInterface
     */
    public function getLastToken()
    {
        return $this->lastToken;
    }

    public function dump($input)
    {
        if ($input instanceof TokenInterface) {
            return $this->dumpToken($input);
        }

        if (!Collection::isIterable($input)) {
            $input = $this->lex((string) $input);
        }

        $dumped = '';
        foreach ($input as $token) {
            $dumped .= $this->dump($token);
        }

        return $dumped;
    }

    private function filterScanner($scanner)
    {
        if (!is_subclass_of($scanner, ScannerInterface::class)) {
            throw new \InvalidArgumentException(
                "Scanner $scanner is not a valid ".ScannerInterface::class.' instance or extended class'
            );
        }
    }

    public function getModuleBaseClassName()
    {
        return LexerModuleInterface::class;
    }
}