packages/sql-parser/src/MySql/Lexer/MySqlLexer.php

1<?php
2
3declare(strict_types=1);
4
5namespace SqlParser\MySql\Lexer;
6
7use SqlParser\Lexer\Cursor;
8use SqlParser\Lexer\Lexeme;
9use SqlParser\Lexer\LexicalException;
10use SqlParser\MySql\MySqlVersion;
11use SqlParser\MySql\SqlMode;
12
13/**
14 * Reads MySQL text into the terminals of its grammar, as `sql_lex.cc` does.
15 *
16 * The lexer is a port of MySQL's own state machine: the state a token leaves
17 * behind decides how the next one is read, `WITH ROLLUP` is one token, and
18 * the input ends with an explicit end-of-input terminal.
19 *
20 * @visibility root
21 */
22final class MySqlLexer
23{
24    /**
25     * @param KeywordTable $keywords Keywords of the release
26     * @param MySqlVersion $version Release the text is read for
27     * @param SqlMode $mode Mode the text is read under
28     * @param TriviaScanner $trivia Skips whitespace and comments
29     * @param QuotedScanner $quoted Reads quoted tokens
30     * @param NumberScanner $numbers Reads numbers
31     * @param WordScanner $words Reads words
32     * @param OperatorScanner $operators Reads operators
33     * @param VariableScanner $variables Reads what follows an at sign
34     */
35    public function __construct(
36        private readonly KeywordTable $keywords,
37        private readonly MySqlVersion $version,
38        private readonly SqlMode $mode = new SqlMode(),
39        private readonly TriviaScanner $trivia = new TriviaScanner(),
40        private readonly QuotedScanner $quoted = new QuotedScanner(),
41        private readonly NumberScanner $numbers = new NumberScanner(),
42        private readonly WordScanner $words = new WordScanner(),
43        private readonly OperatorScanner $operators = new OperatorScanner(),
44        private readonly VariableScanner $variables = new VariableScanner(),
45    ) {
46    }
47
48    /**
49     * Reads SQL text into lexemes, the end-of-input terminal last.
50     *
51     * @param string $sql The SQL text
52     *
53     * @return list<Lexeme> The lexemes in text order
54     *
55     * @throws LexicalException When the text holds something no token starts with
56     */
57    public function scan(string $sql): array
58    {
59        $scan = new Scan(new Cursor($sql), $this->keywords, $this->mode, $this->version);
60        $lexemes = [];
61        while (true) {
62            if ($scan->next === LexerState::Start) {
63                $this->trivia->skip($scan);
64                if ($scan->cursor->eof()) {
65                    break;
66                }
67            }
68            $lexemes[] = $this->next($scan);
69        }
70        $lexemes[] = new Lexeme('END_OF_INPUT', '', strlen($sql));
71
72        return $this->merged($lexemes, $sql);
73    }
74
75    /**
76     * Reads one lexeme in the state the previous one left behind.
77     *
78     * @param Scan $scan The tokenization in progress
79     *
80     * @return Lexeme The lexeme
81     *
82     * @throws LexicalException When the text holds something no token starts with
83     */
84    public function next(Scan $scan): Lexeme
85    {
86        $state = $scan->next;
87        $scan->next = LexerState::Start;
88        $start = $scan->cursor->offset();
89
90        return match ($state) {
91            LexerState::Hostname => $this->variables->hostname($scan),
92            LexerState::SystemVariable => $this->variables->systemVariable($scan),
93            LexerState::IdentifierOrKeyword => $this->variables->identifierOrKeyword($scan, $this->words),
94            LexerState::IdentifierSeparator => $this->operators->separator($scan),
95            LexerState::IdentifierStart => $this->words->identifier($scan, $start),
96            LexerState::Start => $scan->cursor->peek() === '@'
97                ? $this->variables->at($scan)
98                : $this->quoted->scan($scan)
99                    ?? $this->numbers->scan($scan, $this->words)
100                    ?? $this->words->scan($scan)
101                    ?? $this->operators->scan($scan),
102        };
103    }
104
105    /**
106     * Joins `WITH ROLLUP`, and `WITH CUBE` where the release has it, into one lexeme.
107     *
108     * @param list<Lexeme> $lexemes Lexemes as read
109     * @param string $sql The SQL text the joined lexeme spans
110     *
111     * @return list<Lexeme> Lexemes with the pairs joined
112     */
113    public function merged(array $lexemes, string $sql): array
114    {
115        $merged = [];
116        for ($index = 0, $count = count($lexemes); $index < $count; $index++) {
117            $lexeme = $lexemes[$index];
118            $following = $lexemes[$index + 1] ?? null;
119            if ($lexeme->name === 'WITH' && $following !== null) {
120                $joined = match (true) {
121                    $following->name === 'ROLLUP_SYM' => 'WITH_ROLLUP_SYM',
122                    $following->name === 'CUBE_SYM' && $this->version->mergesWithCube() => 'WITH_CUBE_SYM',
123                    default => null,
124                };
125                if ($joined !== null) {
126                    $merged[] = new Lexeme($joined, substr($sql, $lexeme->offset, $following->end() - $lexeme->offset), $lexeme->offset);
127                    $index++;
128                    continue;
129                }
130            }
131            $merged[] = $lexeme;
132        }
133
134        return $merged;
135    }
136}
137