Line data Source code
1 : //! The SQL scanner.
2 : //!
3 : //! The rules of every flex-style start condition are a logos lexer: the longest match wins, and
4 : //! logos priorities break ties. The lexer of the current start condition matches one token at a
5 : //! time, so that actions can switch the start condition and put text back with `yyless`. See
6 : //! docs/design/core_quirks.md for the quirks of the scanner.
7 : //!
8 : //! [`scan`] then applies a lookahead filter, which turns `NOT`, `NULLS`, `WITH` and `.` into their
9 : //! `*_LA` variants depending on the following tokens.
10 :
11 : use crate::keywords::Keyword;
12 : use crate::names::{NameId, NameRegistry};
13 : use crate::span::TextSpan;
14 : use crate::{symbol_kind::*, SymbolKind};
15 : use logos::Logos;
16 : use std::collections::VecDeque;
17 :
18 : /// The semantic value of a scanned symbol
19 : #[derive(Clone, Copy, Debug, PartialEq, Eq)]
20 : pub enum SymbolValue {
21 : None,
22 : /// An identifier
23 : Name(NameId),
24 : /// The text of a keyword
25 : Text(&'static str),
26 : }
27 :
28 : /// A scanned symbol
29 : #[derive(Clone, Copy, Debug, PartialEq, Eq)]
30 : pub struct Symbol {
31 : /// The terminal symbol kind
32 : pub kind: SymbolKind,
33 : /// The byte span in the text
34 : pub location: TextSpan,
35 : /// The semantic value
36 : pub value: SymbolValue,
37 : }
38 :
39 : impl Symbol {
40 155534 : fn new(kind: SymbolKind, location: TextSpan) -> Self {
41 155534 : Self { kind, location, value: SymbolValue::None }
42 155534 : }
43 : }
44 :
45 : /// A literal that Hyper replaced with a typed marker comment, e.g. `/*Integer(4888)*/`
46 : #[derive(Clone, Debug, PartialEq, Eq)]
47 : pub struct ObfuscatedLiteral {
48 : /// The location of the marker
49 : pub location: TextSpan,
50 : /// An executable literal derived from the marker hash, e.g. `18568`
51 : pub replacement: String,
52 : }
53 :
54 : /// A scanner error
55 : #[derive(Clone, Debug, PartialEq, Eq)]
56 : pub struct ScannerError {
57 : /// The byte span of the error
58 : pub location: TextSpan,
59 : /// The error message
60 : pub message: String,
61 : }
62 :
63 : /// The output of the scanner
64 : #[derive(Clone, Debug, Default)]
65 : pub struct ScannedScript {
66 : /// The origin id
67 : pub external_id: u32,
68 : /// The text version
69 : pub text_version: u64,
70 : /// The text
71 : pub text: String,
72 : /// The scanner errors
73 : pub errors: Vec<ScannerError>,
74 : /// The line breaks (`\n`, `\r\n` or `\r`), including those in comments and literals
75 : pub line_breaks: Vec<TextSpan>,
76 : /// The comments
77 : pub comments: Vec<TextSpan>,
78 : /// The obfuscated literals, ordered by offset
79 : pub obfuscated_literals: Vec<ObfuscatedLiteral>,
80 : /// The names
81 : pub names: NameRegistry,
82 : /// The symbols, ending with EOF
83 : pub symbols: Vec<Symbol>,
84 : }
85 :
86 : /// Scan a script.
87 7997 : pub fn scan(text: &str, text_version: u64, external_id: u32) -> ScannedScript {
88 7997 : let mut out = ScannedScript {
89 7997 : external_id,
90 7997 : text_version,
91 7997 : text: text.to_string(),
92 7997 : symbols: Vec::with_capacity(text.len() / 4 + 8),
93 7997 : ..Default::default()
94 7997 : };
95 7997 : out.line_breaks = line_breaks(text.as_bytes());
96 7997 : let mut lexer = Lexer::new(text.as_bytes());
97 7997 : let mut lookahead: VecDeque<Symbol> = VecDeque::new();
98 : loop {
99 350046 : let symbol = next_symbol(&mut lexer, &mut out, &mut lookahead);
100 350046 : out.symbols.push(symbol);
101 350046 : if symbol.kind == S_YYEOF {
102 7997 : break;
103 342049 : }
104 : }
105 7997 : out
106 7997 : }
107 :
108 : /// Find the line breaks of a text, `\r\n` is one line break.
109 7997 : fn line_breaks(text: &[u8]) -> Vec<TextSpan> {
110 7997 : let mut breaks = Vec::new();
111 7997 : let mut i = 0;
112 2003860 : while i < text.len() {
113 1995863 : match text[i] {
114 2 : b'\r' if text.get(i + 1) == Some(&b'\n') => {
115 1 : breaks.push(Lexer::span(i, 2));
116 1 : i += 1;
117 1 : }
118 39047 : b'\n' | b'\r' => breaks.push(Lexer::span(i, 1)),
119 1956815 : _ => {}
120 : }
121 1995863 : i += 1;
122 : }
123 7997 : breaks
124 7997 : }
125 :
126 : /// The lookahead filter of [`scan`].
127 350046 : fn next_symbol(lexer: &mut Lexer, out: &mut ScannedScript, lookahead: &mut VecDeque<Symbol>) -> Symbol {
128 350046 : let current = match lookahead.pop_front() {
129 14937 : Some(symbol) => symbol,
130 335109 : None => lexer.lex(out),
131 : };
132 350046 : match current.kind {
133 14937 : S_NOT | S_NULLS_P | S_WITH | S_DOT => {}
134 335109 : _ => return current,
135 : }
136 14937 : let next = lexer.lex(out);
137 14937 : lookahead.push_back(next);
138 14937 : match (current.kind, next.kind) {
139 90 : (S_NOT, S_BETWEEN | S_IN_P | S_LIKE | S_ILIKE | S_GLOB | S_SIMILAR) => Symbol::new(S_NOT_LA, current.location),
140 1457 : (S_NULLS_P, S_FIRST_P | S_LAST_P) => Symbol::new(S_NULLS_LA, current.location),
141 36 : (S_WITH, S_TIME | S_ORDINALITY) => Symbol::new(S_WITH_LA, current.location),
142 26 : (S_DOT, S_STAR) => Symbol::new(S_DOT_STAR_LA, current.location),
143 13328 : _ => current,
144 : }
145 350046 : }
146 :
147 : /// The flex start conditions
148 : #[derive(Clone, Copy, Debug, PartialEq, Eq)]
149 : enum State {
150 : Initial,
151 : /// Bit string literal
152 : Xb,
153 : /// Backtick-quoted identifier
154 : Xbt,
155 : /// C-style comment
156 : Xc,
157 : /// Double-quoted identifier
158 : Xd,
159 : /// Hexadecimal string literal
160 : Xh,
161 : /// Quoted string
162 : Xq,
163 : /// Dollar-quoted string
164 : Xdq,
165 : }
166 :
167 : // The rules of every start condition form a logos lexer. Logos takes the longest match like flex,
168 : // and resolves a tie by priority. The default priority of a pattern is 2 for every byte or class it
169 : // has to match, which already ranks `{comment}`, `{xcstart}` and the two-character operators above
170 : // `{operator}`. The explicit priorities rank the remaining ties.
171 : //
172 : // Like flex, the lexers match bytes, not characters (`utf8 = false` and byte-string patterns), so
173 : // `\x80-\xFF` are identifier bytes.
174 :
175 : /// The rules of INITIAL
176 : #[derive(Logos, Clone, Copy, Debug, PartialEq, Eq)]
177 : #[logos(utf8 = false)]
178 : #[logos(subpattern space = br"[ \t\n\r\f]")]
179 : #[logos(subpattern op_chars = br"[~!@#^&|+\-*/%<>=]")]
180 : #[logos(subpattern integer = br"[0-9]+")]
181 : #[logos(subpattern decimal = br"[0-9]*\.[0-9]+|[0-9]+\.[0-9]*")]
182 : enum Initial {
183 : /// `{space}+`
184 : #[regex(br"(?&space)+")]
185 : Space,
186 : /// `{comment}`
187 : #[regex(br"--[^\n\r]*", allow_greedy = true)]
188 : Comment,
189 : /// The Hyper obfuscation markers, case-sensitive unlike the other rules
190 : #[regex(br"/\*String\([0-9A-F]{4}\)\*/", |_| S_SCONST)]
191 : #[regex(br"/\*Integer\([0-9A-F]{4}\)\*/", |_| S_ICONST)]
192 : #[regex(br"/\*Float\([0-9A-F]{4}\)\*/", |_| S_FCONST)]
193 : #[regex(br"/\*HexString\([0-9A-F]{4}\)\*/", |_| S_XCONST)]
194 : #[regex(br"/\*BitString\([0-9A-F]{4}\)\*/", |_| S_BCONST)]
195 : ObfuscatedLiteral(SymbolKind),
196 : /// `{xcstart}`: matches as much as `{operator}` would, the action puts the operator back
197 : #[regex(br"/\*(?&op_chars)*")]
198 : CommentStart,
199 : /// `{xbstart}`, `{xhstart}`, `{xqstart}`, `{xdstart}`, `{xbtstart}`
200 : #[regex(b"[bB]'", |_| State::Xb)]
201 : #[regex(b"[xX]'", |_| State::Xh)]
202 : #[token("'", |_| State::Xq)]
203 : #[token("\"", |_| State::Xd)]
204 : #[token("`", |_| State::Xbt)]
205 : #[token("$$", |_| State::Xdq)]
206 : Begin(State),
207 : /// The operators and punctuation with a token of their own
208 : #[token("::", |_| S_TYPECAST)]
209 : #[token("..", |_| S_DOT_DOT)]
210 : #[token(".", |_| S_DOT)]
211 : #[token(":=", |_| S_COLON_EQUALS)]
212 : #[token("=>", |_| S_EQUALS_GREATER)]
213 : #[token("<=", |_| S_LESS_EQUALS)]
214 : #[token(">=", |_| S_GREATER_EQUALS)]
215 : #[token("<>", |_| S_NOT_EQUALS)]
216 : #[token("!=", |_| S_NOT_EQUALS)]
217 : #[token(",", |_| S_COMMA)]
218 : #[token("(", |_| S_LRB)]
219 : #[token(")", |_| S_RRB)]
220 : #[token("[", |_| S_LSB)]
221 : #[token("]", |_| S_RSB)]
222 : #[token("{", |_| S_LCB)]
223 : #[token("}", |_| S_RCB)]
224 : #[token(";", |_| S_SEMICOLON)]
225 : #[token(":", |_| S_COLON)]
226 : #[token("+", |_| S_PLUS)]
227 : #[token("-", |_| S_MINUS)]
228 : #[token("/", |_| S_DIVIDE)]
229 : #[token("*", |_| S_STAR)]
230 : #[token("%", |_| S_MODULO)]
231 : #[token("^", |_| S_CIRCUMFLEX)]
232 : #[token("&", |_| S_AMPERSAND)]
233 : #[token("|", |_| S_PIPE)]
234 : #[token("!", |_| S_EXCLAMATION_MARK)]
235 : #[token("<", |_| S_LESS_THAN)]
236 : #[token(">", |_| S_GREATER_THAN)]
237 : #[token("=", |_| S_EQUALS)]
238 : #[token("$", |_| S_DOLLAR)]
239 : Token(SymbolKind),
240 : /// `{raw_dot_trailing}`, the action puts the space back
241 : #[regex(br"\.(?&space)")]
242 : DotTrailing,
243 : /// `{raw_question_mark}`, the action checks the trailing context
244 : #[token("?")]
245 : QuestionMark,
246 : /// `{operator}`, below the single-character tokens
247 : #[regex(br"(?&op_chars)+", priority = 1)]
248 : Operator,
249 : /// `{integer}`
250 : #[regex(br"(?&integer)")]
251 : Integer,
252 : /// `{decimal}`, `{real}`
253 : #[regex(br"(?&decimal)")]
254 : #[regex(br"((?&integer)|(?&decimal))[Ee][-+]?[0-9]+")]
255 : Float,
256 : /// `{decimalfail}`
257 : #[regex(br"(?&integer)\.\.")]
258 : DecimalFail,
259 : /// `{realfail1}`
260 : #[regex(br"((?&integer)|(?&decimal))[Ee]")]
261 : RealFail1,
262 : /// `{realfail2}`
263 : #[regex(br"((?&integer)|(?&decimal))[Ee][-+]")]
264 : RealFail2,
265 : /// `{identifier}`
266 : #[regex(br"[A-Za-z\x80-\xFF_][A-Za-z\x80-\xFF_0-9$]*")]
267 : Identifier,
268 : /// `{other}`, below everything else
269 : #[regex(br"[^\n]", priority = 0)]
270 : Other,
271 : }
272 :
273 : /// The rules of xc
274 : #[derive(Logos, Clone, Copy, Debug, PartialEq, Eq)]
275 : #[logos(utf8 = false)]
276 : enum Xc {
277 : /// `{xcstart}`
278 : #[regex(br"/\*[~!@#^&|+\-*/%<>=]*")]
279 : CommentStart,
280 : /// `{xcstop}`
281 : #[regex(br"\*+/")]
282 : CommentStop,
283 : /// `{xcinside}`, `{op_chars}`, `\*+`
284 : #[regex(br"[^*/]+")]
285 : #[regex(br"\*+")]
286 : #[token("/")]
287 : Inside,
288 : }
289 :
290 : /// The rules of xb and xh
291 : #[derive(Logos, Clone, Copy, Debug, PartialEq, Eq)]
292 : #[logos(utf8 = false)]
293 : enum XbXh {
294 : /// The closing quote, the action checks for a continuation (`{quotecontinue}`)
295 : #[token("'")]
296 : QuoteStop,
297 : /// `{xbinside}`, `{xhinside}`
298 : #[regex(br"[^']+")]
299 : Inside,
300 : }
301 :
302 : /// The rules of xq
303 : #[derive(Logos, Clone, Copy, Debug, PartialEq, Eq)]
304 : #[logos(utf8 = false)]
305 : enum Xq {
306 : /// The closing quote
307 : #[token("'")]
308 : QuoteStop,
309 : /// `{xqdouble}`, `{xqinside}`
310 : #[token("''")]
311 : #[regex(br"[^']+")]
312 : Inside,
313 : }
314 :
315 : /// The rules of xdq, the text between `$$` is verbatim
316 : #[derive(Logos, Clone, Copy, Debug, PartialEq, Eq)]
317 : #[logos(utf8 = false)]
318 : enum Xdq {
319 : /// The closing `$$`
320 : #[token("$$")]
321 : Stop,
322 : /// A lone `$` or text without one
323 : #[token("$")]
324 : #[regex(br"[^$]+")]
325 : Inside,
326 : }
327 :
328 : /// The rules of xd
329 : #[derive(Logos, Clone, Copy, Debug, PartialEq, Eq)]
330 : #[logos(utf8 = false)]
331 : enum Xd {
332 : /// `{xddouble}`, `{xdinside}`
333 : #[token("\"\"")]
334 : #[regex(br#"[^"]+"#)]
335 : Inside,
336 : /// `{xdstop}`
337 : #[token("\"")]
338 : Stop,
339 : }
340 :
341 : /// The rules of xbt
342 : #[derive(Logos, Clone, Copy, Debug, PartialEq, Eq)]
343 : #[logos(utf8 = false)]
344 : enum Xbt {
345 : /// `{xbtdouble}`, `{xbtinside}`
346 : #[token("``")]
347 : #[regex(br"[^`]+")]
348 : Inside,
349 : /// `{xbtstop}`
350 : #[token("`")]
351 : Stop,
352 : }
353 :
354 : /// `{ident_cont}`
355 50 : fn is_ident_cont(c: u8) -> bool {
356 50 : c.is_ascii_alphanumeric() || c == b'_' || c == b'$' || c >= 0x80
357 50 : }
358 :
359 : /// The token of a single special character
360 54 : fn special_character(c: u8) -> Option<SymbolKind> {
361 54 : Some(match c {
362 0 : b',' => S_COMMA,
363 0 : b'(' => S_LRB,
364 0 : b')' => S_RRB,
365 0 : b'[' => S_LSB,
366 0 : b']' => S_RSB,
367 0 : b'{' => S_LCB,
368 0 : b'}' => S_RCB,
369 0 : b'.' => S_DOT,
370 0 : b';' => S_SEMICOLON,
371 0 : b':' => S_COLON,
372 17 : b'+' => S_PLUS,
373 0 : b'-' => S_MINUS,
374 3 : b'*' => S_STAR,
375 2 : b'/' => S_DIVIDE,
376 0 : b'%' => S_MODULO,
377 0 : b'?' => S_QUESTION_MARK,
378 0 : b'^' => S_CIRCUMFLEX,
379 0 : b'&' => S_AMPERSAND,
380 0 : b'|' => S_PIPE,
381 0 : b'!' => S_EXCLAMATION_MARK,
382 27 : b'<' => S_LESS_THAN,
383 2 : b'>' => S_GREATER_THAN,
384 3 : b'=' => S_EQUALS,
385 0 : _ => return None,
386 : })
387 54 : }
388 :
389 : /// The two-character operator tokens that also match `{operator}`
390 15 : fn two_char_operator(a: u8, b: u8) -> Option<SymbolKind> {
391 15 : Some(match (a, b) {
392 5 : (b'=', b'>') => S_EQUALS_GREATER,
393 2 : (b'<', b'=') => S_LESS_EQUALS,
394 6 : (b'>', b'=') => S_GREATER_EQUALS,
395 2 : (b'<', b'>') | (b'!', b'=') => S_NOT_EQUALS,
396 0 : _ => return None,
397 : })
398 15 : }
399 :
400 : /// The scanner, switches between the lexers of the start conditions like the flex scanner
401 : struct Lexer<'a> {
402 : /// The script text
403 : text: &'a [u8],
404 : /// The byte offset of the next match
405 : pos: usize,
406 : /// The current start condition
407 : state: State,
408 : /// The start of the current extended token (quoted literal, identifier or comment)
409 : ext_begin: TextSpan,
410 : /// The nesting depth of C-style comments
411 : ext_depth: usize,
412 : /// A buffer for lower-casing and unescaping names
413 : buffer: Vec<u8>,
414 : }
415 :
416 : impl<'a> Lexer<'a> {
417 7997 : fn new(text: &'a [u8]) -> Self {
418 7997 : Self {
419 7997 : text,
420 7997 : pos: 0,
421 7997 : state: State::Initial,
422 7997 : ext_begin: TextSpan::default(),
423 7997 : ext_depth: 0,
424 7997 : buffer: Vec::with_capacity(64),
425 7997 : }
426 7997 : }
427 :
428 635818 : fn span(begin: usize, length: usize) -> TextSpan {
429 635818 : TextSpan::new(begin as u32, length as u32)
430 635818 : }
431 :
432 : /// The empty location at the end of the text
433 8031 : fn eof_location(&self) -> TextSpan {
434 8031 : Self::span(self.text.len(), 0)
435 8031 : }
436 :
437 : /// Match the rules of a start condition at the current position and consume the match.
438 588738 : fn next_match<T>(&mut self) -> (T, TextSpan)
439 588738 : where
440 588738 : T: Logos<'a, Source = [u8], Extras = ()>,
441 : {
442 588738 : let start = self.pos;
443 588738 : let mut lexer = T::lexer(&self.text[start..]);
444 : // Every byte starts a match in every start condition.
445 588738 : let Some(Ok(token)) = lexer.next() else {
446 0 : unreachable!("no rule matches at {start}")
447 : };
448 588738 : let length = lexer.span().end;
449 588738 : self.pos = start + length;
450 588738 : (token, Self::span(start, length))
451 588738 : }
452 :
453 : /// Scan the next raw token (before the lookahead filter)
454 350046 : fn lex(&mut self, out: &mut ScannedScript) -> Symbol {
455 : loop {
456 596769 : if self.pos >= self.text.len() {
457 8031 : let eof = self.eof_location();
458 8031 : let message = match self.state {
459 7997 : State::Initial => return Symbol::new(S_YYEOF, eof),
460 5 : State::Xc => "unterminated comment",
461 2 : State::Xb => "unterminated bit string literal",
462 2 : State::Xh => "unterminated hexadecimal bit string literal",
463 19 : State::Xq => "unterminated quoted string",
464 2 : State::Xdq => "unterminated dollar-quoted string",
465 2 : State::Xd => "unterminated delimited identifier",
466 2 : State::Xbt => "unterminated backtick-quoted identifier",
467 : };
468 34 : out.errors.push(ScannerError {
469 34 : location: self.ext_begin.cover(eof),
470 34 : message: message.to_string(),
471 34 : });
472 34 : self.state = State::Initial;
473 34 : continue;
474 588738 : }
475 588738 : let symbol = match self.state {
476 : State::Initial => {
477 564262 : let (token, loc) = self.next_match::<Initial>();
478 564262 : self.initial(out, token, loc)
479 : }
480 1808 : State::Xc => match self.next_match::<Xc>() {
481 15 : (Xc::CommentStart, mut loc) => {
482 15 : self.yyless(&mut loc, 2);
483 15 : self.ext_depth += 1;
484 15 : None
485 : }
486 502 : (Xc::CommentStop, loc) => {
487 502 : self.ext_depth -= 1;
488 502 : if self.ext_depth == 0 {
489 487 : out.comments.push(self.ext_begin.cover(loc));
490 487 : self.state = State::Initial;
491 487 : }
492 502 : None
493 : }
494 1291 : (Xc::Inside, _) => None,
495 : },
496 68 : State::Xb | State::Xh => match self.next_match::<XbXh>() {
497 32 : (XbXh::QuoteStop, loc) => {
498 32 : if self.quote_continue(out) {
499 1 : None
500 : } else {
501 31 : Some(self.quote_stop(loc))
502 : }
503 : }
504 36 : (XbXh::Inside, _) => None,
505 : },
506 20375 : State::Xq => match self.next_match::<Xq>() {
507 10142 : (Xq::QuoteStop, loc) => Some(self.quote_stop(loc)),
508 10233 : (Xq::Inside, _) => None,
509 : },
510 49 : State::Xdq => match self.next_match::<Xdq>() {
511 21 : (Xdq::Stop, loc) => {
512 21 : self.state = State::Initial;
513 21 : Some(Symbol::new(S_DCONST, self.ext_begin.cover(loc)))
514 : }
515 28 : (Xdq::Inside, _) => None,
516 : },
517 1676 : State::Xd => match self.next_match::<Xd>() {
518 819 : (Xd::Stop, loc) => Some(self.identifier_stop(out, loc, b'"', "zero-length delimited identifier")),
519 857 : (Xd::Inside, _) => None,
520 : },
521 500 : State::Xbt => match self.next_match::<Xbt>() {
522 238 : (Xbt::Stop, loc) => {
523 238 : Some(self.identifier_stop(out, loc, b'`', "zero-length backtick-quoted identifier"))
524 : }
525 262 : (Xbt::Inside, _) => None,
526 : },
527 : };
528 588738 : if let Some(symbol) = symbol {
529 342049 : return symbol;
530 246689 : }
531 : }
532 350046 : }
533 :
534 : /// Put back all but the first `n` characters of the current match (`yyless(n)`).
535 597 : fn yyless(&mut self, loc: &mut TextSpan, n: usize) {
536 597 : self.pos = loc.offset as usize + n;
537 597 : loc.length = n as u32;
538 597 : }
539 :
540 : /// Run the action of an INITIAL rule. Returns a symbol if the action returns one.
541 564262 : fn initial(&mut self, out: &mut ScannedScript, token: Initial, mut loc: TextSpan) -> Option<Symbol> {
542 564262 : match token {
543 221128 : Initial::Space => None,
544 : Initial::Comment => {
545 564 : out.comments.push(loc);
546 564 : None
547 : }
548 24 : Initial::ObfuscatedLiteral(kind) => {
549 24 : out.obfuscated_literals.push(obfuscated_literal(self.text, loc));
550 24 : Some(Symbol::new(kind, loc))
551 : }
552 : Initial::CommentStart => {
553 492 : self.yyless(&mut loc, 2);
554 492 : self.ext_begin = loc;
555 492 : self.ext_depth += 1;
556 492 : self.state = State::Xc;
557 492 : None
558 : }
559 11280 : Initial::Begin(state) => {
560 11280 : self.ext_begin = loc;
561 11280 : self.state = state;
562 11280 : None
563 : }
564 116404 : Initial::Token(kind) => Some(Symbol::new(kind, loc)),
565 : Initial::DotTrailing => {
566 17 : self.yyless(&mut loc, 1);
567 17 : Some(Symbol::new(S_DOT_TRAILING, loc))
568 : }
569 : // A standalone ? is an unnamed parameter. ?name, ?1 and ?"name" are not parameters of Hyper,
570 : // the grammar has no rule for QUESTION_MARK, so they are syntax errors.
571 52 : Initial::QuestionMark => match self.text.get(self.pos) {
572 50 : Some(&c) if is_ident_cont(c) || c == b'"' => Some(Symbol::new(S_QUESTION_MARK, loc)),
573 48 : _ => Some(Symbol::new(S_PARAM, loc)),
574 : },
575 225 : Initial::Operator => Some(self.operator(out, loc)),
576 18058 : Initial::Integer => Some(self.number(loc)),
577 948 : Initial::Float => Some(Symbol::new(S_FCONST, loc)),
578 : Initial::DecimalFail | Initial::RealFail1 | Initial::RealFail2 => {
579 : // Put back the dots, or the exponent marker and its sign
580 4 : let putback = if token == Initial::RealFail1 { 1 } else { 2 };
581 4 : let keep = loc.length as usize - putback;
582 4 : self.yyless(&mut loc, keep);
583 4 : Some(self.number(loc))
584 : }
585 195064 : Initial::Identifier => Some(self.read_identifier(out, loc)),
586 2 : Initial::Other => Some(Symbol::new(S_RAW_CHAR, loc)),
587 : }
588 564262 : }
589 :
590 : /// A number literal: integers that do not fit into an `i64` are FCONST, like in Postgres
591 18062 : fn number(&self, loc: TextSpan) -> Symbol {
592 18062 : let text = &self.text[loc.offset as usize..loc.end() as usize];
593 18062 : let is_integer = utf8(text).parse::<i64>().is_ok();
594 18062 : Symbol::new(if is_integer { S_ICONST } else { S_FCONST }, loc)
595 18062 : }
596 :
597 : /// The closing quote of a string, bit string or hex string literal
598 10173 : fn quote_stop(&mut self, loc: TextSpan) -> Symbol {
599 10173 : let kind = self.literal_kind();
600 10173 : self.state = State::Initial;
601 10173 : Symbol::new(kind, self.ext_begin.cover(loc))
602 10173 : }
603 :
604 : /// `{quotecontinue}`: a bit or hex string continues after whitespace with a line break and
605 : /// another quote. Consumes the whitespace and the quote and records the comments in it.
606 32 : fn quote_continue(&mut self, out: &mut ScannedScript) -> bool {
607 32 : let mut i = self.pos;
608 32 : let mut comments = Vec::new();
609 32 : let mut has_line_break = false;
610 49 : while i < self.text.len() {
611 43 : match self.text[i] {
612 15 : b' ' | b'\t' | b'\x0C' => i += 1,
613 1 : b'\n' | b'\r' => {
614 1 : has_line_break = true;
615 1 : i += 1;
616 1 : }
617 11 : b'-' if self.text.get(i + 1) == Some(&b'-') => {
618 5 : let length = self.text[i..].iter().position(|&c| c == b'\n' || c == b'\r');
619 1 : let length = length.unwrap_or(self.text.len() - i);
620 1 : comments.push(Self::span(i, length));
621 1 : i += length;
622 : }
623 1 : b'\'' if has_line_break => {
624 1 : out.comments.extend(comments);
625 1 : self.pos = i + 1;
626 1 : return true;
627 : }
628 25 : _ => break,
629 : }
630 : }
631 31 : false
632 32 : }
633 :
634 : /// The `{xdstop}` and `{xbtstop}` actions
635 1057 : fn identifier_stop(&mut self, out: &mut ScannedScript, loc: TextSpan, quote: u8, message: &str) -> Symbol {
636 1057 : let loc = self.ext_begin.cover(loc);
637 1057 : if loc.length == 2 {
638 3 : out.errors.push(ScannerError {
639 3 : location: loc,
640 3 : message: message.to_string(),
641 3 : });
642 1054 : }
643 1057 : self.state = State::Initial;
644 1057 : self.read_quoted_identifier(out, loc, quote)
645 1057 : }
646 :
647 : /// The token kind of the literal that the current start condition scans
648 10173 : fn literal_kind(&self) -> SymbolKind {
649 10173 : match self.state {
650 18 : State::Xb => S_BCONST,
651 13 : State::Xh => S_XCONST,
652 10142 : _ => S_SCONST,
653 : }
654 10173 : }
655 :
656 1057 : fn read_quoted_identifier(&mut self, out: &mut ScannedScript, loc: TextSpan, quote: u8) -> Symbol {
657 1057 : let text = &self.text[loc.offset as usize..loc.end() as usize];
658 1057 : let inner = &text[1..text.len() - 1];
659 1057 : let name = if inner.contains("e) {
660 32 : let name = &mut self.buffer;
661 32 : name.clear();
662 32 : let mut i = 0;
663 214 : while i < inner.len() {
664 182 : name.push(inner[i]);
665 182 : if inner[i] == quote && i + 1 < inner.len() && inner[i + 1] == quote {
666 32 : i += 1;
667 150 : }
668 182 : i += 1;
669 : }
670 32 : &self.buffer[..]
671 : } else {
672 1025 : inner
673 : };
674 1057 : let id = out.names.register(utf8(name), loc);
675 1057 : Symbol {
676 1057 : kind: S_IDENT,
677 1057 : location: loc,
678 1057 : value: SymbolValue::Name(id),
679 1057 : }
680 1057 : }
681 :
682 195064 : fn read_identifier(&mut self, out: &mut ScannedScript, loc: TextSpan) -> Symbol {
683 195064 : let text = &self.text[loc.offset as usize..loc.end() as usize];
684 195064 : self.buffer.clear();
685 195064 : self.buffer.extend(text.iter().map(u8::to_ascii_lowercase));
686 195064 : let lower = &self.buffer[..];
687 195064 : if let Some(keyword) = Keyword::find(lower) {
688 93887 : return Symbol {
689 93887 : kind: keyword.symbol,
690 93887 : location: loc,
691 93887 : value: SymbolValue::Text(keyword.name),
692 93887 : };
693 101177 : }
694 101177 : let id = out.names.register(utf8(lower), loc);
695 101177 : Symbol {
696 101177 : kind: S_IDENT,
697 101177 : location: loc,
698 101177 : value: SymbolValue::Name(id),
699 101177 : }
700 195064 : }
701 :
702 : /// The `{operator}` action
703 225 : fn operator(&mut self, out: &mut ScannedScript, mut loc: TextSpan) -> Symbol {
704 225 : let start = loc.offset as usize;
705 225 : let yyleng = loc.length as usize;
706 225 : let text = &self.text[start..start + yyleng];
707 :
708 : // Embedded slash-star or dash-dash start a comment, the operator stops there.
709 832 : let find = |pattern: &[u8]| text.windows(2).position(|w| w == pattern);
710 225 : let mut nchars = yyleng;
711 225 : if let Some(i) = [find(b"/*"), find(b"--")].into_iter().flatten().min() {
712 8 : nchars = i;
713 217 : }
714 : // '+' and '-' cannot end a multi-char operator unless it contains chars that are not in
715 : // SQL operators.
716 294 : while nchars > 1 && matches!(text[nchars - 1], b'+' | b'-') {
717 110 : if text[..nchars - 1].iter().any(|c| b"~!@#^&|?%".contains(c)) {
718 10 : break;
719 69 : }
720 69 : nchars -= 1;
721 : }
722 225 : if nchars < yyleng {
723 69 : self.yyless(&mut loc, nchars);
724 69 : if nchars == 1 {
725 54 : if let Some(kind) = special_character(text[0]) {
726 54 : return Symbol::new(kind, loc);
727 0 : }
728 15 : }
729 15 : if nchars == 2 {
730 15 : if let Some(kind) = two_char_operator(text[0], text[1]) {
731 15 : return Symbol::new(kind, loc);
732 0 : }
733 0 : }
734 156 : }
735 156 : if nchars >= 64 {
736 2 : out.errors.push(ScannerError {
737 2 : location: loc,
738 2 : message: "operator too long: operators longer than 64 bytes are not supported".to_string(),
739 2 : });
740 154 : }
741 156 : Symbol::new(S_Op, loc)
742 225 : }
743 : }
744 :
745 : /// Derive an executable literal from the hash of a marker.
746 24 : fn obfuscated_literal(text: &[u8], location: TextSpan) -> ObfuscatedLiteral {
747 24 : let marker = &text[location.offset as usize..location.end() as usize];
748 240 : let open = marker.iter().position(|&c| c == b'(').unwrap();
749 24 : let kind = &marker[2..open];
750 24 : let hash = std::str::from_utf8(&marker[open + 1..open + 5]).unwrap();
751 24 : let value = u32::from_str_radix(hash, 16).unwrap();
752 24 : let replacement = match kind {
753 24 : b"String" => format!("'{hash}'"),
754 22 : b"Integer" => value.to_string(),
755 3 : b"Float" => format!("{value}.0"),
756 2 : b"HexString" => format!("X'{hash}'"),
757 : _ => {
758 16 : let bits: String = (0..16).rev().map(|bit| if (value >> bit) & 1 == 1 { '1' } else { '0' }).collect();
759 1 : format!("B'{bits}'")
760 : }
761 : };
762 24 : ObfuscatedLiteral { location, replacement }
763 24 : }
764 :
765 : /// Names are cut out of valid UTF-8 at ASCII boundaries, so they are valid UTF-8 themselves.
766 120296 : fn utf8(bytes: &[u8]) -> &str {
767 120296 : std::str::from_utf8(bytes).expect("names are cut at ASCII boundaries")
768 120296 : }
769 :
770 : #[cfg(test)]
771 : mod tests {
772 : use super::*;
773 :
774 17 : fn kinds(text: &str) -> Vec<SymbolKind> {
775 17 : scan(text, 0, 0).symbols.iter().map(|s| s.kind).collect()
776 17 : }
777 :
778 4 : fn spans(text: &str) -> Vec<(u32, u32)> {
779 15 : scan(text, 0, 0).symbols.iter().map(|s| (s.location.offset, s.location.length)).collect()
780 4 : }
781 :
782 : #[test]
783 1 : fn scans_keywords_and_identifiers() {
784 1 : let out = scan("SELECT Foo, foo", 0, 0);
785 1 : let kinds: Vec<_> = out.symbols.iter().map(|s| s.kind).collect();
786 1 : assert_eq!(kinds, vec![S_SELECT, S_IDENT, S_COMMA, S_IDENT, S_YYEOF]);
787 1 : assert_eq!(out.symbols[0].value, SymbolValue::Text("select"));
788 1 : assert_eq!(out.symbols[1].value, SymbolValue::Name(0));
789 1 : assert_eq!(out.symbols[3].value, SymbolValue::Name(0));
790 1 : assert_eq!(out.names.get(0).text, "foo");
791 1 : assert_eq!(out.names.get(0).occurrences, 2);
792 1 : assert_eq!(out.symbols[4].location, TextSpan::new(15, 0));
793 1 : }
794 :
795 : #[test]
796 1 : fn scans_numbers() {
797 1 : assert_eq!(kinds("1..2"), vec![S_ICONST, S_DOT_DOT, S_ICONST, S_YYEOF]);
798 1 : assert_eq!(spans("1..2"), vec![(0, 1), (1, 2), (3, 1), (4, 0)]);
799 1 : assert_eq!(kinds("1e5 1.5 .5 1. 1e"), vec![S_FCONST, S_FCONST, S_FCONST, S_FCONST, S_ICONST, S_IDENT, S_YYEOF]);
800 1 : assert_eq!(kinds("1.5e+"), vec![S_FCONST, S_IDENT, S_PLUS, S_YYEOF]);
801 : // Integers that do not fit into an i64 are FCONST
802 1 : assert_eq!(kinds("9223372036854775807 9223372036854775808"), vec![S_ICONST, S_FCONST, S_YYEOF]);
803 1 : }
804 :
805 : #[test]
806 1 : fn scans_trailing_dots() {
807 1 : assert_eq!(kinds("a. b"), vec![S_IDENT, S_DOT_TRAILING, S_IDENT, S_YYEOF]);
808 1 : assert_eq!(spans("a. b")[1], (1, 1));
809 1 : let out = scan("a.\n", 0, 0);
810 1 : assert_eq!(out.line_breaks, vec![TextSpan::new(2, 1)]);
811 1 : }
812 :
813 : #[test]
814 1 : fn trims_trailing_signs_and_comments_from_operators() {
815 : // Like Postgres, the operator keeps the characters before the comment start or trailing sign.
816 1 : assert_eq!(kinds("+-- foo"), vec![S_PLUS, S_YYEOF]);
817 1 : assert_eq!(spans("+-- foo")[0], (0, 1));
818 1 : assert_eq!(kinds("=- <=> !="), vec![S_EQUALS, S_MINUS, S_Op, S_NOT_EQUALS, S_YYEOF]);
819 1 : assert_eq!(kinds("a>=-1"), vec![S_IDENT, S_GREATER_EQUALS, S_MINUS, S_ICONST, S_YYEOF]);
820 1 : assert_eq!(spans("a>=-1")[1], (1, 2));
821 1 : assert_eq!(kinds("a=>-1"), vec![S_IDENT, S_EQUALS_GREATER, S_MINUS, S_ICONST, S_YYEOF]);
822 1 : assert_eq!(kinds("1 +/* c */ 2"), vec![S_ICONST, S_PLUS, S_ICONST, S_YYEOF]);
823 1 : assert_eq!(kinds("*/*"), vec![S_STAR, S_YYEOF]);
824 1 : }
825 :
826 : #[test]
827 1 : fn applies_the_lookahead_filter() {
828 1 : assert_eq!(kinds("not in"), vec![S_NOT_LA, S_IN_P, S_YYEOF]);
829 1 : assert_eq!(kinds("not glob"), vec![S_NOT_LA, S_GLOB, S_YYEOF]);
830 1 : assert_eq!(kinds("not foo"), vec![S_NOT, S_IDENT, S_YYEOF]);
831 1 : assert_eq!(kinds("a.*"), vec![S_IDENT, S_DOT_STAR_LA, S_STAR, S_YYEOF]);
832 1 : }
833 :
834 : #[test]
835 1 : fn scans_literals_and_comments() {
836 1 : let out = scan("'a' --c\n-1", 0, 0);
837 1 : assert_eq!(out.symbols[0].location, TextSpan::new(0, 3));
838 1 : assert_eq!(out.line_breaks, vec![TextSpan::new(7, 1)]);
839 1 : assert_eq!(out.comments, vec![TextSpan::new(4, 3)]);
840 1 : let out = scan("'abc' -- c", 0, 0);
841 1 : assert_eq!(out.symbols[0].location, TextSpan::new(0, 5));
842 1 : assert_eq!(out.comments, vec![TextSpan::new(7, 4)]);
843 1 : let out = scan("'a''b' ", 0, 0);
844 1 : assert_eq!(out.symbols[0].location, TextSpan::new(0, 6));
845 1 : let out = scan("/* a /* b */ c */ 1", 0, 0);
846 1 : assert_eq!(out.comments, vec![TextSpan::new(0, 17)]);
847 1 : let out = scan("'unterminated", 0, 0);
848 1 : assert_eq!(out.errors[0].location, TextSpan::new(0, 13));
849 1 : assert_eq!(out.errors[0].message, "unterminated quoted string");
850 1 : }
851 :
852 : #[test]
853 1 : fn continues_bit_and_hex_strings() {
854 1 : let out = scan("x'AB' -- c\n 'CD' 1", 0, 0);
855 1 : assert_eq!(out.symbols.iter().map(|s| s.kind).collect::<Vec<_>>(), vec![S_XCONST, S_ICONST, S_YYEOF]);
856 1 : assert_eq!(out.symbols[0].location, TextSpan::new(0, 17));
857 1 : assert_eq!(out.comments, vec![TextSpan::new(6, 4)]);
858 : // Without a line break, the next quote starts another literal
859 1 : assert_eq!(kinds("b'01' '1'"), vec![S_BCONST, S_SCONST, S_YYEOF]);
860 : // Strings do not continue
861 1 : assert_eq!(kinds("'a'\n'b'"), vec![S_SCONST, S_SCONST, S_YYEOF]);
862 1 : }
863 :
864 : #[test]
865 1 : fn records_line_breaks() {
866 1 : let out = scan("a\r\nb\rc\n'd\ne' -- f\n", 0, 0);
867 5 : let breaks: Vec<_> = out.line_breaks.iter().map(|s| (s.offset, s.length)).collect();
868 1 : assert_eq!(breaks, vec![(1, 2), (4, 1), (6, 1), (9, 1), (17, 1)]);
869 1 : }
870 :
871 : #[test]
872 1 : fn unescapes_quoted_identifiers() {
873 1 : let out = scan(r#"select "trailing""", """leading", "a""b", `trailing```, `` `a``b`"#, 0, 0);
874 5 : for name in ["trailing\"", "\"leading", "a\"b", "trailing`", "a`b"] {
875 5 : assert!(out.names.find(name).is_some(), "{name}");
876 : }
877 1 : }
878 :
879 : #[test]
880 1 : fn records_obfuscated_literals() {
881 1 : let out = scan("/*Integer(1A2B)*/", 0, 0);
882 1 : assert_eq!(out.symbols[0].kind, S_ICONST);
883 1 : assert_eq!(out.obfuscated_literals[0].replacement, "6699");
884 1 : }
885 : }
|
