From eacffe2820c1cabba1f91e2c2bc14aa3674e01d8 Mon Sep 17 00:00:00 2001
From: Dkx <45234736+dkxmercury@users.noreply.github.com>
Date: Sat, 25 Jul 2026 02:54:09 +0500
Subject: [PATCH] fix(lexer): stop dropping input after an unlexable byte
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

In `Lexer::lex`, when `lex_match` stopped at a byte that no matcher
covers, the fallback ran the last-resort matcher on the original
`str_buff` instead of the unlexed remainder `res.forward_string`, and
broke out of the loop as soon as it produced any element. Since the
last-resort pattern always matches (it can match the empty string), the
loop always broke and the "advance and append" lines were dead code.

The result was silent data loss: everything from the first unlexable
byte to the end of input was discarded. For example `select ф` lexed to
just `select `, so `fix` would silently delete part of the user's SQL
and `lint` parsed a truncated statement.

Run the last-resort matcher on the remainder, append its elements, and
break only when it makes no progress (to avoid an infinite loop). The
unlexable run now becomes an `Unlexable` segment and lexing is lossless.
---
 crates/lib-core/src/parser/lexer.rs | 27 ++++++++++++++++++++++++---
 1 file changed, 24 insertions(+), 3 deletions(-)

diff --git a/crates/lib-core/src/parser/lexer.rs b/crates/lib-core/src/parser/lexer.rs
index 20b8cf77f..bb58cef80 100644
--- a/crates/lib-core/src/parser/lexer.rs
+++ b/crates/lib-core/src/parser/lexer.rs
@@ -506,9 +506,13 @@ impl Lexer {
                 break;
             }
 
-            // If we STILL can't match, then just panic out.
-            let mut resort_res = self.last_resort_lexer.matches(str_buff);
-            if !resort_res.elements.is_empty() {
+            // lex_match stopped at a byte that no matcher covers. Consume that
+            // unlexable run with the last-resort matcher and keep going, instead
+            // of dropping the remainder of the input.
+            let mut resort_res = self.last_resort_lexer.matches(res.forward_string);
+            if resort_res.forward_string.len() == res.forward_string.len() {
+                // The last-resort matcher made no progress; stop to avoid an
+                // infinite loop rather than spin on the same byte.
                 break;
             }
 
@@ -1156,6 +1160,23 @@ mod tests {
         assert_eq!(res.elements.len(), 3);
     }
 
+    #[test]
+    fn lex_preserves_unlexable_input() {
+        // The word matcher only covers ASCII word chars, like real dialects, so
+        // a non-ASCII identifier character is unlexable. The full text must
+        // still round-trip through the returned segments; dropping it would make
+        // `fix` silently delete part of the user's SQL.
+        let matchers = vec![
+            Matcher::regex("whitespace", r"[^\S\r\n]+", SyntaxKind::Whitespace),
+            Matcher::regex("word", r"[0-9a-zA-Z_]+", SyntaxKind::Word),
+        ];
+        let lexer = Lexer::new(&matchers);
+        let tables = Tables::default();
+        let (segments, _) = lexer.lex(&tables, "select ф");
+        let reconstructed: String = segments.iter().map(|s| s.raw().as_str()).collect();
+        assert_eq!(reconstructed, "select ф");
+    }
+
     /// Test the RegexLexer.
     #[test]
     fn test_parser_lexer_regex() {