fix tokenization of {{param}}._

Author: jmhain

src/tokenizer.rs

@@ -1236,7 +1236,7 @@ impl<'a> Tokenizer<'a> {
                     // if the prev token is not a word, then this is not a valid sql
                     // word or number.
                     if ch == '.' && chars.peekable.clone().nth(1) == Some('_') {
-                        if let Some(Token::Word(_)) = prev_token {
+                        if let Some(Token::Word(_) | Token::Mustache(_)) = prev_token {
                             chars.next();
                             return Ok(Some(Token::Period));
                         }
@@ -2495,7 +2495,8 @@ fn take_char_from_hex_digits(
 mod tests {
     use super::*;
     use crate::dialect::{
-        BigQueryDialect, ClickHouseDialect, HiveDialect, MsSqlDialect, MySqlDialect, SQLiteDialect,
+        AnsiDialect, BigQueryDialect, ClickHouseDialect, HiveDialect, MsSqlDialect, MySqlDialect,
+        SQLiteDialect,
     };
     use crate::test_utils::{all_dialects_except, all_dialects_where};
     use core::fmt::Debug;
@@ -4138,4 +4139,19 @@ mod tests {
             panic!("Tokenizer should have failed on {sql}, but it succeeded with {tokens:?}");
         }
     }
+
+    #[test]
+    fn tokenize_mustache_dot_ident() {
+        // Note: Ansi and Hive parse _column into separate `_` and `column` tokens.
+        all_dialects_except(|d| d.is::<HiveDialect>() || d.is::<AnsiDialect>()).tokenizes_to(
+            "SELECT {{schema}}._column",
+            vec![
+                Token::make_keyword("SELECT"),
+                Token::Whitespace(Whitespace::Space),
+                Token::Mustache("schema".to_owned()),
+                Token::Period,
+                Token::make_word("_column", None),
+            ],
+        );
+    }
 }