Skip to content

Commit 1788567

Browse files
benfdkingbenfdking
committed
fix: support unquoted Unicode identifiers in ANSI and Postgres dialects
Replace ASCII-only character classes ([a-zA-Z0-9_]) with Unicode-aware classes (\p{L}, \p{N}) in lexer word patterns and parser identifier regexes for the ANSI and Postgres dialects. This fixes panics when linting SQL with unquoted multibyte identifiers (e.g. Japanese, French, German characters). Closes #2067
1 parent b30abe3 commit 1788567

8 files changed

Lines changed: 230 additions & 8 deletions

File tree

‎crates/lib-dialects/src/ansi.rs‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -356,15 +356,15 @@ pub fn raw_dialect() -> Dialect {
356356
let pattern = reserved_keywords.iter().join("|");
357357
let anti_template = format!("^({pattern})$");
358358

359-
RegexParser::new("[A-Z0-9_]*[A-Z][A-Z0-9_]*", SyntaxKind::NakedIdentifier)
359+
RegexParser::new("[\\p{L}\\p{N}_]*[\\p{L}][\\p{L}\\p{N}_]*", SyntaxKind::NakedIdentifier)
360360
.anti_template(&anti_template)
361361
.to_matchable()
362362
})
363363
.into(),
364364
),
365365
(
366366
"ParameterNameSegment".into(),
367-
RegexParser::new(r#"\"?[A-Z][A-Z0-9_]*\"?"#, SyntaxKind::Parameter)
367+
RegexParser::new(r#"\"?[\p{L}][\p{L}\p{N}_]*\"?"#, SyntaxKind::Parameter)
368368
.to_matchable()
369369
.into(),
370370
),
@@ -383,7 +383,7 @@ pub fn raw_dialect() -> Dialect {
383383
let anti_template = format!("^({})$", "NOT");
384384

385385
one_of(vec![
386-
RegexParser::new("[A-Z_][A-Z0-9_]*", SyntaxKind::DataTypeIdentifier)
386+
RegexParser::new("[\\p{L}_][\\p{L}\\p{N}_]*", SyntaxKind::DataTypeIdentifier)
387387
.anti_template(&anti_template)
388388
.to_matchable(),
389389
Ref::new("SingleIdentifierGrammar")
@@ -5370,7 +5370,7 @@ fn lexer_matchers() -> Vec<Matcher> {
53705370
Matcher::string("end_curly_bracket", "}", SyntaxKind::EndCurlyBracket),
53715371
Matcher::string("colon", ":", SyntaxKind::Colon),
53725372
Matcher::string("semicolon", ";", SyntaxKind::Semicolon),
5373-
Matcher::regex("word", "[0-9a-zA-Z_]+", SyntaxKind::Word),
5373+
Matcher::regex("word", "[\\p{L}\\p{N}_]+", SyntaxKind::Word),
53745374
]
53755375
}
53765376

‎crates/lib-dialects/src/postgres.rs‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -105,7 +105,7 @@ pub fn raw_dialect() -> Dialect {
105105
SyntaxKind::SingleQuote,
106106
),
107107
Matcher::regex("double_quote", r#"(?s)".+?""#, SyntaxKind::DoubleQuote),
108-
Matcher::regex("word", r"[a-zA-Z_][0-9a-zA-Z_$]*", SyntaxKind::Word),
108+
Matcher::regex("word", r"[\p{L}_][\p{L}\p{N}_$]*", SyntaxKind::Word),
109109
]);
110110

111111
let keywords = postgres_keywords();
@@ -402,7 +402,7 @@ pub fn raw_dialect() -> Dialect {
402402
let anti_template = format!("^({pattern})$");
403403

404404
RegexParser::new(
405-
r"([A-Z_]+|[0-9]+[A-Z_$])[A-Z0-9_$]*",
405+
r"([\p{L}_]+|[\p{N}]+[\p{L}_$])[\p{L}\p{N}_$]*",
406406
SyntaxKind::NakedIdentifier,
407407
)
408408
.anti_template(&anti_template)
@@ -412,13 +412,13 @@ pub fn raw_dialect() -> Dialect {
412412
),
413413
(
414414
"ParameterNameSegment".into(),
415-
RegexParser::new(r#"[A-Z_][A-Z0-9_$]*|\"[^\"]*\""#, SyntaxKind::Parameter)
415+
RegexParser::new(r#"[\p{L}_][\p{L}\p{N}_$]*|\"[^\"]*\""#, SyntaxKind::Parameter)
416416
.to_matchable()
417417
.into(),
418418
),
419419
(
420420
"FunctionNameIdentifierSegment".into(),
421-
RegexParser::new(r"[A-Z_][A-Z0-9_$]*", SyntaxKind::FunctionNameIdentifier)
421+
RegexParser::new(r"[\p{L}_][\p{L}\p{N}_$]*", SyntaxKind::FunctionNameIdentifier)
422422
.to_matchable()
423423
.into(),
424424
),

‎crates/lib-dialects/test/fixtures/dialects/ansi/sqruff/unicode_identifiers.sql‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
SELECT 日本語 FROM テーブル;
2+
3+
SELECT 表1.列1 FROM 表1;
4+
5+
SELECT café FROM naïve_table;
6+
7+
SELECT über, straße FROM données;

‎crates/lib-dialects/test/fixtures/dialects/ansi/sqruff/unicode_identifiers.yml‎

Lines changed: 67 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,67 @@
1+
file:
2+
- statement:
3+
- select_statement:
4+
- select_clause:
5+
- keyword: SELECT
6+
- select_clause_element:
7+
- column_reference:
8+
- naked_identifier: 日本語
9+
- from_clause:
10+
- keyword: FROM
11+
- from_expression:
12+
- from_expression_element:
13+
- table_expression:
14+
- table_reference:
15+
- naked_identifier: テーブル
16+
- statement_terminator: ;
17+
- statement:
18+
- select_statement:
19+
- select_clause:
20+
- keyword: SELECT
21+
- select_clause_element:
22+
- column_reference:
23+
- naked_identifier: 表1
24+
- dot: .
25+
- naked_identifier: 列1
26+
- from_clause:
27+
- keyword: FROM
28+
- from_expression:
29+
- from_expression_element:
30+
- table_expression:
31+
- table_reference:
32+
- naked_identifier: 表1
33+
- statement_terminator: ;
34+
- statement:
35+
- select_statement:
36+
- select_clause:
37+
- keyword: SELECT
38+
- select_clause_element:
39+
- column_reference:
40+
- naked_identifier: café
41+
- from_clause:
42+
- keyword: FROM
43+
- from_expression:
44+
- from_expression_element:
45+
- table_expression:
46+
- table_reference:
47+
- naked_identifier: naïve_table
48+
- statement_terminator: ;
49+
- statement:
50+
- select_statement:
51+
- select_clause:
52+
- keyword: SELECT
53+
- select_clause_element:
54+
- column_reference:
55+
- naked_identifier: über
56+
- comma: ','
57+
- select_clause_element:
58+
- column_reference:
59+
- naked_identifier: straße
60+
- from_clause:
61+
- keyword: FROM
62+
- from_expression:
63+
- from_expression_element:
64+
- table_expression:
65+
- table_reference:
66+
- naked_identifier: données
67+
- statement_terminator: ;

‎crates/lib-dialects/test/fixtures/dialects/duckdb/sqruff/unicode_identifiers.sql‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
SELECT 日本語 FROM テーブル;
2+
3+
SELECT 表1.列1 FROM 表1;
4+
5+
SELECT café FROM naïve_table;
6+
7+
SELECT über, straße FROM données;

‎crates/lib-dialects/test/fixtures/dialects/duckdb/sqruff/unicode_identifiers.yml‎

Lines changed: 67 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,67 @@
1+
file:
2+
- statement:
3+
- select_statement:
4+
- select_clause:
5+
- keyword: SELECT
6+
- select_clause_element:
7+
- column_reference:
8+
- naked_identifier: 日本語
9+
- from_clause:
10+
- keyword: FROM
11+
- from_expression:
12+
- from_expression_element:
13+
- table_expression:
14+
- table_reference:
15+
- naked_identifier: テーブル
16+
- statement_terminator: ;
17+
- statement:
18+
- select_statement:
19+
- select_clause:
20+
- keyword: SELECT
21+
- select_clause_element:
22+
- column_reference:
23+
- naked_identifier: 表1
24+
- dot: .
25+
- naked_identifier: 列1
26+
- from_clause:
27+
- keyword: FROM
28+
- from_expression:
29+
- from_expression_element:
30+
- table_expression:
31+
- table_reference:
32+
- naked_identifier: 表1
33+
- statement_terminator: ;
34+
- statement:
35+
- select_statement:
36+
- select_clause:
37+
- keyword: SELECT
38+
- select_clause_element:
39+
- column_reference:
40+
- naked_identifier: café
41+
- from_clause:
42+
- keyword: FROM
43+
- from_expression:
44+
- from_expression_element:
45+
- table_expression:
46+
- table_reference:
47+
- naked_identifier: naïve_table
48+
- statement_terminator: ;
49+
- statement:
50+
- select_statement:
51+
- select_clause:
52+
- keyword: SELECT
53+
- select_clause_element:
54+
- column_reference:
55+
- naked_identifier: über
56+
- comma: ','
57+
- select_clause_element:
58+
- column_reference:
59+
- naked_identifier: straße
60+
- from_clause:
61+
- keyword: FROM
62+
- from_expression:
63+
- from_expression_element:
64+
- table_expression:
65+
- table_reference:
66+
- naked_identifier: données
67+
- statement_terminator: ;

‎crates/lib-dialects/test/fixtures/dialects/postgres/sqruff/unicode_identifiers.sql‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
SELECT 日本語 FROM テーブル;
2+
3+
SELECT 表1.列1 FROM 表1;
4+
5+
SELECT café FROM naïve_table;
6+
7+
SELECT über, straße FROM données;

‎crates/lib-dialects/test/fixtures/dialects/postgres/sqruff/unicode_identifiers.yml‎

Lines changed: 67 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,67 @@
1+
file:
2+
- statement:
3+
- select_statement:
4+
- select_clause:
5+
- keyword: SELECT
6+
- select_clause_element:
7+
- column_reference:
8+
- naked_identifier: 日本語
9+
- from_clause:
10+
- keyword: FROM
11+
- from_expression:
12+
- from_expression_element:
13+
- table_expression:
14+
- table_reference:
15+
- naked_identifier: テーブル
16+
- statement_terminator: ;
17+
- statement:
18+
- select_statement:
19+
- select_clause:
20+
- keyword: SELECT
21+
- select_clause_element:
22+
- column_reference:
23+
- naked_identifier: 表1
24+
- dot: .
25+
- naked_identifier: 列1
26+
- from_clause:
27+
- keyword: FROM
28+
- from_expression:
29+
- from_expression_element:
30+
- table_expression:
31+
- table_reference:
32+
- naked_identifier: 表1
33+
- statement_terminator: ;
34+
- statement:
35+
- select_statement:
36+
- select_clause:
37+
- keyword: SELECT
38+
- select_clause_element:
39+
- column_reference:
40+
- naked_identifier: café
41+
- from_clause:
42+
- keyword: FROM
43+
- from_expression:
44+
- from_expression_element:
45+
- table_expression:
46+
- table_reference:
47+
- naked_identifier: naïve_table
48+
- statement_terminator: ;
49+
- statement:
50+
- select_statement:
51+
- select_clause:
52+
- keyword: SELECT
53+
- select_clause_element:
54+
- column_reference:
55+
- naked_identifier: über
56+
- comma: ','
57+
- select_clause_element:
58+
- column_reference:
59+
- naked_identifier: straße
60+
- from_clause:
61+
- keyword: FROM
62+
- from_expression:
63+
- from_expression_element:
64+
- table_expression:
65+
- table_reference:
66+
- naked_identifier: données
67+
- statement_terminator: ;

0 commit comments

Comments
 (0)