Skip to content

Commit a4ca320

Browse files
chernserchernser
committed
added test to detect new keywords
1 parent 26d0012 commit a4ca320

File tree

3 files changed

+107
-41
lines changed

3 files changed

+107
-41
lines changed

‎jdbc-v2/src/main/java/com/clickhouse/jdbc/internal/parser/javacc/ClickHouseSqlUtils.java‎

Lines changed: 45 additions & 41 deletions
Original file line numberDiff line numberDiff line change
@@ -13,50 +13,54 @@ private static Set<String> initAllowedKeywordAliases() {
1313
return buildKeywordSet(
1414
"ACCESS", "ACTION", "ADD", "ADMIN", "AFTER", "ALGORITHM", "ALIAS", "ALLOWED_LATENESS", "ALTER",
1515
"AND", "APPEND", "APPLY", "ASC", "ASCENDING", "ASSUME", "AST", "ASYNC", "ATTACH",
16-
"AUTHENTICATION", "AUTO_INCREMENT", "AZURE", "BACKUP", "BCRYPT_HASH", "BCRYPT_PASSWORD", "BEGIN",
17-
"BIDIRECTIONAL", "BOTH", "BY", "CACHE", "CACHES", "CASCADE", "CASE", "CAST", "CHANGE",
18-
"CHANGEABLE_IN_READONLY", "CHANGED", "CHAR", "CHARACTER", "CHECK", "CLEANUP", "CLEAR", "CLONE",
19-
"CLUSTER", "CLUSTERS", "CN", "CODEC", "COLLATE", "COLLECTION", "COLUMN", "COLUMNS", "COMMENT",
20-
"COMMIT", "COMPRESSION", "CONNECTIONS", "CONST", "CONSTRAINT", "COPY", "CREATE", "CUBE",
21-
"CURRENT", "CURRENT_USER", "CURRENTUSER", "DATA", "DATABASE", "DATABASES", "DATE", "DAY", "DAYS",
22-
"DDL", "DEALLOCATE", "DEDUPLICATE", "DEFAULT", "DEFINER", "DELAY", "DELETE", "DELETED", "DEPENDS",
23-
"DESC", "DESCENDING", "DESCRIBE", "DETACH", "DETACHED", "DICTIONARIES", "DICTIONARY", "DISK",
24-
"DISTINCT", "DIV", "DOUBLE_SHA1_HASH", "DOUBLE_SHA1_PASSWORD", "DROP", "EMPTY", "ENABLED", "END",
25-
"ENFORCED", "ENGINE", "ENGINES", "EPHEMERAL", "ESTIMATE", "EVENT", "EVENTS", "EVERY", "EXCHANGE",
26-
"EXECUTE", "EXISTS", "EXPLAIN", "EXPRESSION", "EXTENDED", "EXTERNAL", "FAKE", "FALSE", "FETCH",
27-
"FIELDS", "FILE", "FILESYSTEM", "FILL", "FILTER", "FIRST", "FOLLOWING", "FOR", "FORCE", "FOREIGN",
16+
"AUTHENTICATION", "AUTO_INCREMENT", "AZURE", "BACKUP", "BAGEXPANSION", "BASE_BACKUP",
17+
"BCRYPT_HASH", "BCRYPT_PASSWORD", "BEGIN", "BIDIRECTIONAL", "BOTH", "BY", "CACHE", "CACHES",
18+
"CASCADE", "CASE", "CAST", "CHANGE", "CHANGEABLE_IN_READONLY", "CHANGED", "CHAR", "CHARACTER",
19+
"CHECK", "CLEANUP", "CLEAR", "CLONE", "CLUSTER", "CLUSTERS", "CLUSTER_HOST_IDS", "CN", "CODEC",
20+
"COLLATE", "COLLECTION", "COLUMN", "COLUMNS", "COMMENT", "COMMIT", "COMPRESSION", "CONNECTIONS",
21+
"CONST", "CONSTRAINT", "COPY", "CREATE", "CUBE", "CURRENT", "CURRENT_USER", "CURRENTUSER",
22+
"D", "DATA", "DATABASE", "DATABASES", "DATE", "DAY", "DAYS", "DD", "DDL", "DEALLOCATE",
23+
"DEDUPLICATE", "DEFAULT", "DEFINER", "DELAY", "DELETE", "DELETED", "DEPENDS", "DESC",
24+
"DESCENDING", "DESCRIBE", "DETACH", "DETACHED", "DICTIONARIES", "DICTIONARY", "DISK", "DISTINCT",
25+
"DIV", "DOUBLE_SHA1_HASH", "DOUBLE_SHA1_PASSWORD", "DROP", "EMPTY", "ENABLED", "END", "ENFORCED",
26+
"ENGINE", "ENGINES", "EPHEMERAL", "ESTIMATE", "EVENT", "EVENTS", "EVERY", "EXCHANGE", "EXECUTE",
27+
"EXISTS", "EXPLAIN", "EXPRESSION", "EXTENDED", "EXTERNAL", "FAKE", "FALSE", "FETCH", "FIELDS",
28+
"FILE", "FILES", "FILESYSTEM", "FILL", "FILTER", "FIRST", "FOLLOWING", "FOR", "FORCE", "FOREIGN",
2829
"FORGET", "FREEZE", "FULLTEXT", "FUNCTION", "FUNCTIONS", "GRANT", "GRANTEES", "GRANTS",
29-
"GRANULARITY", "GROUPING", "GROUPS", "HASH", "HDFS", "HIERARCHICAL", "HOST", "HOUR", "HOURS",
30-
"HTTP", "ID", "IDENTIFIED", "IF", "IGNORE", "IMPLICIT", "IN", "INDEX", "INDEXES", "INDICES",
31-
"INFILE", "INHERIT", "INJECTIVE", "INSERT", "INTERPOLATE", "INTERVAL", "INVISIBLE", "INVOKER",
32-
"IP", "IS", "IS_OBJECT_ID", "JWT", "KERBEROS", "KEY", "KEYED", "KEYS", "KILL", "KIND", "LARGE",
33-
"LAST", "LAYOUT", "LDAP", "LEADING", "LESS", "LEVEL", "LIFETIME", "LIGHTWEIGHT", "LIMITS",
34-
"LINEAR", "LIST", "LIVE", "LOCAL", "MASK", "MASTER", "MATCH", "MATERIALIZE", "MATERIALIZED", "MAX",
35-
"MEMORY", "MERGES", "METHODS", "METRICS", "MICROSECOND", "MICROSECONDS", "MILLISECOND",
36-
"MILLISECONDS", "MIN", "MINUTE", "MINUTES", "MOD", "MODIFY", "MONTH", "MONTHS", "MOVE", "MUTATION",
37-
"NAME", "NAMED", "NANOSECOND", "NANOSECONDS", "NEW", "NEXT", "NO", "NO_AUTHENTICATION", "NONE",
38-
"NO_PASSWORD", "NULL", "NULLS", "OBJECT", "OPTIMIZE", "OPTION", "OR", "OUTER", "OUTFILE", "OVER",
39-
"OVERRIDABLE", "OVERRIDE", "PART", "PARTIAL", "PARTITION", "PARTITIONS", "PART_MOVE_TO_SHARD",
40-
"PARTS", "PATCHES", "PERIODIC", "PERMANENTLY", "PERMISSIVE", "PERSISTENT", "PIPELINE", "PLAN",
30+
"GRANULARITY", "GROUPING", "GROUPS", "H", "HASH", "HDFS", "HH", "HIERARCHICAL", "HOST", "HOUR",
31+
"HOURS", "HTTP", "ID", "IDENTIFIED", "IF", "IGNORE", "IMPLICIT", "IN", "INDEX", "INDEXES",
32+
"INDICES", "INFILE", "INHERIT", "INJECTIVE", "INSERT", "INTERPOLATE", "INTERVAL", "INVISIBLE",
33+
"INVOKER", "IP", "IS", "IS_OBJECT_ID", "JWT", "KERBEROS", "KEY", "KEYED", "KEYS", "KILL", "KIND",
34+
"LARGE", "LAST", "LAYOUT", "LDAP", "LEADING", "LESS", "LEVEL", "LIFETIME", "LIGHTWEIGHT",
35+
"LIMITS", "LINEAR", "LIST", "LIVE", "LOCAL", "M", "MASK", "MASKING", "MASTER", "MATCH",
36+
"MATERIALIZE", "MATERIALIZED", "MAX", "MCS", "MEMORY", "MERGES", "METHODS", "METRICS", "MI",
37+
"MICROSECOND", "MICROSECONDS", "MILLISECOND", "MILLISECONDS", "MIN", "MINUTE", "MINUTES", "MM",
38+
"MOD", "MODIFY", "MONTH", "MONTHS", "MOVE", "MS", "MUTATION", "N", "NAME", "NAMED", "NANOSECOND",
39+
"NANOSECONDS", "NEW", "NEXT", "NO", "NO_AUTHENTICATION", "NONE", "NO_PASSWORD", "NS", "NULL",
40+
"NULLS", "OBJECT", "OPTIMIZE", "OPTION", "OR", "OUTER", "OUTFILE", "OVER", "OVERRIDABLE",
41+
"OVERRIDE", "PART", "PARTIAL", "PARTITION", "PARTITIONS", "PART_MOVE_TO_SHARD", "PARTS",
42+
"PATCHES", "PAUSE", "PERIODIC", "PERMANENTLY", "PERMISSIVE", "PERSISTENT", "PIPELINE", "PLAN",
4143
"PLAINTEXT_PASSWORD", "POLICY", "POPULATE", "PRECEDING", "PRECISION", "PREFIX", "PREPARE",
42-
"PRIMARY", "PRIVILEGES", "PROCESSLIST", "PROFILE", "PROFILES", "PROJECTION", "PULL", "QUARTER",
43-
"QUARTERS", "QUERY", "QUOTA", "RANDOMIZE", "RANDOMIZED", "RANGE", "READ", "READONLY", "REALM",
44-
"RECOMPRESS", "RECURSIVE", "REFERENCES", "REFRESH", "REGEXP", "REMOVE", "RENAME", "REPLACE",
45-
"REPLICATED", "RESET", "RESOURCE", "RESPECT", "RESTORE", "RESTRICT", "RESTRICTIVE", "RESUME",
46-
"REVOKE", "REWRITE", "ROLE", "ROLES", "ROLLBACK", "ROLLUP", "ROW", "ROWS", "S3", "SALT", "SAN",
47-
"SCHEME", "SCRAM_SHA256_HASH", "SCRAM_SHA256_PASSWORD", "SECOND", "SECONDS", "SECURITY", "SELECT",
48-
"SEQUENTIAL", "SERVER", "SET", "SETS", "SETTING", "SHA256_HASH", "SHA256_PASSWORD", "SHARD", "SHOW",
49-
"SIGNED", "SIMPLE", "SKIP", "SNAPSHOT", "SOURCE", "SPATIAL", "SQL", "SQL_TSI_DAY", "SQL_TSI_HOUR",
44+
"PRIMARY", "PRIORITY", "PRIVILEGES", "PROCESSLIST", "PROFILE", "PROFILES", "PROJECTION",
45+
"PROTOBUF", "PULL", "Q", "QQ", "QUARTER", "QUARTERS", "QUERY", "QUOTA", "RANDOMIZE",
46+
"RANDOMIZED", "RANGE", "READ", "READONLY", "REALM", "RECOMPRESS", "RECURSIVE", "REFERENCES",
47+
"REFRESH", "REGEXP", "REMOVE", "RENAME", "REPLACE", "REPLICATED", "RESET", "RESOURCE", "RESPECT",
48+
"RESTORE", "RESTRICT", "RESTRICTIVE", "RESUME", "REVOKE", "REWRITE", "ROLE", "ROLES", "ROLLBACK",
49+
"ROLLUP", "ROW", "ROWS", "S", "S3", "SALT", "SAN", "SCHEME", "SCRAM_SHA256_HASH",
50+
"SCRAM_SHA256_PASSWORD", "SECOND", "SECONDS", "SECURITY", "SELECT", "SEQUENTIAL", "SERVER",
51+
"SET", "SETS", "SETTING", "SHA256_HASH", "SHA256_PASSWORD", "SHARD", "SHOW", "SIGNED", "SIMPLE",
52+
"SKIP", "SNAPSHOT", "SOURCE", "SPATIAL", "SQL", "SQL_TSI_DAY", "SQL_TSI_HOUR",
5053
"SQL_TSI_MICROSECOND", "SQL_TSI_MILLISECOND", "SQL_TSI_MINUTE", "SQL_TSI_MONTH",
51-
"SQL_TSI_NANOSECOND", "SQL_TSI_QUARTER", "SQL_TSI_SECOND", "SQL_TSI_WEEK", "SQL_TSI_YEAR", "SSH_KEY",
52-
"SSL_CERTIFICATE", "STALENESS", "START", "STATISTIC", "STATISTICS", "STDOUT", "STEP", "STORAGE",
53-
"STRICT", "STRICTLY_ASCENDING", "SUBPARTITION", "SUBPARTITIONS", "SUSPEND", "SYNC", "SYNTAX",
54-
"SYSTEM", "TABLE", "TABLES", "TAG", "TAGS", "TEMPORARY", "TEST", "THAN", "THEN", "THREAD", "TIES",
55-
"TIME", "TIMESTAMP", "TO", "TOP", "TOTALS", "TRACKING", "TRAILING", "TRANSACTION", "TREE", "TRIGGER",
56-
"TRUE", "TRUNCATE", "TTL", "TYPE", "TYPEOF", "UNBOUNDED", "UNDROP", "UNFREEZE", "UNIQUE", "UNLOCK",
57-
"UNSET", "UNSIGNED", "UNTIL", "UPDATE", "URL", "USE", "USER", "VALID", "VALUES", "VARYING", "VIEW",
58-
"VISIBLE", "VOLUME", "WATCH", "WATERMARK", "WEEK", "WEEKS", "WHEN", "WITH_ITEMINDEX", "WORKER",
59-
"WORKLOAD", "WRITABLE", "WRITE", "YEAR", "YEARS", "ZKPATH");
54+
"SQL_TSI_NANOSECOND", "SQL_TSI_QUARTER", "SQL_TSI_SECOND", "SQL_TSI_WEEK", "SQL_TSI_YEAR", "SS",
55+
"SSH_KEY", "SSL_CERTIFICATE", "STALENESS", "START", "STATISTIC", "STATISTICS", "STDOUT", "STEP",
56+
"STORAGE", "STRICT", "STRICTLY_ASCENDING", "SUBPARTITION", "SUBPARTITIONS", "SUSPEND", "SYNC",
57+
"SYNTAX", "SYSTEM", "TABLE", "TABLES", "TAG", "TAGS", "TEMPORARY", "TEST", "THAN", "THEN",
58+
"THREAD", "TIES", "TIME", "TIMESTAMP", "TO", "TOP", "TOTALS", "TRACKING", "TRAILING",
59+
"TRANSACTION", "TREE", "TRIGGER", "TRUE", "TRUNCATE", "TTL", "TYPE", "TYPEOF", "UNBOUNDED",
60+
"UNDROP", "UNFREEZE", "UNIQUE", "UNLOCK", "UNSET", "UNSIGNED", "UNTIL", "UPDATE", "URL", "USE",
61+
"USER", "VALID", "VALUES", "VARYING", "VIEW", "VISIBLE", "VOLUME", "WATCH", "WATERMARK", "WEEK",
62+
"WEEKS", "WHEN", "WITH_ITEMINDEX", "WK", "WORKER", "WORKLOAD", "WRITABLE", "WRITE", "WW",
63+
"YEAR", "YEARS", "YY", "YYYY", "ZKPATH");
6064
}
6165

6266
private static Set<String> buildKeywordSet(String... values) {

‎jdbc-v2/src/test/java/com/clickhouse/jdbc/StatementSQLTest.java‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,8 @@
55
import org.testng.annotations.DataProvider;
66
import org.testng.annotations.Test;
77

8+
import com.clickhouse.jdbc.internal.parser.javacc.ClickHouseSqlUtils;
9+
810
import java.io.BufferedReader;
911
import java.io.InputStream;
1012
import java.io.InputStreamReader;
@@ -15,6 +17,8 @@
1517
import java.util.ArrayList;
1618
import java.util.List;
1719
import java.util.Map;
20+
import java.util.Set;
21+
import java.util.TreeSet;
1822
import java.util.concurrent.atomic.AtomicBoolean;
1923

2024
/**
@@ -110,6 +114,37 @@ public void testKeywordsAsTableNames() throws Exception {
110114
"All keywords should be categorized");
111115
}
112116

117+
@Test(groups = {"integration"})
118+
public void testAllowedKeywordAliasesMatchSystemKeywords() throws Exception {
119+
Set<String> reservedKeywords = new TreeSet<>(loadKeywordsFromResource("reserved_keywords.txt"));
120+
121+
Set<String> systemKeywords = new TreeSet<>();
122+
try (Connection connection = getJdbcConnection();
123+
Statement stmt = connection.createStatement();
124+
ResultSet rs = stmt.executeQuery("SELECT keyword FROM system.keywords")) {
125+
while (rs.next()) {
126+
for (String word : rs.getString(1).toUpperCase().split("\\s+")) {
127+
systemKeywords.add(word);
128+
}
129+
}
130+
}
131+
132+
Assert.assertFalse(systemKeywords.isEmpty(), "system.keywords should not be empty");
133+
134+
Set<String> nonReservedSystemKeywords = new TreeSet<>(systemKeywords);
135+
nonReservedSystemKeywords.removeAll(reservedKeywords);
136+
137+
Set<String> allowedAliases = ClickHouseSqlUtils.getKeywordGroup(
138+
ClickHouseSqlUtils.KEYWORD_GROUP_ALLOWED_ALIASES);
139+
140+
Set<String> missingFromAllowed = new TreeSet<>(nonReservedSystemKeywords);
141+
missingFromAllowed.removeAll(allowedAliases);
142+
143+
Assert.assertTrue(missingFromAllowed.isEmpty(),
144+
"New keywords found in system.keywords (non-reserved) that must be added to "
145+
+ "ALLOWED_KEYWORD_ALIASES or reserved_keywords.txt: " + missingFromAllowed);
146+
}
147+
113148
private List<String> loadKeywordsFromResource(String resourceName) throws Exception {
114149
List<String> keywords = new ArrayList<>();
115150
try (InputStream is = getClass().getClassLoader().getResourceAsStream(resourceName)) {

‎jdbc-v2/src/test/resources/sql-keywords.txt‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,8 @@ AUTHENTICATION
3030
AUTO_INCREMENT
3131
AZURE
3232
BACKUP
33+
BAGEXPANSION
34+
BASE_BACKUP
3335
BCRYPT_HASH
3436
BCRYPT_PASSWORD
3537
BEGIN
@@ -53,6 +55,7 @@ CLEAR
5355
CLONE
5456
CLUSTER
5557
CLUSTERS
58+
CLUSTER_HOST_IDS
5659
CN
5760
CODEC
5861
COLLATE
@@ -72,12 +75,14 @@ CUBE
7275
CURRENT
7376
CURRENT_USER
7477
CURRENTUSER
78+
D
7579
DATA
7680
DATABASE
7781
DATABASES
7882
DATE
7983
DAY
8084
DAYS
85+
DD
8186
DDL
8287
DEALLOCATE
8388
DEDUPLICATE
@@ -125,6 +130,7 @@ FALSE
125130
FETCH
126131
FIELDS
127132
FILE
133+
FILES
128134
FILESYSTEM
129135
FILL
130136
FILTER
@@ -150,9 +156,11 @@ GRANULARITY
150156
GROUP
151157
GROUPING
152158
GROUPS
159+
H
153160
HASH
154161
HAVING
155162
HDFS
163+
HH
156164
HIERARCHICAL
157165
HOST
158166
HOUR
@@ -207,29 +215,36 @@ LINEAR
207215
LIST
208216
LIVE
209217
LOCAL
218+
M
210219
MASK
220+
MASKING
211221
MASTER
212222
MATCH
213223
MATERIALIZE
214224
MATERIALIZED
215225
MAX
226+
MCS
216227
MEMORY
217228
MERGES
218229
METHODS
219230
METRICS
231+
MI
220232
MICROSECOND
221233
MICROSECONDS
222234
MILLISECOND
223235
MILLISECONDS
224236
MIN
225237
MINUTE
226238
MINUTES
239+
MM
227240
MOD
228241
MODIFY
229242
MONTH
230243
MONTHS
231244
MOVE
245+
MS
232246
MUTATION
247+
N
233248
NAME
234249
NAMED
235250
NANOSECOND
@@ -241,6 +256,7 @@ NO_AUTHENTICATION
241256
NONE
242257
NO_PASSWORD
243258
NOT
259+
NS
244260
NULL
245261
NULLS
246262
OBJECT
@@ -265,6 +281,7 @@ PART_MOVE_TO_SHARD
265281
PARTS
266282
PASTE
267283
PATCHES
284+
PAUSE
268285
PERIODIC
269286
PERMANENTLY
270287
PERMISSIVE
@@ -280,12 +297,16 @@ PREFIX
280297
PREPARE
281298
PREWHERE
282299
PRIMARY
300+
PRIORITY
283301
PRIVILEGES
284302
PROCESSLIST
285303
PROFILE
286304
PROFILES
287305
PROJECTION
306+
PROTOBUF
288307
PULL
308+
Q
309+
QQ
289310
QUALIFY
290311
QUARTER
291312
QUARTERS
@@ -322,6 +343,7 @@ ROLLBACK
322343
ROLLUP
323344
ROW
324345
ROWS
346+
S
325347
S3
326348
SALT
327349
SAMPLE
@@ -362,6 +384,7 @@ SQL_TSI_QUARTER
362384
SQL_TSI_SECOND
363385
SQL_TSI_WEEK
364386
SQL_TSI_YEAR
387+
SS
365388
SSH_KEY
366389
SSL_CERTIFICATE
367390
STALENESS
@@ -434,10 +457,14 @@ WHERE
434457
WINDOW
435458
WITH
436459
WITH_ITEMINDEX
460+
WK
437461
WORKER
438462
WORKLOAD
439463
WRITABLE
440464
WRITE
465+
WW
441466
YEAR
442467
YEARS
468+
YY
469+
YYYY
443470
ZKPATH

0 commit comments

Comments
 (0)