Skip to content

Commit 19d297f

Browse files
nenenene
authored
Merge PR #403: Remove Tokens from AST
2 parents ee3898f + 35e8c0f commit 19d297f

File tree

11 files changed

+506
-766
lines changed

11 files changed

+506
-766
lines changed

‎src/formatter/ExpressionFormatter.ts

Lines changed: 176 additions & 200 deletions
Large diffs are not rendered by default.

‎src/formatter/Formatter.ts

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ import Params from 'src/formatter/Params';
44
import Tokenizer from 'src/lexer/Tokenizer';
55

66
import { createParser } from 'src/parser/createParser';
7-
import { Statement } from 'src/parser/ast';
7+
import { StatementNode } from 'src/parser/ast';
88

99
import formatCommaPositions from './formatCommaPositions';
1010
import formatAliasPositions from './formatAliasPositions';
@@ -53,17 +53,17 @@ export default class Formatter {
5353
return finalQuery.trimEnd();
5454
}
5555

56-
private parse(query: string): Statement[] {
56+
private parse(query: string): StatementNode[] {
5757
return createParser(this.cachedTokenizer()).parse(query, this.cfg.paramTypes || {});
5858
}
5959

60-
private formatAst(statements: Statement[]): string {
60+
private formatAst(statements: StatementNode[]): string {
6161
return statements
6262
.map(stat => this.formatStatement(stat))
6363
.join('\n'.repeat(this.cfg.linesBetweenQueries + 1));
6464
}
6565

66-
private formatStatement(statement: Statement): string {
66+
private formatStatement(statement: StatementNode): string {
6767
const layout = new ExpressionFormatter({
6868
cfg: this.cfg,
6969
params: this.params,

‎src/formatter/InlineBlock.ts

Lines changed: 30 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,11 @@
1-
import { type Token, TokenType, isLogicalOperator } from 'src/lexer/token';
2-
import { AstNode, BetweenPredicate, NodeType, Parenthesis } from 'src/parser/ast';
1+
import { TokenType, isLogicalOperator } from 'src/lexer/token';
2+
import {
3+
AstNode,
4+
BetweenPredicateNode,
5+
KeywordNode,
6+
NodeType,
7+
ParenthesisNode,
8+
} from 'src/parser/ast';
39

410
/**
511
* Bookkeeper for inline blocks.
@@ -15,11 +21,11 @@ export default class InlineBlock {
1521
* Check if this should be an inline parentheses block
1622
* Examples are "NOW()", "COUNT(*)", "int(10)", key(`somecolumn`), DECIMAL(7,2)
1723
*/
18-
public isInlineBlock(parenthesis: Parenthesis): boolean {
24+
public isInlineBlock(parenthesis: ParenthesisNode): boolean {
1925
return this.inlineParenthesisWidth(parenthesis) <= this.expressionWidth;
2026
}
2127

22-
private inlineParenthesisWidth(parenthesis: Parenthesis): number {
28+
private inlineParenthesisWidth(parenthesis: ParenthesisNode): number {
2329
// +2 for the two parenthesis
2430
return this.inlineWidth(parenthesis.children) + 2;
2531
}
@@ -30,10 +36,10 @@ export default class InlineBlock {
3036
for (const node of nodes) {
3137
switch (node.type) {
3238
case NodeType.function_call:
33-
length += node.nameToken.text.length + this.inlineParenthesisWidth(node.parenthesis);
39+
length += node.name.text.length + this.inlineParenthesisWidth(node.parenthesis);
3440
break;
3541
case NodeType.array_subscript:
36-
length += node.arrayToken.text.length + this.inlineParenthesisWidth(node.parenthesis);
42+
length += node.array.text.length + this.inlineParenthesisWidth(node.parenthesis);
3743
break;
3844
case NodeType.parenthesis:
3945
length += this.inlineParenthesisWidth(node);
@@ -44,13 +50,22 @@ export default class InlineBlock {
4450
case NodeType.clause:
4551
case NodeType.limit_clause:
4652
case NodeType.set_operation:
53+
case NodeType.line_comment:
54+
case NodeType.block_comment:
4755
return Infinity;
4856
case NodeType.all_columns_asterisk:
57+
case NodeType.comma:
4958
length += 1;
5059
break;
51-
case NodeType.token:
52-
length += node.token.text.length;
53-
if (this.isForbiddenToken(node.token)) {
60+
case NodeType.literal:
61+
case NodeType.identifier:
62+
case NodeType.parameter:
63+
case NodeType.operator:
64+
length += node.text.length;
65+
break;
66+
case NodeType.keyword:
67+
length += node.text.length;
68+
if (this.isForbiddenKeyword(node)) {
5469
return Infinity;
5570
}
5671
break;
@@ -64,23 +79,17 @@ export default class InlineBlock {
6479
return length;
6580
}
6681

67-
private betweenWidth(node: BetweenPredicate): number {
82+
private betweenWidth(node: BetweenPredicateNode): number {
6883
return (
69-
node.betweenToken.text.length +
84+
node.between.text.length +
7085
this.inlineWidth(node.expr1) +
71-
node.andToken.text.length +
86+
node.and.text.length +
7287
this.inlineWidth(node.expr2)
7388
);
7489
}
7590

76-
// Reserved words that cause newlines, comments and semicolons
77-
// are not allowed inside inline parentheses block
78-
private isForbiddenToken(token: Token) {
79-
return (
80-
isLogicalOperator(token) ||
81-
token.type === TokenType.LINE_COMMENT ||
82-
token.type === TokenType.BLOCK_COMMENT ||
83-
token.type === TokenType.CASE // CASE cannot have inline blocks
84-
);
91+
// Reserved words that cause newlines are not allowed inside inline parentheses block
92+
private isForbiddenKeyword(node: KeywordNode) {
93+
return isLogicalOperator(node.tokenType) || node.tokenType === TokenType.CASE;
8594
}
8695
}

‎src/formatter/Params.ts

Lines changed: 1 addition & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,3 @@
1-
import { type Token } from '../lexer/token';
2-
31
export type ParamItems = { [k: string]: string };
42

53
/**
@@ -19,10 +17,8 @@ export default class Params {
1917

2018
/**
2119
* Returns param value that matches given placeholder with param key.
22-
* @param {Token} token
23-
* @return {string} param or token.text when params are missing
2420
*/
25-
get({ key, text }: Token): string {
21+
get({ key, text }: { key?: string; text: string }): string {
2622
if (!this.params) {
2723
return text;
2824
}

‎src/formatter/tabularStyle.ts

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
import type { IndentStyle } from 'src/FormatOptions';
2-
import { isLogicalOperator, Token, TokenType } from 'src/lexer/token';
2+
import { isLogicalOperator, TokenType } from 'src/lexer/token';
33

44
/**
55
* When tabular style enabled,
@@ -28,13 +28,13 @@ export default function toTabularFormat(tokenText: string, indentStyle: IndentSt
2828
/**
2929
* True when the token can be formatted in tabular style
3030
*/
31-
export function isTabularToken(token: Token): boolean {
31+
export function isTabularToken(type: TokenType): boolean {
3232
return (
33-
isLogicalOperator(token) ||
34-
token.type === TokenType.RESERVED_DEPENDENT_CLAUSE ||
35-
token.type === TokenType.RESERVED_COMMAND ||
36-
token.type === TokenType.RESERVED_SELECT ||
37-
token.type === TokenType.RESERVED_SET_OPERATION ||
38-
token.type === TokenType.RESERVED_JOIN
33+
isLogicalOperator(type) ||
34+
type === TokenType.RESERVED_DEPENDENT_CLAUSE ||
35+
type === TokenType.RESERVED_COMMAND ||
36+
type === TokenType.RESERVED_SELECT ||
37+
type === TokenType.RESERVED_SET_OPERATION ||
38+
type === TokenType.RESERVED_JOIN
3939
);
4040
}

‎src/languages/plsql/plsql.formatter.ts

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -119,7 +119,7 @@ function postProcess(tokens: Token[]) {
119119
return { ...token, type: TokenType.RESERVED_KEYWORD };
120120
}
121121

122-
if (isReserved(token)) {
122+
if (isReserved(token.type)) {
123123
previousReservedToken = token;
124124
}
125125

‎src/lexer/token.ts

Lines changed: 19 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -76,30 +76,23 @@ export const isToken = {
7676
};
7777

7878
/** Checks if token is any Reserved Keyword or Command */
79-
export const isReserved = (token: Token): boolean =>
80-
token.type === TokenType.RESERVED_KEYWORD ||
81-
token.type === TokenType.RESERVED_FUNCTION_NAME ||
82-
token.type === TokenType.RESERVED_PHRASE ||
83-
token.type === TokenType.RESERVED_DEPENDENT_CLAUSE ||
84-
token.type === TokenType.RESERVED_COMMAND ||
85-
token.type === TokenType.RESERVED_SELECT ||
86-
token.type === TokenType.RESERVED_SET_OPERATION ||
87-
token.type === TokenType.RESERVED_JOIN ||
88-
token.type === TokenType.ARRAY_KEYWORD ||
89-
token.type === TokenType.CASE ||
90-
token.type === TokenType.END ||
91-
token.type === TokenType.LIMIT ||
92-
token.type === TokenType.BETWEEN ||
93-
token.type === TokenType.AND ||
94-
token.type === TokenType.OR ||
95-
token.type === TokenType.XOR;
79+
export const isReserved = (type: TokenType): boolean =>
80+
type === TokenType.RESERVED_KEYWORD ||
81+
type === TokenType.RESERVED_FUNCTION_NAME ||
82+
type === TokenType.RESERVED_PHRASE ||
83+
type === TokenType.RESERVED_DEPENDENT_CLAUSE ||
84+
type === TokenType.RESERVED_COMMAND ||
85+
type === TokenType.RESERVED_SELECT ||
86+
type === TokenType.RESERVED_SET_OPERATION ||
87+
type === TokenType.RESERVED_JOIN ||
88+
type === TokenType.ARRAY_KEYWORD ||
89+
type === TokenType.CASE ||
90+
type === TokenType.END ||
91+
type === TokenType.LIMIT ||
92+
type === TokenType.BETWEEN ||
93+
type === TokenType.AND ||
94+
type === TokenType.OR ||
95+
type === TokenType.XOR;
9696

97-
/** checks if token is one of the parameter tokens */
98-
export const isParameter = (token: Token): boolean =>
99-
token.type === TokenType.NUMBERED_PARAMETER ||
100-
token.type === TokenType.NAMED_PARAMETER ||
101-
token.type === TokenType.POSITIONAL_PARAMETER ||
102-
token.type === TokenType.QUOTED_PARAMETER;
103-
104-
export const isLogicalOperator = (token: Token): boolean =>
105-
token.type === TokenType.AND || token.type === TokenType.OR || token.type === TokenType.XOR;
97+
export const isLogicalOperator = (type: TokenType): boolean =>
98+
type === TokenType.AND || type === TokenType.OR || type === TokenType.XOR;

0 commit comments

Comments
 (0)