Skip to content

Commit 7077d45

Browse files
crisptrutskicrisptrutski
committed
simple ast for transpiling
1 parent b34720c commit 7077d45

File tree

3 files changed

+337
-2
lines changed

3 files changed

+337
-2
lines changed

‎java/com/metabase/macaw/SimpleParser.java‎

Lines changed: 294 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,294 @@
1+
package com.metabase.macaw;
2+
3+
import clojure.lang.Keyword;
4+
import net.sf.jsqlparser.expression.Expression;
5+
import net.sf.jsqlparser.expression.Function;
6+
import net.sf.jsqlparser.expression.LongValue;
7+
import net.sf.jsqlparser.expression.operators.relational.ComparisonOperator;
8+
import net.sf.jsqlparser.expression.operators.relational.EqualsTo;
9+
import net.sf.jsqlparser.expression.operators.relational.GreaterThan;
10+
import net.sf.jsqlparser.expression.operators.relational.GreaterThanEquals;
11+
import net.sf.jsqlparser.schema.Column;
12+
import net.sf.jsqlparser.schema.Table;
13+
import net.sf.jsqlparser.statement.Statement;
14+
import net.sf.jsqlparser.statement.select.*;
15+
16+
import java.util.ArrayList;
17+
import java.util.HashMap;
18+
import java.util.List;
19+
import java.util.Map;
20+
21+
/**
22+
* Return a simplified query representation we can work with further, if possible.
23+
*/
24+
@SuppressWarnings({
25+
"rawtypes", // will let us return Persistent datastructures eventually
26+
"unchecked", // lets us use raw types without casting
27+
"PatternVariableCanBeUsed", "IfCanBeSwitch"} // don't force a newer JVM version
28+
)
29+
public final class SimpleParser {
30+
31+
public static Map maybeParse(Statement statement) {
32+
try {
33+
if (statement instanceof Select) {
34+
return maybeParse((Select) statement);
35+
}
36+
// This is not a query.
37+
return null;
38+
} catch (IllegalArgumentException e) {
39+
// This query uses features that we do not yet support translating.
40+
System.out.println(e.getMessage());
41+
return null;
42+
}
43+
}
44+
45+
private static Map maybeParse(Select select) {
46+
PlainSelect ps = select.getPlainSelect();
47+
if (ps != null) {
48+
return maybeParse(ps);
49+
}
50+
// We don't support more complex kinds of select statements yet.
51+
throw new IllegalArgumentException("Unsupported query type " + select.getClass().getName());
52+
}
53+
54+
private static Map maybeParse(PlainSelect select) {
55+
// any of these - nope out
56+
if (select.getDistinct() != null ||
57+
select.getFetch() != null ||
58+
select.getFirst() != null ||
59+
select.getForClause() != null ||
60+
select.getForMode() != null ||
61+
select.getForUpdateTable() != null ||
62+
select.getForXmlPath() != null ||
63+
select.getHaving() != null ||
64+
select.getIntoTables() != null ||
65+
select.getIsolation() != null ||
66+
select.getKsqlWindow() != null ||
67+
select.getLateralViews() != null ||
68+
select.getLimitBy() != null ||
69+
select.getMySqlHintStraightJoin() ||
70+
select.getMySqlSqlCacheFlag() != null ||
71+
select.getOffset() != null ||
72+
select.getOptimizeFor() != null ||
73+
select.getOracleHierarchical() != null ||
74+
select.getOracleHint() != null ||
75+
select.getSkip() != null ||
76+
select.getTop() != null ||
77+
select.getWait() != null ||
78+
select.getWindowDefinitions() != null ||
79+
select.getWithItemsList() != null) {
80+
throw new IllegalArgumentException("Unsupported query feature(s)");
81+
}
82+
83+
Map m = new HashMap();
84+
m.put("select", select.getSelectItems().stream().map(SimpleParser::parse).toList());
85+
86+
if (select.getFromItem() != null) {
87+
ArrayList from = new ArrayList();
88+
from.add(parse(select.getFromItem()));
89+
List<Join> joins = select.getJoins();
90+
if (joins != null) {
91+
joins.stream().map(SimpleParser::parse).forEach(from::add);
92+
}
93+
m.put("from", from);
94+
}
95+
96+
Expression where = select.getWhere();
97+
if (where != null) {
98+
m.put("where", parseWhere(where));
99+
}
100+
GroupByElement gbe = select.getGroupBy();
101+
if (gbe != null) {
102+
m.put("group-by", parse(gbe));
103+
}
104+
List<OrderByElement> obe = select.getOrderByElements();
105+
if (obe != null) {
106+
m.put("order-by", obe.stream().map(SimpleParser::parse).toList());
107+
}
108+
Limit limit = select.getLimit();
109+
if (limit != null) {
110+
m.put("limit", parse(limit));
111+
}
112+
return m;
113+
}
114+
115+
private static Map parse(Join join) {
116+
if (join.isApply() ||
117+
join.isCross() ||
118+
join.isGlobal() ||
119+
join.isSemi() ||
120+
join.isStraight() ||
121+
join.isWindowJoin() ||
122+
join.getJoinHint() != null ||
123+
join.getJoinWindow() != null ||
124+
!join.getUsingColumns().isEmpty()) {
125+
throw new IllegalArgumentException("Unsupported join expression");
126+
}
127+
assert(join.isSimple());
128+
129+
if (join.isFull() ||
130+
join.isLeft() ||
131+
join.isRight()) {
132+
// TODO
133+
throw new IllegalArgumentException("Join type not supported yet");
134+
}
135+
assert(join.isInnerJoin());
136+
137+
if (!join.getOnExpressions().isEmpty()) {
138+
throw new IllegalArgumentException("Only unconditional joins supported for now");
139+
}
140+
141+
return parse(join.getFromItem());
142+
}
143+
144+
private static Map parse(FromItem fromItem) {
145+
// We don't support table aliases yet - which is fine since pMBQL doesn't generate them
146+
// fromItem.getAlias();
147+
if (fromItem instanceof Table) {
148+
return parse((Table) fromItem);
149+
}
150+
throw new IllegalArgumentException("Unsupported from clause");
151+
}
152+
153+
private static Long parse(Limit limit) {
154+
Expression rc = limit.getRowCount();
155+
if (limit.getOffset() != null || limit.getByExpressions() != null || !(rc instanceof LongValue)) {
156+
throw new IllegalArgumentException("Unsupported limit clause");
157+
}
158+
return ((LongValue) limit.getRowCount()).getValue();
159+
}
160+
161+
private static Map parse(OrderByElement elem) {
162+
if (elem.getNullOrdering() != null) {
163+
throw new IllegalArgumentException("Unsupported order by clause(s)");
164+
}
165+
Expression e = elem.getExpression();
166+
if (e instanceof Column) {
167+
return parse((Column) e);
168+
}
169+
throw new IllegalArgumentException("Unsupported order by clause(s)");
170+
}
171+
172+
private static List parseWhere(Expression where) {
173+
// oh my lord, what a mission to convert all these, definitely some clojure metaprogramming would be nice
174+
if (where instanceof ComparisonOperator) {
175+
ComparisonOperator co = (ComparisonOperator) where;
176+
if (co.getOldOracleJoinSyntax() > 0 || co.getOraclePriorPosition() > 0) {
177+
throw new IllegalArgumentException("Unsupported where clause");
178+
}
179+
ArrayList form = new ArrayList();
180+
// if we handle ComparisonOperator then we could get the private field "operator" and rely on that.
181+
if (co instanceof EqualsTo) {
182+
form.add(Keyword.find("="));
183+
} else if (co instanceof GreaterThan) {
184+
form.add(Keyword.find("<"));
185+
} else if (co instanceof GreaterThanEquals) {
186+
form.add(Keyword.find("<"));
187+
}
188+
189+
form.add(parseComparisonExpression(co.getLeftExpression()));
190+
form.add(parseComparisonExpression(co.getRightExpression()));
191+
return form;
192+
}
193+
194+
throw new IllegalArgumentException("Unsupported where clause");
195+
}
196+
197+
private static Object parseComparisonExpression(Expression expr) {
198+
if (expr instanceof Column) {
199+
return parse((Column) expr);
200+
} else if (expr instanceof LongValue) {
201+
return ((LongValue) expr).getValue();
202+
}
203+
throw new IllegalArgumentException("Unsupported expression in comparison");
204+
}
205+
206+
private static List<Map> parse(GroupByElement groupBy) {
207+
if (groupBy == null) {
208+
return null;
209+
}
210+
if (groupBy.getGroupingSets() != null && !groupBy.getGroupingSets().isEmpty()) {
211+
throw new IllegalArgumentException("Unsupported group by clause(s)");
212+
}
213+
return groupBy.getGroupByExpressionList().stream().map(SimpleParser::parseGroupByExpr).toList();
214+
}
215+
216+
private static Map parseGroupByExpr(Object o) {
217+
if (o instanceof Column) {
218+
return parse((Column) o);
219+
}
220+
throw new IllegalArgumentException("Unsupported group by expression(s)");
221+
}
222+
223+
private static final Map STAR = new HashMap();
224+
225+
static {
226+
STAR.put("type", "*");
227+
}
228+
229+
private static Map parse(AllColumns expr) {
230+
if (expr.getExceptColumns() != null || expr.getReplaceExpressions() != null) {
231+
throw new IllegalArgumentException("Unsupported expression:" + expr);
232+
}
233+
return STAR;
234+
}
235+
236+
private static Map parse(Table t) {
237+
Map m = new HashMap();
238+
String s = t.getSchemaName();
239+
if (s != null) {
240+
m.put("schema", s);
241+
}
242+
m.put("table", t.getName());
243+
return m;
244+
}
245+
246+
private static Map parse(Column c) {
247+
Map m = new HashMap();
248+
m.put("type", "column");
249+
Table t = c.getTable();
250+
if (t != null) {
251+
String s = t.getSchemaName();
252+
if (s != null) {
253+
m.put("schema", s);
254+
}
255+
m.put("table", t.getName());
256+
}
257+
m.put("column", c.getColumnName());
258+
return m;
259+
}
260+
261+
private static Map parse(SelectItem item) {
262+
// We ignore the alias for now, but could use this in future to create a custom expression with the given name.
263+
// item.getAlias();
264+
265+
Expression exp = item.getExpression();
266+
if (exp instanceof AllColumns) {
267+
return parse((AllColumns) exp);
268+
} else if (exp instanceof Column) {
269+
return parse((Column) exp);
270+
} else if (exp instanceof Function) {
271+
Function f = (Function) exp;
272+
if (f.getName().equalsIgnoreCase("COUNT")) {
273+
Map m = new HashMap();
274+
if (f.getParameters().size() != 1) {
275+
throw new IllegalArgumentException("Malformed COUNT expression");
276+
}
277+
Expression p = f.getParameters().getFirst();
278+
if (p instanceof AllColumns) {
279+
m.put("type", "count");
280+
m.put("column", "*");
281+
return m;
282+
}
283+
// If there's a concrete column given, we can add an implicit non-null clause for it.
284+
// For now, we simply don't support more complex cases.
285+
}
286+
// Fall through if it's not supported
287+
}
288+
289+
// The next step would be looking at the full list of expressions that we support.
290+
throw new IllegalArgumentException("Unsupported expression(s) in select");
291+
}
292+
293+
294+
}

‎src/macaw/scope_experiments.clj‎

Lines changed: 18 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,10 +3,26 @@
33
[macaw.core :as m]
44
[macaw.walk :as mw])
55
(:import
6-
(net.sf.jsqlparser.schema Column Table)))
6+
(com.metabase.macaw SimpleParser)
7+
(java.util List Map)
8+
(net.sf.jsqlparser.schema Column Table)
9+
(net.sf.jsqlparser.statement.select SelectItem)))
10+
11+
(defn- java->clj
12+
"Recursively converts Java ArrayList and HashMap to Clojure vector and map."
13+
[java-obj]
14+
(condp instance? java-obj
15+
List (mapv java->clj java-obj)
16+
Map (into {} (for [[k v] java-obj]
17+
[(keyword k) (java->clj v)]))
18+
java-obj))
19+
20+
(defn query-map [sql]
21+
(java->clj (SimpleParser/maybeParse (m/parsed-query sql))))
722

823
(defn- node->clj [node]
924
(cond
25+
(instance? SelectItem node) [:select-item (.getAlias node) (.getExpression node)]
1026
(instance? Column node) [:column
1127
(some-> (.getTable node) .getName)
1228
(.getColumnName node)]
@@ -32,7 +48,7 @@
3248
(update :parents assoc id parent-id)
3349
(update-in [:children parent-id] (fnil conj #{}) id))
3450
acc')))
35-
(update :sequence (fnil conj []) [id node]))))}
51+
(update :sequence (fnil conj []) [id node #_(mapv m/scope-label (reverse ctx))]))))}
3652
{:scopes {} ;; id -> {:path [labels], :children [nodes]}
3753
:parents {} ;; what scope is this inside?
3854
:children {} ;; what scopes are inside?

‎test/macaw/scope_experiments_test.clj‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,31 @@
33
[clojure.test :refer :all]
44
[macaw.scope-experiments :as mse]))
55

6+
(set! *warn-on-reflection* true)
7+
8+
(deftest ^:parallel query-map-test
9+
(is (= (mse/query-map "SELECT x FROM t")
10+
{:select [{:column "x", :type "column"}]
11+
:from [{:table "t"}]}))
12+
13+
(is (= (mse/query-map "SELECT x FROM t WHERE y = 1")
14+
{:select [{:column "x", :type "column"}]
15+
:from [{:table "t"}]
16+
:where [:=
17+
{:column "y", :type "column"}
18+
1]}))
19+
20+
(is (= (mse/query-map "SELECT x, z FROM t WHERE y = 1 GROUP BY z ORDER BY x DESC LIMIT 1")
21+
{:select [{:column "x", :type "column"} {:column "z", :type "column"}],
22+
:from [{:table "t"}],
23+
:where [:= {:column "y", :type "column"} 1]
24+
:group-by [{:column "z", :type "column"}],
25+
:order-by [{:column "x", :type "column"}],
26+
:limit 1,}))
27+
28+
(is (= (mse/query-map "SELECT x FROM t1, t2")
29+
{:select [{:column "x", :type "column"}], :from [{:table "t1"} {:table "t2"}]})))
30+
631
(deftest ^:parallel semantic-map-test
732
(is (= (mse/semantic-map "select x from t, u, v left join w on w.id = v.id where t.id = u.id and u.id = v.id limit 3")
833
{:scopes {1 {:path ["SELECT"], :children [[:column nil "x"]]},

0 commit comments

Comments
 (0)