Skip to content

Commit 41b1802

Browse files
crisptrutskicrisptrutski
committed
extract and stuff
1 parent fc13675 commit 41b1802

File tree

2 files changed

+142
-81
lines changed

2 files changed

+142
-81
lines changed

‎src/macaw/scope_experiments.clj‎

Lines changed: 139 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,139 @@
1+
(ns macaw.scope-experiments
2+
(:require
3+
[macaw.core :as m]
4+
[macaw.walk :as mw])
5+
(:import
6+
(net.sf.jsqlparser.schema Column Table)))
7+
8+
(defn- node->clj [node]
9+
(cond
10+
(instance? Column node) [:column
11+
(some-> (.getTable node) .getName)
12+
(.getColumnName node)]
13+
(instance? Table node) [:table (.getName node)]
14+
:else [(type node) node]))
15+
16+
(defn- semantic-map [sql]
17+
(mw/fold-query (m/parsed-query sql)
18+
{:every-node (fn [acc node ctx]
19+
(let [id (m/scope-id (first ctx))
20+
node (node->clj node)]
21+
(-> acc
22+
(update-in [:scopes id]
23+
(fn [scope]
24+
(-> scope
25+
(update :path #(or % (mapv m/scope-label (reverse ctx))))
26+
(update :children (fnil conj []) node))))
27+
((fn [acc']
28+
(if-let [parent-id (some-> (second ctx) m/scope-id)]
29+
(-> acc'
30+
(update :parents assoc id parent-id)
31+
(update-in [:children parent-id] (fnil conj #{}) id))
32+
acc')))
33+
(update :sequence (fnil conj []) [id node]))))}
34+
{:scopes {} ;; id -> {:path [labels], :children [nodes]}
35+
:parents {} ;; what scope is this inside?
36+
:children {} ;; what scopes are inside?
37+
:sequence []})) ;; [scope-id, node]
38+
39+
(comment
40+
(semantic-map "select x from t, u, v left join w on w.id = v.id where t.id = u.id and u.id = v.id limit 3")
41+
;{:scopes {1 {:path ["SELECT"], :children [[:column "x"]]},
42+
; 2 {:path ["SELECT" "FROM"], :children [[:table "t"]]},
43+
; 4 {:path ["SELECT" "JOIN" "FROM"], :children [[:table "u"]]},
44+
; 5 {:path ["SELECT" "JOIN" "FROM"], :children [[:table "v"]]},
45+
; 6 {:path ["SELECT" "JOIN" "FROM"], :children [[:table "w"]]},
46+
; 3 {:path ["SELECT" "JOIN"], :children [[:column "id"] [:table "w"] [:column "id"] [:table "v"]]},
47+
; 7 {:path ["SELECT" "WHERE"],
48+
; :children [[:column "id"]
49+
; [:table "t"]
50+
; [:column "id"]
51+
; [:table "u"]
52+
; [:column "id"]
53+
; [:table "u"]
54+
; [:column "id"]
55+
; [:table "v"]]}},
56+
; :parents {2 1, 4 3, 5 3, 6 3, 3 1, 7 1},
57+
; :children {1 #{7 3 2}, 3 #{4 6 5}},
58+
; :sequence [[1 [:column "x"]]
59+
; [2 [:table "t"]]
60+
; [4 [:table "u"]]
61+
; [5 [:table "v"]]
62+
; [6 [:table "w"]]
63+
; [3 [:column "id"]]
64+
; [3 [:table "w"]]
65+
; [3 [:column "id"]]
66+
; [3 [:table "v"]]
67+
; [7 [:column "id"]]
68+
; [7 [:table "t"]]
69+
; [7 [:column "id"]]
70+
; [7 [:table "u"]]
71+
; [7 [:column "id"]]
72+
; [7 [:table "u"]]
73+
; [7 [:column "id"]]
74+
; [7 [:table "v"]]]}
75+
76+
77+
(semantic-map "select t.a,b,c,d from t")
78+
;{:scopes {1 {:path ["select"], :children [[:column "a"] [:column "b"] [:column "c"] [:column "d"]]},
79+
; 2 {:path ["select" "from"], :children [[:table "t"]]}},
80+
; :parents {2 1},
81+
; :children {1 #{2}},
82+
; :sequence [[1 [:column "a"]] [1 [:column "b"]] [1 [:column "c"]] [1 [:column "d"]] [2 [:table "t"]]]}
83+
)
84+
85+
(defn- get-descendants-map [parent-children-map]
86+
(letfn [(get-all-descendants [parent]
87+
(let [children (get parent-children-map parent [])]
88+
(into #{} (concat children
89+
(mapcat #(get-all-descendants %)
90+
children)))))]
91+
(into {}
92+
(for [parent (keys parent-children-map)]
93+
[parent (get-all-descendants parent)]))))
94+
95+
(defn fields->tables-in-scope [sql]
96+
(let [sm (semantic-map sql)
97+
tables (filter (comp #{:table} first second) (:sequence sm))
98+
scope->tables (reduce
99+
(fn [m [scope-id [_ table-name]]]
100+
(update m scope-id (fnil conj #{}) table-name))
101+
{}
102+
tables)
103+
scope->descendants (get-descendants-map (:children sm))
104+
scope->nested-tables (reduce
105+
(fn [m parent-id]
106+
(assoc m parent-id
107+
(into (set (scope->tables parent-id)) (mapcat scope->tables (scope->descendants parent-id)))))
108+
{}
109+
(keys (:scopes sm)))
110+
columns (filter (comp #{:column} first second) (:sequence sm))]
111+
112+
(vec (distinct (for [[scope-id [_ table-name column-name]] columns]
113+
[[scope-id column-name]
114+
(if table-name
115+
#{table-name}
116+
(scope->nested-tables scope-id))])))))
117+
118+
(defn- fields-to-search [f->ts]
119+
(into (sorted-set)
120+
(mapcat (fn [[[_ column-name] table-names]]
121+
(map #(vector :table % :column column-name) table-names)))
122+
123+
f->ts))
124+
125+
(comment
126+
;; like source-columns, but understands scope
127+
(fields-to-search
128+
(fields->tables-in-scope "select x from t, u, v left join w on w.a = v.a where t.b = u.b and u.c = v.c limit 3"))
129+
;#{[:table "t" :column "b"]
130+
; [:table "t" :column "x"]
131+
; [:table "u" :column "b"]
132+
; [:table "u" :column "c"]
133+
; [:table "u" :column "x"]
134+
; [:table "v" :column "a"]
135+
; [:table "v" :column "c"]
136+
; [:table "v" :column "x"]
137+
; [:table "w" :column "a"]
138+
; [:table "w" :column "x"]}
139+
)

‎test/macaw/core_test.clj‎

Lines changed: 3 additions & 81 deletions
Original file line numberDiff line numberDiff line change
@@ -618,7 +618,7 @@ from foo")
618618
raw-components)))))
619619

620620
(comment
621-
(require 'user) ;; kondo, really
621+
(require 'user) ;; kondo, really
622622
(require '[clj-async-profiler.core :as prof])
623623
(prof/serve-ui 8080)
624624

@@ -632,93 +632,15 @@ from foo")
632632

633633
(user/time+ (simple-benchmark))
634634
(prof/profile {:event :alloc}
635-
(dotimes [_ 1000] (simple-benchmark)))
635+
(dotimes [_ 1000] (simple-benchmark)))
636636

637637
(user/time+ (complex-benchmark))
638638
(prof/profile {:event :alloc}
639-
(dotimes [_ 100] (complex-benchmark)))
639+
(dotimes [_ 100] (complex-benchmark)))
640640

641641
(anonymize-query "SELECT x FROM a")
642642
(anonymize-fixture :snowflakelet)
643643

644-
(defn- node->clj [node]
645-
(cond
646-
(instance? Column node) [:column
647-
(some-> (.getTable node) .getName)
648-
(.getColumnName node)]
649-
(instance? Table node) [:table (.getName node)]
650-
:else [(type node) node]))
651-
652-
(mw/fold-query (m/parsed-query
653-
;"select x from t, u, v left join w on w.id = v.id where t.id = u.id and u.id = v.id limit 3"
654-
"select t.a,b,c,d from t"
655-
)
656-
{:every-node (fn [acc node ctx]
657-
(let [id (m/scope-id (first ctx))
658-
node (node->clj node)]
659-
(-> acc
660-
(update-in [:scopes id]
661-
(fn [scope]
662-
(-> scope
663-
(update :path #(or % (mapv m/scope-label (reverse ctx))))
664-
(update :children (fnil conj []) node))))
665-
((fn [acc']
666-
(if-let [parent-id (some-> (second ctx) m/scope-id)]
667-
(-> acc'
668-
(update :parents assoc id parent-id)
669-
(update-in [:children parent-id] (fnil conj #{}) id))
670-
acc')))
671-
(update :sequence (fnil conj []) [id node]))))}
672-
{:scopes {} ;; id -> {:path [labels], :children [nodes]}
673-
:parents {} ;; what scope is this inside?
674-
:children {} ;; what scopes are inside?
675-
:sequence []}) ;; [scope-id, node]
676-
677-
;"select x from t, u, v left join w on w.id = v.id where t.id = u.id and u.id = v.id limit 3"
678-
;
679-
;{:scopes {1 {:path ["SELECT"], :children [[:column "x"]]},
680-
; 2 {:path ["SELECT" "FROM"], :children [[:table "t"]]},
681-
; 4 {:path ["SELECT" "JOIN" "FROM"], :children [[:table "u"]]},
682-
; 5 {:path ["SELECT" "JOIN" "FROM"], :children [[:table "v"]]},
683-
; 6 {:path ["SELECT" "JOIN" "FROM"], :children [[:table "w"]]},
684-
; 3 {:path ["SELECT" "JOIN"], :children [[:column "id"] [:table "w"] [:column "id"] [:table "v"]]},
685-
; 7 {:path ["SELECT" "WHERE"],
686-
; :children [[:column "id"]
687-
; [:table "t"]
688-
; [:column "id"]
689-
; [:table "u"]
690-
; [:column "id"]
691-
; [:table "u"]
692-
; [:column "id"]
693-
; [:table "v"]]}},
694-
; :parents {2 1, 4 3, 5 3, 6 3, 3 1, 7 1},
695-
; :children {1 #{7 3 2}, 3 #{4 6 5}},
696-
; :sequence [[1 [:column "x"]]
697-
; [2 [:table "t"]]
698-
; [4 [:table "u"]]
699-
; [5 [:table "v"]]
700-
; [6 [:table "w"]]
701-
; [3 [:column "id"]]
702-
; [3 [:table "w"]]
703-
; [3 [:column "id"]]
704-
; [3 [:table "v"]]
705-
; [7 [:column "id"]]
706-
; [7 [:table "t"]]
707-
; [7 [:column "id"]]
708-
; [7 [:table "u"]]
709-
; [7 [:column "id"]]
710-
; [7 [:table "u"]]
711-
; [7 [:column "id"]]
712-
; [7 [:table "v"]]]}
713-
714-
715-
;"select a,b,c,d from t"
716-
;
717-
;{:scopes {1 {:path ["SELECT"], :children [[:column "a"] [:column "b"] [:column "c"] [:column "d"]]},
718-
; 2 {:path ["SELECT" "FROM"], :children [[:table "t"]]}},
719-
; :parents {2 1},
720-
; :children {1 #{2}},
721-
; :sequence [[1 [:column "a"]] [1 [:column "b"]] [1 [:column "c"]] [1 [:column "d"]] [2 [:table "t"]]]}
722644

723645
(require 'virgil)
724646
(require 'clojure.tools.namespace.repl)

0 commit comments

Comments
 (0)