Skip to content

Commit b61f470

Browse files
dihedrondihedron
authored
Merge pull request #3 from dihedron/feature/multiple-tables
Add support for separate nested table field in spec
2 parents 01c65cd + ec07ed9 commit b61f470

File tree

11 files changed

+752
-512
lines changed

11 files changed

+752
-512
lines changed

‎_test/import-csv.yaml‎

Lines changed: 78 additions & 51 deletions
Original file line numberDiff line numberDiff line change
@@ -1,37 +1,63 @@
11
---
2-
# kind: source
3-
# spec:
4-
# name: test1
5-
# registry: local
6-
# path: /data/workspaces/gomods/cq-source-file/dist/cq-source-file_linux_amd64_v1/cq-source-file
7-
# version: v1.0.0
8-
# tables:
9-
# ["*"]
10-
# destinations:
11-
# - sqlite
12-
# spec:
13-
# file: ./test.csv
14-
# format: csv
15-
# table: T1
16-
# separator: ","
17-
# filter: _.color startsWith 'b'
18-
# columns:
19-
# - name: color
20-
# type: string
21-
# key: true
22-
# unique: true
23-
# notnull: true
24-
# - name: value
25-
# type: string
26-
# unique: true
27-
# notnull: true
28-
# - name: optimized
29-
# type: boolean
30-
# notnull: true
31-
# - name: count
32-
# type: integer
33-
# notnull: true
34-
# ---
2+
kind: source
3+
spec:
4+
name: test1
5+
registry: local
6+
path: /data/workspaces/gomods/cq-source-file/dist/cq-source-file_linux_amd64_v1/cq-source-file
7+
version: v1.0.0
8+
tables:
9+
["*"]
10+
destinations:
11+
- sqlite
12+
spec:
13+
file: ./test.csv
14+
format: csv
15+
separator: ","
16+
table:
17+
name: T1
18+
filter: _.color startsWith 'b'
19+
columns:
20+
- name: color
21+
type: string
22+
key: true
23+
unique: true
24+
notnull: true
25+
- name: value
26+
type: string
27+
unique: true
28+
notnull: true
29+
- name: optimized
30+
type: boolean
31+
notnull: true
32+
- name: count
33+
type: integer
34+
notnull: true
35+
relations:
36+
- name: T1_UPPER
37+
filter: _.color startsWith 'blu'
38+
columns:
39+
- name: upper_color
40+
type: string
41+
key: true
42+
unique: true
43+
notnull: true
44+
transform: "{{index .Row \"color\" | toString | upper}}"
45+
- name: upper_value
46+
type: string
47+
transform: "{{index .Row \"value\" | toString | upper}}"
48+
- name: T1_UPPER
49+
filter: _.color startsWith 'bla'
50+
columns:
51+
- name: upper_color
52+
type: string
53+
key: true
54+
unique: true
55+
notnull: true
56+
transform: "{{index .Row \"color\" | toString | upper}}"
57+
- name: upper_value
58+
type: string
59+
transform: "{{index .Row \"value\" | toString | upper}}"
60+
---
3561
kind: source
3662
spec:
3763
name: test2
@@ -45,25 +71,26 @@ spec:
4571
spec:
4672
file: ./test.csv
4773
format: csv
48-
table: T2
4974
separator: ","
50-
filter: _.color not startsWith 'b'
51-
columns:
52-
- name: color
53-
type: string
54-
key: true
55-
unique: true
56-
notnull: true
57-
- name: value
58-
type: string
59-
unique: true
60-
notnull: true
61-
- name: optimized
62-
type: boolean
63-
notnull: true
64-
- name: count
65-
type: integer
66-
notnull: true
75+
table:
76+
name: T2
77+
filter: _.color not startsWith 'b'
78+
columns:
79+
- name: color
80+
type: string
81+
key: true
82+
unique: true
83+
notnull: true
84+
- name: value
85+
type: string
86+
unique: true
87+
notnull: true
88+
- name: optimized
89+
type: boolean
90+
notnull: true
91+
- name: count
92+
type: integer
93+
notnull: true
6794
---
6895
kind: destination
6996
spec:

‎_test/import-json.yaml‎

Lines changed: 63 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -12,24 +12,50 @@ spec:
1212
spec:
1313
file: ./test.json
1414
format: json
15-
table: T1
16-
filter: _.color startsWith 'b'
17-
columns:
18-
- name: color
19-
type: string
20-
key: true
21-
unique: true
22-
notnull: true
23-
- name: value
24-
type: string
25-
unique: true
26-
notnull: true
27-
- name: optimized
28-
type: boolean
29-
notnull: true
30-
- name: count
31-
type: integer
32-
notnull: true
15+
table:
16+
name: T1
17+
filter: _.color startsWith 'b'
18+
columns:
19+
- name: color
20+
type: string
21+
key: true
22+
unique: true
23+
notnull: true
24+
- name: value
25+
type: string
26+
unique: true
27+
notnull: true
28+
- name: optimized
29+
type: boolean
30+
notnull: true
31+
- name: count
32+
type: integer
33+
notnull: true
34+
relations:
35+
- name: T1_UPPER
36+
filter: _.color startsWith 'blu'
37+
columns:
38+
- name: upper_color
39+
type: string
40+
key: true
41+
unique: true
42+
notnull: true
43+
transform: "{{index .Row \"color\" | toString | upper}}"
44+
- name: upper_value
45+
type: string
46+
transform: "{{index .Row \"value\" | toString | upper}}"
47+
- name: T1_UPPER
48+
filter: _.color startsWith 'bla'
49+
columns:
50+
- name: upper_color
51+
type: string
52+
key: true
53+
unique: true
54+
notnull: true
55+
transform: "{{index .Row \"color\" | toString | upper}}"
56+
- name: upper_value
57+
type: string
58+
transform: "{{index .Row \"value\" | toString | upper}}"
3359
---
3460
kind: source
3561
spec:
@@ -44,24 +70,25 @@ spec:
4470
spec:
4571
file: ./test.json
4672
format: json
47-
table: T2
48-
filter: "_.color not startsWith 'b'"
49-
columns:
50-
- name: color
51-
type: string
52-
key: true
53-
unique: true
54-
notnull: true
55-
- name: value
56-
type: string
57-
unique: true
58-
notnull: true
59-
- name: optimized
60-
type: boolean
61-
notnull: true
62-
- name: count
63-
type: integer
64-
notnull: true
73+
table:
74+
name: T2
75+
filter: _.color not startsWith 'b'
76+
columns:
77+
- name: color
78+
type: string
79+
key: true
80+
unique: true
81+
notnull: true
82+
- name: value
83+
type: string
84+
unique: true
85+
notnull: true
86+
- name: optimized
87+
type: boolean
88+
notnull: true
89+
- name: count
90+
type: integer
91+
notnull: true
6592
---
6693
kind: destination
6794
spec:
@@ -70,4 +97,3 @@ spec:
7097
version: v2.0.5
7198
spec:
7299
connection_string: ./import.sqlite
73-

‎_test/import-xlsx.yaml‎

Lines changed: 63 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -12,24 +12,50 @@ spec:
1212
spec:
1313
file: ./test.xlsx
1414
format: xlsx
15-
table: T1
16-
filter: _.color startsWith 'b'
17-
columns:
18-
- name: color
19-
type: string
20-
key: true
21-
unique: true
22-
notnull: true
23-
- name: value
24-
type: string
25-
unique: true
26-
notnull: true
27-
- name: optimized
28-
type: boolean
29-
notnull: true # comment to get a null
30-
- name: count
31-
type: integer
32-
notnull: true
15+
table:
16+
name: T1
17+
filter: _.color startsWith 'b'
18+
columns:
19+
- name: color
20+
type: string
21+
key: true
22+
unique: true
23+
notnull: true
24+
- name: value
25+
type: string
26+
unique: true
27+
notnull: true
28+
- name: optimized
29+
type: boolean
30+
notnull: true
31+
- name: count
32+
type: integer
33+
notnull: true
34+
relations:
35+
- name: T1_UPPER
36+
filter: _.color startsWith 'blu'
37+
columns:
38+
- name: upper_color
39+
type: string
40+
key: true
41+
unique: true
42+
notnull: true
43+
transform: "{{index .Row \"color\" | toString | upper}}"
44+
- name: upper_value
45+
type: string
46+
transform: "{{index .Row \"value\" | toString | upper}}"
47+
- name: T1_UPPER
48+
filter: _.color startsWith 'bla'
49+
columns:
50+
- name: upper_color
51+
type: string
52+
key: true
53+
unique: true
54+
notnull: true
55+
transform: "{{index .Row \"color\" | toString | upper}}"
56+
- name: upper_value
57+
type: string
58+
transform: "{{index .Row \"value\" | toString | upper}}"
3359
---
3460
kind: source
3561
spec:
@@ -44,24 +70,25 @@ spec:
4470
spec:
4571
file: ./test.xlsx
4672
format: xlsx
47-
table: T2
48-
filter: _.color not startsWith 'b'
49-
columns:
50-
- name: color
51-
type: string
52-
key: true
53-
unique: true
54-
notnull: true
55-
- name: value
56-
type: string
57-
unique: true
58-
notnull: true
59-
- name: optimized
60-
type: boolean
61-
notnull: true # comment to get a null
62-
- name: count
63-
type: integer
64-
#notnull: true
73+
table:
74+
name: T2
75+
filter: _.color not startsWith 'b'
76+
columns:
77+
- name: color
78+
type: string
79+
key: true
80+
unique: true
81+
notnull: true
82+
- name: value
83+
type: string
84+
unique: true
85+
notnull: true
86+
- name: optimized
87+
type: boolean
88+
notnull: true
89+
- name: count
90+
type: integer
91+
notnull: false
6592
---
6693
kind: destination
6794
spec:

0 commit comments

Comments
 (0)