Skip to content

Commit 120ee9c

Browse files
dihedrondihedron
authored
Merge pull request #4 from dihedron/main
Pull into main
2 parents 33d876a + 5ab757c commit 120ee9c

File tree

9 files changed

+337
-53
lines changed

9 files changed

+337
-53
lines changed

‎_test/import-csv.yaml‎

Lines changed: 40 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,40 @@
11
---
2+
# kind: source
3+
# spec:
4+
# name: test1
5+
# registry: local
6+
# path: /data/workspaces/gomods/cq-source-file/dist/cq-source-file_linux_amd64_v1/cq-source-file
7+
# version: v1.0.0
8+
# tables:
9+
# ["*"]
10+
# destinations:
11+
# - sqlite
12+
# spec:
13+
# file: ./test.csv
14+
# format: csv
15+
# table: T1
16+
# separator: ","
17+
# filter: _.color startsWith 'b'
18+
# columns:
19+
# - name: color
20+
# type: string
21+
# key: true
22+
# unique: true
23+
# notnull: true
24+
# - name: value
25+
# type: string
26+
# unique: true
27+
# notnull: true
28+
# - name: optimized
29+
# type: boolean
30+
# notnull: true
31+
# - name: count
32+
# type: integer
33+
# notnull: true
34+
# ---
235
kind: source
336
spec:
4-
name: local_csv
37+
name: test2
538
registry: local
639
path: /data/workspaces/gomods/cq-source-file/dist/cq-source-file_linux_amd64_v1/cq-source-file
740
version: v1.0.0
@@ -12,20 +45,25 @@ spec:
1245
spec:
1346
file: ./test.csv
1447
format: csv
15-
table: my_csv_data
48+
table: T2
1649
separator: ","
50+
filter: _.color not startsWith 'b'
1751
columns:
1852
- name: color
1953
type: string
2054
key: true
2155
unique: true
56+
notnull: true
2257
- name: value
2358
type: string
2459
unique: true
60+
notnull: true
2561
- name: optimized
2662
type: boolean
63+
notnull: true
2764
- name: count
2865
type: integer
66+
notnull: true
2967
---
3068
kind: destination
3169
spec:

‎_test/import-json.yaml‎

Lines changed: 39 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
---
22
kind: source
33
spec:
4-
name: local_json
4+
name: test1
55
registry: local
66
path: /data/workspaces/gomods/cq-source-file/dist/cq-source-file_linux_amd64_v1/cq-source-file
77
version: v1.0.0
@@ -12,19 +12,56 @@ spec:
1212
spec:
1313
file: ./test.json
1414
format: json
15-
table: my_json_data
15+
table: T1
16+
filter: _.color startsWith 'b'
1617
columns:
1718
- name: color
1819
type: string
1920
key: true
2021
unique: true
22+
notnull: true
2123
- name: value
2224
type: string
2325
unique: true
26+
notnull: true
2427
- name: optimized
2528
type: boolean
29+
notnull: true
2630
- name: count
2731
type: integer
32+
notnull: true
33+
---
34+
kind: source
35+
spec:
36+
name: test2
37+
registry: local
38+
path: /data/workspaces/gomods/cq-source-file/dist/cq-source-file_linux_amd64_v1/cq-source-file
39+
version: v1.0.0
40+
tables:
41+
["*"]
42+
destinations:
43+
- sqlite
44+
spec:
45+
file: ./test.json
46+
format: json
47+
table: T2
48+
filter: "_.color not startsWith 'b'"
49+
columns:
50+
- name: color
51+
type: string
52+
key: true
53+
unique: true
54+
notnull: true
55+
- name: value
56+
type: string
57+
unique: true
58+
notnull: true
59+
- name: optimized
60+
type: boolean
61+
notnull: true
62+
- name: count
63+
type: integer
64+
notnull: true
2865
---
2966
kind: destination
3067
spec:

‎_test/import-xlsx.yaml‎

Lines changed: 38 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
---
22
kind: source
33
spec:
4-
name: local_xlsx
4+
name: test1
55
registry: local
66
path: /data/workspaces/gomods/cq-source-file/dist/cq-source-file_linux_amd64_v1/cq-source-file
77
version: v1.0.0
@@ -12,20 +12,56 @@ spec:
1212
spec:
1313
file: ./test.xlsx
1414
format: xlsx
15-
table: my_xlsx_data
15+
table: T1
16+
filter: _.color startsWith 'b'
1617
columns:
1718
- name: color
1819
type: string
1920
key: true
2021
unique: true
22+
notnull: true
2123
- name: value
2224
type: string
2325
unique: true
26+
notnull: true
2427
- name: optimized
2528
type: boolean
2629
notnull: true # comment to get a null
2730
- name: count
2831
type: integer
32+
notnull: true
33+
---
34+
kind: source
35+
spec:
36+
name: test2
37+
registry: local
38+
path: /data/workspaces/gomods/cq-source-file/dist/cq-source-file_linux_amd64_v1/cq-source-file
39+
version: v1.0.0
40+
tables:
41+
["*"]
42+
destinations:
43+
- sqlite
44+
spec:
45+
file: ./test.xlsx
46+
format: xlsx
47+
table: T2
48+
filter: _.color not startsWith 'b'
49+
columns:
50+
- name: color
51+
type: string
52+
key: true
53+
unique: true
54+
notnull: true
55+
- name: value
56+
type: string
57+
unique: true
58+
notnull: true
59+
- name: optimized
60+
type: boolean
61+
notnull: true # comment to get a null
62+
- name: count
63+
type: integer
64+
#notnull: true
2965
---
3066
kind: destination
3167
spec:

‎_test/import-yaml.yaml‎

Lines changed: 39 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
---
22
kind: source
33
spec:
4-
name: local_yaml
4+
name: test1
55
registry: local
66
path: /data/workspaces/gomods/cq-source-file/dist/cq-source-file_linux_amd64_v1/cq-source-file
77
version: v1.0.0
@@ -12,19 +12,56 @@ spec:
1212
spec:
1313
file: ./test.yaml
1414
format: yaml
15-
table: my_yaml_data
15+
table: T1
16+
filter: _.color startsWith 'b'
1617
columns:
1718
- name: color
1819
type: string
1920
key: true
21+
notnull: true
2022
unique: true
2123
- name: value
2224
type: string
2325
unique: true
26+
notnull: true
2427
- name: optimized
2528
type: boolean
29+
notnull: true
2630
- name: count
2731
type: integer
32+
notnull: true
33+
---
34+
kind: source
35+
spec:
36+
name: test2
37+
registry: local
38+
path: /data/workspaces/gomods/cq-source-file/dist/cq-source-file_linux_amd64_v1/cq-source-file
39+
version: v1.0.0
40+
tables:
41+
["*"]
42+
destinations:
43+
- sqlite
44+
spec:
45+
file: ./test.yaml
46+
format: yaml
47+
table: T2
48+
filter: _.color not startsWith 'b'
49+
columns:
50+
- name: color
51+
type: string
52+
key: true
53+
notnull: true
54+
unique: true
55+
- name: value
56+
type: string
57+
unique: true
58+
notnull: true
59+
- name: optimized
60+
type: boolean
61+
notnull: true
62+
- name: count
63+
type: integer
64+
notnull: true
2865
---
2966
kind: destination
3067
spec:

‎client/client.go‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,6 @@ import (
1414
type Client struct {
1515
Logger zerolog.Logger
1616
Specs *Spec
17-
//Data []map[string]any
1817
}
1918

2019
func (c *Client) ID() string {

‎client/spec.go‎

Lines changed: 22 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -1,18 +1,28 @@
11
package client
22

3+
import (
4+
"text/template"
5+
6+
"github.com/antonmedv/expr/vm"
7+
)
8+
39
type Column struct {
4-
Name string `json:"name,omitempty" yaml:"name,omitempty"`
5-
Description *string `json:"description,omitempty" yaml:"description,omitempty"`
6-
Type string `json:"type,omitempty" yaml:"type,omitempty"`
7-
Key bool `json:"key,omitempty" yaml:"pk,omitempty"`
8-
Unique bool `json:"unique,omitempty" yaml:"unique,omitempty"`
9-
NotNull bool `json:"notnull,omitempty" yaml:"notnull,omitempty"`
10+
Name string `json:"name,omitempty" yaml:"name,omitempty"`
11+
Description *string `json:"description,omitempty" yaml:"description,omitempty"`
12+
Type string `json:"type,omitempty" yaml:"type,omitempty"`
13+
Key bool `json:"key,omitempty" yaml:"pk,omitempty"`
14+
Unique bool `json:"unique,omitempty" yaml:"unique,omitempty"`
15+
NotNull bool `json:"notnull,omitempty" yaml:"notnull,omitempty"`
16+
Transform *string `json:"transform,omitempty" yaml:"transform,omitempty"`
17+
Template *template.Template `json:"-" yaml:"-"`
1018
}
1119
type Spec struct {
12-
File string `json:"file,omitempty" yaml:"file,omitempty"`
13-
Format string `json:"format,omitempty" yaml:"format,omitempty"`
14-
Table string `json:"table,omitempty" yaml:"table,omitempty"`
15-
Columns []Column `json:"columns,omitempty" yaml:"columns,omitempty"`
16-
Separator *string `json:"separator,omitempty" yaml:"separator,omitempty"` // CSV only
17-
Sheet *string `json:"sheet,omitempty" yaml:"sheet,omitempty"` // XLSX only
20+
File string `json:"file,omitempty" yaml:"file,omitempty"`
21+
Format string `json:"format,omitempty" yaml:"format,omitempty"`
22+
Table string `json:"table,omitempty" yaml:"table,omitempty"`
23+
Filter *string `json:"filter,omitempty" yaml:"filter,omitempty"`
24+
Evaluator *vm.Program `json:"-,omitempty" yaml:"-,omitempty"`
25+
Columns []*Column `json:"columns,omitempty" yaml:"columns,omitempty"`
26+
Separator *string `json:"separator,omitempty" yaml:"separator,omitempty"` // CSV only
27+
Sheets []string `json:"sheets,omitempty" yaml:"sheets,omitempty"` // XLSX only
1828
}

‎go.mod‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,8 @@ module github.com/dihedron/cq-source-file
33
go 1.20
44

55
require (
6+
github.com/Masterminds/sprig/v3 v3.2.3
7+
github.com/antonmedv/expr v1.12.5
68
github.com/cloudquery/plugin-sdk v1.45.0
79
github.com/dihedron/cq-plugin-utils v0.0.0-20230515115130-eb4d7b050d1c
810
github.com/rs/zerolog v1.29.1
@@ -11,20 +13,27 @@ require (
1113
)
1214

1315
require (
16+
github.com/Masterminds/goutils v1.1.1 // indirect
17+
github.com/Masterminds/semver/v3 v3.2.0 // indirect
1418
github.com/davecgh/go-spew v1.1.1 // indirect
1519
github.com/getsentry/sentry-go v0.20.0 // indirect
1620
github.com/ghodss/yaml v1.0.0 // indirect
1721
github.com/golang/protobuf v1.5.3 // indirect
1822
github.com/google/uuid v1.3.0 // indirect
1923
github.com/grpc-ecosystem/go-grpc-middleware/providers/zerolog/v2 v2.0.0-rc.3 // indirect
2024
github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.0.0-rc.3 // indirect
25+
github.com/huandu/xstrings v1.3.3 // indirect
26+
github.com/imdario/mergo v0.3.11 // indirect
2127
github.com/inconshreveable/mousetrap v1.1.0 // indirect
2228
github.com/mattn/go-colorable v0.1.13 // indirect
2329
github.com/mattn/go-isatty v0.0.18 // indirect
30+
github.com/mitchellh/copystructure v1.0.0 // indirect
31+
github.com/mitchellh/reflectwalk v1.0.0 // indirect
2432
github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 // indirect
2533
github.com/pmezard/go-difflib v1.0.0 // indirect
2634
github.com/richardlehane/mscfb v1.0.4 // indirect
2735
github.com/richardlehane/msoleps v1.0.3 // indirect
36+
github.com/shopspring/decimal v1.2.0 // indirect
2837
github.com/spf13/cast v1.5.0 // indirect
2938
github.com/spf13/cobra v1.7.0 // indirect
3039
github.com/spf13/pflag v1.0.5 // indirect

0 commit comments

Comments
 (0)