Skip to content

Commit c1eee41

Browse files
dihedrondihedron
committed
Add support for XLSX to spec-typed columns
1 parent 221f74d commit c1eee41

File tree

3 files changed

+61
-147
lines changed

3 files changed

+61
-147
lines changed

‎_test/import-xlsx.yaml‎

Lines changed: 12 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -13,10 +13,18 @@ spec:
1313
file: ./test.xlsx
1414
format: xlsx
1515
table: my_xlsx_data
16-
keys: [color]
17-
types:
18-
value: string
19-
color: string
16+
columns:
17+
- name: color
18+
type: string
19+
key: true
20+
unique: true
21+
- name: value
22+
type: string
23+
unique: true
24+
- name: optimized
25+
type: boolean
26+
- name: count
27+
type: integer
2028
---
2129
kind: destination
2230
spec:

‎_test/test.xlsx‎

130 Bytes
Binary file not shown.

‎resources/data.go‎

Lines changed: 49 additions & 143 deletions
Original file line numberDiff line numberDiff line change
@@ -13,9 +13,12 @@ import (
1313
"github.com/dihedron/cq-plugin-utils/format"
1414
"github.com/dihedron/cq-plugin-utils/pointer"
1515
"github.com/dihedron/cq-source-file/client"
16-
"gopkg.in/yaml.v2"
16+
"github.com/xuri/excelize/v2"
17+
"gopkg.in/yaml.v3"
1718
)
1819

20+
// GetTable uses data in the spec section of the client configuration to
21+
// dynamically build the information about the columns being imported.
1922
func GetTables(ctx context.Context, meta schema.ClientMeta) (schema.Tables, error) {
2023
client := meta.(*client.Client)
2124

@@ -60,150 +63,12 @@ func GetTables(ctx context.Context, meta schema.ClientMeta) (schema.Tables, erro
6063
Columns: columns,
6164
},
6265
}, nil
63-
64-
// client.Logger.Debug().Str("file", client.Specs.File).Msg("reading input from file")
65-
66-
// client.Data = []map[string]any{}
67-
// switch strings.ToLower(client.Specs.Format) {
68-
// case "json":
69-
// data, err := os.ReadFile(client.Specs.File)
70-
// if err != nil {
71-
// client.Logger.Error().Err(err).Str("file", client.Specs.File).Msg("error reading input file")
72-
// return nil, fmt.Errorf("error reading input file %q: %w", client.Specs.File, err)
73-
// }
74-
// client.Logger.Debug().Str("file", client.Specs.File).Msg("input file read")
75-
// if err := json.Unmarshal(data, &client.Data); err != nil {
76-
// client.Logger.Error().Err(err).Msg("error unmarshalling data from JSON")
77-
// return nil, fmt.Errorf("error unmarshalling data from JSON: %w", err)
78-
// }
79-
// case "yaml", "yml":
80-
// data, err := os.ReadFile(client.Specs.File)
81-
// if err != nil {
82-
// client.Logger.Error().Err(err).Str("file", client.Specs.File).Msg("error reading input file")
83-
// return nil, fmt.Errorf("error reading input file %q: %w", client.Specs.File, err)
84-
// }
85-
// client.Logger.Debug().Str("file", client.Specs.File).Msg("input file read")
86-
// if err := yaml.Unmarshal(data, &client.Data); err != nil {
87-
// client.Logger.Error().Err(err).Msg("error unmarshalling data from JSON")
88-
// return nil, fmt.Errorf("error unmarshalling data from JSON: %w", err)
89-
// }
90-
// case "csv":
91-
// data, err := os.ReadFile(client.Specs.File)
92-
// if err != nil {
93-
// client.Logger.Error().Err(err).Str("file", client.Specs.File).Msg("error reading input file")
94-
// return nil, fmt.Errorf("error reading input file %q: %w", client.Specs.File, err)
95-
// }
96-
// client.Logger.Debug().Str("file", client.Specs.File).Msg("input file read")
97-
// if client.Specs.Separator == nil {
98-
// client.Specs.Separator = pointer.To(",")
99-
// }
100-
// scanner := bufio.NewScanner(bytes.NewReader(data))
101-
// var keys []string
102-
// client.Data = []map[string]any{}
103-
// first := true
104-
// for scanner.Scan() {
105-
// line := scanner.Text()
106-
// client.Logger.Debug().Str("line", line).Msg("read line from input file")
107-
// if first {
108-
// first = false
109-
// keys = strings.Split(line, *client.Specs.Separator)
110-
// } else {
111-
// values := strings.Split(line, *client.Specs.Separator)
112-
// entry := map[string]any{}
113-
// for i := 0; i < len(keys); i++ {
114-
// entry[keys[i]] = values[i]
115-
// }
116-
// client.Data = append(client.Data, entry)
117-
// }
118-
// }
119-
// case "xsl", "xlsx", "excel":
120-
// xls, err := excelize.OpenFile(client.Specs.File)
121-
// if err != nil {
122-
// client.Logger.Error().Err(err).Str("file", client.Specs.File).Msg("error reading input file")
123-
// return nil, fmt.Errorf("error reading input file %q: %w", client.Specs.File, err)
124-
// }
125-
// defer func() {
126-
// if err := xls.Close(); err != nil {
127-
// client.Logger.Error().Err(err).Str("file", client.Specs.File).Msg("error reading input file")
128-
// }
129-
// }()
130-
// // Get all the rows in the Sheet1.
131-
// if client.Specs.Sheet == nil {
132-
// // get the currently active sheet in the file
133-
// client.Specs.Sheet = pointer.To(xls.GetSheetName(xls.GetActiveSheetIndex()))
134-
// }
135-
// client.Logger.Debug().Str("sheet", *client.Specs.Sheet).Msg("getting data from sheet")
136-
// rows, err := xls.GetRows(*client.Specs.Sheet)
137-
// if err != nil {
138-
// client.Logger.Error().Err(err).Str("file", client.Specs.File).Msg("error getting rows")
139-
// return nil, fmt.Errorf("error getting rows from input file %q: %w", client.Specs.File, err)
140-
// }
141-
142-
// var keys []string
143-
// client.Data = []map[string]any{}
144-
// first := true
145-
// for _, row := range rows {
146-
// if first {
147-
// first = false
148-
// keys = row
149-
// } else {
150-
// values := row
151-
// entry := map[string]any{}
152-
// for i := 0; i < len(keys); i++ {
153-
// entry[keys[i]] = values[i]
154-
// }
155-
// client.Data = append(client.Data, entry)
156-
// }
157-
// }
158-
// default:
159-
// client.Logger.Error().Str("format", client.Specs.Format).Msg("unsupported format")
160-
// return nil, fmt.Errorf("unsupported format: %q", client.Specs.Format)
161-
// }
162-
163-
// if len(client.Data) > 0 {
164-
// columns := []schema.Column{}
165-
// for name := range client.Data[0] {
166-
// client.Logger.Debug().Str("name", name).Msg("adding column")
167-
// column := schema.Column{
168-
// Name: name,
169-
// Description: fmt.Sprintf("The column mapping the %q field from the input data", name),
170-
// Resolver: fetchColumn,
171-
// }
172-
// for _, v := range client.Specs.Keys {
173-
// if name == v {
174-
// client.Logger.Debug().Str("name", name).Msg("column is primary key")
175-
// column.CreationOptions.PrimaryKey = true
176-
// break
177-
// }
178-
// }
179-
// switch strings.ToLower(client.Specs.Types[name]) {
180-
// case "string", "str", "s":
181-
// client.Logger.Debug().Str("name", name).Msg("column is of type string")
182-
// column.Type = schema.TypeString
183-
// case "integer", "int", "i":
184-
// client.Logger.Debug().Str("name", name).Msg("column is of type int")
185-
// column.Type = schema.TypeInt
186-
// case "boolean", "bool", "b":
187-
// client.Logger.Debug().Str("name", name).Msg("column is of type bool")
188-
// column.Type = schema.TypeBool
189-
// default:
190-
// client.Logger.Debug().Str("name", name).Msg("column is of unmapped type, assuming string")
191-
// column.Type = schema.TypeString
192-
// }
193-
// columns = append(columns, column)
194-
// }
195-
// client.Logger.Debug().Msg("returning table")
196-
// return []*schema.Table{
197-
// {
198-
// Name: client.Specs.Table,
199-
// Resolver: fetchData,
200-
// Columns: columns,
201-
// },
202-
// }, nil
203-
// }
204-
// return nil, errors.New("no data in file")
20566
}
20667

68+
// fetchData reads the input file and unmarshals it into a set of rows using
69+
// format-specific mechanisms, then encodes the information as a map[string]any
70+
// per row and returns it; fetchColumn knows how to pick the data out of this
71+
// map and set it into the resource being returned to ClouqQuery.
20772
func fetchData(ctx context.Context, meta schema.ClientMeta, parent *schema.Resource, res chan<- interface{}) error {
20873
client := meta.(*client.Client)
20974

@@ -265,6 +130,45 @@ func fetchData(ctx context.Context, meta schema.ClientMeta, parent *schema.Resou
265130
rows = append(rows, row)
266131
}
267132
}
133+
case "xsl", "xlsx", "excel":
134+
xls, err := excelize.OpenFile(client.Specs.File)
135+
if err != nil {
136+
client.Logger.Error().Err(err).Str("file", client.Specs.File).Msg("error reading input file")
137+
return fmt.Errorf("error reading input file %q: %w", client.Specs.File, err)
138+
}
139+
defer func() {
140+
if err := xls.Close(); err != nil {
141+
client.Logger.Error().Err(err).Str("file", client.Specs.File).Msg("error reading input file")
142+
}
143+
}()
144+
// get all the rows in the requested (or the active) sheet
145+
if client.Specs.Sheet == nil {
146+
// get the currently active sheet in the file
147+
client.Specs.Sheet = pointer.To(xls.GetSheetName(xls.GetActiveSheetIndex()))
148+
}
149+
client.Logger.Debug().Str("sheet", *client.Specs.Sheet).Msg("getting data from sheet")
150+
xlsrows, err := xls.GetRows(*client.Specs.Sheet)
151+
if err != nil {
152+
client.Logger.Error().Err(err).Str("file", client.Specs.File).Msg("error getting rows")
153+
return fmt.Errorf("error getting rows from input file %q: %w", client.Specs.File, err)
154+
}
155+
156+
var keys []string
157+
first := true
158+
for _, xlsrow := range xlsrows {
159+
if first {
160+
first = false
161+
keys = xlsrow
162+
} else {
163+
values := xlsrow
164+
row := map[string]any{}
165+
for i := 0; i < len(keys); i++ {
166+
row[keys[i]] = values[i]
167+
}
168+
rows = append(rows, row)
169+
}
170+
}
171+
268172
// TODO: add more formats
269173
default:
270174
client.Logger.Error().Str("format", client.Specs.Format).Msg("unsupported format")
@@ -279,6 +183,8 @@ func fetchData(ctx context.Context, meta schema.ClientMeta, parent *schema.Resou
279183
return nil
280184
}
281185

186+
// fetchColumn picks the value under the right key from the map[string]any
187+
// and sets it into the resource being returned to CloudQuery.
282188
func fetchColumn(ctx context.Context, meta schema.ClientMeta, resource *schema.Resource, c schema.Column) error {
283189
client := meta.(*client.Client)
284190
client.Logger.Debug().Str("resource", format.ToJSON(resource)).Str("column", format.ToJSON(c)).Str("item type", fmt.Sprintf("%T", resource.Item)).Msg("fetching column...")

0 commit comments

Comments
 (0)