Fix panic on null cells

Author: dihedron

_test/import-xlsx.yaml

@@ -23,6 +23,7 @@ spec:
         unique: true
       - name: optimized
         type: boolean
+        notnull: true # comment to get a null
       - name: count
         type: integer
 ---

_test/test.csv

@@ -5,4 +5,6 @@ blue,#00f,false,-345
 cyan,#0ff,true,678
 magenta,#f0f,false,-678
 yellow,#ff0,true,901
-black,#000,false,-901
+black,#000,false,-901
+grey,#aaa,false,
+white,#fff,,-222

_test/test.xlsx

@@ -7,6 +7,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"os"
+	"reflect"
 	"strings"
 
 	"github.com/cloudquery/plugin-sdk/schema"
@@ -118,16 +119,20 @@ func fetchData(ctx context.Context, meta schema.ClientMeta, parent *schema.Resou
 				keys = strings.Split(line, *client.Specs.Separator)
 			} else {
 				values := strings.Split(line, *client.Specs.Separator)
-				row := map[string]any{}
-				//for i := 0; i < len(client.Specs.Columns); i++ {
-				for i := 0; i < len(keys); i++ {
-					for _, column := range client.Specs.Columns {
-						if keys[i] == column.Name {
-							row[client.Specs.Columns[i].Name] = values[i]
+				if len(values) >= len(keys) {
+					row := map[string]any{}
+					for i := 0; i < len(keys); i++ {
+						for _, column := range client.Specs.Columns {
+
+							if keys[i] == column.Name {
+								row[client.Specs.Columns[i].Name] = values[i]
+							}
 						}
 					}
+					rows = append(rows, row)
+				} else {
+					client.Logger.Warn().Str("file", client.Specs.File).Str("line", line).Int("expected", len(keys)).Int("actual", len(values)).Msg("invalid number of columns")
 				}
-				rows = append(rows, row)
 			}
 		}
 	case "xsl", "xlsx", "excel":
@@ -163,7 +168,13 @@ func fetchData(ctx context.Context, meta schema.ClientMeta, parent *schema.Resou
 				values := xlsrow
 				row := map[string]any{}
 				for i := 0; i < len(keys); i++ {
-					row[keys[i]] = values[i]
+					if i < len(values) {
+						// XLSX rows can be sparse, in which case all TRAILING empty cells are removed
+						// from the returned slice; empty cells in the middle are still valid
+						row[keys[i]] = values[i]
+					} else {
+						row[keys[i]] = nil
+					}
 				}
 				rows = append(rows, row)
 			}
@@ -187,7 +198,38 @@ func fetchData(ctx context.Context, meta schema.ClientMeta, parent *schema.Resou
 // and sets it into the resource being returned to CloudQuery.
 func fetchColumn(ctx context.Context, meta schema.ClientMeta, resource *schema.Resource, c schema.Column) error {
 	client := meta.(*client.Client)
-	client.Logger.Debug().Str("resource", format.ToJSON(resource)).Str("column", format.ToJSON(c)).Str("item type", fmt.Sprintf("%T", resource.Item)).Msg("fetching column...")
+	// client.Logger.Debug().Str("resource", format.ToJSON(resource)).Str("column", format.ToJSON(c)).Str("item type", fmt.Sprintf("%T", resource.Item)).Msg("fetching column...")
 	item := resource.Item.(map[string]any)
-	return resource.Set(c.Name, item[c.Name])
+	value := item[c.Name]
+	client.Logger.Debug().Str("value", fmt.Sprintf("%v", value)).Str("type", fmt.Sprintf("%T", value)).Msg("checking value type")
+	if value == nil {
+		client.Logger.Warn().Msg("value is nil")
+		if c.CreationOptions.NotNull {
+			err := fmt.Errorf("invalid nil value for non-nullable column %s", c.Name)
+			client.Logger.Error().Err(err).Str("name", c.Name).Msg("error setting column")
+			return err
+		}
+	} else {
+		client.Logger.Warn().Msg("value is NOT nil")
+		if reflect.ValueOf(value).IsZero() {
+			if !c.CreationOptions.NotNull {
+				// column is nullable, let's null it
+				client.Logger.Warn().Str("name", c.Name).Msg("nulling column value")
+				value = nil
+			} else {
+				client.Logger.Warn().Msg("set default value for type")
+				switch c.Type {
+				case schema.TypeBool:
+					value = false
+				case schema.TypeInt:
+					value = 0
+				case schema.TypeString:
+					value = ""
+				}
+			}
+		}
+	}
+	// in XLSX some values may be null, in which case we must
+	// be sure we're not asking cloudQuery to parse invalid values
+	return resource.Set(c.Name, value)
 }