Merge pull request #205 from behrica/fixAsDoubleArray

fix :as-double-array with "nan+nil"

Author: genmeblog

src/tablecloth/api/dataset.clj

@@ -9,7 +9,8 @@
    [tech.v3.dataset.print :as p]
    [tech.v3.dataset.protocols :as prot]
    [tech.v3.dataset.tensor :as ds-tensor]
-   [tech.v3.tensor :as tensor])
+   [tech.v3.tensor :as tensor]
+   [ham-fisted.api :as hf])
   (:import
    [java.io FileNotFoundException]))
 
@@ -156,7 +157,7 @@
    (let [cols (ds/columns ds)]
      (case result-type
        :as-map (zipmap (ds/column-names ds) cols)
-       :as-double-arrays (into-array (map double-array (ds/columns ds)))
+       :as-double-arrays (into-array (map (fn [row] (hf/double-array row)) (ds/columns ds)))
        :as-seqs cols
        cols))))
 
@@ -177,7 +178,7 @@
    (let [options (assoc options :nil-missing? nil-missing?)]
      (case result-type
        :as-maps (ds/mapseq-reader ds options)
-       :as-double-arrays (into-array (map double-array (ds/value-reader ds)))
+       :as-double-arrays (into-array (map (fn [row] (hf/double-array row)) (ds/value-reader ds)))
        :as-seqs (ds/value-reader ds options)
        :as-vecs (ds/rowvecs ds options)
        (ds/value-reader ds options)))))

test/tablecloth/api/dataset_test.clj

@@ -6,8 +6,7 @@
    [tablecloth.column.api :as tcc]
    [tablecloth.common-test :refer [DS]]
    [tech.v3.libs.arrow :as arrow]
-   [tech.v3.libs.fastexcel :as fastexcel]
-   [tablecloth.api :as tc])
+   [tech.v3.libs.fastexcel :as fastexcel])
   (:import
    [java.io FileNotFoundException]))
 
@@ -126,15 +125,34 @@
            {:name :V4, :n-elems 9, :categorical? true, :datatype :string}]))
 
 (fact "as-double-arrays"
-      (tabular (fact (-> (api/dataset {:a [1 2 3]
-                                       :b [5 6 7]})
-                         (?f :as-double-arrays)
-                         (->> (map seq)))
-                     = ?v)
+      (tabular (fact
+                (-> (api/dataset {:a [1 2 3]
+                                  :b [5 6 7]})
+                    (?f :as-double-arrays)
+                    (->> (map seq)))
+                => ?v)
                ?f ?v
                api/rows  '((1.0 5.0) (2.0 6.0) (3.0 7.0))
                api/columns '((1.0 2.0 3.0) (5.0 6.0 7.0))))
 
+(fact "as-double-arrays--nil/nan"
+      (let [rows
+            (->
+             (api/dataset {:a [1.0 nil 3.0]
+                           :b [Double/NaN 5.0 7.0]})
+             (api/rows :as-double-arrays))
+            cols
+            (->
+             (api/dataset {:a [1.0 nil 3.0]
+                           :b [Double/NaN 5.0 7.0]})
+             (api/columns :as-double-arrays))]
+        
+        (-> rows first second Double/isNaN) => true
+        (-> rows second first Double/isNaN) => true
+        (-> cols first second Double/isNaN) => true
+        (-> cols second first Double/isNaN) => true))
+
+
 (fact "let-dataset"
       (fact (api/let-dataset [x (range 4) y 10 z (tcc/+ x y)])
             => (api/dataset {:x [0 1 2 3]