Merge pull request #37 from doug-martin/master

v0.3.0

Author: doug-martin

.jshintrc

@@ -33,7 +33,6 @@
     "boss": false,
     "debug": false,
     "eqnull": true,
-    "es5": true,
     "esnext": true,
     "evil": false,
     "expr": true,

History.md

@@ -1,3 +1,9 @@
+# v0.3.0
+
+* You can now specify `objectMode` when parsing a csv which will cause `data` events to have an object emitted.
+* You can now pipe directly to the stream returned from `createWriteStream`
+* You can now transform csvs by piping output from parsing into a formatter.
+
 # v0.2.5
 
 * Fixed issue where not all rows are emitted when using `pause` and `resume`

README.md

@@ -1,7 +1,4 @@
-<a name="top"></a>
-
-
-  [![build status](https://secure.travis-ci.org/C2FO/fast-csv.png)](http://travis-ci.org/C2FO/fast-csv)
+[![build status](https://secure.travis-ci.org/C2FO/fast-csv.png)](http://travis-ci.org/C2FO/fast-csv)
 # Fast-csv
 
 This is a library that provides CSV parsing and formatting.
@@ -18,6 +15,7 @@ This is a library that provides CSV parsing and formatting.
 
 All methods accept the following `options`
 
+* `objectMode=true`: Ensure that `data` events have an object emitted rather than the stringified version set to false to have a stringified buffer.
 * `headers=false`: Ste to true if you expect the first line of your `CSV` to contain headers, alternatly you can specify an array of headers to use.
 * `ignoreEmpty=false`: If you wish to ignore empty rows.
 * `delimiter=','`: If your data uses an alternate delimiter such as `;` or `\t`.
@@ -33,7 +31,6 @@ All methods accept the following `options`
 
 **events**
 
-`parse-error`: Emitted if there was an error parsing a row.
 `record`: Emitted when a record is parsed.
 `data-invalid`: Emitted if there was invalid row encounted, **only emitted if the `validate` function is used**.
 `data`: Emitted with the `stringified` version of a record.
@@ -56,7 +53,7 @@ var csvStream = csv()
 stream.pipe(csvStream);
 ```
 
-**`.fromPath(path[, options])**
+**`.fromPath(path[, options])`**
 
 This method parses a file from the specified path.
 
@@ -73,7 +70,7 @@ csv
  });
 ```
 
-**`.fromString(string[, options])**
+**`.fromString(string[, options])`**
 
 This method parses a string
 
@@ -94,7 +91,7 @@ csv
  });
 ```
 
-**`.fromStream(stream[, options])**
+**`.fromStream(stream[, options])`**
 
 This accepted a readable stream to parse data from.
 
@@ -223,7 +220,7 @@ This is the lowest level of the write methods, it creates a stream that can be u
 
 ```javascript
 var csvStream = csv.createWriteStream({headers: true}),
-    writableStream = fs.createWritableStream("my.csv");
+    writableStream = fs.createWriteStream("my.csv");
 
 writableStream.on("finish", function(){
   console.log("DONE!");
@@ -333,6 +330,37 @@ csv.writeToString([
 ], {headers: true}); //"a,b\na1,b1\na2,b2\n"
 ```
 
+## Piping from Parser to Writer
+
+You can use `fast-csv` to pipe the output from a parsed CSV to a transformed CSV by setting the parser to `objectMode` and using `createWriteStream`.
+
+```javascript
+csv
+   .fromPath("in.csv", {headers: true})
+   .pipe(csv.createWriteStream({headers: true}))
+   .pipe(fs.createWriteStream("out.csv", {encoding: "utf8"}));
+```
+
+When piping from a parser to a formatter the transforms are maintained also.
+
+
+```javascript
+csv
+   .fromPath("in.csv", {headers: true})
+   .transform(function(obj){
+        return {
+            name: obj.Name,
+            address: obj.Address,
+            emailAddress: obj.Email_Address,
+            verified: obj.Verified
+        };
+   })
+   .pipe(csv.createWriteStream({headers: true}))
+   .pipe(fs.createWriteStream("out.csv", {encoding: "utf8"}));
+```
+
+The output will contain formatted result from the transform function.
+
 ## Benchmarks
 
 `Parsing 20000 records AVG over 3 runs`
@@ -371,11 +399,3 @@ MIT <https://github.com/C2FO/fast-csv/raw/master/LICENSE>
 * Code: `git clone git://github.com/C2FO/fast-csv.git`
 * Website: <http://c2fo.com>
 * Twitter: [http://twitter.com/c2fo](http://twitter.com/c2fo) - 877.465.4045
-
-##Namespaces
-
-
-
-
-
-##Classes

docs/History.html

@@ -176,6 +176,12 @@
 
 
 
+<h1>v0.3.0</h1>
+<ul>
+<li>You can now specify <code>objectMode</code> when parsing a csv which will cause <code>data</code> events to have an object emitted.</li>
+<li>You can now pipe directly to the stream returned from <code>createWriteStream</code></li>
+<li>You can now transform csvs by piping output from parsing into a formatter.</li>
+</ul>
 <h1>v0.2.5</h1>
 <ul>
 <li>Fixed issue where not all rows are emitted when using <code>pause</code> and <code>resume</code></li>

docs/index.html

@@ -177,8 +177,7 @@
 
 
 
-  <p><a name="top"></a></p>
-<p>  <a href="http://travis-ci.org/C2FO/fast-csv"><img src="https://secure.travis-ci.org/C2FO/fast-csv.png" alt="build status"></a></p>
+  <p><a href="http://travis-ci.org/C2FO/fast-csv"><img src="https://secure.travis-ci.org/C2FO/fast-csv.png" alt="build status"></a></p>
 <h1>Fast-csv</h1>
 <p>This is a library that provides CSV parsing and formatting.</p>
 <p><strong>NOTE</strong> As of v0.2.0 <code>fast-csv</code> supports multi-line values.</p>
@@ -188,6 +187,7 @@ <h2>Usage</h2>
 <h3>Parsing</h3>
 <p>All methods accept the following <code>options</code></p>
 <ul>
+<li><code>objectMode=true</code>: Ensure that <code>data</code> events have an object emitted rather than the stringified version set to false to have a stringified buffer.</li>
 <li><code>headers=false</code>: Ste to true if you expect the first line of your <code>CSV</code> to contain headers, alternatly you can specify an array of headers to use.</li>
 <li><code>ignoreEmpty=false</code>: If you wish to ignore empty rows.</li>
 <li><code>delimiter=&#39;,&#39;</code>: If your data uses an alternate delimiter such as <code>;</code> or <code>\t</code>.<ul>
@@ -207,8 +207,7 @@ <h3>Parsing</h3>
 </li>
 </ul>
 <p><strong>events</strong></p>
-<p><code>parse-error</code>: Emitted if there was an error parsing a row.
-<code>record</code>: Emitted when a record is parsed.
+<p><code>record</code>: Emitted when a record is parsed.
 <code>data-invalid</code>: Emitted if there was invalid row encounted, <strong>only emitted if the <code>validate</code> function is used</strong>.
 <code>data</code>: Emitted with the <code>stringified</code> version of a record.</p>
 <p><strong>([options])</strong></p>
@@ -224,7 +223,7 @@ <h3>Parsing</h3>
  });
 
 stream.pipe(csvStream);</code></pre>
-<p><strong>`.fromPath(path[, options])</strong></p>
+<p><strong><code>.fromPath(path[, options])</code></strong></p>
 <p>This method parses a file from the specified path.</p>
 <pre class='prettyprint linenums lang-js'><code class="lang-javascript">var csv = require(&quot;fast-csv&quot;);
 
@@ -236,7 +235,7 @@ <h3>Parsing</h3>
  .on(&quot;end&quot;, function(){
      console.log(&quot;done&quot;);
  });</code></pre>
-<p><strong>`.fromString(string[, options])</strong></p>
+<p><strong><code>.fromString(string[, options])</code></strong></p>
 <p>This method parses a string</p>
 <pre class='prettyprint linenums lang-js'><code class="lang-javascript">var csv = require(&quot;fast-csv&quot;);
 
@@ -252,7 +251,7 @@ <h3>Parsing</h3>
  .on(&quot;end&quot;, function(){
      console.log(&quot;done&quot;);
  });</code></pre>
-<p><strong>`.fromStream(stream[, options])</strong></p>
+<p><strong><code>.fromStream(stream[, options])</code></strong></p>
 <p>This accepted a readable stream to parse data from.</p>
 <pre class='prettyprint linenums lang-js'><code class="lang-javascript">var stream = fs.createReadStream(&quot;my.csv&quot;);
 
@@ -342,7 +341,7 @@ <h3>Formatting</h3>
 <p><strong><code>createWriteStream(options)</code></strong></p>
 <p>This is the lowest level of the write methods, it creates a stream that can be used to create a csv of unknown size and pipe to an output csv.</p>
 <pre class='prettyprint linenums lang-js'><code class="lang-javascript">var csvStream = csv.createWriteStream({headers: true}),
-    writableStream = fs.createWritableStream(&quot;my.csv&quot;);
+    writableStream = fs.createWriteStream(&quot;my.csv&quot;);
 
 writableStream.on(&quot;finish&quot;, function(){
   console.log(&quot;DONE!&quot;);
@@ -417,6 +416,26 @@ <h3>Formatting</h3>
    {a: &quot;a1&quot;, b: &quot;b1&quot;},
    {a: &quot;a2&quot;, b: &quot;b2&quot;}
 ], {headers: true}); //&quot;a,b\na1,b1\na2,b2\n&quot;</code></pre>
+<h2>Piping from Parser to Writer</h2>
+<p>You can use <code>fast-csv</code> to pipe the output from a parsed CSV to a transformed CSV by setting the parser to <code>objectMode</code> and using <code>createWriteStream</code>.</p>
+<pre class='prettyprint linenums lang-js'><code class="lang-javascript">csv
+   .fromPath(&quot;in.csv&quot;, {headers: true})
+   .pipe(csv.createWriteStream({headers: true}))
+   .pipe(fs.createWriteStream(&quot;out.csv&quot;, {encoding: &quot;utf8&quot;}));</code></pre>
+<p>When piping from a parser to a formatter the transforms are maintained also.</p>
+<pre class='prettyprint linenums lang-js'><code class="lang-javascript">csv
+   .fromPath(&quot;in.csv&quot;, {headers: true})
+   .transform(function(obj){
+        return {
+            name: obj.Name,
+            address: obj.Address,
+            emailAddress: obj.Email_Address,
+            verified: obj.Verified
+        };
+   })
+   .pipe(csv.createWriteStream({headers: true}))
+   .pipe(fs.createWriteStream(&quot;out.csv&quot;, {encoding: &quot;utf8&quot;}));</code></pre>
+<p>The output will contain formatted result from the transform function.</p>
 <h2>Benchmarks</h2>
 <p><code>Parsing 20000 records AVG over 3 runs</code></p>
 <pre class='prettyprint linenums lang-js'><code>fast-csv: 198.67ms
@@ -438,8 +457,6 @@ <h2>Meta</h2>
 <li>Website: <a href="http://c2fo.com">http://c2fo.com</a></li>
 <li>Twitter: <a href="http://twitter.com/c2fo"><a href="http://twitter.com/c2fo">http://twitter.com/c2fo</a></a> - 877.465.4045</li>
 </ul>
-<h2>Namespaces</h2>
-<h2>Classes</h2>
 
 
 <hr>

lib/formatter.js

@@ -1,8 +1,10 @@
 var fs = require("fs"),
+    util = require("util"),
     extended = require("./extended"),
     isUndefinedOrNull = extended.isUndefinedOrNull,
     hash = extended.hash,
     stream = require("stream"),
+    Transform = stream.Transform,
     LINE_BREAK = extended.LINE_BREAK;
 
 function createFormatter(options) {
@@ -73,7 +75,7 @@ function wrapWriter(writer, options) {
         hasHeaders = extended.has(options, "headers") ? options.headers : true,
         parsedHeaders = hasHeaders ? false : true,
         headersLength = 0, i = -1,
-        writerWrite = writer.push, headers,
+        writerWrite = writer.write, headers,
         buffer = [],
         totalCount = 0,
         MAX_BUFFER_SIZE = options.maxBuffer || 100000;
@@ -82,7 +84,7 @@ function wrapWriter(writer, options) {
         if (item) {
             var isHash = !extended.isArray(item), vals;
             if (!parsedHeaders) {
-                totalCount++
+                totalCount++;
                 parsedHeaders = true;
                 if (isHash) {
                     headers = hash.keys(item);
@@ -117,17 +119,32 @@ function wrapWriter(writer, options) {
                 writerWrite.call(writer, new Buffer(buffer.join("")).toString("utf8"));
                 buffer.length = 0;
             }
-            writerWrite.call(writer, null);
+            writer.end();
         }
     };
     return writer;
 }
 
+function CsvTransformStream(opts) {
+    Transform.call(this, opts);
+    wrapWriter(this, opts);
+}
+
+util.inherits(CsvTransformStream, Transform);
+
+extended(CsvTransformStream).extend({
+
+    _transform: function (str, encoding, cb) {
+        cb(null, str);
+    },
+    _flush: function (cb) {
+        this.write(null);
+        cb(null);
+    }
+});
+
 function createWriteStream(options) {
-    var writer = new stream.Readable();
-    writer._read = function () {
-    };
-    return wrapWriter(writer, options);
+    return new CsvTransformStream(options);
 }
 
 function write(arr, options) {