Skip to content

Commit b60acd2

Browse files
doug-martindoug-martin
committed
v0.1.1
* Added trim, ltrim, and rtrim to parsing options
1 parent 4401b0a commit b60acd2

File tree

12 files changed

+250
-102
lines changed

12 files changed

+250
-102
lines changed

‎README.md‎

Lines changed: 46 additions & 41 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,11 @@ All methods accept the following `options`
2121
* `ignoreEmpty=false`: If you wish to ignore empty rows.
2222
* `delimiter=','`: If your data uses an alternate delimiter such as `;` or `\t`.
2323
* **NOTE** When specifying an alternate `delimiter` you may only pass in a single character delimeter
24+
* The following are options for parsing only.
25+
* `trim=false`: If you want to trim all values parsed set to true.
26+
* `rtrim=false`: If you want to right trim all values parsed set to true.
27+
* `ltrim=false`: If you want to left trim all values parsed set to true.
28+
2429

2530
**events**
2631

@@ -223,22 +228,22 @@ Create a readable stream to read data from.
223228
```javascript
224229
var ws = fs.createWritableStream("my.csv");
225230
csv
226-
.write([
227-
["a", "b"],
228-
["a1", "b1"],
229-
["a2", "b2"]
230-
], {headers: true})
231-
.pipe(ws);
231+
.write([
232+
["a", "b"],
233+
["a1", "b1"],
234+
["a2", "b2"]
235+
], {headers: true})
236+
.pipe(ws);
232237
```
233238
234239
```javascript
235240
var ws = fs.createWritableStream("my.csv");
236241
csv
237-
.write([
238-
{a: "a1", b: "b1"},
239-
{a: "a2", b: "b2"}
240-
], {headers: true})
241-
.pipe(ws);
242+
.write([
243+
{a: "a1", b: "b1"},
244+
{a: "a2", b: "b2"}
245+
], {headers: true})
246+
.pipe(ws);
242247
```
243248
244249
**`writeToStream(stream,arr[, options])`**
@@ -247,20 +252,20 @@ Write an array of values to a `WritableStream`
247252
248253
```javascript
249254
csv
250-
.writeToStream(fs.createWritableStream("my.csv"), [
251-
["a", "b"],
252-
["a1", "b1"],
253-
["a2", "b2"]
254-
], {headers: true});
255+
.writeToStream(fs.createWritableStream("my.csv"), [
256+
["a", "b"],
257+
["a1", "b1"],
258+
["a2", "b2"]
259+
], {headers: true});
255260
```
256261
257262
```javascript
258263
csv
259-
.writeToStream(fs.createWritableStream("my.csv"), [
260-
{a: "a1", b: "b1"},
261-
{a: "a2", b: "b2"}
262-
], {headers: true})
263-
.pipe(ws);
264+
.writeToStream(fs.createWritableStream("my.csv"), [
265+
{a: "a1", b: "b1"},
266+
{a: "a2", b: "b2"}
267+
], {headers: true})
268+
.pipe(ws);
264269
```
265270
266271
**`writeToPath(arr[, options])`**
@@ -269,41 +274,41 @@ Write an array of values to the specified path
269274
270275
```javascript
271276
csv
272-
.writeToPath("my.csv", [
273-
["a", "b"],
274-
["a1", "b1"],
275-
["a2", "b2"]
276-
], {headers: true})
277-
.on("finish", function(){
278-
console.log("done!");
279-
});
277+
.writeToPath("my.csv", [
278+
["a", "b"],
279+
["a1", "b1"],
280+
["a2", "b2"]
281+
], {headers: true})
282+
.on("finish", function(){
283+
console.log("done!");
284+
});
280285
```
281286
282287
```javascript
283288
csv
284-
.writeToStream("my.csv", [
285-
{a: "a1", b: "b1"},
286-
{a: "a2", b: "b2"}
287-
], {headers: true})
288-
.on("finish", function(){
289-
console.log("done!");
290-
});
289+
.writeToStream("my.csv", [
290+
{a: "a1", b: "b1"},
291+
{a: "a2", b: "b2"}
292+
], {headers: true})
293+
.on("finish", function(){
294+
console.log("done!");
295+
});
291296
```
292297
293298
**`writeToString(arr[, options])`**
294299
295300
```javascript
296301
csv.writeToString([
297-
["a", "b"],
298-
["a1", "b1"],
299-
["a2", "b2"]
302+
["a", "b"],
303+
["a1", "b1"],
304+
["a2", "b2"]
300305
], {headers: true}); //"a,b\na1,b1\na2,b2\n"
301306
```
302307
303308
```javascript
304309
csv.writeToString([
305-
{a: "a1", b: "b1"},
306-
{a: "a2", b: "b2"}
310+
{a: "a1", b: "b1"},
311+
{a: "a2", b: "b2"}
307312
], {headers: true}); //"a,b\na1,b1\na2,b2\n"
308313
```
309314

‎benchmark/benchmark.js‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
var fastCsv = require("../lib"),
22
csv = require("csv"),
33
path = require("path"),
4-
COUNT = 1000000,
4+
COUNT = 20000,
55
TEST_FILE = path.resolve(__dirname, "./assets/" + COUNT + ".csv");
66

77

‎docs/index.html‎

Lines changed: 47 additions & 41 deletions
Original file line numberDiff line numberDiff line change
@@ -185,6 +185,12 @@ <h3>Parsing</h3>
185185
<li><strong>NOTE</strong> When specifying an alternate <code>delimiter</code> you may only pass in a single character delimeter</li>
186186
</ul>
187187
</li>
188+
<li>The following are options for parsing only.<ul>
189+
<li><code>trim=false</code>: If you want to trim all values parsed set to true.</li>
190+
<li><code>rtrim=false</code>: If you want to right trim all values parsed set to true.</li>
191+
<li><code>ltrim=false</code>: If you want to left trim all values parsed set to true.</li>
192+
</ul>
193+
</li>
188194
</ul>
189195
<p><strong>events</strong></p>
190196
<p><code>parse-error</code>: Emitted if there was an error parsing a row.
@@ -332,61 +338,61 @@ <h3>Formatting</h3>
332338
<p>Create a readable stream to read data from.</p>
333339
<pre class='prettyprint linenums lang-js'><code class="lang-javascript">var ws = fs.createWritableStream(&quot;my.csv&quot;);
334340
csv
335-
.write([
336-
[&quot;a&quot;, &quot;b&quot;],
337-
[&quot;a1&quot;, &quot;b1&quot;],
338-
[&quot;a2&quot;, &quot;b2&quot;]
339-
], {headers: true})
340-
.pipe(ws);</code></pre>
341+
.write([
342+
[&quot;a&quot;, &quot;b&quot;],
343+
[&quot;a1&quot;, &quot;b1&quot;],
344+
[&quot;a2&quot;, &quot;b2&quot;]
345+
], {headers: true})
346+
.pipe(ws);</code></pre>
341347
<pre class='prettyprint linenums lang-js'><code class="lang-javascript">var ws = fs.createWritableStream(&quot;my.csv&quot;);
342348
csv
343-
.write([
344-
{a: &quot;a1&quot;, b: &quot;b1&quot;},
345-
{a: &quot;a2&quot;, b: &quot;b2&quot;}
346-
], {headers: true})
347-
.pipe(ws);</code></pre>
349+
.write([
350+
{a: &quot;a1&quot;, b: &quot;b1&quot;},
351+
{a: &quot;a2&quot;, b: &quot;b2&quot;}
352+
], {headers: true})
353+
.pipe(ws);</code></pre>
348354
<p><strong><code>writeToStream(stream,arr[, options])</code></strong></p>
349355
<p>Write an array of values to a <code>WritableStream</code></p>
350356
<pre class='prettyprint linenums lang-js'><code class="lang-javascript">csv
351-
.writeToStream(fs.createWritableStream(&quot;my.csv&quot;), [
352-
[&quot;a&quot;, &quot;b&quot;],
353-
[&quot;a1&quot;, &quot;b1&quot;],
354-
[&quot;a2&quot;, &quot;b2&quot;]
355-
], {headers: true});</code></pre>
357+
.writeToStream(fs.createWritableStream(&quot;my.csv&quot;), [
358+
[&quot;a&quot;, &quot;b&quot;],
359+
[&quot;a1&quot;, &quot;b1&quot;],
360+
[&quot;a2&quot;, &quot;b2&quot;]
361+
], {headers: true});</code></pre>
356362
<pre class='prettyprint linenums lang-js'><code class="lang-javascript">csv
357-
.writeToStream(fs.createWritableStream(&quot;my.csv&quot;), [
358-
{a: &quot;a1&quot;, b: &quot;b1&quot;},
359-
{a: &quot;a2&quot;, b: &quot;b2&quot;}
360-
], {headers: true})
361-
.pipe(ws);</code></pre>
363+
.writeToStream(fs.createWritableStream(&quot;my.csv&quot;), [
364+
{a: &quot;a1&quot;, b: &quot;b1&quot;},
365+
{a: &quot;a2&quot;, b: &quot;b2&quot;}
366+
], {headers: true})
367+
.pipe(ws);</code></pre>
362368
<p><strong><code>writeToPath(arr[, options])</code></strong></p>
363369
<p>Write an array of values to the specified path</p>
364370
<pre class='prettyprint linenums lang-js'><code class="lang-javascript">csv
365-
.writeToPath(&quot;my.csv&quot;, [
366-
[&quot;a&quot;, &quot;b&quot;],
367-
[&quot;a1&quot;, &quot;b1&quot;],
368-
[&quot;a2&quot;, &quot;b2&quot;]
369-
], {headers: true})
370-
.on(&quot;finish&quot;, function(){
371-
console.log(&quot;done!&quot;);
372-
});</code></pre>
371+
.writeToPath(&quot;my.csv&quot;, [
372+
[&quot;a&quot;, &quot;b&quot;],
373+
[&quot;a1&quot;, &quot;b1&quot;],
374+
[&quot;a2&quot;, &quot;b2&quot;]
375+
], {headers: true})
376+
.on(&quot;finish&quot;, function(){
377+
console.log(&quot;done!&quot;);
378+
});</code></pre>
373379
<pre class='prettyprint linenums lang-js'><code class="lang-javascript">csv
374-
.writeToStream(&quot;my.csv&quot;, [
375-
{a: &quot;a1&quot;, b: &quot;b1&quot;},
376-
{a: &quot;a2&quot;, b: &quot;b2&quot;}
377-
], {headers: true})
378-
.on(&quot;finish&quot;, function(){
379-
console.log(&quot;done!&quot;);
380-
});</code></pre>
380+
.writeToStream(&quot;my.csv&quot;, [
381+
{a: &quot;a1&quot;, b: &quot;b1&quot;},
382+
{a: &quot;a2&quot;, b: &quot;b2&quot;}
383+
], {headers: true})
384+
.on(&quot;finish&quot;, function(){
385+
console.log(&quot;done!&quot;);
386+
});</code></pre>
381387
<p><strong><code>writeToString(arr[, options])</code></strong></p>
382388
<pre class='prettyprint linenums lang-js'><code class="lang-javascript">csv.writeToString([
383-
[&quot;a&quot;, &quot;b&quot;],
384-
[&quot;a1&quot;, &quot;b1&quot;],
385-
[&quot;a2&quot;, &quot;b2&quot;]
389+
[&quot;a&quot;, &quot;b&quot;],
390+
[&quot;a1&quot;, &quot;b1&quot;],
391+
[&quot;a2&quot;, &quot;b2&quot;]
386392
], {headers: true}); //&quot;a,b\na1,b1\na2,b2\n&quot;</code></pre>
387393
<pre class='prettyprint linenums lang-js'><code class="lang-javascript">csv.writeToString([
388-
{a: &quot;a1&quot;, b: &quot;b1&quot;},
389-
{a: &quot;a2&quot;, b: &quot;b2&quot;}
394+
{a: &quot;a1&quot;, b: &quot;b1&quot;},
395+
{a: &quot;a2&quot;, b: &quot;b2&quot;}
390396
], {headers: true}); //&quot;a,b\na1,b1\na2,b2\n&quot;</code></pre>
391397
<h2>Benchmarks</h2>
392398
<p><code>Parsing 20000 records AVG over 3 runs</code></p>

‎lib/index.js‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,11 @@
2222
* * `ignoreEmpty=false`: If you wish to ignore empty rows.
2323
* * `delimiter=','`: If your data uses an alternate delimiter such as `;` or `\t`.
2424
* * **NOTE** When specifying an alternate `delimiter` you may only pass in a single character delimeter
25+
* * The following are options for parsing only.
26+
* * `trim=false`: If you want to trim all values parsed set to true.
27+
* * `rtrim=false`: If you want to right trim all values parsed set to true.
28+
* * `ltrim=false`: If you want to left trim all values parsed set to true.
29+
*
2530
*
2631
* **events**
2732
*

‎lib/parser.js‎

Lines changed: 51 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,31 @@
1-
var SINGLE_QUOTE = "'",
1+
var extended = require("./extended"),
2+
trim = extended.trim,
3+
trimLeft = extended.trimLeft,
4+
trimRight = extended.trimRight,
5+
SINGLE_QUOTE = "'",
26
DOUBLE_QUOTE = '"';
37

4-
function createParser(delimiter) {
5-
6-
var VALUE_REGEXP = new RegExp("([^" + delimiter + "'\"\\s\\\\]*(?:\\s+[^" + delimiter + "'\"\\s\\\\]+)*)"),
8+
function createParser(options) {
9+
options = options || {};
10+
var delimiter = options.delimiter || ",",
11+
doLtrim = options.ltrim || false,
12+
doRtrim = options.rtrim || false,
13+
doTrim = options.trim || false,
14+
VALUE_REGEXP = new RegExp("([^" + delimiter + "'\"\\s\\\\]*(?:\\s+[^" + delimiter + "'\"\\s\\\\]+)*)"),
715
SEARCH_REGEXP = new RegExp("[^\\\\]" + delimiter),
8-
ESCAPE_CHAR = "\\";
16+
ESCAPE_CHAR = "\\",
17+
WHITE_SPACE = /\s/;
18+
19+
function formatItem(item) {
20+
if (doTrim) {
21+
item = trim(item);
22+
} else if (doLtrim) {
23+
item = trimLeft(item);
24+
} else if (doRtrim) {
25+
item = trimRight(item);
26+
}
27+
return item;
28+
}
929

1030
function getTokensBetween(str, start, items, cursor) {
1131
var depth = 0, ret = [];
@@ -35,10 +55,10 @@ function createParser(delimiter) {
3555
++cursor;
3656
}
3757
}
38-
if (++cursor < str.length && str[cursor].search(delimiter) !== 0) {
58+
if (++cursor < str.length && getNextToken(str, cursor).token.search(delimiter) !== 0) {
3959
throw new Error("Invalid row " + str);
4060
}
41-
items.push(ret.join(""));
61+
items.push(formatItem(ret.join("")));
4262
return ++cursor;
4363
}
4464

@@ -52,7 +72,7 @@ function createParser(delimiter) {
5272
nextIndex = searchStr.length - 1;
5373
}
5474
}
55-
items.push(searchStr.substr(0, nextIndex + 1));
75+
items.push(formatItem(searchStr.substr(0, nextIndex + 1)));
5676
return cursor + (nextIndex + 2);
5777
}
5878

@@ -69,20 +89,39 @@ function createParser(delimiter) {
6989
return findNextToken(line, items, cursor);
7090
}
7191

92+
function getNextToken(line, cursor) {
93+
var l = line.length, ret, token;
94+
do {
95+
token = line[cursor];
96+
if (token === delimiter || !WHITE_SPACE.test(token)) {
97+
ret = token;
98+
} else {
99+
token = null;
100+
}
101+
102+
} while (!token && cursor++ < l);
103+
if (!token) {
104+
throw new Error("Invalid row " + line);
105+
}
106+
return {token: token, cursor: cursor};
107+
}
108+
72109
return function parseLine(line) {
73-
var i = 0, l = line.length, items = [], token;
110+
var i = 0, l = line.length, items = [], token, nextToken;
74111
while (i < l) {
75-
token = line[i];
112+
nextToken = getNextToken(line, i);
113+
token = nextToken.token;
76114
if (token === delimiter) {
77115
items.push("");
78116
i++;
79117
} else if (token === SINGLE_QUOTE) {
80-
i = parseSingleQuoteItem(line, items, i);
118+
i = parseSingleQuoteItem(line, items, nextToken.cursor);
81119
} else if (token === DOUBLE_QUOTE) {
82-
i = parseDoubleQuoteItem(line, items, i);
120+
i = parseDoubleQuoteItem(line, items, nextToken.cursor);
83121
} else {
84122
i = parseItem(line, items, i);
85123
}
124+
86125
}
87126
return items;
88127
};

‎lib/parser_stream.js‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,8 @@ function ParserStream(options) {
2727
} else {
2828
delimiter = DEFAULT_DELIMITER;
2929
}
30-
this.parser = createParser(delimiter);
30+
options.delimiter = delimiter;
31+
this.parser = createParser(options);
3132
this._headers = options.headers;
3233
this._ignoreEmpty = options.ignoreEmpty;
3334
return this;

‎package.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
{
22
"name": "fast-csv",
3-
"version": "0.1.0",
3+
"version": "0.1.1",
44
"description": "CSV parser for node.js",
55
"main": "index.js",
66
"scripts": {

0 commit comments

Comments
 (0)