From 949f0e1276d8dbe0bd65abdb95763f44e3f0d8e5 Mon Sep 17 00:00:00 2001 From: Benjamin Ettori Date: Tue, 23 Feb 2016 19:21:17 -0500 Subject: [PATCH 01/10] add travis yaml file --- .travis.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 .travis.yml diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..c9ff5fe --- /dev/null +++ b/.travis.yml @@ -0,0 +1,10 @@ +language: node_js +node_js: + - "4.1" + - "4.0" + - "0.12" + - "0.11" + - "0.10" + - "0.8" + - "0.6" + - "iojs" \ No newline at end of file From 106f9163c14767aa51f4d12577cde460c95d0967 Mon Sep 17 00:00:00 2001 From: Benjamin Ettori Date: Tue, 23 Feb 2016 19:25:41 -0500 Subject: [PATCH 02/10] remove 0.8 and 0.6 versions of node from travis yml file --- .travis.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index c9ff5fe..6160d82 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,6 +5,4 @@ node_js: - "0.12" - "0.11" - "0.10" - - "0.8" - - "0.6" - "iojs" \ No newline at end of file From 1f1c46010f2f6647eb5476a8a9b4475cafa93c9d Mon Sep 17 00:00:00 2001 From: Benjamin Ettori Date: Tue, 23 Feb 2016 19:53:18 -0500 Subject: [PATCH 03/10] Adding travis ci image url to readme --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 14aceae..562df2f 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,7 @@ # dat jawn: 'Git for Tabular Data' +https://travis-ci.org/benjaminettori/jawn.svg?branch=master + Jawn is a node.js module that allows _distributed version control of Tabular Data_. It's connected to the [dat](https://github.com/maxogden/dat) project. It allows you to import tabular data (rows and columns like CSV or TSV) and track how those data change over time. _Do you have non-tabular data? read this:_ [What about Non Tabular Data?](https://github.com/CfABrigadePhiladelphia/jawn/wiki/What-about-Non-Tabular-Data%3F) The key features for jawn are to: From 7e0edbeeb95d550621075baa3f35dfbfde0aa0d9 Mon Sep 17 00:00:00 2001 From: Benjamin Ettori Date: Tue, 23 Feb 2016 20:07:21 -0500 Subject: [PATCH 04/10] change travis ci image to markdown --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 562df2f..b50d2fc 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # dat jawn: 'Git for Tabular Data' -https://travis-ci.org/benjaminettori/jawn.svg?branch=master +[![Build Status](https://travis-ci.org/benjaminettori/jawn.svg?branch=master)](https://travis-ci.org/benjaminettori/jawn) Jawn is a node.js module that allows _distributed version control of Tabular Data_. It's connected to the [dat](https://github.com/maxogden/dat) project. It allows you to import tabular data (rows and columns like CSV or TSV) and track how those data change over time. _Do you have non-tabular data? read this:_ [What about Non Tabular Data?](https://github.com/CfABrigadePhiladelphia/jawn/wiki/What-about-Non-Tabular-Data%3F) From ab93f72ae8cebc6743f4ea1422b9a1c9e0f1ff23 Mon Sep 17 00:00:00 2001 From: Benjamin Ettori Date: Mon, 29 Feb 2016 22:10:36 -0500 Subject: [PATCH 05/10] Parse csv file and transform to JSON --- csvToJsonReader/app.js | 4 ++ csvToJsonReader/csvToJson.js | 71 ++++++++++++++++++++++++++++++++++++ csvToJsonReader/testfile.csv | 3 ++ 3 files changed, 78 insertions(+) create mode 100644 csvToJsonReader/app.js create mode 100644 csvToJsonReader/csvToJson.js create mode 100644 csvToJsonReader/testfile.csv diff --git a/csvToJsonReader/app.js b/csvToJsonReader/app.js new file mode 100644 index 0000000..47c4438 --- /dev/null +++ b/csvToJsonReader/app.js @@ -0,0 +1,4 @@ +var csvJson = require('./csvToJson'); +console.log('starting'); +data = csvJson().readFile('testfile.csv'); +console.log(data); \ No newline at end of file diff --git a/csvToJsonReader/csvToJson.js b/csvToJsonReader/csvToJson.js new file mode 100644 index 0000000..dc8847e --- /dev/null +++ b/csvToJsonReader/csvToJson.js @@ -0,0 +1,71 @@ +(function() { + var csvReader = require('fast-csv'); + var fs = require('fs'); + + module.exports = function(){ + + var methods = {}; + + methods.readFileAsync = function(filename) { + var stream = fs.createReadStream(filename); + + var lineNumber = 0; + var headers = {}; + var csvString = ''; + csvReader.fromStream(stream) + .transform(function(data) { + if(lineNumber === 0) { + headers = data; + lineNumber++; + return; + } + + lineNumber++; + csvString = csvString + methods.csvJsonLine(data, headers); + //console.log(csvString); + return csvString; + }) + .on('data', function(data) { + //console.log(data); + }) + .on('end', function() { + console.log('done'); + }); + }; + + methods.csvJsonLine = function(csvLine, headers) { + var obj = {}; + + for(var j = 0; j < headers.length; j++) { + obj[headers[j]] = csvLine[j]; + } + + return JSON.stringify(obj); + }; + + methods.readFile = function(filename) { + var buffer = fs.readFileSync(filename); + + var lines = buffer.toString().split('\n'); + + var headers = lines[0].split(',').map(function(d) { + return d.replace(/[\r]/g, ''); + }); + var numberOfLines = lines.length; + var objArray = []; + + for(var j = 1; j < numberOfLines; j++) { + var line = lines[j].split(',').map(function(d) { + return d.replace(/[\r]/g, ''); + }) + var jsonObj = methods.csvJsonLine(line, headers); + objArray.push(jsonObj); + } + + return objArray; + } + + return methods; + }; + +}()); diff --git a/csvToJsonReader/testfile.csv b/csvToJsonReader/testfile.csv new file mode 100644 index 0000000..0e4847f --- /dev/null +++ b/csvToJsonReader/testfile.csv @@ -0,0 +1,3 @@ +header1, header2, header3 +one, two, three +four, five, six \ No newline at end of file From a45f4a636842da6d9a3762bb39324e0f5f026a9c Mon Sep 17 00:00:00 2001 From: Benjamin Ettori Date: Tue, 1 Mar 2016 19:33:27 -0500 Subject: [PATCH 06/10] Updates to method. Added simple tests via tape. --- csvToJsonReader/csvToJson.js | 35 +++--------------------- csvToJsonReader/tests/testcsvJsonLine.js | 13 +++++++++ csvToJsonReader/tests/testreadFile.js | 9 ++++++ 3 files changed, 26 insertions(+), 31 deletions(-) create mode 100644 csvToJsonReader/tests/testcsvJsonLine.js create mode 100644 csvToJsonReader/tests/testreadFile.js diff --git a/csvToJsonReader/csvToJson.js b/csvToJsonReader/csvToJson.js index dc8847e..1235845 100644 --- a/csvToJsonReader/csvToJson.js +++ b/csvToJsonReader/csvToJson.js @@ -1,38 +1,10 @@ (function() { - var csvReader = require('fast-csv'); var fs = require('fs'); module.exports = function(){ var methods = {}; - methods.readFileAsync = function(filename) { - var stream = fs.createReadStream(filename); - - var lineNumber = 0; - var headers = {}; - var csvString = ''; - csvReader.fromStream(stream) - .transform(function(data) { - if(lineNumber === 0) { - headers = data; - lineNumber++; - return; - } - - lineNumber++; - csvString = csvString + methods.csvJsonLine(data, headers); - //console.log(csvString); - return csvString; - }) - .on('data', function(data) { - //console.log(data); - }) - .on('end', function() { - console.log('done'); - }); - }; - methods.csvJsonLine = function(csvLine, headers) { var obj = {}; @@ -49,15 +21,16 @@ var lines = buffer.toString().split('\n'); var headers = lines[0].split(',').map(function(d) { - return d.replace(/[\r]/g, ''); + return d.replace(/[\r]/g, '').trim(); }); + var numberOfLines = lines.length; var objArray = []; for(var j = 1; j < numberOfLines; j++) { var line = lines[j].split(',').map(function(d) { - return d.replace(/[\r]/g, ''); - }) + return d.replace(/[\r]/g, '').trim(); + }); var jsonObj = methods.csvJsonLine(line, headers); objArray.push(jsonObj); } diff --git a/csvToJsonReader/tests/testcsvJsonLine.js b/csvToJsonReader/tests/testcsvJsonLine.js new file mode 100644 index 0000000..4cf2594 --- /dev/null +++ b/csvToJsonReader/tests/testcsvJsonLine.js @@ -0,0 +1,13 @@ +var test = require('tape'); +var csvJson = require('../csvToJson'); + +test('oneLineFile', function (t) { + var headers = ['header1', 'header2']; + var line = ['one', 'two']; + + var result = csvJson().csvJsonLine(line, headers); + + t.equal(result, '{"header1":"one","header2":"two"}'); + + t.end(); +}); \ No newline at end of file diff --git a/csvToJsonReader/tests/testreadFile.js b/csvToJsonReader/tests/testreadFile.js new file mode 100644 index 0000000..f7babb1 --- /dev/null +++ b/csvToJsonReader/tests/testreadFile.js @@ -0,0 +1,9 @@ +var test = require('tape'); +var csvJson = require('../csvToJson'); + +test('simpleFile', function(t) { + var expectedResult = ['{"header1":"one","header2":"two","header3":"three"}','{"header1":"four","header2":"five","header3":"six"}']; + var result = csvJson().readFile('../testfile.csv'); + t.deepEqual(result, expectedResult); + t.end(); +}); \ No newline at end of file From eab8737c79d2ab90ea8276445530cea4ba1745bc Mon Sep 17 00:00:00 2001 From: Benjamin Ettori Date: Wed, 2 Mar 2016 18:25:18 -0500 Subject: [PATCH 07/10] small change to app.js --- csvToJsonReader/app.js | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/csvToJsonReader/app.js b/csvToJsonReader/app.js index 47c4438..ec7029a 100644 --- a/csvToJsonReader/app.js +++ b/csvToJsonReader/app.js @@ -1,4 +1,6 @@ +'use strict'; + var csvJson = require('./csvToJson'); console.log('starting'); -data = csvJson().readFile('testfile.csv'); -console.log(data); \ No newline at end of file +var data = csvJson().readFile('testfile.csv') +console.log(data) From 91cb5188f7372442f91e7a9d6222311f2dcd2e64 Mon Sep 17 00:00:00 2001 From: Benjamin Ettori Date: Wed, 2 Mar 2016 18:33:48 -0500 Subject: [PATCH 08/10] Update package.json and test files to run tests correctly --- csvToJsonReader/tests/testreadFile.js | 2 +- package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/csvToJsonReader/tests/testreadFile.js b/csvToJsonReader/tests/testreadFile.js index f7babb1..5ad18af 100644 --- a/csvToJsonReader/tests/testreadFile.js +++ b/csvToJsonReader/tests/testreadFile.js @@ -3,7 +3,7 @@ var csvJson = require('../csvToJson'); test('simpleFile', function(t) { var expectedResult = ['{"header1":"one","header2":"two","header3":"three"}','{"header1":"four","header2":"five","header3":"six"}']; - var result = csvJson().readFile('../testfile.csv'); + var result = csvJson().readFile('./csvToJsonReader/testfile.csv'); t.deepEqual(result, expectedResult); t.end(); }); \ No newline at end of file diff --git a/package.json b/package.json index 8930853..213108a 100644 --- a/package.json +++ b/package.json @@ -4,7 +4,7 @@ "description": "Tabular Data support for Dat/hypercore", "main": "index.js", "scripts": { - "test": "standard && tape test/*.js" + "test": "tape csvToJsonReader/tests/*.js" }, "repository": { "type": "git", From cf5897c93f8614927e7bc3bd0719e1ec44b0169e Mon Sep 17 00:00:00 2001 From: "benjamin.ettori" Date: Sat, 5 Mar 2016 11:10:40 -0500 Subject: [PATCH 09/10] Asynchronous conversion method using stream. --- csvToJsonReader/app.js | 3 ++ csvToJsonReader/csvToJson.js | 37 ++++++++++++++++++++++-- csvToJsonReader/tests/testcsvJsonLine.js | 16 +++++----- package.json | 2 +- 4 files changed, 47 insertions(+), 11 deletions(-) diff --git a/csvToJsonReader/app.js b/csvToJsonReader/app.js index ec7029a..518c3aa 100644 --- a/csvToJsonReader/app.js +++ b/csvToJsonReader/app.js @@ -4,3 +4,6 @@ var csvJson = require('./csvToJson'); console.log('starting'); var data = csvJson().readFile('testfile.csv') console.log(data) + +console.log('Asynchronous conversion using streams') +csvJson().readFileAsync('testfile.csv') \ No newline at end of file diff --git a/csvToJsonReader/csvToJson.js b/csvToJsonReader/csvToJson.js index 1235845..c509b53 100644 --- a/csvToJsonReader/csvToJson.js +++ b/csvToJsonReader/csvToJson.js @@ -1,10 +1,43 @@ -(function() { - var fs = require('fs'); +(function () { + var fs = require('fs') + var csvReader = require('fast-csv') module.exports = function(){ var methods = {}; + methods.readFileAsync = function(filename) { + var stream = fs.createReadStream(filename); + + var lineNumber = 0; + var headers = {}; + var csvString = ''; + csvReader.fromStream(stream) + .transform(function(data) { + if(lineNumber === 0) { + headers = data.map(function(d) { + return d.trim(); + }); + lineNumber++; + return; + } + + lineNumber++; + + var line = data.map(function(d) { + return d.trim(); + }) + csvString = csvString + ' ' + methods.csvJsonLine(line, headers); + console.log(csvString); + }) + .on('data', function(data) { + //console.log(data); + }) + .on('end', function() { + console.log('done'); + }); + }; + methods.csvJsonLine = function(csvLine, headers) { var obj = {}; diff --git a/csvToJsonReader/tests/testcsvJsonLine.js b/csvToJsonReader/tests/testcsvJsonLine.js index 4cf2594..c03d2f7 100644 --- a/csvToJsonReader/tests/testcsvJsonLine.js +++ b/csvToJsonReader/tests/testcsvJsonLine.js @@ -1,13 +1,13 @@ -var test = require('tape'); -var csvJson = require('../csvToJson'); +var test = require('tape') +var csvJson = require('../csvToJson') test('oneLineFile', function (t) { - var headers = ['header1', 'header2']; - var line = ['one', 'two']; + var headers = ['header1', 'header2'] + var line = ['one', 'two'] - var result = csvJson().csvJsonLine(line, headers); + var result = csvJson().csvJsonLine(line, headers) - t.equal(result, '{"header1":"one","header2":"two"}'); + t.equal(result, '{"header1":"one","header2":"two"}') - t.end(); -}); \ No newline at end of file + t.end() +}) diff --git a/package.json b/package.json index 213108a..869be75 100644 --- a/package.json +++ b/package.json @@ -25,6 +25,6 @@ }, "devDependencies": { "standard": "^6.0.5", - "tape": "^4.4.0" + "tape": "^4.5.0" } } From 8101ef3adfa85ef31964baabf618b9ce0536c715 Mon Sep 17 00:00:00 2001 From: "benjamin.ettori" Date: Sat, 5 Mar 2016 11:20:57 -0500 Subject: [PATCH 10/10] Adding fast-csv package dependency. --- package.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/package.json b/package.json index 869be75..d0208c2 100644 --- a/package.json +++ b/package.json @@ -25,6 +25,7 @@ }, "devDependencies": { "standard": "^6.0.5", - "tape": "^4.5.0" + "tape": "^4.5.0", + "fast-csv": "^1.0.0" } }