diff --git a/README.md b/README.md index 2abee8d..bc513f5 100644 --- a/README.md +++ b/README.md @@ -1,13 +1,26 @@ -## JSON to CSV Converter +## Complex JSON to CSV Converter A simple JSON to CSV converter that handles objects and nested documents. -Conversion happens inside the browser, in straight JavaScript. It may choke on large files. +* In the **web** version, conversion happens inside the browser, in straight JavaScript. It may choke on large files. +* With complex-csv2json.js, it can be run via the **command line**. It uses jsonsplit.sh to deal with large files. -Please file all bugs [in the issue tracker](https://github.com/konklone/json/issues). +To install and run via the command line: +* `npm install -g complex-json2csv` +* type the name of the command and provide an input file + * `complex-json2csv inputfile.json` - this will print the output to the screen. + * `complex-json2csv inputfile.json > outputfile.csv` - this will print output to a csv file + * `jsonsplit inputfile.json [records]` - this will split the file into records based on the [records] size + * If you do not specify size, it defaults to splitting by 100000 + +If not yet already done, also remember to install `json` via the command line: + * `npm install -g json` -Read more about the converter and why I built it: "[Making JSON as simple as a spreadsheet](http://sunlightfoundation.com/blog/2014/03/11/making-json-as-simple-as-a-spreadsheet/)". +(Web tool originally from https://github.com/konklone/json; command line tool complex-csv2json and jsonsplit by [@DanaMLewis](https://github.com/danamlewis).) + +Please file all bugs [in the issue tracker](https://github.com/konklone/json/issues). +Read more about the converter and why I (@konklone) built it: "[Making JSON as simple as a spreadsheet](http://sunlightfoundation.com/blog/2014/03/11/making-json-as-simple-as-a-spreadsheet/)". ## Public domain @@ -17,4 +30,4 @@ All **other files** in this project are [dedicated to the public domain](LICENSE > The project is in the public domain within the United States, and copyright and related rights in the work worldwide are waived through the [CC0 1.0 Universal public domain dedication](http://creativecommons.org/publicdomain/zero/1.0/). -> All contributions to this project will be released under the CC0 dedication. By submitting a pull request, you are agreeing to comply with this waiver of copyright interest. \ No newline at end of file +> All contributions to this project will be released under the CC0 dedication. By submitting a pull request, you are agreeing to comply with this waiver of copyright interest. diff --git a/assets/site.js b/assets/site.js index 0ea624c..6429981 100644 --- a/assets/site.js +++ b/assets/site.js @@ -13,7 +13,7 @@ Events = { } -function getParam(name) { +getParam = function(name) { name = name.replace(/[\[]/, "\\[").replace(/[\]]/, "\\]"); var regex = new RegExp("[\\?&]" + name + "=([^&#]*)"), results = regex.exec(location.search); @@ -25,7 +25,7 @@ function getParam(name) { // depends on jquery and jquery-csv (for now) -function parse_object(obj, path) { +parse_object = function(obj, path) { if (path == undefined) path = ""; @@ -56,8 +56,9 @@ function parse_object(obj, path) { // otherwise, just find the first one -function arrayFrom(json) { +arrayFrom = function(json) { var queue = [], next = json; + while (next !== undefined) { if ($.type(next) == "array") { @@ -82,15 +83,15 @@ function arrayFrom(json) { // adapted from Mattias Petter Johanssen: // https://www.quora.com/How-can-I-parse-unquoted-JSON-with-JavaScript/answer/Mattias-Petter-Johansson -function quoteKeys(input) { +quoteKeys = function(input) { return input.replace(/(['"])?([a-zA-Z0-9_]+)(['"])?:/g, '"$2": '); } -function removeSmartQuotes(input) { +removeSmartQuotes = function(input) { return input.replace(/[“”]/g, "\""); } -function removeTrailingComma(input) { +removeTrailingComma = function(input) { if (input.slice(-1) == ",") return input.slice(0,-1); else @@ -100,19 +101,19 @@ function removeTrailingComma(input) { // Rudimentary, imperfect detection of JSON Lines (http://jsonlines.org): // // Is there a closing brace and an opening brace with only whitespace between? -function isJSONLines(string) { +isJSONLines = function(string) { return !!(string.match(/\}\s+\{/)) } // To convert JSON Lines to JSON: // * Add a comma between spaced braces // * Surround with array brackets -function linesToJSON(string) { +linesToJSON = function(string) { return "[" + string.replace(/\}\s+\{/g, "}, {") + "]"; } // todo: add graceful error handling -function jsonFrom(input) { +jsonFrom = function(input) { var string = $.trim(input); if (!string) return; @@ -158,4 +159,4 @@ function jsonFrom(input) { console.log("Nope: that didn't work either. No good.") return result; -} \ No newline at end of file +} diff --git a/complex-json2csv.js b/complex-json2csv.js new file mode 100755 index 0000000..8c984d1 --- /dev/null +++ b/complex-json2csv.js @@ -0,0 +1,156 @@ +#!/usr/bin/env node + +var input; + +function usage ( ) { + console.log('usage: ', process.argv.slice(0, 2), 'inputfile.json'); +} + +if (!module.parent) { + + var jsdom = require("jsdom").jsdom; + global.window = jsdom().defaultView; + global.jQuery = global.$ = require("jquery"); + + require('./assets/jquery-2.1.1.min.js'); + require('./assets/jquery.csv.js'); + require('./assets/site.js'); + var inputFileName = process.argv.slice(2, 3).pop(); + + if ([null, '--help', '-h', 'help'].indexOf(inputFileName) > 0) { + usage( ); + process.exit(0) + } + if (!inputFileName) { + usage( ) + process.exit(1); + } + + var fs = require('fs'); + var cwd = process.cwd() + try { + var inputData = JSON.parse(fs.readFileSync(inputFileName, 'utf8')); + } catch (e) { + return console.error("Could not parse input file: ", e); + } + + //console.error(JSON.stringify(inputData)); + //console.error("About to convert",inputData.length,"records to CSV"); + doCSV(inputData); +} + +function doJSON() { + // just in case + $(".drop").hide(); + + // get input JSON, try to parse it + var newInput = $(".json textarea").val(); + if (newInput == input) return; + + input = newInput; + if (!input) { + // wipe the rendered version too + $(".json code").html(""); + return; + } + + var json = jsonFrom(input); + + // if succeeded, prettify and highlight it + // highlight shows when textarea loses focus + if (json) { + // Reset any error message from previous failed parses. + $("div.error").hide(); + $("div.warning").show(); + + var pretty = JSON.stringify(json, undefined, 2); + $(".json code").html(pretty); + if (pretty.length < (50 * 1024)) + hljs.highlightBlock($(".json code").get(0)); + + // convert to CSV, make available + doCSV(json); + } else { + // Show error. + $("div.warning").hide(); + $("div.error").show(); + $(".json code").html(""); + } + + // Either way, update the error-reporting link to include the latest. + setErrorReporting(null, input); + + return true; +} + + +function showCSV(rendered) { + if (rendered) { + if ($(".csv table").html()) { + $(".csv .rendered").show(); + $(".csv .editing").hide(); + } + } else { + $(".csv .rendered").hide(); + $(".csv .editing").show().focus(); + } +} + +// takes an array of flat JSON objects, converts them to arrays +// renders them into a small table as an example +function renderCSV(objects) { + var rows = $.csv.fromObjects(objects, {justArrays: true}); + if (rows.length < 1) return; + + // find CSV table + var table = $(".csv table")[0]; + $(table).html(""); + + // render header row + var thead = document.createElement("thead"); + var tr = document.createElement("tr"); + var header = rows[0]; + for (field in header) { + var th = document.createElement("th"); + $(th).html(header[field]) + tr.appendChild(th); + } + thead.appendChild(tr); + + // render body of table + var tbody = document.createElement("tbody"); + for (var i=1; i&1 | grep -v "No such file" || echo -n "" + +#converts chunked files into json array +cd $partsdir/ +ls | while read file; do echo -n "." > /dev/stderr; done; echo > /dev/stderr +echo "Grouping split records into valid json..." +ls | while read file; do + cat $file | json -g > $file.json + echo -n "-" > /dev/stderr +done +echo > /dev/stderr +cd .. diff --git a/package.json b/package.json new file mode 100644 index 0000000..923d5d1 --- /dev/null +++ b/package.json @@ -0,0 +1,19 @@ +{ + "name": "complex-json2csv", + "preferGlobal": true, + "version": "0.0.3", + "author": "Dana M. Lewis", + "description": "command line json2csv converter that supports super large files and complex, unknown json schemas", + "license": "MIT", + "engines": { + "node": ">=0.10" + }, + "bin": { + "complex-json2csv": "./complex-json2csv.js", + "jsonsplit": "./jsonsplit.sh" + }, + "dependencies": { + "jquery": "~3.1.1", + "jsdom": "~9.11.0" + } +}