mirror of https://github.com/gchq/CyberChef.git
'JSON to CSV' operation now escapes characters correctly. Added tests for CSV/JSON operations.
This commit is contained in:
n1474335
parent
863bdffa84
commit
3a979b6cda
|
|
@ -20,7 +20,7 @@ class JSONToCSV extends Operation {
|
|||
|
||||
this.name = "JSON to CSV";
|
||||
this.module = "Default";
|
||||
this.description = "Converts JSON data to a CSV.";
|
||||
this.description = "Converts JSON data to a CSV based on the definition in RFC 4180.";
|
||||
this.infoURL = "https://wikipedia.org/wiki/Comma-separated_values";
|
||||
this.inputType = "JSON";
|
||||
this.outputType = "string";
|
||||
|
|
@ -46,27 +46,67 @@ class JSONToCSV extends Operation {
|
|||
run(input, args) {
|
||||
const [cellDelim, rowDelim] = args;
|
||||
|
||||
this.cellDelim = cellDelim;
|
||||
this.rowDelim = rowDelim;
|
||||
const self = this;
|
||||
|
||||
// TODO: Escape cells correctly.
|
||||
|
||||
try {
|
||||
// If the JSON is an array of arrays, this is easy
|
||||
if (input[0] instanceof Array) {
|
||||
return input.map(row => row.join(cellDelim)).join(rowDelim) + rowDelim;
|
||||
return input
|
||||
.map(row => row
|
||||
.map(self.escapeCellContents.bind(self))
|
||||
.join(cellDelim)
|
||||
)
|
||||
.join(rowDelim) +
|
||||
rowDelim;
|
||||
}
|
||||
|
||||
// If it's an array of dictionaries...
|
||||
const header = Object.keys(input[0]);
|
||||
return header.join(cellDelim) +
|
||||
return header
|
||||
.map(self.escapeCellContents.bind(self))
|
||||
.join(cellDelim) +
|
||||
rowDelim +
|
||||
input.map(
|
||||
row => header.map(
|
||||
h => row[h]
|
||||
).join(cellDelim)
|
||||
).join(rowDelim) +
|
||||
input
|
||||
.map(row => header
|
||||
.map(h => row[h])
|
||||
.map(self.escapeCellContents.bind(self))
|
||||
.join(cellDelim)
|
||||
)
|
||||
.join(rowDelim) +
|
||||
rowDelim;
|
||||
} catch (err) {
|
||||
throw new OperationError("Unable to parse JSON to CSV: " + err);
|
||||
throw new OperationError("Unable to parse JSON to CSV: " + err.toString());
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Correctly escapes a cell's contents based on the cell and row delimiters.
|
||||
*
|
||||
* @param {string} data
|
||||
* @returns {string}
|
||||
*/
|
||||
escapeCellContents(data) {
|
||||
// Double quotes should be doubled up
|
||||
data = data.replace(/"/g, '""');
|
||||
|
||||
// If the cell contains a cell or row delimiter or a double quote, it mut be enclosed in double quotes
|
||||
if (
|
||||
data.indexOf(this.cellDelim) >= 0 ||
|
||||
data.indexOf(this.rowDelim) >= 0 ||
|
||||
data.indexOf("\n") >= 0 ||
|
||||
data.indexOf("\r") >= 0 ||
|
||||
data.indexOf('"') >= 0
|
||||
) {
|
||||
data = `"${data}"`;
|
||||
}
|
||||
|
||||
return data;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
export default JSONToCSV;
|
||||
|
|
|
|||
|
|
@ -39,6 +39,7 @@ import "./tests/operations/Comment";
|
|||
import "./tests/operations/Compress";
|
||||
import "./tests/operations/ConditionalJump";
|
||||
import "./tests/operations/Crypt";
|
||||
import "./tests/operations/CSV";
|
||||
import "./tests/operations/DateTime";
|
||||
import "./tests/operations/ExtractEmailAddresses";
|
||||
import "./tests/operations/Fork";
|
||||
|
|
@ -126,12 +127,12 @@ function handleTestResult(testResult) {
|
|||
|
||||
|
||||
/**
|
||||
* Fail if the process takes longer than 10 seconds.
|
||||
* Fail if the process takes longer than 60 seconds.
|
||||
*/
|
||||
setTimeout(function() {
|
||||
console.log("Tests took longer than 10 seconds to run, returning.");
|
||||
console.log("Tests took longer than 60 seconds to run, returning.");
|
||||
process.exit(1);
|
||||
}, 10 * 1000);
|
||||
}, 60 * 1000);
|
||||
|
||||
|
||||
TestRegister.runTests()
|
||||
|
|
|
|||
|
|
@ -0,0 +1,179 @@
|
|||
/**
|
||||
* CSV tests.
|
||||
*
|
||||
* @author n1474335 [n1474335@gmail.com]
|
||||
*
|
||||
* @copyright Crown Copyright 2018
|
||||
* @license Apache-2.0
|
||||
*/
|
||||
import TestRegister from "../../TestRegister";
|
||||
|
||||
const EXAMPLE_CSV = `A,B,C,D,E,F\r
|
||||
1,2,3,4,5,6\r
|
||||
",",;,',"""",,\r
|
||||
"""hello""","a""1","multi\r
|
||||
line",,,end\r
|
||||
`;
|
||||
|
||||
TestRegister.addTests([
|
||||
{
|
||||
name: "CSV to JSON: Array of dictionaries",
|
||||
input: EXAMPLE_CSV,
|
||||
expectedOutput: JSON.stringify([
|
||||
{
|
||||
"A": "1",
|
||||
"B": "2",
|
||||
"C": "3",
|
||||
"D": "4",
|
||||
"E": "5",
|
||||
"F": "6"
|
||||
},
|
||||
{
|
||||
"A": ",",
|
||||
"B": ";",
|
||||
"C": "'",
|
||||
"D": "\"",
|
||||
"E": "",
|
||||
"F": ""
|
||||
},
|
||||
{
|
||||
"A": "\"hello\"",
|
||||
"B": "a\"1",
|
||||
"C": "multi\r\nline",
|
||||
"D": "",
|
||||
"E": "",
|
||||
"F": "end"
|
||||
}
|
||||
], null, 4),
|
||||
recipeConfig: [
|
||||
{
|
||||
op: "CSV to JSON",
|
||||
args: [",", "\r\n", "Array of dictionaries"],
|
||||
}
|
||||
],
|
||||
},
|
||||
{
|
||||
name: "CSV to JSON: Array of arrays",
|
||||
input: EXAMPLE_CSV,
|
||||
expectedOutput: JSON.stringify([
|
||||
[
|
||||
"A",
|
||||
"B",
|
||||
"C",
|
||||
"D",
|
||||
"E",
|
||||
"F"
|
||||
],
|
||||
[
|
||||
"1",
|
||||
"2",
|
||||
"3",
|
||||
"4",
|
||||
"5",
|
||||
"6"
|
||||
],
|
||||
[
|
||||
",",
|
||||
";",
|
||||
"'",
|
||||
"\"",
|
||||
"",
|
||||
""
|
||||
],
|
||||
[
|
||||
"\"hello\"",
|
||||
"a\"1",
|
||||
"multi\r\nline",
|
||||
"",
|
||||
"",
|
||||
"end"
|
||||
]
|
||||
], null, 4),
|
||||
recipeConfig: [
|
||||
{
|
||||
op: "CSV to JSON",
|
||||
args: [",", "\r\n", "Array of arrays"],
|
||||
}
|
||||
],
|
||||
},
|
||||
{
|
||||
name: "JSON to CSV: Array of dictionaries",
|
||||
input: JSON.stringify([
|
||||
{
|
||||
"A": "1",
|
||||
"B": "2",
|
||||
"C": "3",
|
||||
"D": "4",
|
||||
"E": "5",
|
||||
"F": "6"
|
||||
},
|
||||
{
|
||||
"A": ",",
|
||||
"B": ";",
|
||||
"C": "'",
|
||||
"D": "\"",
|
||||
"E": "",
|
||||
"F": ""
|
||||
},
|
||||
{
|
||||
"A": "\"hello\"",
|
||||
"B": "a\"1",
|
||||
"C": "multi\r\nline",
|
||||
"D": "",
|
||||
"E": "",
|
||||
"F": "end"
|
||||
}
|
||||
]),
|
||||
expectedOutput: EXAMPLE_CSV,
|
||||
recipeConfig: [
|
||||
{
|
||||
op: "JSON to CSV",
|
||||
args: [",", "\r\n"],
|
||||
}
|
||||
],
|
||||
},
|
||||
{
|
||||
name: "JSON to CSV: Array of arrays",
|
||||
input: JSON.stringify([
|
||||
[
|
||||
"A",
|
||||
"B",
|
||||
"C",
|
||||
"D",
|
||||
"E",
|
||||
"F"
|
||||
],
|
||||
[
|
||||
"1",
|
||||
"2",
|
||||
"3",
|
||||
"4",
|
||||
"5",
|
||||
"6"
|
||||
],
|
||||
[
|
||||
",",
|
||||
";",
|
||||
"'",
|
||||
"\"",
|
||||
"",
|
||||
""
|
||||
],
|
||||
[
|
||||
"\"hello\"",
|
||||
"a\"1",
|
||||
"multi\r\nline",
|
||||
"",
|
||||
"",
|
||||
"end"
|
||||
]
|
||||
]),
|
||||
expectedOutput: EXAMPLE_CSV,
|
||||
recipeConfig: [
|
||||
{
|
||||
op: "JSON to CSV",
|
||||
args: [",", "\r\n"],
|
||||
}
|
||||
],
|
||||
},
|
||||
]);
|
||||
Loading…
Reference in New Issue