Mirror
/
CyberChef
mirror of https://github.com/gchq/CyberChef.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'feature-protobuf'

This commit is contained in:
n1474335 2019-04-12 16:16:03 +01:00
parent 786e50c3c3 f6977ea264
commit 5797786a75
7 changed files with 463 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
src/core/config/Categories.json
View File

@ -169,6 +169,9 @@
"Parse URI",
"URL Encode",
"URL Decode",
"Protobuf Decode",
"VarInt Encode",
"VarInt Decode",
"Format MAC addresses",
"Change IP format",
"Group IP addresses",

285
src/core/lib/Protobuf.mjs Normal file
View File

@ -0,0 +1,285 @@
import Utils from "../Utils";
/**
* Protobuf lib. Contains functions to decode protobuf serialised
* data without a schema or .proto file.
*
* Provides utility functions to encode and decode variable length
* integers (varint).
*
* @author GCHQ Contributor [3]
* @copyright Crown Copyright 2019
* @license Apache-2.0
*/
class Protobuf {
/**
* Protobuf constructor
*
* @param {byteArray} data
*/
constructor(data) {
// Check we have a byteArray
if (data instanceof Array) {
this.data = data;
} else {
throw new Error("Protobuf input must be a byteArray");
}
// Set up masks
this.TYPE = 0x07;
this.NUMBER = 0x78;
this.MSB = 0x80;
this.VALUE = 0x7f;
// Declare offset and length
this.offset = 0;
this.LENGTH = data.length;
}
// Public Functions
/**
* Encode a varint from a number
*
* @param {number} number
* @returns {byteArray}
*/
static varIntEncode(number) {
const MSB = 0x80,
VALUE = 0x7f,
MSBALL = ~VALUE,
INT = Math.pow(2, 31);
const out = [];
let offset = 0;
while (number >= INT) {
out[offset++] = (number & 0xff) | MSB;
number /= 128;
}
while (number & MSBALL) {
out[offset++] = (number & 0xff) | MSB;
number >>>= 7;
}
out[offset] = number | 0;
return out;
}
/**
* Decode a varint from the byteArray
*
* @param {byteArray} input
* @returns {number}
*/
static varIntDecode(input) {
const pb = new Protobuf(input);
return pb._varInt();
}
/**
* Parse Protobuf data
*
* @param {byteArray} input
* @returns {Object}
*/
static decode(input) {
const pb = new Protobuf(input);
return pb._parse();
}
// Private Class Functions
/**
* Main private parsing function
*
* @private
* @returns {Object}
*/
_parse() {
let object = {};
// Continue reading whilst we still have data
while (this.offset < this.LENGTH) {
const field = this._parseField();
object = this._addField(field, object);
}
// Throw an error if we have gone beyond the end of the data
if (this.offset > this.LENGTH) {
throw new Error("Exhausted Buffer");
}
return object;
}
/**
* Add a field read from the protobuf data into the Object. As
* protobuf fields can appear multiple times, if the field already
* exists we need to add the new field into an array of fields
* for that key.
*
* @private
* @param {Object} field
* @param {Object} object
* @returns {Object}
*/
_addField(field, object) {
// Get the field key/values
const key = field.key;
const value = field.value;
object[key] = object.hasOwnProperty(key) ?
object[key] instanceof Array ?
object[key].concat([value]) :
[object[key], value] :
value;
return object;
}
/**
* Parse a field and return the Object read from the record
*
* @private
* @returns {Object}
*/
_parseField() {
// Get the field headers
const header = this._fieldHeader();
const type = header.type;
const key = header.key;
switch (type) {
// varint
case 0:
return { "key": key, "value": this._varInt() };
// fixed 64
case 1:
return { "key": key, "value": this._uint64() };
// length delimited
case 2:
return { "key": key, "value": this._lenDelim() };
// fixed 32
case 5:
return { "key": key, "value": this._uint32() };
// unknown type
default:
throw new Error("Unknown type 0x" + type.toString(16));
}
}
/**
* Parse the field header and return the type and key
*
* @private
* @returns {Object}
*/
_fieldHeader() {
// Make sure we call type then number to preserve offset
return { "type": this._fieldType(), "key": this._fieldNumber() };
}
/**
* Parse the field type from the field header. Type is stored in the
* lower 3 bits of the tag byte. This does not move the offset on as
* we need to read the field number from the tag byte too.
*
* @private
* @returns {number}
*/
_fieldType() {
// Field type stored in lower 3 bits of tag byte
return this.data[this.offset] & this.TYPE;
}
/**
* Parse the field number (i.e. the key) from the field header. The
* field number is stored in the upper 5 bits of the tag byte - but
* is also varint encoded so the follow on bytes may need to be read
* when field numbers are > 15.
*
* @private
* @returns {number}
*/
_fieldNumber() {
let shift = -3;
let fieldNumber = 0;
do {
fieldNumber += shift < 28 ?
shift === -3 ?
(this.data[this.offset] & this.NUMBER) >> -shift :
(this.data[this.offset] & this.VALUE) << shift :
(this.data[this.offset] & this.VALUE) * Math.pow(2, shift);
shift += 7;
} while ((this.data[this.offset++] & this.MSD) === this.MSB);
return fieldNumber;
}
// Field Parsing Functions
/**
* Read off a varint from the data
*
* @private
* @returns {number}
*/
_varInt() {
let value = 0;
let shift = 0;
// Keep reading while upper bit set
do {
value += shift < 28 ?
(this.data[this.offset] & this.VALUE) << shift :
(this.data[this.offset] & this.VALUE) * Math.pow(2, shift);
shift += 7;
} while ((this.data[this.offset++] & this.MSB) === this.MSB);
return value;
}
/**
* Read off a 64 bit unsigned integer from the data
*
* @private
* @returns {number}
*/
_uint64() {
// Read off a Uint64
let num = this.data[this.offset++] * 0x1000000 + (this.data[this.offset++] << 16) + (this.data[this.offset++] << 8) + this.data[this.offset++];
num = num * 0x100000000 + this.data[this.offset++] * 0x1000000 + (this.data[this.offset++] << 16) + (this.data[this.offset++] << 8) + this.data[this.offset++];
return num;
}
/**
* Read off a length delimited field from the data
*
* @private
* @returns {Object|string}
*/
_lenDelim() {
// Read off the field length
const length = this._varInt();
const fieldBytes = this.data.slice(this.offset, this.offset + length);
let field;
try {
// Attempt to parse as a new Protobuf Object
const pbObject = new Protobuf(fieldBytes);
field = pbObject._parse();
} catch (err) {
// Otherwise treat as bytes
field = Utils.byteArrayToChars(fieldBytes);
}
// Move the offset and return the field
this.offset += length;
return field;
}
/**
* Read a 32 bit unsigned integer from the data
*
* @private
* @returns {number}
*/
_uint32() {
// Use a dataview to read off the integer
const dataview = new DataView(new Uint8Array(this.data.slice(this.offset, this.offset + 4)).buffer);
const value = dataview.getUint32(0);
this.offset += 4;
return value;
}
}
export default Protobuf;

46
src/core/operations/ProtobufDecode.mjs Normal file
View File

@ -0,0 +1,46 @@
/**
* @author GCHQ Contributor [3]
* @copyright Crown Copyright 2019
* @license Apache-2.0
*/
import Operation from "../Operation";
import OperationError from "../errors/OperationError";
import Protobuf from "../lib/Protobuf";
/**
* Protobuf Decode operation
*/
class ProtobufDecode extends Operation {
/**
* ProtobufDecode constructor
*/
constructor() {
super();
this.name = "Protobuf Decode";
this.module = "Default";
this.description = "Decodes any Protobuf encoded data to a JSON representation of the data using the field number as the field key.";
this.infoURL = "https://wikipedia.org/wiki/Protocol_Buffers";
this.inputType = "byteArray";
this.outputType = "JSON";
this.args = [];
}
/**
* @param {byteArray} input
* @param {Object[]} args
* @returns {JSON}
*/
run(input, args) {
try {
return Protobuf.decode(input);
} catch (err) {
throw new OperationError(err);
}
}
}
export default ProtobufDecode;

46
src/core/operations/VarIntDecode.mjs Normal file
View File

@ -0,0 +1,46 @@
/**
* @author GCHQ Contributor [3]
* @copyright Crown Copyright 2019
* @license Apache-2.0
*/
import Operation from "../Operation";
import OperationError from "../errors/OperationError";
import Protobuf from "../lib/Protobuf";
/**
* VarInt Decode operation
*/
class VarIntDecode extends Operation {
/**
* VarIntDecode constructor
*/
constructor() {
super();
this.name = "VarInt Decode";
this.module = "Default";
this.description = "Decodes a VarInt encoded integer. VarInt is an efficient way of encoding variable length integers and is commonly used with Protobuf.";
this.infoURL = "https://developers.google.com/protocol-buffers/docs/encoding#varints";
this.inputType = "byteArray";
this.outputType = "number";
this.args = [];
}
/**
* @param {byteArray} input
* @param {Object[]} args
* @returns {number}
*/
run(input, args) {
try {
return Protobuf.varIntDecode(input);
} catch (err) {
throw new OperationError(err);
}
}
}
export default VarIntDecode;

46
src/core/operations/VarIntEncode.mjs Normal file
View File

@ -0,0 +1,46 @@
/**
* @author GCHQ Contributor [3]
* @copyright Crown Copyright 2019
* @license Apache-2.0
*/
import Operation from "../Operation";
import OperationError from "../errors/OperationError";
import Protobuf from "../lib/Protobuf";
/**
* VarInt Encode operation
*/
class VarIntEncode extends Operation {
/**
* VarIntEncode constructor
*/
constructor() {
super();
this.name = "VarInt Encode";
this.module = "Default";
this.description = "Encodes a Vn integer as a VarInt. VarInt is an efficient way of encoding variable length integers and is commonly used with Protobuf.";
this.infoURL = "https://developers.google.com/protocol-buffers/docs/encoding#varints";
this.inputType = "number";
this.outputType = "byteArray";
this.args = [];
}
/**
* @param {number} input
* @param {Object[]} args
* @returns {byteArray}
*/
run(input, args) {
try {
return Protobuf.varIntEncode(input);
} catch (err) {
throw new OperationError(err);
}
}
}
export default VarIntEncode;

1
tests/operations/index.mjs
View File

@ -89,6 +89,7 @@ import "./tests/MultipleBombe";
import "./tests/Typex";
import "./tests/BLAKE2b";
import "./tests/BLAKE2s";
import "./tests/Protobuf";
// Cannot test operations that use the File type yet
//import "./tests/SplitColourChannels";

36
tests/operations/tests/Protobuf.mjs Normal file
View File

@ -0,0 +1,36 @@
/**
* Protobuf tests.
*
* @author n1474335 [n1474335@gmail.com]
* @copyright Crown Copyright 2019
* @license Apache-2.0
*/
import TestRegister from "../TestRegister";
TestRegister.addTests([
{
name: "Protobuf Decode",
input: "0d1c0000001203596f751a024d65202b2a0a0a066162633132331200",
expectedOutput: JSON.stringify({
"1": 469762048,
"2": "You",
"3": "Me",
"4": 43,
"5": {
"1": "abc123",
"2": {}
}
}, null, 4),
recipeConfig: [
{
"op": "From Hex",
"args": ["Auto"]
},
{
"op": "Protobuf Decode",
"args": []
}
]
},
]);