mirror of https://github.com/gchq/CyberChef.git
563 lines
18 KiB
JavaScript
563 lines
18 KiB
JavaScript
import Utils from "../Utils.mjs";
|
|
import protobuf from "protobufjs";
|
|
|
|
/**
|
|
* Protobuf lib. Contains functions to decode protobuf serialised
|
|
* data without a schema or .proto file.
|
|
*
|
|
* Provides utility functions to encode and decode variable length
|
|
* integers (varint).
|
|
*
|
|
* @author GCHQ Contributor [3]
|
|
* @copyright Crown Copyright 2019
|
|
* @license Apache-2.0
|
|
*/
|
|
class Protobuf {
|
|
|
|
/**
|
|
* Protobuf constructor
|
|
*
|
|
* @param {byteArray|Uint8Array} data
|
|
*/
|
|
constructor(data) {
|
|
// Check we have a byteArray or Uint8Array
|
|
if (data instanceof Array || data instanceof Uint8Array) {
|
|
this.data = data;
|
|
} else {
|
|
throw new Error("Protobuf input must be a byteArray or Uint8Array");
|
|
}
|
|
|
|
// Set up masks
|
|
this.TYPE = 0x07;
|
|
this.NUMBER = 0x78;
|
|
this.MSB = 0x80;
|
|
this.VALUE = 0x7f;
|
|
|
|
// Declare offset, length, and field type object
|
|
this.offset = 0;
|
|
this.LENGTH = data.length;
|
|
this.fieldTypes = {};
|
|
}
|
|
|
|
// Public Functions
|
|
|
|
/**
|
|
* Encode a varint from a number
|
|
*
|
|
* @param {number} number
|
|
* @returns {byteArray}
|
|
*/
|
|
static varIntEncode(number) {
|
|
const MSB = 0x80,
|
|
VALUE = 0x7f,
|
|
MSBALL = ~VALUE,
|
|
INT = Math.pow(2, 31);
|
|
const out = [];
|
|
let offset = 0;
|
|
|
|
while (number >= INT) {
|
|
out[offset++] = (number & 0xff) | MSB;
|
|
number /= 128;
|
|
}
|
|
while (number & MSBALL) {
|
|
out[offset++] = (number & 0xff) | MSB;
|
|
number >>>= 7;
|
|
}
|
|
out[offset] = number | 0;
|
|
return out;
|
|
}
|
|
|
|
/**
|
|
* Decode a varint from the byteArray
|
|
*
|
|
* @param {byteArray} input
|
|
* @returns {number}
|
|
*/
|
|
static varIntDecode(input) {
|
|
const pb = new Protobuf(input);
|
|
return pb._varInt();
|
|
}
|
|
|
|
/**
|
|
* Encode input JSON according to the given schema
|
|
*
|
|
* @param {Object} input
|
|
* @param {Object []} args
|
|
* @returns {Object}
|
|
*/
|
|
static encode(input, args) {
|
|
this.updateProtoRoot(args[0]);
|
|
if (!this.mainMessageName) {
|
|
throw new Error("Schema Error: Schema not defined");
|
|
}
|
|
const message = this.parsedProto.root.nested[this.mainMessageName];
|
|
|
|
// Convert input into instance of message, and verify instance
|
|
input = message.fromObject(input);
|
|
const error = message.verify(input);
|
|
if (error) {
|
|
throw new Error("Input Error: " + error);
|
|
}
|
|
// Encode input
|
|
const output = message.encode(input).finish();
|
|
return new Uint8Array(output).buffer;
|
|
}
|
|
|
|
/**
|
|
* Parse Protobuf data
|
|
*
|
|
* @param {byteArray} input
|
|
* @returns {Object}
|
|
*/
|
|
static decode(input, args) {
|
|
this.updateProtoRoot(args[0]);
|
|
this.showUnknownFields = args[1];
|
|
this.showTypes = args[2];
|
|
return this.mergeDecodes(input);
|
|
}
|
|
|
|
/**
|
|
* Update the parsedProto, throw parsing errors
|
|
*
|
|
* @param {string} protoText
|
|
*/
|
|
static updateProtoRoot(protoText) {
|
|
try {
|
|
this.parsedProto = protobuf.parse(protoText);
|
|
if (this.parsedProto.package) {
|
|
this.parsedProto.root = this.parsedProto.root.nested[this.parsedProto.package];
|
|
}
|
|
this.updateMainMessageName();
|
|
} catch (error) {
|
|
throw new Error("Schema " + error);
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Set mainMessageName to the first instance of a message defined in the schema that is not a submessage
|
|
*
|
|
*/
|
|
static updateMainMessageName() {
|
|
const messageNames = [];
|
|
const fieldTypes = [];
|
|
this.parsedProto.root.nestedArray.forEach(block => {
|
|
if (block instanceof protobuf.Type) {
|
|
messageNames.push(block.name);
|
|
this.parsedProto.root.nested[block.name].fieldsArray.forEach(field => {
|
|
fieldTypes.push(field.type);
|
|
});
|
|
}
|
|
});
|
|
|
|
if (messageNames.length === 0) {
|
|
this.mainMessageName = null;
|
|
} else {
|
|
// for (const name of messageNames) {
|
|
// if (!fieldTypes.includes(name)) {
|
|
// this.mainMessageName = name;
|
|
// break;
|
|
// }
|
|
// }
|
|
this.mainMessageName = messageNames[0];
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Decode input using Protobufjs package and raw methods, compare, and merge results
|
|
*
|
|
* @param {byteArray} input
|
|
* @returns {Object}
|
|
*/
|
|
static mergeDecodes(input) {
|
|
const pb = new Protobuf(input);
|
|
let rawDecode = pb._parse();
|
|
let message;
|
|
|
|
if (this.showTypes) {
|
|
rawDecode = this.showRawTypes(rawDecode, pb.fieldTypes);
|
|
this.parsedProto.root = this.appendTypesToFieldNames(this.parsedProto.root);
|
|
}
|
|
|
|
try {
|
|
message = this.parsedProto.root.nested[this.mainMessageName];
|
|
const packageDecode = message.toObject(message.decode(input), {
|
|
bytes: String,
|
|
longs: Number,
|
|
enums: String,
|
|
defaults: true
|
|
});
|
|
const output = {};
|
|
|
|
if (this.showUnknownFields) {
|
|
output[message.name] = packageDecode;
|
|
output["Unknown Fields"] = this.compareFields(rawDecode, message);
|
|
return output;
|
|
} else {
|
|
return packageDecode;
|
|
}
|
|
|
|
} catch (error) {
|
|
if (message) {
|
|
throw new Error("Input " + error);
|
|
} else {
|
|
return rawDecode;
|
|
}
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Replace fieldnames with fieldname and type
|
|
*
|
|
* @param {Object} schemaRoot
|
|
* @returns {Object}
|
|
*/
|
|
static appendTypesToFieldNames(schemaRoot) {
|
|
for (const block of schemaRoot.nestedArray) {
|
|
if (block instanceof protobuf.Type) {
|
|
for (const [fieldName, fieldData] of Object.entries(block.fields)) {
|
|
schemaRoot.nested[block.name].remove(block.fields[fieldName]);
|
|
schemaRoot.nested[block.name].add(new protobuf.Field(`${fieldName} (${fieldData.type})`, fieldData.id, fieldData.type, fieldData.rule));
|
|
}
|
|
}
|
|
}
|
|
return schemaRoot;
|
|
}
|
|
|
|
/**
|
|
* Add field type to field name for fields in the raw decoded output
|
|
*
|
|
* @param {Object} rawDecode
|
|
* @param {Object} fieldTypes
|
|
* @returns {Object}
|
|
*/
|
|
static showRawTypes(rawDecode, fieldTypes) {
|
|
for (const [fieldNum, value] of Object.entries(rawDecode)) {
|
|
const fieldType = fieldTypes[fieldNum];
|
|
let outputFieldValue;
|
|
let outputFieldType;
|
|
|
|
// Submessages
|
|
if (isNaN(fieldType)) {
|
|
outputFieldType = 2;
|
|
|
|
// Repeated submessages
|
|
if (Array.isArray(value)) {
|
|
const fieldInstances = [];
|
|
for (const instance of Object.keys(value)) {
|
|
if (typeof(value[instance]) !== "string") {
|
|
fieldInstances.push(this.showRawTypes(value[instance], fieldType));
|
|
} else {
|
|
fieldInstances.push(value[instance]);
|
|
}
|
|
}
|
|
outputFieldValue = fieldInstances;
|
|
|
|
// Single submessage
|
|
} else {
|
|
outputFieldValue = this.showRawTypes(value, fieldType);
|
|
}
|
|
|
|
// Non-submessage field
|
|
} else {
|
|
outputFieldType = fieldType;
|
|
outputFieldValue = value;
|
|
}
|
|
|
|
// Substitute fieldNum with field number and type
|
|
rawDecode[`field #${fieldNum}: ${this.getTypeInfo(outputFieldType)}`] = outputFieldValue;
|
|
delete rawDecode[fieldNum];
|
|
}
|
|
return rawDecode;
|
|
}
|
|
|
|
/**
|
|
* Compare raw decode to package decode and return discrepancies
|
|
*
|
|
* @param rawDecodedMessage
|
|
* @param schemaMessage
|
|
* @returns {Object}
|
|
*/
|
|
static compareFields(rawDecodedMessage, schemaMessage) {
|
|
// Define message data using raw decode output and schema
|
|
const schemaFieldProperties = {};
|
|
const schemaFieldNames = Object.keys(schemaMessage.fields);
|
|
schemaFieldNames.forEach(field => schemaFieldProperties[schemaMessage.fields[field].id] = field);
|
|
|
|
// Loop over each field present in the raw decode output
|
|
for (const fieldName in rawDecodedMessage) {
|
|
let fieldId;
|
|
if (isNaN(fieldName)) {
|
|
fieldId = fieldName.match(/^field #(\d+)/)[1];
|
|
} else {
|
|
fieldId = fieldName;
|
|
}
|
|
|
|
// Check if this field is defined in the schema
|
|
if (fieldId in schemaFieldProperties) {
|
|
const schemaFieldName = schemaFieldProperties[fieldId];
|
|
|
|
// Extract the current field data from the raw decode and schema
|
|
const rawFieldData = rawDecodedMessage[fieldName];
|
|
const schemaField = schemaMessage.fields[schemaFieldName];
|
|
|
|
// Check for repeated fields
|
|
if (Array.isArray(rawFieldData) && !schemaField.repeated) {
|
|
rawDecodedMessage[`(${schemaMessage.name}) ${schemaFieldName} is a repeated field`] = rawFieldData;
|
|
}
|
|
|
|
// Check for submessage fields
|
|
if (schemaField.resolvedType instanceof protobuf.Type) {
|
|
const subMessageType = schemaMessage.fields[schemaFieldName].type;
|
|
const schemaSubMessage = this.parsedProto.root.nested[subMessageType];
|
|
const rawSubMessages = rawDecodedMessage[fieldName];
|
|
let rawDecodedSubMessage = {};
|
|
|
|
// Squash multiple submessage instances into one submessage
|
|
if (Array.isArray(rawSubMessages)) {
|
|
rawSubMessages.forEach(subMessageInstance => {
|
|
const instanceFields = Object.entries(subMessageInstance);
|
|
instanceFields.forEach(subField => {
|
|
rawDecodedSubMessage[subField[0]] = subField[1];
|
|
});
|
|
});
|
|
} else {
|
|
rawDecodedSubMessage = rawSubMessages;
|
|
}
|
|
|
|
// Treat submessage as own message and compare its fields
|
|
rawDecodedSubMessage = Protobuf.compareFields(rawDecodedSubMessage, schemaSubMessage);
|
|
if (Object.entries(rawDecodedSubMessage).length !== 0) {
|
|
rawDecodedMessage[`${schemaFieldName} (${subMessageType}) has missing fields`] = rawDecodedSubMessage;
|
|
}
|
|
}
|
|
delete rawDecodedMessage[fieldName];
|
|
}
|
|
}
|
|
return rawDecodedMessage;
|
|
}
|
|
|
|
/**
|
|
* Returns wiretype information for input wiretype number
|
|
*
|
|
* @param {number} wireType
|
|
* @returns {string}
|
|
*/
|
|
static getTypeInfo(wireType) {
|
|
switch (wireType) {
|
|
case 0:
|
|
return "VarInt (e.g. int32, bool)";
|
|
case 1:
|
|
return "64-Bit (e.g. fixed64, double)";
|
|
case 2:
|
|
return "L-delim (e.g. string, message)";
|
|
case 5:
|
|
return "32-Bit (e.g. fixed32, float)";
|
|
}
|
|
}
|
|
|
|
// Private Class Functions
|
|
|
|
/**
|
|
* Main private parsing function
|
|
*
|
|
* @private
|
|
* @returns {Object}
|
|
*/
|
|
_parse() {
|
|
let object = {};
|
|
// Continue reading whilst we still have data
|
|
while (this.offset < this.LENGTH) {
|
|
const field = this._parseField();
|
|
object = this._addField(field, object);
|
|
}
|
|
// Throw an error if we have gone beyond the end of the data
|
|
if (this.offset > this.LENGTH) {
|
|
throw new Error("Exhausted Buffer");
|
|
}
|
|
return object;
|
|
}
|
|
|
|
/**
|
|
* Add a field read from the protobuf data into the Object. As
|
|
* protobuf fields can appear multiple times, if the field already
|
|
* exists we need to add the new field into an array of fields
|
|
* for that key.
|
|
*
|
|
* @private
|
|
* @param {Object} field
|
|
* @param {Object} object
|
|
* @returns {Object}
|
|
*/
|
|
_addField(field, object) {
|
|
// Get the field key/values
|
|
const key = field.key;
|
|
const value = field.value;
|
|
object[key] = Object.prototype.hasOwnProperty.call(object, key) ?
|
|
object[key] instanceof Array ?
|
|
object[key].concat([value]) :
|
|
[object[key], value] :
|
|
value;
|
|
return object;
|
|
}
|
|
|
|
/**
|
|
* Parse a field and return the Object read from the record
|
|
*
|
|
* @private
|
|
* @returns {Object}
|
|
*/
|
|
_parseField() {
|
|
// Get the field headers
|
|
const header = this._fieldHeader();
|
|
const type = header.type;
|
|
const key = header.key;
|
|
|
|
if (typeof(this.fieldTypes[key]) !== "object") {
|
|
this.fieldTypes[key] = type;
|
|
}
|
|
|
|
switch (type) {
|
|
// varint
|
|
case 0:
|
|
return { "key": key, "value": this._varInt() };
|
|
// fixed 64
|
|
case 1:
|
|
return { "key": key, "value": this._uint64() };
|
|
// length delimited
|
|
case 2:
|
|
return { "key": key, "value": this._lenDelim(key) };
|
|
// fixed 32
|
|
case 5:
|
|
return { "key": key, "value": this._uint32() };
|
|
// unknown type
|
|
default:
|
|
throw new Error("Unknown type 0x" + type.toString(16));
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Parse the field header and return the type and key
|
|
*
|
|
* @private
|
|
* @returns {Object}
|
|
*/
|
|
_fieldHeader() {
|
|
// Make sure we call type then number to preserve offset
|
|
return { "type": this._fieldType(), "key": this._fieldNumber() };
|
|
}
|
|
|
|
/**
|
|
* Parse the field type from the field header. Type is stored in the
|
|
* lower 3 bits of the tag byte. This does not move the offset on as
|
|
* we need to read the field number from the tag byte too.
|
|
*
|
|
* @private
|
|
* @returns {number}
|
|
*/
|
|
_fieldType() {
|
|
// Field type stored in lower 3 bits of tag byte
|
|
return this.data[this.offset] & this.TYPE;
|
|
}
|
|
|
|
/**
|
|
* Parse the field number (i.e. the key) from the field header. The
|
|
* field number is stored in the upper 5 bits of the tag byte - but
|
|
* is also varint encoded so the follow on bytes may need to be read
|
|
* when field numbers are > 15.
|
|
*
|
|
* @private
|
|
* @returns {number}
|
|
*/
|
|
_fieldNumber() {
|
|
let shift = -3;
|
|
let fieldNumber = 0;
|
|
do {
|
|
fieldNumber += shift < 28 ?
|
|
shift === -3 ?
|
|
(this.data[this.offset] & this.NUMBER) >> -shift :
|
|
(this.data[this.offset] & this.VALUE) << shift :
|
|
(this.data[this.offset] & this.VALUE) * Math.pow(2, shift);
|
|
shift += 7;
|
|
} while ((this.data[this.offset++] & this.MSB) === this.MSB);
|
|
return fieldNumber;
|
|
}
|
|
|
|
// Field Parsing Functions
|
|
|
|
/**
|
|
* Read off a varint from the data
|
|
*
|
|
* @private
|
|
* @returns {number}
|
|
*/
|
|
_varInt() {
|
|
let value = 0;
|
|
let shift = 0;
|
|
// Keep reading while upper bit set
|
|
do {
|
|
value += shift < 28 ?
|
|
(this.data[this.offset] & this.VALUE) << shift :
|
|
(this.data[this.offset] & this.VALUE) * Math.pow(2, shift);
|
|
shift += 7;
|
|
} while ((this.data[this.offset++] & this.MSB) === this.MSB);
|
|
return value;
|
|
}
|
|
|
|
/**
|
|
* Read off a 64 bit unsigned integer from the data
|
|
*
|
|
* @private
|
|
* @returns {number}
|
|
*/
|
|
_uint64() {
|
|
// Read off a Uint64 with little-endian
|
|
const lowerHalf = this.data[this.offset++] + (this.data[this.offset++] * 0x100) + (this.data[this.offset++] * 0x10000) + this.data[this.offset++] * 0x1000000;
|
|
const upperHalf = this.data[this.offset++] + (this.data[this.offset++] * 0x100) + (this.data[this.offset++] * 0x10000) + this.data[this.offset++] * 0x1000000;
|
|
return upperHalf * 0x100000000 + lowerHalf;
|
|
}
|
|
|
|
/**
|
|
* Read off a length delimited field from the data
|
|
*
|
|
* @private
|
|
* @returns {Object|string}
|
|
*/
|
|
_lenDelim(fieldNum) {
|
|
// Read off the field length
|
|
const length = this._varInt();
|
|
const fieldBytes = this.data.slice(this.offset, this.offset + length);
|
|
let field;
|
|
try {
|
|
// Attempt to parse as a new Protobuf Object
|
|
const pbObject = new Protobuf(fieldBytes);
|
|
field = pbObject._parse();
|
|
|
|
// Set field types object
|
|
this.fieldTypes[fieldNum] = {...this.fieldTypes[fieldNum], ...pbObject.fieldTypes};
|
|
|
|
} catch (err) {
|
|
// Otherwise treat as bytes
|
|
field = Utils.byteArrayToChars(fieldBytes);
|
|
}
|
|
// Move the offset and return the field
|
|
this.offset += length;
|
|
return field;
|
|
}
|
|
|
|
/**
|
|
* Read a 32 bit unsigned integer from the data
|
|
*
|
|
* @private
|
|
* @returns {number}
|
|
*/
|
|
_uint32() {
|
|
// Use a dataview to read off the integer
|
|
const dataview = new DataView(new Uint8Array(this.data.slice(this.offset, this.offset + 4)).buffer);
|
|
const value = dataview.getUint32(0, true);
|
|
this.offset += 4;
|
|
return value;
|
|
}
|
|
}
|
|
|
|
export default Protobuf;
|