~kiito/bare-js

63ba7027c492964c0690bf722e6b34748ec8b8a4 — Emma 4 months ago 348810d
Use js BigInt for variable length, and 64 bit integer types
3 files changed, 125 insertions(+), 51 deletions(-)

M README.md
M example.js
M lib-bare.js
M README.md => README.md +12 -9
@@ 4,22 4,25 @@ This is a work-in-progress JavaScript/Node.js implementation of [BARE](https://b

The idea so far is, that the parser and converter will run with node.js, while the resulting JavaScript classes should be usable in any JavaScript environment.

Have a look at `examples.js` on how to create type definitions in code for now.  
Have a look at `examples.js` on how to create type definitions in code for now.\
Or peek inside the `lib-bare.js`, where all conversion classes are located.
These are to be used directly when defining your own type.
These are to be used directly when defining your own type.\
Whenever a type takes some sort of arguments, they are static variables right at the top,
just make sure to set them correctly since there are, as of now, no integrity checks on them. 
just make sure to set them correctly since there are, as of now, no integrity checks on them.

###What is still missing
 * The schema to js translator
 * Union and Enum types  
 * Union and Enum types\
   These are difficult to translate to js, so I'm going to have to think about use cases and how to make them convenient to use
 * Conversion error handling and integrity verification
 * Unit tests, these will come last

###Known Issues
Javascript is a wonderful language, and as such it doesn't actually have an integer type.
###A note about Number and 64 bits
Javascript is a wonderful language, and as such it doesn't use an integer type for numbers.
Instead, every single number is stored as a double precision float.
This limits the usable range of integers to 53 bits, which means the maximum unsigned value is just over 9 quadrillion `(10^15)`
and about ±4.5e15 for signed integers, though that could technically be expanded to ±9e15 since the sign is technically stored outside of the 53 bits.  
In my use case this is not a problem, but for proper adherence to spec this should be addressed.
\ No newline at end of file
This limits the usable range of integers to 53 bits, which means the maximum unsigned value is just over 9 quadrillion `(10^15)`.\
There is the [BitInt](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/BigInt) type that allows arbitrary large integers.
But it has limitations; you can't use them in Math functions or beside regular Numbers.\
Keep this in mind when using the variable length, and the 64 bit integer types, since these return a BigInt by default.\
The lib provides `safeNumber` which will convert a BigInt to Number if it fits into 53 bits, or throw an Error if it does not fit if you just need a little more headroom.


M example.js => example.js +41 -7
@@ 34,7 34,7 @@ class Test2 extends BARE.Struct {
			static type = BARE.Int;
		}],
		['uint', BARE.UInt],
	]
	];
}

let test2 = {


@@ 42,13 42,47 @@ let test2 = {
	'uint': 365555,
};

(() => {
class Address extends BARE.Struct {
	static entries = [
		['address', class extends BARE.ArrayFixed {
			static length = 4;
			static type = BARE.String;
		}],
		['city', BARE.String],
		['state', BARE.String],
		['country', BARE.String],
	];
}

let addrTest = Uint8Array.from([
	0o016, 0o101, 0o144, 0o144, 0o162, 0o145, 0o163, 0o163, 0o040, 0o154, 0o151, 0o156, 0o145, 0o040, 0o061, 0o000,
	0o000, 0o000, 0o014, 0o124, 0o150, 0o145, 0o040, 0o142, 0o151, 0o147, 0o040, 0o143, 0o151, 0o164, 0o171, 0o007,
	0o104, 0o162, 0o145, 0o156, 0o164, 0o150, 0o145, 0o017, 0o124, 0o150, 0o145, 0o040, 0o116, 0o145, 0o164, 0o150,
	0o145, 0o162, 0o154, 0o141, 0o156, 0o144, 0o163,
]);

/**
 * hexdump -b to array:
 * find: '([0-7]{3}) ?'
 * replace: '0o$1, '
 */

	console.log(tst);

	let res = Test2.pack(tst);
	console.log(res);
(() => {
	console.log(test1);
	let test1_bin = Test1.pack(test1);
	console.log(test1_bin);
	let test1_un = Test1.unpack(test1_bin);
	console.log(test1_un);

	console.log("-------------------");
	console.log(test2);
	let test2_bin = Test2.pack(test2);
	console.log(test2_bin);
	let test2_un = Test2.unpack(test2_bin);
	console.log(test2_un);

	let back = Test2.unpack(res);
	console.log(back);
	console.log("-------------------");
	let addr = Address.unpack(addrTest);
	console.log(addr);
})();
\ No newline at end of file

M lib-bare.js => lib-bare.js +72 -35
@@ 19,6 19,28 @@ function twoWayMap(pairs) {
	return map;
}

function safeNumber(bigInt) {
	if (bigInt > MAX_U53 || bigInt < -MAX_U53) {
		throw RangeError("BigInt value out of double precision range (53 bits)");
	} else {
		return Number(bigInt);
	}
}

const MAX_U32 = 2n ** 32n - 1n;
const MAX_U53 = 2n ** 53n - 1n; // same as Number.MAX_SAFE_INTEGER
const MAX_U64 = 2n ** 64n - 1n;

// this is the maximum string length in spec,
// some browsers support less (eg. firefox with 2^30 - 2)
const MAX_STRING_LENGTH = MAX_U53;

// this is the maximum array length in spec,
// used for underlying data structure
const MAX_DATA_LENGTH = MAX_U32;
const MAX_ARRAY_LENGTH = MAX_U32;
const MAX_MAP_LENGTH = MAX_U32;

/* PRIMITIVE TYPES */

class BareType {


@@ 43,46 65,53 @@ class BarePrimitive extends BareType {

class BareUInt extends BarePrimitive {
	static pack(value) {
		value = BigInt(value);
		if (value > MAX_U64) {
			throw RangeError("Unsigned value out of 64-bit range");
		} else if (value < 0) {
			throw RangeError("Passed signed value to unsigned field");
		}
		let bytes = [];
		while (value >= 0x80) {
			bytes.push((value & 0xFF) | 0x80);
			// shift 7 bits to the right (>> 7)
			// js caps the value to 32bit when using binary operators, so I use division
			value = Math.floor(value / 0x80);
		while (value >= 0x80n) {
			bytes.push(Number((value & 0xFFn) | 0x80n));
			value = value >> 7n;
		}
		bytes.push(value);
		bytes.push(Number(value));
		return Uint8Array.from(bytes);
	}

	static unpack(raw) {
		let value = 0;
		let value = 0n;
		for (let i = 0;; i++) {
			let byte = raw.getUint8(i);
			if (byte < 0x80) {
				value += byte * Math.pow(0x80, i);
			let byte = BigInt(raw.getUint8(i));
			if (byte < 0x80n) {
				value += byte << BigInt(7 * i);
				return [value, i + 1];
			}
			// shifted i*7 bits to the left, same story
			value += (byte & 0x7F) * Math.pow(0x80, i);
			value += (byte & 0x7Fn) << BigInt(7 * i);
		}
	}
}
class BareInt extends BarePrimitive {
	static pack(value) {
		value = BigInt(value);
		if (value < 0) {
			value = (2 * Math.abs(value)) - 1;
			value = ~(2n * value);
		} else {
			value = 2 * value;
			value = 2n * value;
		}
		if (value > MAX_U64) {
			throw RangeError("Signed value out of 64-bit range");
		}
		return BareUInt.pack(value);
	}

	static unpack(raw) {
		let [value, length] = BareUInt.unpack(raw);
		let sign = value % 2;
		value = Math.floor(value / 2);
		let sign = value % 2n;
		value = value / 2n;
		if (sign) {
			value = -1 * (value + 1)
			value = -(value + 1n);
		}
		return [value, length];
	}


@@ 131,7 160,7 @@ class BareU64 extends BarePrimitive {
	static pack(value) {
		let bin = new Uint8Array(8);
		let view = new DataView(bin.buffer);
		view.setBigUint64(0, value, true);
		view.setBigUint64(0, BigInt(value), true);
		return bin;
	}



@@ 184,7 213,7 @@ class BareI64 extends BarePrimitive {
	static pack(value) {
		let bin = new Uint8Array(8);
		let view = new DataView(bin.buffer);
		view.setBigInt64(0, value, true);
		view.setBigInt64(0, BigInt(value), true);
		return bin;
	}



@@ 259,12 288,16 @@ const BareUTF8Decoder = new TextDecoder();
class BareString extends BarePrimitive {
	static pack(value) {
		let bytes = BareUTF8Encoder.encode(value);
		let length = BareUInt.pack(bytes.length);
		let length = BareUInt.pack(BigInt(bytes.length));
		return joinUint8Arrays(length, bytes);
	}

	static unpack(raw) {
		let [length, lenBytes] = BareUInt.unpack(raw);
		if (length > MAX_STRING_LENGTH) {
			throw RangeError("Invalid string length");
		}
		length = Number(length);
		let bytes = new DataView(raw.buffer, raw.byteOffset + lenBytes, length);
		let value = BareUTF8Decoder.decode(bytes);
		return [value, length + lenBytes];


@@ 285,12 318,16 @@ class BareDataFixed extends BarePrimitive {
}
class BareData extends BarePrimitive {
	static pack(value) {
		let length = BareUInt.pack(value.length);
		let length = BareUInt.pack(BigInt(value.length));
		return joinUint8Arrays(length, value);
	}

	static unpack(raw) {
		let [length, lenBytes] = BareUInt.unpack(raw);
		if (length > MAX_DATA_LENGTH) {
			throw RangeError("Invalid array length");
		}
		length = Number(length);
		let value = raw.buffer.slice(raw.byteOffset + lenBytes, raw.byteOffset + lenBytes + length);
		return [value, length + lenBytes];
	}


@@ 365,7 402,7 @@ class BareArray extends BareType {
	static type;

	static pack(obj) {
		let bin = BareUInt.pack(obj.length);
		let bin = BareUInt.pack(BigInt(obj.length));
		for (let i = 0; i < obj.length; i++) {
			let bytes = this.type.pack(obj[i]);
			bin = joinUint8Arrays(bin, bytes);


@@ 376,6 413,10 @@ class BareArray extends BareType {
	static unpack(raw) {
		let obj = [];
		let [numElements, length] = BareUInt.unpack(raw);
		if (numElements > MAX_ARRAY_LENGTH) {
			throw RangeError("Invalid array length");
		}
		numElements = Number(numElements);
		for (let i = 0; i < numElements; i++) {
			let view = new DataView(raw.buffer, raw.byteOffset + length);
			let [elem, bytes] = this.type.unpack(view);


@@ 393,7 434,7 @@ class BareMap extends BareType {

	static pack(obj) {
		let keys = Object.keys(obj);
		let bin = BareUInt.pack(keys.length);
		let bin = BareUInt.pack(BigInt(keys.length));
		for (let i = 0; i < keys.length; i++) {
			let keyBytes = this.keyType.pack(keys[i]);
			bin = joinUint8Arrays(bin, keyBytes);


@@ 406,6 447,10 @@ class BareMap extends BareType {
	static unpack(raw) {
		let obj = {};
		let [numEntries, length] = BareUInt.unpack(raw);
		if (numEntries > MAX_MAP_LENGTH) {
			throw RangeError("Invalid array length");
		}
		numEntries = Number(numEntries);
		for (let i = 0; i < numEntries; i++) {
			let keyView = new DataView(raw.buffer, raw.byteOffset + length);
			let [key, keyBytes] = this.type.unpack(keyView);


@@ 424,6 469,8 @@ class BareUnion extends BareType {
	// INVARIANT: has at least one type
	static types; // = twoWayMap([[class, i], ...])

	// TODO use BigInt

	static pack(obj) {
		let unionIndex = this.types[obj.constructor];
		if (!unionIndex) {


@@ 472,6 519,8 @@ class BareStruct extends BareType {
}

module.exports = {
	twoWayMap: twoWayMap,
	safeNumber: safeNumber,
	UInt: BareUInt,
	Int: BareInt,
	U8: BareU8,


@@ 497,15 546,3 @@ module.exports = {
	Union: BareUnion, // TODO
	Struct: BareStruct,
}

/**
 * Things to be aware of:
 *
 * JavaScript represents all numbers as double precision floating point,
 * therefore the maximum precision for "integers" is clipped to 53-bit.
 * (This means the maximum value is 0x1F_FFFF_FFFF_FFFF or about 9 quadrillion (10^15))
 * This is an issue out of my scope, if you need that kind of precision,
 * there are some libraries which add a proper 64-bit long type.
 * The types UInt, Int, U64 and I64 will require some modification
 * to make use of such a library.
 */