Fixed reading pointers in buffers when not starting at the beginning

client-test/server.js

@@ -10,6 +10,12 @@ const type = new sb.ArrayType(
 		id: new sb.UnsignedShortType
 	})
 );
+const downloadType = new sb.ArrayType(
+	new sb.StructType({
+		name: new sb.StringType,
+		id: new sb.PointerType(new sb.UnsignedShortType)
+	})
+);
 const VALUE = [{name: 'John', id: 2}, {name: 'Jane', id: 10}];
 
 const PORT = 8080;
@@ -28,7 +34,7 @@ http.createServer((req, res) => {
 		});
 	}
 	else if (req.url.startsWith('/downloadtest')) {
-		sb.httpRespond({req, res, type, value: VALUE});
+		sb.httpRespond({req, res, type: downloadType, value: VALUE});
 		console.log('Sent download');
 	}
 	else {

doc/r.html

@@ -70,7 +70,7 @@ <h2>r</h2>
 
     <dt class="tag-source">Source:</dt>
     <dd class="tag-source"><ul class="dummy"><li>
-        <a href="read.js.html">read.js</a>, <a href="read.js.html#line419">line 419</a>
+        <a href="read.js.html">read.js</a>, <a href="read.js.html#line430">line 430</a>
     </li></ul></dd>
 
 
@@ -262,7 +262,7 @@ <h5>Parameters:</h5>
 
     <dt class="tag-source">Source:</dt>
     <dd class="tag-source"><ul class="dummy"><li>
-        <a href="read.js.html">read.js</a>, <a href="read.js.html#line432">line 432</a>
+        <a href="read.js.html">read.js</a>, <a href="read.js.html#line443">line 443</a>
     </li></ul></dd>
 
 
@@ -555,7 +555,7 @@ <h6>Properties</h6>
 
     <dt class="tag-source">Source:</dt>
     <dd class="tag-source"><ul class="dummy"><li>
-        <a href="read.js.html">read.js</a>, <a href="read.js.html#line445">line 445</a>
+        <a href="read.js.html">read.js</a>, <a href="read.js.html#line456">line 456</a>
     </li></ul></dd>
 
 

doc/read.js.html

@@ -144,7 +144,13 @@ <h1 class="page-title">Source: read.js</h1>
 			length++;
 			const values = [];
 			for (let i = 0; i &lt; valueCount; i++) {
-				const value = consumeValue({buffer: typeBuffer, offset: offset + length, type: enumType.value}); //reading values rather than types
+				const valueLocation = offset + length;
+				const value = consumeValue({ //reading values rather than types
+					buffer: typeBuffer,
+					pointerStart: valueLocation,
+					offset: valueLocation,
+					type: enumType.value
+				});
 				length += value.length;
 				values[i] = value.value;
 			}
@@ -212,9 +218,14 @@ <h1 class="page-title">Source: read.js</h1>
 	}
 	return {value, length: byteLength};
 }
-//Reads a value from the specified bytes at the specified offset, given a type
-//Returns the value that was read and the number of bytes consumed (excepting any values being pointed to)
-function consumeValue({buffer, offset, type}) {
+/*
+	Reads a value from the specified bytes at the specified offset, given a type
+	Returns the value that was read and the number of bytes consumed (excepting any values being pointed to)
+	Pointer start refers to the position in the buffer where the root value starts
+	(pointers will be relative to this location)
+*/
+function consumeValue({buffer, pointerStart, offset, type}) {
+	assert.instanceOf(pointerStart, Number);
 	let value, length;
 	switch (type.constructor) {
 		case t.ByteType:
@@ -346,7 +357,7 @@ <h1 class="page-title">Source: read.js</h1>
 			length = 0;
 			value = new Array(type.length);
 			for (let i = 0; i &lt; type.length; i++) {
-				const tupleElement = consumeValue({buffer, offset: offset + length, type: type.type});
+				const tupleElement = consumeValue({buffer, pointerStart, offset: offset + length, type: type.type});
 				length += tupleElement.length;
 				value[i] = tupleElement.value;
 			}
@@ -357,7 +368,7 @@ <h1 class="page-title">Source: read.js</h1>
 			for (let field of type.fields) {
 				const fieldName = field.name;
 				const fieldType = field.type;
-				const readField = consumeValue({buffer, offset: offset + length, type: fieldType});
+				const readField = consumeValue({buffer, pointerStart, offset: offset + length, type: fieldType});
 				value[fieldName] = readField.value;
 				length += readField.length;
 			}
@@ -367,7 +378,7 @@ <h1 class="page-title">Source: read.js</h1>
 			length = arrayLength.length;
 			value = new Array(arrayLength.value);
 			for (let i = 0; i &lt; value.length; i++) {
-				const arrayElement = consumeValue({buffer, offset: offset + length, type: type.type});
+				const arrayElement = consumeValue({buffer, pointerStart, offset: offset + length, type: type.type});
 				length += arrayElement.length;
 				value[i] = arrayElement.value;
 			}
@@ -377,7 +388,7 @@ <h1 class="page-title">Source: read.js</h1>
 			length = setLength.length;
 			value = new Set;
 			for (let i = 0; i &lt; setLength.value; i++) {
-				const setElement = consumeValue({buffer, offset: offset + length, type: type.type});
+				const setElement = consumeValue({buffer, pointerStart, offset: offset + length, type: type.type});
 				length += setElement.length;
 				value.add(setElement.value);
 			}
@@ -387,9 +398,9 @@ <h1 class="page-title">Source: read.js</h1>
 			length = mapSize.length;
 			value = new Map;
 			for (let i = 0; i &lt; mapSize.value; i++) {
-				const keyElement = consumeValue({buffer, offset: offset + length, type: type.keyType});
+				const keyElement = consumeValue({buffer, pointerStart, offset: offset + length, type: type.keyType});
 				length += keyElement.length;
-				const valueElement = consumeValue({buffer, offset: offset + length, type: type.valueType});
+				const valueElement = consumeValue({buffer, pointerStart, offset: offset + length, type: type.valueType});
 				length += valueElement.length;
 				value.set(keyElement.value, valueElement.value);
 			}
@@ -405,7 +416,7 @@ <h1 class="page-title">Source: read.js</h1>
 			length = 1;
 			assert.assert(buffer.byteLength >= offset + length, NOT_LONG_ENOUGH);
 			const typeIndex = new Uint8Array(buffer)[offset];
-			const subValue = consumeValue({buffer, offset: offset + length, type: type.types[typeIndex]});
+			const subValue = consumeValue({buffer, pointerStart, offset: offset + length, type: type.types[typeIndex]});
 			length += subValue.length;
 			value = subValue.value;
 			break;
@@ -415,7 +426,7 @@ <h1 class="page-title">Source: read.js</h1>
 			const optionalByte = new Uint8Array(buffer)[offset];
 			assert.assert(optionalByte === 0x00 || optionalByte === 0xFF, '0x' + optionalByte.toString(16) + ' is an invalid Optional byte');
 			if (optionalByte) {
-				const subValue = consumeValue({buffer, offset: offset + length, type: type.type});
+				const subValue = consumeValue({buffer, pointerStart, offset: offset + length, type: type.type});
 				length += subValue.length;
 				value = subValue.value;
 			}
@@ -424,7 +435,7 @@ <h1 class="page-title">Source: read.js</h1>
 		case t.PointerType:
 			const location = readLengthBuffer(buffer, offset);
 			length = location.length;
-			value = consumeValue({buffer, offset: location.value, type: type.type}).value;
+			value = consumeValue({buffer, pointerStart, offset: pointerStart + location.value, type: type.type}).value;
 			break;
 		default:
 			assert.fail('Not a structure type: ' + util.inspect(type));
@@ -436,7 +447,7 @@ <h1 class="page-title">Source: read.js</h1>
 	assert.instanceOf(type, t.Type);
 	if (offset === undefined) offset = 0; //for some reason, isparta doesn't like default parameters inside destructuring
 	assert.instanceOf(offset, Number);
-	const {value} = consumeValue({buffer, offset, type});
+	const {value} = consumeValue({buffer, offset, type, pointerStart: offset});
 	//no length validation because bytes being pointed to don't get counted in the length
 	return value;
 }

package.json

@@ -1,6 +1,6 @@
 {
 	"name": "structure-bytes",
-	"version": "2.9.8",
+	"version": "2.10.8",
 	"description": "Efficiently stores data of any shape by separating it into structure and value",
 	"main": "index.js",
 	"dependencies": {

read.js

@@ -116,7 +116,13 @@ function consumeType(typeBuffer, offset) {
 			length++;
 			const values = [];
 			for (let i = 0; i < valueCount; i++) {
-				const value = consumeValue({buffer: typeBuffer, offset: offset + length, type: enumType.value}); //reading values rather than types
+				const valueLocation = offset + length;
+				const value = consumeValue({ //reading values rather than types
+					buffer: typeBuffer,
+					pointerStart: valueLocation,
+					offset: valueLocation,
+					type: enumType.value
+				});
 				length += value.length;
 				values[i] = value.value;
 			}
@@ -184,9 +190,13 @@ function readBooleans({buffer, offset, count}) { //complement to writeBooleans()
 	}
 	return {value, length: byteLength};
 }
-//Reads a value from the specified bytes at the specified offset, given a type
-//Returns the value that was read and the number of bytes consumed (excepting any values being pointed to)
-function consumeValue({buffer, offset, type}) {
+/*
+	Reads a value from the specified bytes at the specified offset, given a type
+	Returns the value that was read and the number of bytes consumed (excepting any values being pointed to)
+	Pointer start refers to the position in the buffer where the root value starts
+	(pointers will be relative to this location)
+*/
+function consumeValue({buffer, pointerStart, offset, type}) {
 	let value, length;
 	switch (type.constructor) {
 		case t.ByteType:
@@ -318,7 +328,7 @@ function consumeValue({buffer, offset, type}) {
 			length = 0;
 			value = new Array(type.length);
 			for (let i = 0; i < type.length; i++) {
-				const tupleElement = consumeValue({buffer, offset: offset + length, type: type.type});
+				const tupleElement = consumeValue({buffer, pointerStart, offset: offset + length, type: type.type});
 				length += tupleElement.length;
 				value[i] = tupleElement.value;
 			}
@@ -329,7 +339,7 @@ function consumeValue({buffer, offset, type}) {
 			for (let field of type.fields) {
 				const fieldName = field.name;
 				const fieldType = field.type;
-				const readField = consumeValue({buffer, offset: offset + length, type: fieldType});
+				const readField = consumeValue({buffer, pointerStart, offset: offset + length, type: fieldType});
 				value[fieldName] = readField.value;
 				length += readField.length;
 			}
@@ -339,7 +349,7 @@ function consumeValue({buffer, offset, type}) {
 			length = arrayLength.length;
 			value = new Array(arrayLength.value);
 			for (let i = 0; i < value.length; i++) {
-				const arrayElement = consumeValue({buffer, offset: offset + length, type: type.type});
+				const arrayElement = consumeValue({buffer, pointerStart, offset: offset + length, type: type.type});
 				length += arrayElement.length;
 				value[i] = arrayElement.value;
 			}
@@ -349,7 +359,7 @@ function consumeValue({buffer, offset, type}) {
 			length = setLength.length;
 			value = new Set;
 			for (let i = 0; i < setLength.value; i++) {
-				const setElement = consumeValue({buffer, offset: offset + length, type: type.type});
+				const setElement = consumeValue({buffer, pointerStart, offset: offset + length, type: type.type});
 				length += setElement.length;
 				value.add(setElement.value);
 			}
@@ -359,9 +369,9 @@ function consumeValue({buffer, offset, type}) {
 			length = mapSize.length;
 			value = new Map;
 			for (let i = 0; i < mapSize.value; i++) {
-				const keyElement = consumeValue({buffer, offset: offset + length, type: type.keyType});
+				const keyElement = consumeValue({buffer, pointerStart, offset: offset + length, type: type.keyType});
 				length += keyElement.length;
-				const valueElement = consumeValue({buffer, offset: offset + length, type: type.valueType});
+				const valueElement = consumeValue({buffer, pointerStart, offset: offset + length, type: type.valueType});
 				length += valueElement.length;
 				value.set(keyElement.value, valueElement.value);
 			}
@@ -377,7 +387,7 @@ function consumeValue({buffer, offset, type}) {
 			length = 1;
 			assert.assert(buffer.byteLength >= offset + length, NOT_LONG_ENOUGH);
 			const typeIndex = new Uint8Array(buffer)[offset];
-			const subValue = consumeValue({buffer, offset: offset + length, type: type.types[typeIndex]});
+			const subValue = consumeValue({buffer, pointerStart, offset: offset + length, type: type.types[typeIndex]});
 			length += subValue.length;
 			value = subValue.value;
 			break;
@@ -387,7 +397,7 @@ function consumeValue({buffer, offset, type}) {
 			const optionalByte = new Uint8Array(buffer)[offset];
 			assert.assert(optionalByte === 0x00 || optionalByte === 0xFF, '0x' + optionalByte.toString(16) + ' is an invalid Optional byte');
 			if (optionalByte) {
-				const subValue = consumeValue({buffer, offset: offset + length, type: type.type});
+				const subValue = consumeValue({buffer, pointerStart, offset: offset + length, type: type.type});
 				length += subValue.length;
 				value = subValue.value;
 			}
@@ -396,7 +406,7 @@ function consumeValue({buffer, offset, type}) {
 		case t.PointerType:
 			const location = readLengthBuffer(buffer, offset);
 			length = location.length;
-			value = consumeValue({buffer, offset: location.value, type: type.type}).value;
+			value = consumeValue({buffer, pointerStart, offset: pointerStart + location.value, type: type.type}).value;
 			break;
 		default:
 			assert.fail('Not a structure type: ' + util.inspect(type));
@@ -408,7 +418,7 @@ function value({buffer, type, offset}) {
 	assert.instanceOf(type, t.Type);
 	if (offset === undefined) offset = 0; //for some reason, isparta doesn't like default parameters inside destructuring
 	assert.instanceOf(offset, Number);
-	const {value} = consumeValue({buffer, offset, type});
+	const {value} = consumeValue({buffer, offset, type, pointerStart: offset});
 	//no length validation because bytes being pointed to don't get counted in the length
 	return value;
 }

test/io/type-and-value.js

@@ -2,10 +2,10 @@
 const fs = require('fs');
 const OUT_FILE = 'type-value-out';
 
-let type = new t.MapType(new t.StringType, new t.UnsignedIntType);
-let value = new Map().set('abc', 123).set('def', 4560).set('—ݥ—', 7512112);
+let type = new t.MapType(new t.StringType, new t.PointerType(new t.UnsignedIntType));
+let value = new Map().set('abc', 4560).set('def', 4560).set('—ݥ—', 4560);
 let outStream = fs.createWriteStream(OUT_FILE);
-const BUFFER = bufferFrom([0x54, 0x41, 0x13, 0, 0, 0, 3, 0x61, 0x62, 0x63, 0, 0, 0, 0, 0x7b, 0x64, 0x65, 0x66, 0, 0, 0, 0x11, 0xd0, 0xe2, 0x80, 0x94, 0xdd, 0xa5, 0xe2, 0x80, 0x94, 0, 0, 0x72, 0xa0, 0x30]);
+const BUFFER = bufferFrom([0x54, 0x41, 0x70, 0x13, 0, 0, 0, 3, 0x61, 0x62, 0x63, 0, 0, 0, 0, 33, 0x64, 0x65, 0x66, 0, 0, 0, 0, 33, 0xe2, 0x80, 0x94, 0xdd, 0xa5, 0xe2, 0x80, 0x94, 0, 0, 0, 0, 33, 0, 0, 0x11, 0xd0]);
 let s = new Simultaneity;
 s.addTask(() => {
 	io.writeTypeAndValue({type, value, outStream}, err => {