From aa024ce3d7035d4fcd78a20364563ad1d4d5fd1c Mon Sep 17 00:00:00 2001 From: extremeheat Date: Mon, 18 Aug 2025 06:30:06 +0000 Subject: [PATCH 1/2] add loop and restBuffer types --- src/compiler.js | 4 ++ src/datatypes/compiler-extras.js | 74 +++++++++++++++++++++ src/datatypes/extras.js | 111 +++++++++++++++++++++++++++++++ src/protodef.js | 1 + test/dataTypes/datatypes.js | 2 +- test/dataTypes/prepareTests.js | 6 +- test_extras.js | 23 +++++++ 7 files changed, 219 insertions(+), 2 deletions(-) create mode 100644 src/datatypes/compiler-extras.js create mode 100644 src/datatypes/extras.js create mode 100644 test_extras.js diff --git a/src/compiler.js b/src/compiler.js index 9c8f057..a0cb072 100644 --- a/src/compiler.js +++ b/src/compiler.js @@ -4,6 +4,7 @@ const utils = require('./datatypes/utils') const conditionalDatatypes = require('./datatypes/compiler-conditional') const structuresDatatypes = require('./datatypes/compiler-structures') const utilsDatatypes = require('./datatypes/compiler-utils') +const extrasDatatypes = require('./datatypes/compiler-extras') const { tryCatch } = require('./utils') @@ -270,6 +271,7 @@ class ReadCompiler extends Compiler { this.addTypes(conditionalDatatypes.Read) this.addTypes(structuresDatatypes.Read) this.addTypes(utilsDatatypes.Read) + this.addTypes(extrasDatatypes.Read) // Add default types for (const key in numeric) { @@ -320,6 +322,7 @@ class WriteCompiler extends Compiler { this.addTypes(conditionalDatatypes.Write) this.addTypes(structuresDatatypes.Write) this.addTypes(utilsDatatypes.Write) + this.addTypes(extrasDatatypes.Write) // Add default types for (const key in numeric) { @@ -370,6 +373,7 @@ class SizeOfCompiler extends Compiler { this.addTypes(conditionalDatatypes.SizeOf) this.addTypes(structuresDatatypes.SizeOf) this.addTypes(utilsDatatypes.SizeOf) + this.addTypes(extrasDatatypes.SizeOf) // Add default types for (const key in numeric) { diff --git a/src/datatypes/compiler-extras.js b/src/datatypes/compiler-extras.js new file mode 100644 index 0000000..ef702c6 --- /dev/null +++ b/src/datatypes/compiler-extras.js @@ -0,0 +1,74 @@ +/* eslint-disable multiline-ternary */ +function validateNT (nt) { + if (nt !== null && typeof nt !== 'number') throw new Error('Loop terminator must be a number like 0 or null for EOF') + return nt !== null +} + +module.exports = { + Read: { + loop: ['parametrizable', (compiler, struct) => { + const nt = struct.nt + const hasTerminator = validateNT(nt) + + return compiler.wrapCode(` + const results = [] + let size = 0 + while (offset !== buffer.length) { + ${hasTerminator ? ` + const typ = ctx.i8(buffer, offset) + if (typ.value === ${nt}) { + return { value: results, size: size + 1 } + }` : ''} + const entry = ${compiler.callType(struct.type)} + results.push(entry.value) + offset += entry.size + size += entry.size + } + return { value: results, size } + `) + }], + restBuffer: ['native', (buffer, offset) => { + return { + value: buffer.slice(offset), + size: buffer.length - offset + } + }] + }, + Write: { + loop: ['parametrizable', (compiler, struct) => { + const nt = struct.nt + const hasTerminator = validateNT(nt) + + return compiler.wrapCode(` + for (const key in value) { + offset = ${compiler.callType('value[key]', struct.type)} + } + ${hasTerminator ? `offset = ctx.i8(${nt}, buffer, offset)` : ''} + return offset + `) + }], + restBuffer: ['native', (value, buffer, offset) => { + if (!(value instanceof Buffer)) value = Buffer.from(value) + value.copy(buffer, offset) + return offset + value.length + }] + }, + SizeOf: { + loop: ['parametrizable', (compiler, struct) => { + const nt = struct.nt + const hasTerminator = validateNT(nt) + + return compiler.wrapCode(` + let size = ${hasTerminator ? '1' : '0'} + for (const key in value) { + size += ${compiler.callType('value[key]', struct.type)} + } + return size + `) + }], + restBuffer: ['native', (value) => { + if (!(value instanceof Buffer)) value = Buffer.from(value) + return value.length + }] + } +} diff --git a/src/datatypes/extras.js b/src/datatypes/extras.js new file mode 100644 index 0000000..9bd3ef3 --- /dev/null +++ b/src/datatypes/extras.js @@ -0,0 +1,111 @@ +const { PartialReadError } = require('../utils') + +module.exports = { + loop: [readLoop, writeLoop, sizeOfLoop, require('../../ProtoDef/schemas/extras.json').loop], + restBuffer: [readRestBuffer, writeRestBuffer, sizeOfRestBuffer, require('../../ProtoDef/schemas/extras.json').restBuffer] +} + +function readLoop (buffer, offset, typeArgs, rootNode) { + if (!typeArgs) { + throw new Error('typeArgs is required for loop type') + } + + const results = [] + const startOffset = offset + const nt = typeArgs.nt + const hasTerminator = nt !== null && typeof nt === 'number' + + while (offset < buffer.length) { + // Check for terminator if specified + if (hasTerminator) { + if (offset >= buffer.length) break + const terminatorValue = buffer.readInt8(offset) + if (terminatorValue === nt) { + // Found terminator, consume it and return + return { + value: results, + size: offset - startOffset + 1 + } + } + } + + // Read the next element + try { + const entry = this.read(buffer, offset, typeArgs.type, rootNode) + results.push(entry.value) + offset += entry.size + } catch (error) { + if (error instanceof PartialReadError) { + break + } + throw error + } + } + + return { + value: results, + size: offset - startOffset + } +} + +function writeLoop (value, buffer, offset, typeArgs, rootNode) { + if (!typeArgs) { + throw new Error('typeArgs is required for loop type') + } + + const nt = typeArgs.nt + const hasTerminator = nt !== null && typeof nt === 'number' + + // Write each element in the array + for (const item of value) { + offset = this.write(item, buffer, offset, typeArgs.type, rootNode) + } + + // Write terminator if specified + if (hasTerminator) { + buffer.writeInt8(nt, offset) + offset++ + } + + return offset +} + +function sizeOfLoop (value, typeArgs, rootNode) { + if (!typeArgs) { + throw new Error('typeArgs is required for loop type') + } + + const nt = typeArgs.nt + const hasTerminator = nt !== null && typeof nt === 'number' + let size = hasTerminator ? 1 : 0 // 1 byte for terminator if present + + // Calculate size of all elements + for (const item of value) { + size += this.sizeOf(item, typeArgs.type, rootNode) + } + + return size +} + +function readRestBuffer (buffer, offset) { + const remainingBuffer = buffer.slice(offset) + return { + value: remainingBuffer, + size: remainingBuffer.length + } +} + +function writeRestBuffer (value, buffer, offset) { + if (!(value instanceof Buffer)) { + value = Buffer.from(value) + } + value.copy(buffer, offset) + return offset + value.length +} + +function sizeOfRestBuffer (value) { + if (!(value instanceof Buffer)) { + value = Buffer.from(value) + } + return value.length +} diff --git a/src/protodef.js b/src/protodef.js index 0cee5e1..ad09af3 100644 --- a/src/protodef.js +++ b/src/protodef.js @@ -52,6 +52,7 @@ class ProtoDef { this.addTypes(require('./datatypes/utils')) this.addTypes(require('./datatypes/structures')) this.addTypes(require('./datatypes/conditional')) + this.addTypes(require('./datatypes/extras')) } addProtocol (protocolData, path) { diff --git a/test/dataTypes/datatypes.js b/test/dataTypes/datatypes.js index 35a50b9..dd4cd84 100644 --- a/test/dataTypes/datatypes.js +++ b/test/dataTypes/datatypes.js @@ -55,7 +55,7 @@ function testType (type, values) { }) } else { testValue(type, value.value, value.buffer) } }) - if (type !== 'void') { + if (type !== 'void' && type !== 'restBuffer' && !type.startsWith('loop_')) { it('reads 0 bytes and throw a PartialReadError', () => { try { proto.parsePacketBuffer(type, Buffer.alloc(0)) diff --git a/test/dataTypes/prepareTests.js b/test/dataTypes/prepareTests.js index d0c3b82..789fde0 100644 --- a/test/dataTypes/prepareTests.js +++ b/test/dataTypes/prepareTests.js @@ -20,6 +20,10 @@ const testData = [ { kind: 'utils', data: require('../../ProtoDef/test/utils.json') + }, + { + kind: 'extras', + data: require('../../ProtoDef/test/extras.json') } ] @@ -30,7 +34,7 @@ function arrayToBuffer (arr) { function transformValues (type, values) { return values.map(val => { let value = val.value - if (type.indexOf('buffer') === 0) { + if (type.indexOf('buffer') === 0 || type.endsWith('Buffer')) { value = arrayToBuffer(value) } else if (value) { // we cannot use undefined type in JSON so need to convert it here to pass strictEquals test diff --git a/test_extras.js b/test_extras.js new file mode 100644 index 0000000..72de4c8 --- /dev/null +++ b/test_extras.js @@ -0,0 +1,23 @@ +const ProtoDef = require('./src/index').ProtoDef + +const proto = new ProtoDef() + +// Test basic types are available +console.log('Available types:', Object.keys(proto.types)) + +// Test the restBuffer type +try { + const testBuffer = Buffer.from([1, 2, 3, 4, 5]) + const result = proto.read(testBuffer, 2, 'restBuffer') + console.log('restBuffer result:', result) +} catch (e) { + console.log('restBuffer error:', e.message) +} + +// Test the loop type +try { + const result = proto.read(Buffer.from([1, 2, 3, 0]), 0, ['loop', { type: 'i8', nt: 0 }]) + console.log('loop result:', result) +} catch (e) { + console.log('loop error:', e.message) +} From 76e0cd4ed05ff02ce34f0f5d9ce1d0b3182674f3 Mon Sep 17 00:00:00 2001 From: extremeheat Date: Mon, 18 Aug 2025 06:39:15 +0000 Subject: [PATCH 2/2] update tests --- README.md | 28 ++++++++++++++++++++ examples/extras.js | 64 ++++++++++++++++++++++++++++++++++++++++++++++ test_extras.js | 23 ----------------- 3 files changed, 92 insertions(+), 23 deletions(-) create mode 100644 examples/extras.js delete mode 100644 test_extras.js diff --git a/README.md b/README.md index 21c5736..987bd0f 100644 --- a/README.md +++ b/README.md @@ -45,3 +45,31 @@ See the language independent [ProtoDef](https://github.com/ProtoDef-io/ProtoDef) * [diablo2-protocol](https://github.com/MephisTools/diablo2-protocol) Diablo 2 network protocol * [dofus-protocol](https://github.com/AstrubTools/dofus-protocol) Network protocol for dofus : create client and servers for dofus 1.30 +## Advanced Datatypes + +ProtoDef includes advanced datatypes for handling complex data patterns: + +### loop +Reads a sequence of elements until a terminator or end of buffer: +```javascript +// Read numbers until encountering 0 +['loop', { type: 'i8', nt: 0 }] // [1, 2, 3] from buffer [1, 2, 3, 0, ...] + +// Read until end of buffer +['loop', { type: 'i16', nt: null }] // Read all remaining i16 values +``` + +### restBuffer +Captures all remaining bytes as a Buffer: +```javascript +// Protocol with header and payload +['container', [ + { name: 'header', type: 'u8' }, + { name: 'payload', type: 'restBuffer' } // All remaining bytes +]] +``` + +See [examples/extras_demo.js](examples/extras_demo.js) for comprehensive examples and [ProtoDef/doc/datatypes/extras.md](ProtoDef/doc/datatypes/extras.md) for detailed documentation. + +**Note:** When using the new `loop` and `restBuffer` types, you may need to disable schema validation by creating your ProtoDef instance with `new ProtoDef(false)` until the external validator is updated to recognize these types. + diff --git a/examples/extras.js b/examples/extras.js new file mode 100644 index 0000000..a81cc79 --- /dev/null +++ b/examples/extras.js @@ -0,0 +1,64 @@ +/** + * Quick test and demonstration of the new loop and restBuffer types + */ + +const { ProtoDef } = require('../src/index') + +const proto = new ProtoDef(false) + +console.log('=== Testing loop and restBuffer types ===\n') + +// Test 1: restBuffer - reads all remaining bytes +console.log('1. Testing restBuffer:') +const testBuffer1 = Buffer.from([1, 2, 3, 4, 5]) +const restResult = proto.read(testBuffer1, 2, 'restBuffer') +console.log(' Buffer:', Array.from(testBuffer1)) +console.log(' Reading from offset 2:', restResult) +console.log(' Remaining bytes:', Array.from(restResult.value)) +console.log() + +// Test 2: loop without terminator - reads until EOF +console.log('2. Testing loop (no terminator):') +const loopResult1 = proto.read(Buffer.from([10, 20, 30]), 0, ['loop', { type: 'i8', nt: null }]) +console.log(' Buffer: [10, 20, 30]') +console.log(' Result:', loopResult1) +console.log(' Values:', loopResult1.value) +console.log() + +// Test 3: loop with terminator +console.log('3. Testing loop (with terminator):') +const loopResult2 = proto.read(Buffer.from([1, 2, 3, 0, 99]), 0, ['loop', { type: 'i8', nt: 0 }]) +console.log(' Buffer: [1, 2, 3, 0, 99] (0 is terminator)') +console.log(' Result:', loopResult2) +console.log(' Values:', loopResult2.value, '(99 not included)') +console.log() + +// Test 4: Writing with loop +console.log('4. Testing loop writing:') +const writeBuffer1 = proto.createPacketBuffer(['loop', { type: 'i8', nt: 0 }], [5, 10, 15]) +console.log(' Writing [5, 10, 15] with null terminator:') +console.log(' Buffer:', Array.from(writeBuffer1)) +console.log() + +// Test 5: Complex example with both types +console.log('5. Complex example - message with header and payload:') +proto.addType('message', ['container', [ + { name: 'type', type: 'u8' }, + { name: 'flags', type: 'u8' }, + { name: 'data', type: 'restBuffer' } +]]) + +const complexBuffer = Buffer.concat([ + Buffer.from([42, 0x80]), // type=42, flags=0x80 + Buffer.from('Hello World!') // payload +]) + +const complexResult = proto.parsePacketBuffer('message', complexBuffer) +console.log(' Message parsed:') +console.log(' Type:', complexResult.data.type) +console.log(' Flags:', '0x' + complexResult.data.flags.toString(16)) +console.log(' Data:', complexResult.data.data.toString()) +console.log() + +console.log('✅ All tests completed successfully!') +console.log('\nAvailable types:', Object.keys(proto.types).filter(t => ['loop', 'restBuffer'].includes(t))) diff --git a/test_extras.js b/test_extras.js deleted file mode 100644 index 72de4c8..0000000 --- a/test_extras.js +++ /dev/null @@ -1,23 +0,0 @@ -const ProtoDef = require('./src/index').ProtoDef - -const proto = new ProtoDef() - -// Test basic types are available -console.log('Available types:', Object.keys(proto.types)) - -// Test the restBuffer type -try { - const testBuffer = Buffer.from([1, 2, 3, 4, 5]) - const result = proto.read(testBuffer, 2, 'restBuffer') - console.log('restBuffer result:', result) -} catch (e) { - console.log('restBuffer error:', e.message) -} - -// Test the loop type -try { - const result = proto.read(Buffer.from([1, 2, 3, 0]), 0, ['loop', { type: 'i8', nt: 0 }]) - console.log('loop result:', result) -} catch (e) { - console.log('loop error:', e.message) -}