Add loop and restBuffer types

README.md

@@ -45,3 +45,31 @@ See the language independent [ProtoDef](https://github.com/ProtoDef-io/ProtoDef)
 * [diablo2-protocol](https://github.com/MephisTools/diablo2-protocol) Diablo 2 network protocol
 * [dofus-protocol](https://github.com/AstrubTools/dofus-protocol) Network protocol for dofus : create client and servers for dofus 1.30
 
+## Advanced Datatypes
+
+ProtoDef includes advanced datatypes for handling complex data patterns:
+
+### loop
+Reads a sequence of elements until a terminator or end of buffer:
+```javascript
+// Read numbers until encountering 0
+['loop', { type: 'i8', nt: 0 }]  // [1, 2, 3] from buffer [1, 2, 3, 0, ...]
+
+// Read until end of buffer  
+['loop', { type: 'i16', nt: null }]  // Read all remaining i16 values
+```
+
+### restBuffer
+Captures all remaining bytes as a Buffer:
+```javascript
+// Protocol with header and payload
+['container', [
+  { name: 'header', type: 'u8' },
+  { name: 'payload', type: 'restBuffer' }  // All remaining bytes
+]]
+```
+
+See [examples/extras_demo.js](examples/extras_demo.js) for comprehensive examples and [ProtoDef/doc/datatypes/extras.md](ProtoDef/doc/datatypes/extras.md) for detailed documentation.
+
+**Note:** When using the new `loop` and `restBuffer` types, you may need to disable schema validation by creating your ProtoDef instance with `new ProtoDef(false)` until the external validator is updated to recognize these types.
+

examples/extras.js

@@ -0,0 +1,64 @@
+/**
+ * Quick test and demonstration of the new loop and restBuffer types
+ */
+
+const { ProtoDef } = require('../src/index')
+
+const proto = new ProtoDef(false)
+
+console.log('=== Testing loop and restBuffer types ===\n')
+
+// Test 1: restBuffer - reads all remaining bytes
+console.log('1. Testing restBuffer:')
+const testBuffer1 = Buffer.from([1, 2, 3, 4, 5])
+const restResult = proto.read(testBuffer1, 2, 'restBuffer')
+console.log('   Buffer:', Array.from(testBuffer1))
+console.log('   Reading from offset 2:', restResult)
+console.log('   Remaining bytes:', Array.from(restResult.value))
+console.log()
+
+// Test 2: loop without terminator - reads until EOF
+console.log('2. Testing loop (no terminator):')
+const loopResult1 = proto.read(Buffer.from([10, 20, 30]), 0, ['loop', { type: 'i8', nt: null }])
+console.log('   Buffer: [10, 20, 30]')
+console.log('   Result:', loopResult1)
+console.log('   Values:', loopResult1.value)
+console.log()
+
+// Test 3: loop with terminator
+console.log('3. Testing loop (with terminator):')
+const loopResult2 = proto.read(Buffer.from([1, 2, 3, 0, 99]), 0, ['loop', { type: 'i8', nt: 0 }])
+console.log('   Buffer: [1, 2, 3, 0, 99] (0 is terminator)')
+console.log('   Result:', loopResult2)
+console.log('   Values:', loopResult2.value, '(99 not included)')
+console.log()
+
+// Test 4: Writing with loop
+console.log('4. Testing loop writing:')
+const writeBuffer1 = proto.createPacketBuffer(['loop', { type: 'i8', nt: 0 }], [5, 10, 15])
+console.log('   Writing [5, 10, 15] with null terminator:')
+console.log('   Buffer:', Array.from(writeBuffer1))
+console.log()
+
+// Test 5: Complex example with both types
+console.log('5. Complex example - message with header and payload:')
+proto.addType('message', ['container', [
+  { name: 'type', type: 'u8' },
+  { name: 'flags', type: 'u8' },
+  { name: 'data', type: 'restBuffer' }
+]])
+
+const complexBuffer = Buffer.concat([
+  Buffer.from([42, 0x80]), // type=42, flags=0x80
+  Buffer.from('Hello World!') // payload
+])
+
+const complexResult = proto.parsePacketBuffer('message', complexBuffer)
+console.log('   Message parsed:')
+console.log('   Type:', complexResult.data.type)
+console.log('   Flags:', '0x' + complexResult.data.flags.toString(16))
+console.log('   Data:', complexResult.data.data.toString())
+console.log()
+
+console.log('✅ All tests completed successfully!')
+console.log('\nAvailable types:', Object.keys(proto.types).filter(t => ['loop', 'restBuffer'].includes(t)))

src/compiler.js

@@ -4,6 +4,7 @@ const utils = require('./datatypes/utils')
 const conditionalDatatypes = require('./datatypes/compiler-conditional')
 const structuresDatatypes = require('./datatypes/compiler-structures')
 const utilsDatatypes = require('./datatypes/compiler-utils')
+const extrasDatatypes = require('./datatypes/compiler-extras')
 
 const { tryCatch } = require('./utils')
 
@@ -270,6 +271,7 @@ class ReadCompiler extends Compiler {
     this.addTypes(conditionalDatatypes.Read)
     this.addTypes(structuresDatatypes.Read)
     this.addTypes(utilsDatatypes.Read)
+    this.addTypes(extrasDatatypes.Read)
 
     // Add default types
     for (const key in numeric) {
@@ -320,6 +322,7 @@ class WriteCompiler extends Compiler {
     this.addTypes(conditionalDatatypes.Write)
     this.addTypes(structuresDatatypes.Write)
     this.addTypes(utilsDatatypes.Write)
+    this.addTypes(extrasDatatypes.Write)
 
     // Add default types
     for (const key in numeric) {
@@ -370,6 +373,7 @@ class SizeOfCompiler extends Compiler {
     this.addTypes(conditionalDatatypes.SizeOf)
     this.addTypes(structuresDatatypes.SizeOf)
     this.addTypes(utilsDatatypes.SizeOf)
+    this.addTypes(extrasDatatypes.SizeOf)
 
     // Add default types
     for (const key in numeric) {

src/datatypes/compiler-extras.js

@@ -0,0 +1,74 @@
+/* eslint-disable multiline-ternary */
+function validateNT (nt) {
+  if (nt !== null && typeof nt !== 'number') throw new Error('Loop terminator must be a number like 0 or null for EOF')
+  return nt !== null
+}
+
+module.exports = {
+  Read: {
+    loop: ['parametrizable', (compiler, struct) => {
+      const nt = struct.nt
+      const hasTerminator = validateNT(nt)
+
+      return compiler.wrapCode(`
+        const results = []
+        let size = 0
+        while (offset !== buffer.length) {
+          ${hasTerminator ? `
+          const typ = ctx.i8(buffer, offset)
+          if (typ.value === ${nt}) {
+            return { value: results, size: size + 1 }
+          }` : ''}
+          const entry = ${compiler.callType(struct.type)}
+          results.push(entry.value)
+          offset += entry.size
+          size += entry.size
+        }
+        return { value: results, size }
+      `)
+    }],
+    restBuffer: ['native', (buffer, offset) => {
+      return {
+        value: buffer.slice(offset),
+        size: buffer.length - offset
+      }
+    }]
+  },
+  Write: {
+    loop: ['parametrizable', (compiler, struct) => {
+      const nt = struct.nt
+      const hasTerminator = validateNT(nt)
+
+      return compiler.wrapCode(`
+        for (const key in value) {
+          offset = ${compiler.callType('value[key]', struct.type)}
+        }
+        ${hasTerminator ? `offset = ctx.i8(${nt}, buffer, offset)` : ''}
+        return offset
+      `)
+    }],
+    restBuffer: ['native', (value, buffer, offset) => {
+      if (!(value instanceof Buffer)) value = Buffer.from(value)
+      value.copy(buffer, offset)
+      return offset + value.length
+    }]
+  },
+  SizeOf: {
+    loop: ['parametrizable', (compiler, struct) => {
+      const nt = struct.nt
+      const hasTerminator = validateNT(nt)
+
+      return compiler.wrapCode(`
+        let size = ${hasTerminator ? '1' : '0'}
+        for (const key in value) {
+          size += ${compiler.callType('value[key]', struct.type)}
+        }
+        return size
+      `)
+    }],
+    restBuffer: ['native', (value) => {
+      if (!(value instanceof Buffer)) value = Buffer.from(value)
+      return value.length
+    }]
+  }
+}

src/datatypes/extras.js

@@ -0,0 +1,111 @@
+const { PartialReadError } = require('../utils')
+
+module.exports = {
+  loop: [readLoop, writeLoop, sizeOfLoop, require('../../ProtoDef/schemas/extras.json').loop],
+  restBuffer: [readRestBuffer, writeRestBuffer, sizeOfRestBuffer, require('../../ProtoDef/schemas/extras.json').restBuffer]
+}
+
+function readLoop (buffer, offset, typeArgs, rootNode) {
+  if (!typeArgs) {
+    throw new Error('typeArgs is required for loop type')
+  }
+
+  const results = []
+  const startOffset = offset
+  const nt = typeArgs.nt
+  const hasTerminator = nt !== null && typeof nt === 'number'
+
+  while (offset < buffer.length) {
+    // Check for terminator if specified
+    if (hasTerminator) {
+      if (offset >= buffer.length) break
+      const terminatorValue = buffer.readInt8(offset)
+      if (terminatorValue === nt) {
+        // Found terminator, consume it and return
+        return {
+          value: results,
+          size: offset - startOffset + 1
+        }
+      }
+    }
+
+    // Read the next element
+    try {
+      const entry = this.read(buffer, offset, typeArgs.type, rootNode)
+      results.push(entry.value)
+      offset += entry.size
+    } catch (error) {
+      if (error instanceof PartialReadError) {
+        break
+      }
+      throw error
+    }
+  }
+
+  return {
+    value: results,
+    size: offset - startOffset
+  }
+}
+
+function writeLoop (value, buffer, offset, typeArgs, rootNode) {
+  if (!typeArgs) {
+    throw new Error('typeArgs is required for loop type')
+  }
+
+  const nt = typeArgs.nt
+  const hasTerminator = nt !== null && typeof nt === 'number'
+
+  // Write each element in the array
+  for (const item of value) {
+    offset = this.write(item, buffer, offset, typeArgs.type, rootNode)
+  }
+
+  // Write terminator if specified
+  if (hasTerminator) {
+    buffer.writeInt8(nt, offset)
+    offset++
+  }
+
+  return offset
+}
+
+function sizeOfLoop (value, typeArgs, rootNode) {
+  if (!typeArgs) {
+    throw new Error('typeArgs is required for loop type')
+  }
+
+  const nt = typeArgs.nt
+  const hasTerminator = nt !== null && typeof nt === 'number'
+  let size = hasTerminator ? 1 : 0 // 1 byte for terminator if present
+
+  // Calculate size of all elements
+  for (const item of value) {
+    size += this.sizeOf(item, typeArgs.type, rootNode)
+  }
+
+  return size
+}
+
+function readRestBuffer (buffer, offset) {
+  const remainingBuffer = buffer.slice(offset)
+  return {
+    value: remainingBuffer,
+    size: remainingBuffer.length
+  }
+}
+
+function writeRestBuffer (value, buffer, offset) {
+  if (!(value instanceof Buffer)) {
+    value = Buffer.from(value)
+  }
+  value.copy(buffer, offset)
+  return offset + value.length
+}
+
+function sizeOfRestBuffer (value) {
+  if (!(value instanceof Buffer)) {
+    value = Buffer.from(value)
+  }
+  return value.length
+}

src/protodef.js

@@ -52,6 +52,7 @@ class ProtoDef {
     this.addTypes(require('./datatypes/utils'))
     this.addTypes(require('./datatypes/structures'))
     this.addTypes(require('./datatypes/conditional'))
+    this.addTypes(require('./datatypes/extras'))
   }
 
   addProtocol (protocolData, path) {

test/dataTypes/datatypes.js

@@ -55,7 +55,7 @@ function testType (type, values) {
       })
     } else { testValue(type, value.value, value.buffer) }
   })
-  if (type !== 'void') {
+  if (type !== 'void' && type !== 'restBuffer' && !type.startsWith('loop_')) {
     it('reads 0 bytes and throw a PartialReadError', () => {
       try {
         proto.parsePacketBuffer(type, Buffer.alloc(0))

test/dataTypes/prepareTests.js

@@ -20,6 +20,10 @@ const testData = [
   {
     kind: 'utils',
     data: require('../../ProtoDef/test/utils.json')
+  },
+  {
+    kind: 'extras',
+    data: require('../../ProtoDef/test/extras.json')
   }
 ]
 
@@ -30,7 +34,7 @@ function arrayToBuffer (arr) {
 function transformValues (type, values) {
   return values.map(val => {
     let value = val.value
-    if (type.indexOf('buffer') === 0) {
+    if (type.indexOf('buffer') === 0 || type.endsWith('Buffer')) {
       value = arrayToBuffer(value)
     } else if (value) {
       // we cannot use undefined type in JSON so need to convert it here to pass strictEquals test