Skip to content

Commit e1c295c

Browse files
authored
unpack char star with lengthOf (#1)
1 parent c0d86e0 commit e1c295c

4 files changed

Lines changed: 285 additions & 10 deletions

File tree

examples/16-cstring-vs-char-star.ts

Lines changed: 44 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -23,16 +23,56 @@ const cstringPacked = CStringStruct.pack(cstringData)
2323
console.log(" Input:", cstringData)
2424
console.log(" Packed size:", cstringPacked.byteLength, "bytes")
2525
console.log(" Automatically adds null terminator")
26+
console.log(" Note: cstring unpacking not yet implemented (returns pointer)")
2627

27-
console.log("\n2. char* (length-prefixed):")
28+
console.log("\n2. char* with lengthOf (automatic unpacking):")
2829
const charStarData = {
2930
id: 2,
30-
nameData: "world",
31+
nameData: "world 🌍",
3132
}
3233

3334
const charStarPacked = CharStarStruct.pack(charStarData)
3435
console.log(" Input:", charStarData)
3536
console.log(" Packed size:", charStarPacked.byteLength, "bytes")
36-
console.log(" Length stored separately in nameLength field")
37+
console.log(" Length stored in nameLength field:", Buffer.byteLength("world 🌍"), "bytes")
3738

38-
console.log("\n✓ Use cstring for null-terminated, char* for length-prefixed strings!")
39+
const charStarUnpacked = CharStarStruct.unpack(charStarPacked)
40+
console.log("\n Unpacked:")
41+
console.log(" id:", charStarUnpacked.id)
42+
console.log(" nameData:", charStarUnpacked.nameData)
43+
console.log(" nameLength:", charStarUnpacked.nameLength)
44+
console.log(" ✓ char* with lengthOf automatically unpacks to string!")
45+
46+
console.log("\n3. char* without lengthOf (returns pointer):")
47+
const PointerStruct = defineStruct([
48+
["id", "u32"],
49+
["data", "char*"],
50+
] as const)
51+
52+
const pointerData = {
53+
id: 3,
54+
data: "test",
55+
}
56+
57+
const pointerPacked = PointerStruct.pack(pointerData)
58+
const pointerUnpacked = PointerStruct.unpack(pointerPacked)
59+
console.log(" Input:", pointerData)
60+
console.log(" Unpacked data (pointer):", pointerUnpacked.data)
61+
console.log(" Type:", typeof pointerUnpacked.data)
62+
console.log(" Note: Without lengthOf, char* returns the raw pointer value")
63+
64+
console.log("\n4. Unicode and emoji support:")
65+
const UnicodeStruct = defineStruct([
66+
["text", "char*"],
67+
["textLength", "u32", { lengthOf: "text" }],
68+
] as const)
69+
70+
const testStrings = ["Hello", "Café", "Hello 🌍🎉✨", "👨‍👩‍👧‍👦", "日本語"]
71+
72+
console.log(" Testing various unicode strings:")
73+
for (const str of testStrings) {
74+
const packed = UnicodeStruct.pack({ text: str })
75+
const unpacked = UnicodeStruct.unpack(packed)
76+
const match = unpacked.text === str
77+
console.log(` "${str}" -> ${match ? "✓" : "✗"} (${unpacked.textLength} bytes)`)
78+
}

src/structs_ffi.ts

Lines changed: 27 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -260,6 +260,7 @@ export function packObjectArray(val: (PointyObject | null)[]) {
260260
}
261261

262262
const encoder = new TextEncoder()
263+
const decoder = new TextDecoder()
263264

264265
// Define Struct
265266
export function defineStruct<const Fields extends readonly StructField[], const Opts extends StructDefOptions = {}>(
@@ -272,7 +273,7 @@ export function defineStruct<const Fields extends readonly StructField[], const
272273
const lengthOfFields: Record<string, StructLayoutField> = {}
273274
const lengthOfRequested: {
274275
requester: StructLayoutField
275-
def: EnumDef<any> | PrimitiveType
276+
def: EnumDef<any> | PrimitiveType | "char*"
276277
}[] = []
277278
const arrayFieldsMetadata: Record<string, ArrayFieldMetadata> = {}
278279

@@ -314,11 +315,12 @@ export function defineStruct<const Fields extends readonly StructField[], const
314315
const bufPtr = val ? ptr(encoder.encode(val)) : null // No null terminator
315316
pointerPacker(view, off, bufPtr)
316317
}
318+
// Initial unpack returns pointer; will be replaced if lengthOf field exists
317319
unpack = (view: DataView, off: number) => {
318-
// TODO: Unpack char* requires length info, typically from another field
319320
const ptrVal = pointerUnpacker(view, off)
320-
return ptrVal // Returning pointer for now
321+
return ptrVal
321322
}
323+
needsLengthOf = true // Mark for later resolution
322324
// Enum
323325
} else if (isEnum(typeOrStruct)) {
324326
const base = typeOrStruct.type
@@ -553,8 +555,10 @@ export function defineStruct<const Fields extends readonly StructField[], const
553555
lengthOfFields[options.lengthOf] = layoutField
554556
}
555557
if (needsLengthOf) {
556-
if (!lengthOfDef) fatalError(`Internal error: needsLengthOf=true but lengthOfDef is null for ${name}`)
557-
lengthOfRequested.push({ requester: layoutField, def: lengthOfDef })
558+
// For char*, pass "char*" as the def; for arrays, pass the actual def
559+
const def = typeof typeOrStruct === "string" && typeOrStruct === "char*" ? "char*" : lengthOfDef
560+
if (!def) fatalError(`Internal error: needsLengthOf=true but def is null for ${name}`)
561+
lengthOfRequested.push({ requester: layoutField, def })
558562
}
559563

560564
offset += size
@@ -564,11 +568,28 @@ export function defineStruct<const Fields extends readonly StructField[], const
564568
// Resolve lengthOf fields
565569
for (const { requester, def } of lengthOfRequested) {
566570
const lengthOfField = lengthOfFields[requester.name]
571+
567572
if (!lengthOfField) {
573+
if (def === "char*") {
574+
continue
575+
}
568576
throw new Error(`lengthOf field not found for array field ${requester.name}`)
569577
}
570578

571-
if (isPrimitiveType(def)) {
579+
if (def === "char*") {
580+
requester.unpack = (view, off) => {
581+
const ptrAddress = pointerUnpacker(view, off)
582+
const length = lengthOfField.unpack(view, lengthOfField.offset)
583+
584+
if (ptrAddress === 0 || length === 0) {
585+
return null
586+
}
587+
588+
const byteLength = typeof length === "bigint" ? Number(length) : length
589+
const buffer = toArrayBuffer(ptrAddress, 0, byteLength)
590+
return decoder.decode(buffer)
591+
}
592+
} else if (isPrimitiveType(def)) {
572593
const elemSize = typeSizes[def]
573594
const { unpack: primitiveUnpack } = primitivePackers(def)
574595

Lines changed: 198 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,198 @@
1+
import { expect, describe, it } from "bun:test"
2+
import { defineStruct } from "../structs_ffi"
3+
4+
describe("char* automatic unpacking", () => {
5+
it("should automatically unpack char* to string when lengthOf field exists", () => {
6+
const StringStruct = defineStruct([
7+
["data", "char*"],
8+
["length", "u64", { lengthOf: "data" }],
9+
] as const)
10+
11+
const testString = "Hello, World!"
12+
const packed = StringStruct.pack({ data: testString })
13+
const unpacked = StringStruct.unpack(packed)
14+
15+
expect(typeof unpacked.data).toBe("string")
16+
expect(unpacked.data).toBe(testString)
17+
expect(unpacked.length).toBe(BigInt(Buffer.byteLength(testString)))
18+
})
19+
20+
it("should handle emoji strings correctly", () => {
21+
const StringStruct = defineStruct([
22+
["data", "char*"],
23+
["length", "u32", { lengthOf: "data" }],
24+
] as const)
25+
26+
const testString = "Hello 🌍🎉✨"
27+
const packed = StringStruct.pack({ data: testString })
28+
const unpacked = StringStruct.unpack(packed)
29+
30+
expect(unpacked.data).toBe(testString)
31+
})
32+
33+
it("should handle complex unicode grapheme clusters", () => {
34+
const StringStruct = defineStruct([
35+
["data", "char*"],
36+
["length", "u64", { lengthOf: "data" }],
37+
] as const)
38+
39+
const testString = "👨‍👩‍👧‍👦🏳️‍🌈🇺🇸"
40+
const packed = StringStruct.pack({ data: testString })
41+
const unpacked = StringStruct.unpack(packed)
42+
43+
expect(unpacked.data).toBe(testString)
44+
})
45+
46+
it("should handle null char* pointers", () => {
47+
const StringStruct = defineStruct([
48+
["data", "char*", { optional: true }],
49+
["length", "u64", { lengthOf: "data" }],
50+
] as const)
51+
52+
const packed = StringStruct.pack({ data: null })
53+
const unpacked = StringStruct.unpack(packed)
54+
55+
expect(unpacked.data).toBeNull()
56+
expect(unpacked.length).toBe(0n)
57+
})
58+
59+
it("should handle empty strings", () => {
60+
const StringStruct = defineStruct([
61+
["data", "char*"],
62+
["length", "u64", { lengthOf: "data" }],
63+
] as const)
64+
65+
const packed = StringStruct.pack({ data: "" })
66+
const unpacked = StringStruct.unpack(packed)
67+
68+
expect(unpacked.data).toBeNull()
69+
expect(unpacked.length).toBe(0n)
70+
})
71+
72+
it("should handle multiple char* fields in one struct", () => {
73+
const MultiStringStruct = defineStruct([
74+
["field1", "char*"],
75+
["length1", "u32", { lengthOf: "field1" }],
76+
["field2", "char*"],
77+
["length2", "u32", { lengthOf: "field2" }],
78+
["field3", "char*"],
79+
["length3", "u32", { lengthOf: "field3" }],
80+
] as const)
81+
82+
const packed = MultiStringStruct.pack({
83+
field1: "ASCII text",
84+
field2: "Emoji 🎉",
85+
field3: "Combined é",
86+
})
87+
const unpacked = MultiStringStruct.unpack(packed)
88+
89+
expect(unpacked.field1).toBe("ASCII text")
90+
expect(unpacked.field2).toBe("Emoji 🎉")
91+
expect(unpacked.field3).toBe("Combined é")
92+
})
93+
94+
it("should handle char* in nested structs", () => {
95+
const InnerStruct = defineStruct([
96+
["content", "char*"],
97+
["contentLength", "u64", { lengthOf: "content" }],
98+
] as const)
99+
100+
const OuterStruct = defineStruct([
101+
["id", "u32"],
102+
["message", InnerStruct],
103+
["timestamp", "u64"],
104+
] as const)
105+
106+
const packed = OuterStruct.pack({
107+
id: 42,
108+
message: {
109+
content: "Hello! 👋 How are you? 😊",
110+
},
111+
timestamp: 1234567890n,
112+
})
113+
const unpacked = OuterStruct.unpack(packed)
114+
115+
expect(unpacked.message.content).toBe("Hello! 👋 How are you? 😊")
116+
expect(unpacked.id).toBe(42)
117+
expect(unpacked.timestamp).toBe(1234567890n)
118+
})
119+
120+
it("should handle char* with various length field types", () => {
121+
const testString = "Test string 🎉"
122+
123+
// Test with u32
124+
const Struct32 = defineStruct([
125+
["data", "char*"],
126+
["length", "u32", { lengthOf: "data" }],
127+
] as const)
128+
129+
const packed32 = Struct32.pack({ data: testString })
130+
const unpacked32 = Struct32.unpack(packed32)
131+
expect(unpacked32.data).toBe(testString)
132+
133+
// Test with u64
134+
const Struct64 = defineStruct([
135+
["data", "char*"],
136+
["length", "u64", { lengthOf: "data" }],
137+
] as const)
138+
139+
const packed64 = Struct64.pack({ data: testString })
140+
const unpacked64 = Struct64.unpack(packed64)
141+
expect(unpacked64.data).toBe(testString)
142+
143+
// Test with u16
144+
const Struct16 = defineStruct([
145+
["data", "char*"],
146+
["length", "u16", { lengthOf: "data" }],
147+
] as const)
148+
149+
const packed16 = Struct16.pack({ data: testString })
150+
const unpacked16 = Struct16.unpack(packed16)
151+
expect(unpacked16.data).toBe(testString)
152+
})
153+
154+
it("should return pointer for char* without lengthOf field", () => {
155+
const PointerStruct = defineStruct([
156+
["data", "char*"],
157+
["someOtherField", "u32"],
158+
] as const)
159+
160+
const packed = PointerStruct.pack({
161+
data: "test",
162+
someOtherField: 42,
163+
})
164+
const unpacked = PointerStruct.unpack(packed)
165+
166+
// Without lengthOf, should return the pointer value (number)
167+
expect(typeof unpacked.data).toBe("number")
168+
expect(unpacked.data).toBeGreaterThan(0)
169+
expect(unpacked.someOtherField).toBe(42)
170+
})
171+
172+
it("should handle very long strings", () => {
173+
const StringStruct = defineStruct([
174+
["data", "char*"],
175+
["length", "u32", { lengthOf: "data" }],
176+
] as const)
177+
178+
const testString = "a".repeat(10000) + "🎉".repeat(1000)
179+
const packed = StringStruct.pack({ data: testString })
180+
const unpacked = StringStruct.unpack(packed)
181+
182+
expect(unpacked.data).toBe(testString)
183+
expect(unpacked.length).toBe(Buffer.byteLength(testString))
184+
})
185+
186+
it("should handle strings with mixed ascii and unicode", () => {
187+
const StringStruct = defineStruct([
188+
["data", "char*"],
189+
["length", "u64", { lengthOf: "data" }],
190+
] as const)
191+
192+
const testString = "Hello Café 日本語 🌍 Привет"
193+
const packed = StringStruct.pack({ data: testString })
194+
const unpacked = StringStruct.unpack(packed)
195+
196+
expect(unpacked.data).toBe(testString)
197+
})
198+
})

0 commit comments

Comments
 (0)