SPICE-0021 Implement Pkl binary renderer and parser

docs/modules/bindings-specification/pages/binary-encoding.adoc

@@ -36,7 +36,9 @@ For example, value `8` gets encoded as MessagePack `int8` format.
 == Non-primitives
 
 All non-primitive values are encoded as MessagePack arrays.
-The first slot of the array designates the value's type. The remaining slots have fixed meanings depending on the type.
+The first slot of the array designates the value's type.
+The remaining slots have fixed meanings depending on the type.
+Additional slots may be added to types in future Pkl releases. Decoders *must* be designed to defensively discard values beyond the number of known slots for a type or provide meaningful error messages.
 
 The array's length is the number of slots that are filled. For example, xref:{uri-stdlib-List}[List] is encoded as an MessagePack array with two elements.
 
@@ -46,7 +48,7 @@ The array's length is the number of slots that are filled. For example, xref:{ur
 ||code |type |description |type |description |type |description
 
 |link:{uri-stdlib-Typed}[Typed], link:{uri-stdlib-Dynamic}[Dynamic]
-|`0x1`
+|`0x01`
 |link:{uri-messagepack-str}[str]
 |Fully qualified class name
 |link:{uri-messagepack-str}[str]
@@ -55,7 +57,7 @@ The array's length is the number of slots that are filled. For example, xref:{ur
 |Array of <<object-members,object members>>
 
 |link:{uri-stdlib-Map}[Map]
-|`0x2`
+|`0x02`
 |link:{uri-messagepack-map}[map]
 |Map of `<value>` to `<value>`
 |
@@ -64,7 +66,7 @@ The array's length is the number of slots that are filled. For example, xref:{ur
 |
 
 |link:{uri-stdlib-Mapping}[Mapping]
-|`0x3`
+|`0x03`
 |link:{uri-messagepack-map}[map]
 |Map of `<value>` to `<value>`
 |
@@ -73,7 +75,7 @@ The array's length is the number of slots that are filled. For example, xref:{ur
 |
 
 |link:{uri-stdlib-List}[List]
-|`0x4`
+|`0x04`
 |link:{uri-messagepack-array}[array]
 |Array of `<value>`
 |
@@ -82,7 +84,7 @@ The array's length is the number of slots that are filled. For example, xref:{ur
 |
 
 |link:{uri-stdlib-Listing}[Listing]
-|`0x5`
+|`0x05`
 |link:{uri-messagepack-array}[array]
 |Array of `<value>`
 |
@@ -91,7 +93,7 @@ The array's length is the number of slots that are filled. For example, xref:{ur
 |
 
 |link:{uri-stdlib-Set}[Set]
-|`0x6`
+|`0x06`
 |link:{uri-messagepack-array}[array]
 |Array of `<value>`
 |
@@ -100,7 +102,7 @@ The array's length is the number of slots that are filled. For example, xref:{ur
 |
 
 |link:{uri-stdlib-Duration}[Duration]
-|`0x7`
+|`0x07`
 |{uri-messagepack-float}[float64]
 |Duration value
 |link:{uri-messagepack-str}[str]
@@ -109,7 +111,7 @@ The array's length is the number of slots that are filled. For example, xref:{ur
 |
 
 |link:{uri-stdlib-DataSize}[DataSize]
-|`0x8`
+|`0x08`
 |link:{uri-messagepack-float}[float64]
 |Value (float64)
 |link:{uri-messagepack-str}[str]
@@ -118,7 +120,7 @@ The array's length is the number of slots that are filled. For example, xref:{ur
 |
 
 |link:{uri-stdlib-Pair}[Pair]
-|`0x9`
+|`0x09`
 |`<value>`
 |First value
 |`<value>`
@@ -127,7 +129,7 @@ The array's length is the number of slots that are filled. For example, xref:{ur
 |
 
 |link:{uri-stdlib-IntSeq}[IntSeq]
-|`0xA`
+|`0x0A`
 |link:{uri-messagepack-int}[int]
 |Start
 |link:{uri-messagepack-int}[int]
@@ -136,7 +138,7 @@ The array's length is the number of slots that are filled. For example, xref:{ur
 |Step
 
 |link:{uri-stdlib-Regex}[Regex]
-|`0xB`
+|`0x0B`
 |link:{uri-messagepack-str}[str]
 |Regex string representation
 |
@@ -145,25 +147,25 @@ The array's length is the number of slots that are filled. For example, xref:{ur
 |
 
 |link:{uri-stdlib-Class}[Class]
-|`0xC`
-|
-|
-|
-|
+|`0x0C`
+|link:{uri-messagepack-str}[str]
+|Module URI
+|link:{uri-messagepack-str}[str]
+|Qualified name
 |
 |
 
 |link:{uri-stdlib-TypeAlias}[TypeAlias]
-|`0xD`
-|
-|
-|
-|
+|`0x0D`
+|link:{uri-messagepack-str}[str]
+|Module URI
+|link:{uri-messagepack-str}[str]
+|Qualified name
 |
 |
 
 |link:{uri-stdlib-Function}[Function]
-|`0xE`
+|`0x0E`
 |
 |
 |
@@ -172,7 +174,7 @@ The array's length is the number of slots that are filled. For example, xref:{ur
 |
 
 |link:{uri-stdlib-Bytes}[Bytes]
-|`0xF`
+|`0x0F`
 |link:{uri-messagepack-bin}[bin]
 |Binary contents
 |
@@ -212,4 +214,3 @@ Like non-primitive values, object members are encoded as MessagePack arrays, whe
 |`<value>`
 |element value
 |===
-

pkl-commons-test/gradle.lockfile

@@ -32,5 +32,6 @@ org.junit.platform:junit-platform-commons:1.13.4=apiDependenciesMetadata,compile
 org.junit.platform:junit-platform-engine:1.13.4=apiDependenciesMetadata,compileClasspath,implementationDependenciesMetadata,runtimeClasspath,testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath
 org.junit.platform:junit-platform-launcher:1.13.4=testRuntimeClasspath
 org.junit:junit-bom:5.13.4=apiDependenciesMetadata,compileClasspath,implementationDependenciesMetadata,runtimeClasspath,testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath
+org.msgpack:msgpack-core:0.9.8=compileClasspath
 org.opentest4j:opentest4j:1.3.0=apiDependenciesMetadata,compileClasspath,implementationDependenciesMetadata,runtimeClasspath,testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath
 empty=annotationProcessor,compileOnlyDependenciesMetadata,intransitiveDependenciesMetadata,kotlinCompilerPluginClasspath,kotlinNativeCompilerPluginClasspath,kotlinScriptDefExtensions,sourcesJar,testAnnotationProcessor,testApiDependenciesMetadata,testCompileOnlyDependenciesMetadata,testIntransitiveDependenciesMetadata,testKotlinScriptDefExtensions

pkl-commons-test/pkl-commons-test.gradle.kts

@@ -28,6 +28,7 @@ dependencies {
   api(libs.junitParams)
   api(projects.pklCommons) // for convenience
   implementation(libs.assertj)
+  implementation(libs.msgpack)
 }
 
 /**

pkl-server/src/test/kotlin/org/pkl/server/MessagePackDebugRenderer.kt → pkl-commons-test/src/main/kotlin/org/pkl/commons/test/MessagePackDebugRenderer.kt

@@ -1,5 +1,5 @@
 /*
- * Copyright © 2024-2025 Apple Inc. and the Pkl project authors. All rights reserved.
+ * Copyright © 2025 Apple Inc. and the Pkl project authors. All rights reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -13,21 +13,19 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.pkl.server
+package org.pkl.commons.test
 
-import java.lang.IllegalStateException
+import java.util.Base64
 import org.msgpack.core.MessagePack
 import org.msgpack.core.MessageUnpacker
 import org.msgpack.value.ValueType
-import org.pkl.core.util.yaml.YamlEmitter
 
 /** Renders MessagePack structures in YAML. */
 class MessagePackDebugRenderer(bytes: ByteArray) {
   private val unpacker: MessageUnpacker = MessagePack.newDefaultUnpacker(bytes)
   private val currIndent = StringBuilder("")
   private val sb = StringBuilder()
   private val indent = "  "
-  private val yamlEmitter = YamlEmitter.create(sb, "1.2", indent)
 
   private fun incIndent() {
     currIndent.append(indent)
@@ -45,7 +43,7 @@ class MessagePackDebugRenderer(bytes: ByteArray) {
   private fun renderKey() {
     val mf = unpacker.nextFormat
     when (mf.valueType!!) {
-      ValueType.STRING -> yamlEmitter.emit(unpacker.unpackString(), currIndent, true)
+      ValueType.STRING -> emitString(unpacker.unpackString())
       ValueType.MAP,
       ValueType.ARRAY -> {
         sb.append("? ")
@@ -66,14 +64,14 @@ class MessagePackDebugRenderer(bytes: ByteArray) {
       ValueType.FLOAT,
       ValueType.BOOLEAN,
       ValueType.NIL -> sb.append(unpacker.unpackValue().toJson())
-      ValueType.STRING -> yamlEmitter.emit(unpacker.unpackString(), currIndent, false)
+      ValueType.STRING -> emitString(unpacker.unpackString())
       ValueType.ARRAY -> {
         val size = unpacker.unpackArrayHeader()
         if (size == 0) {
           sb.append("[]")
           return
         }
-        for (i in 0 until size) {
+        repeat(size) {
           newline()
           sb.append("- ")
           incIndent()
@@ -87,15 +85,20 @@ class MessagePackDebugRenderer(bytes: ByteArray) {
           sb.append("{}")
           return
         }
-        for (i in 0 until size) {
+        repeat(size) {
           newline()
           renderKey()
           incIndent()
           renderValue()
           decIndent()
         }
       }
-      ValueType.BINARY,
+      ValueType.BINARY -> {
+        // https://yaml.org/type/binary.html
+        sb.append("!!binary ")
+        val size = unpacker.unpackBinaryHeader()
+        emitString(Base64.getEncoder().encodeToString(unpacker.readPayload(size)))
+      }
       ValueType.EXTENSION -> throw IllegalStateException("Unexpected value type ${mf.valueType}")
     }
   }
@@ -104,4 +107,29 @@ class MessagePackDebugRenderer(bytes: ByteArray) {
     renderValue()
     sb.toString().removePrefix("\n")
   }
+
+  // always single quote stirngs for simplicity
+  // adapted from org.pkl.core.util.yaml.YamlEmitter.emitSingleQuotedString
+  fun emitString(str: String) {
+    sb.append('\'')
+
+    val singleQuoteIndex = str.indexOfFirst { it == '\'' }
+    if (singleQuoteIndex == -1) {
+      sb.append(str)
+    } else {
+      var start = 0
+      val length = str.length
+      for (i in singleQuoteIndex..<length) {
+        if (str[i] == '\'') {
+          sb.append(str, start, i).append("''")
+          start = i + 1
+        }
+      }
+      if (start < length) {
+        sb.append(str, start, length)
+      }
+    }
+
+    sb.append('\'')
+  }
 }