Merge pull request #15 from sparsetech/feat/language-extensions

Introduce language extension for multi-line inline tables

Author: tindzk

README.md

@@ -133,6 +133,20 @@ scalaDeps = [
 ]
 ```
 
+#### Language Extensions
+toml-scala supports the following language extensions which are disabled by default:
+
+* [New lines and trailing commas in inline tables](https://github.com/toml-lang/toml/issues/516)
+
+To enable them, pass in a set of extensions to the `parse()` or `parseAs()` function as a second argument:
+
+```scala
+toml.Toml.parse("""key = {
+  a = 23,
+  b = 42,
+}""", Set(toml.Extension.MultiLineInlineTables))
+```
+
 ## Links
 * [ScalaDoc](https://www.javadoc.io/doc/tech.sparse/toml-scala_2.12/)
 

jvm/src/main/scala/toml/PlatformRules.scala

@@ -4,7 +4,7 @@ import java.time._
 
 import scala.meta.internal.fastparse.all._
 
-trait PlatformRules { this: Rules.type =>
+trait PlatformRules { this: Rules =>
   private val TenPowers =
     List(1, 10, 100, 1000, 10000, 100000, 1000000, 10000000, 100000000)
 
@@ -39,4 +39,4 @@ trait PlatformRules { this: Rules.type =>
   }
 
   val date = P(offsetDateTime | localDateTime | localDate | localTime)
-}
+}

shared/src/main/scala/toml/Rules.scala

@@ -9,26 +9,31 @@ private[toml] case class NamedFunction[T, V](f: T => V, name: String)
   override def toString: String = name
 }
 
-object Rules extends PlatformRules {
+sealed trait Extension
+object Extension {
+  case object MultiLineInlineTables extends Extension
+}
+
+case object Rules extends toml.Rules(Set())
+
+class Rules(extensions: Set[Extension]) extends PlatformRules {
   import Constants._
+  import Extension._
 
   val UntilNewline = NamedFunction(!CrLf.contains(_: Char), "UntilNewline")
 
   val newLine    = P(StringIn(CrLf, Lf))
   val charsChunk = P(CharsWhile(UntilNewline))
   val comment    = P("#" ~ charsChunk.? ~ &(newLine | End))
+  val whitespace = P(CharIn(WhitespaceChars.toList))
 
-  val whitespace  = P(CharIn(WhitespaceChars.toList))
-  val whitespaces = P(whitespace.rep(1))
+  val skip   = P(NoCut(NoTrace((whitespace | comment | newLine).rep)))
+  val skipWs = P(NoCut(NoTrace(whitespace.rep)))
 
-  val skip = P(NoCut(NoTrace((whitespaces | comment | newLine).rep)))
-
-  val letter  = P(CharIn(LettersRange))
-  val letters = P(letter.rep(1))
-  val digit   = P(CharIn(NumbersRange))
-  val digits  = P(digit.rep(1))
-
-  val skipSpaces = P(CharsWhile(_.isWhitespace).?)
+  val letter = P(CharIn(LettersRange))
+  val digit  = P(CharIn(NumbersRange))
+  val digits = P(digit.rep(1))
+  val dash   = P(CharIn(Dashes.toList))
 
   val StringChars = NamedFunction(!"\"\\".contains(_: Char), "StringChars")
   val strChars    = P(CharsWhile(StringChars))
@@ -52,14 +57,14 @@ object Rules extends PlatformRules {
   val multiLineBasicStr: Parser[Value.Str] =
     P(
       MultiLineDoubleQuote ~/
-      skipSpaces ~
+      newLine.? ~
       (!MultiLineDoubleQuote ~ AnyChar).rep.! ~
       MultiLineDoubleQuote
     ).map(str => Value.Str(Unescape.unescapeJavaString(str)))
   val multiLineLiteralStr: Parser[Value.Str] =
     P(
       MultiLineSingleQuote ~/
-      skipSpaces ~
+      newLine.? ~
       (!MultiLineSingleQuote ~ AnyChar).rep.! ~
       MultiLineSingleQuote
     ).map(Value.Str)
@@ -98,44 +103,42 @@ object Rules extends PlatformRules {
   val `false` = P("false").map(_ => Value.Bool(false))
   val boolean = P(`true` | `false`)
 
-  val dashes = P(CharIn(Dashes.toList))
-  val bareKey = P((letters | digits | dashes).rep(min = 1)).!
+  val bareKey = P((letter | digit | dash).rep(min = 1)).!
   val validKey: Parser[String] =
     P(NoCut(basicStr.map(_.value)) | NoCut(literalStr.map(_.value)) | bareKey)
   val pair: Parser[(String, Value)] =
-    P(validKey ~ whitespaces.? ~ "=" ~ whitespaces.? ~ elem)
+    P(validKey ~ skipWs ~ "=" ~ skipWs ~ elem)
   val array: Parser[Value.Arr] =
     P("[" ~ skip ~ elem.rep(sep = "," ~ skip) ~ ",".? ~ skip ~ "]")
       .map(l => Value.Arr(l.toList))
   val inlineTable: Parser[Value.Tbl] =
-    P("{" ~ skip ~ pair.rep(sep = "," ~ skip) ~ ",".? ~ skip ~ "}")
-      .map(p => Value.Tbl(p.toMap))
+    (if (extensions.contains(MultiLineInlineTables))
+      P("{" ~ skip ~ pair.rep(sep = "," ~ skip) ~ ",".? ~ skip ~ "}")
+     else
+      P("{" ~ skipWs ~ pair.rep(sep = "," ~ skipWs) ~ skipWs ~ "}")
+    ).map(p => Value.Tbl(p.toMap))
 
   val tableIds: Parser[Seq[String]] =
-    P(validKey.rep(min = 1, sep = whitespaces.? ~ "." ~ whitespaces.?).map(_.toSeq))
+    P(validKey.rep(min = 1, sep = skipWs ~ "." ~ skipWs).map(_.toSeq))
   val tableDef: Parser[Seq[String]] =
-    P("[" ~ whitespaces.? ~ tableIds ~ whitespaces.? ~ "]")
+    P("[" ~ skipWs ~ tableIds ~ skipWs ~ "]")
   val tableArrayDef: Parser[Seq[String]] =
-    P("[[" ~ whitespaces.? ~ tableIds ~ whitespaces.? ~ "]]")
+    P("[[" ~ skipWs ~ tableIds ~ skipWs ~ "]]")
 
   val pairNode: Parser[Node.Pair] = pair.map { case (k, v) => Node.Pair(k, v) }
   val table: Parser[Node.NamedTable] =
-    P(skip ~ tableDef ~ skip ~ pair.rep(sep = skip)).map { case (a, b) =>
+    P(tableDef ~ skip ~ pair.rep(sep = skip)).map { case (a, b) =>
       Node.NamedTable(a.toList, b.toList)
     }
   val tableArray: Parser[Node.NamedArray] =
-    P(skip ~ tableArrayDef ~ skip ~ pair.rep(sep = skip)).map { case (a, b) =>
+    P(tableArrayDef ~ skip ~ pair.rep(sep = skip)).map { case (a, b) =>
       Node.NamedArray(a.toList, b.toList)
     }
 
-  lazy val elem: Parser[Value] = P {
-    skip ~
-    (date | string | boolean | double | integer | array | inlineTable) ~
-    skip
-  }
-
-  lazy val node: Parser[Node] = P(skip ~ (pairNode | table | tableArray))
+  lazy val elem: Parser[Value] =
+    P(date | string | boolean | double | integer | array | inlineTable)
 
-  val root: Parser[Root] = P(node.rep(sep = skip) ~ skip ~ End)
+  val node: Parser[Node] = P(pairNode | table | tableArray)
+  val root: Parser[Root] = P(skip ~ node.rep(sep = skip) ~ skip ~ End)
     .map(nodes => Root(nodes.toList))
 }

shared/src/main/scala/toml/Toml.scala

@@ -5,8 +5,8 @@ import shapeless._
 import scala.meta.internal.fastparse.core.Parsed._
 
 object Toml {
-  def parse(toml: String): Either[Parse.Error, Value.Tbl] =
-    Rules.root.parse(toml) match {
+  def parse(toml: String, extensions: Set[Extension] = Set()): Either[Parse.Error, Value.Tbl] =
+    new Rules(extensions).root.parse(toml) match {
       case Success(v, _)    => Embed.root(v)
       case f: Failure[_, _] => Left(List() -> f.msg)
     }
@@ -24,23 +24,37 @@ object Toml {
       codec(table, d, 0).right.map(generic.from)
     }
 
-    def apply[D <: HList, R <: HList](toml: String)(implicit
+    def apply[D <: HList, R <: HList](
+      toml      : String,
+      extensions: Set[Extension]
+    )(implicit
       generic      : LabelledGeneric.Aux[A, R],
       defaults     : Default.AsRecord.Aux[A, D],
       defaultMapper: util.RecordToMap[D],
       codec        : Codec[R]
     ): Either[Parse.Error, A] = {
       val d = defaultMapper(defaults())
-      parse(toml).right.flatMap(codec(_, d, 0).right.map(generic.from))
+      parse(toml, extensions)
+        .right
+        .flatMap(codec(_, d, 0).right.map(generic.from))
     }
+
+    def apply[D <: HList, R <: HList](toml: String)(
+      implicit
+        generic      : LabelledGeneric.Aux[A, R],
+        defaults     : Default.AsRecord.Aux[A, D],
+        defaultMapper: util.RecordToMap[D],
+        codec        : Codec[R]
+    ): Either[Parse.Error, A] = apply(toml, Set())
   }
 
   class CodecHelperValue[A] {
     def apply(value: Value)(implicit codec: Codec[A]): Either[Parse.Error, A] =
       codec(value, Map(), 0)
 
-    def apply(toml: String)(implicit codec: Codec[A]): Either[Parse.Error, A] =
-      parse(toml).right.flatMap(codec(_, Map(), 0))
+    def apply(toml: String, extensions: Set[Extension] = Set())
+             (implicit codec: Codec[A]): Either[Parse.Error, A] =
+      parse(toml, extensions).right.flatMap(codec(_, Map(), 0))
   }
 
   def parseAs     [T]: CodecHelperGeneric[T] = new CodecHelperGeneric[T]

shared/src/test/scala/toml/CodecSpec.scala

@@ -62,7 +62,7 @@ class CodecSpec extends FunSuite {
     case class Pair(a: Int)
 
     val pair = """a = 1"""
-    assert(Toml.parseAs[Pair](pair) == Right(Pair(1)))
+    assert(Toml.parseAs[Pair](pair, Set()) == Right(Pair(1)))
 
     case class Pairs(a: Int, b: Int)
     val pairs =

shared/src/test/scala/toml/GeneratedSpec.scala

@@ -63,10 +63,10 @@ class GeneratedSpec extends PropSpec with PropertyChecks with Matchers {
     }
   }
 
-  property("Parse pairs (with `node` parser)") {
+  property("Parse pairs (with `root` parser)") {
     import Generators.Tables._
     forAll(pairGen) { s: String =>
-      shouldBeSuccess(Rules.node.parse(s))
+      shouldBeSuccess(Rules.root.parse(s))
     }
   }
 
@@ -80,14 +80,14 @@ class GeneratedSpec extends PropSpec with PropertyChecks with Matchers {
   property("Parse tables") {
     import Generators.Tables._
     forAll(tableGen) { s: String =>
-      shouldBeSuccess[Node.NamedTable](Rules.table.parse(s))
+      shouldBeSuccess[Node.NamedTable]((Rules.skip ~ Rules.table).parse(s))
     }
   }
 
-  property("Parse tables (with `node` parser)") {
+  property("Parse tables (with `root` parser)") {
     import Generators.Tables._
     forAll(tableGen) { s: String =>
-      shouldBeSuccess(Rules.node.parse(s))
+      shouldBeSuccess(Rules.root.parse(s))
     }
   }
 }

shared/src/test/scala/toml/ParseSpec.scala

@@ -6,6 +6,27 @@ import Node._
 import org.scalatest.FunSuite
 
 class ParseSpec extends FunSuite {
+  test("Parse strings") {
+    val toml =
+      """
+        |lines = '''
+        |
+        |The first newline is
+        |trimmed in raw strings.
+        |  All other whitespace
+        |  is preserved.
+        |'''
+        |""".stripMargin
+
+    val result = Toml.parse(toml)
+    assert(result == Right(Tbl(Map("lines" -> Value.Str(
+      "\n" +
+      "The first newline is\n" +
+      "trimmed in raw strings.\n" +
+      "  All other whitespace\n" +
+      "  is preserved.\n")))))
+  }
+
   test("Redefine value on root level") {
     val toml =
       """a = 23
@@ -34,4 +55,14 @@ class ParseSpec extends FunSuite {
     val result = Toml.parse(toml)
     assert(result == Left((List("a", "b"), "Cannot redefine value")))
   }
+
+  test("Extension: Parse inline tables with trailing comma") {
+    val result = Toml.parse("""key = {
+      a = 23,
+      b = 42,
+    }""", Set(Extension.MultiLineInlineTables))
+
+    assert(result == Right(Tbl(
+      Map("key" -> Tbl(Map("a" -> Num(23), "b" -> Num(42)))))))
+  }
 }

shared/src/test/scala/toml/RulesSpec.scala

@@ -63,15 +63,51 @@ class RulesSpec extends FunSuite with Matchers {
     testSuccess(example)
   }
 
-  test("Parse inline tables") {
+  test("Parse inline tables (1)") {
+    val example =
+      """
+        |a = { name = "value", name2 = "value2" }
+        |b = { name = "value" }
+      """.stripMargin
+
+    testSuccess(example)
+  }
+
+  test("Extension: Parse inline tables with new line") {
+    // See https://github.com/toml-lang/toml/issues/516
+    val example =
+      """
+        |a = { name = "value", name2 = "value2" }
+        |b = { name = "value"
+        |    }
+      """.stripMargin
+
+    testFailure(example)
+    testSuccess(example, new Rules(Set(Extension.MultiLineInlineTables)))
+  }
+
+  test("Extension: Parse inline tables with trailing comma") {
+    val example =
+      """
+        |a = { name = "value", name2 = "value2" }
+        |b = { name = "value", }
+      """.stripMargin
+
+    testFailure(example)
+    testSuccess(example, new Rules(Set(Extension.MultiLineInlineTables)))
+  }
+
+  test("Extension: Parse inline tables with trailing comma and comment") {
     val example =
       """
         |a = { name = "value", name2 = "value2" }
         |b = { name = "value",
         |      # Trailing comma
         |    }
       """.stripMargin
-    testSuccess(example)
+
+    testFailure(example)
+    testSuccess(example, new Rules(Set(Extension.MultiLineInlineTables)))
   }
 
   test("Parse complex table keys") {

shared/src/test/scala/toml/TestHelpers.scala

@@ -8,14 +8,14 @@ import org.scalatest.Matchers
 object TestHelpers {
   import Matchers._
 
-  def testSuccess(example: String): Root =
-    Rules.root.parse(example) match {
+  def testSuccess(example: String, rules: Rules = Rules): Root =
+    rules.root.parse(example) match {
       case Success(v, _)    => v
       case f: Failure[_, _] => fail(s"Failed to parse `$example`: ${f.msg}")
     }
 
-  def testFailure(example: String): Unit =
-    Rules.root.parse(example) match {
+  def testFailure(example: String, rules: Rules = Rules): Unit =
+    rules.root.parse(example) match {
       case Success(_, _) => fail(s"Did not fail: $example")
       case _: Failure[_, _] =>
     }
@@ -29,4 +29,4 @@ object TestHelpers {
     case s: Success[T, _, _] => fail(s"$r is not a Failure.")
     case f: Failure[_, _]    =>
   }
-}
+}