Verbatim parsing

cmd/astro-wasm/astro-wasm.go

@@ -6,7 +6,7 @@
 	"fmt"
 	"strings"
 	"sync"
 	"syscall/js"
 	"unicode"
 
 	"github.com/norunners/vert"
@@ -143,21 +143,27 @@
 		experimentalScriptOrder = true
 	}
 
+	experimentalExactParsingThingy := false
+	if jsBool(options.Get("experimentalExactParsingThingy")) {
+		experimentalExactParsingThingy = true
+	}
+
 	return transform.TransformOptions{
-		Filename:                filename,
-		NormalizedFilename:      normalizedFilename,
-		InternalURL:             internalURL,
-		SourceMap:               sourcemap,
-		AstroGlobalArgs:         astroGlobalArgs,
-		Compact:                 compact,
-		ResolvePath:             resolvePathFn,
-		PreprocessStyle:         preprocessStyle,
-		ResultScopedSlot:        scopedSlot,
-		ScopedStyleStrategy:     scopedStyleStrategy,
-		TransitionsAnimationURL: transitionsAnimationURL,
-		AnnotateSourceFile:      annotateSourceFile,
-		RenderScript:            renderScript,
-		ExperimentalScriptOrder: experimentalScriptOrder,
+		Filename:                       filename,
+		NormalizedFilename:             normalizedFilename,
+		InternalURL:                    internalURL,
+		SourceMap:                      sourcemap,
+		AstroGlobalArgs:                astroGlobalArgs,
+		Compact:                        compact,
+		ResolvePath:                    resolvePathFn,
+		PreprocessStyle:                preprocessStyle,
+		ResultScopedSlot:               scopedSlot,
+		ScopedStyleStrategy:            scopedStyleStrategy,
+		TransitionsAnimationURL:        transitionsAnimationURL,
+		AnnotateSourceFile:             annotateSourceFile,
+		RenderScript:                   renderScript,
+		ExperimentalScriptOrder:        experimentalScriptOrder,
+		ExperimentalExactParsingThingy: experimentalExactParsingThingy,
 	}
 }
 
@@ -257,7 +263,7 @@
 		h := handler.NewHandler(source, parseOptions.Filename)
 
 		var doc *astro.Node
-		doc, err := astro.ParseWithOptions(strings.NewReader(source), astro.ParseOptionWithHandler(h), astro.ParseOptionEnableLiteral(true))
+		doc, err := astro.ParseWithOptions(strings.NewReader(source), astro.ParseOptionWithHandler(h), astro.ParseOptionEnableLiteral(true), astro.ParseOptionExperimentalBetterLiteralThingy(transformOptions.ExperimentalExactParsingThingy))
 		if err != nil {
 			h.AppendError(err)
 		}
@@ -266,7 +272,7 @@
 		// AFTER printing, exec transformations to pickup any errors/warnings
 		transform.Transform(doc, transformOptions, h)
 
 		return vert.ValueOf(ParseResult{
 			AST:         string(result.Output),
 			Diagnostics: h.Diagnostics(),
 		}).Value
@@ -281,7 +287,7 @@
 		h := handler.NewHandler(source, transformOptions.Filename)
 
 		var doc *astro.Node
-		doc, err := astro.ParseWithOptions(strings.NewReader(source), astro.ParseOptionWithHandler(h), astro.ParseOptionEnableLiteral(true))
+		doc, err := astro.ParseWithOptions(strings.NewReader(source), astro.ParseOptionWithHandler(h), astro.ParseOptionEnableLiteral(true), astro.ParseOptionExperimentalBetterLiteralThingy(transformOptions.ExperimentalExactParsingThingy))
 		if err != nil {
 			h.AppendError(err)
 		}
@@ -300,7 +306,7 @@
 			code += "\n" + inlineSourcemap
 		}
 
 		return vert.ValueOf(TSXResult{
 			Code:        code,
 			Map:         sourcemapString,
 			Diagnostics: h.Diagnostics(),
@@ -335,7 +341,7 @@
 					}
 				}()
 
-				doc, err := astro.ParseWithOptions(strings.NewReader(source), astro.ParseOptionWithHandler(h))
+				doc, err := astro.ParseWithOptions(strings.NewReader(source), astro.ParseOptionWithHandler(h), astro.ParseOptionExperimentalBetterLiteralThingy(transformOptions.ExperimentalExactParsingThingy))
 				if err != nil {
 					reject.Invoke(wasm_utils.ErrorToJSError(h, err))
 					return
@@ -512,13 +518,13 @@
 }`, sourcemap.Sources[0], sourcemap.SourcesContent[0], sourcemap.Mappings)
 }
 
 func createExternalSourceMap(source string, transformResult *TransformResult, result printer.PrintResult, transformOptions transform.TransformOptions) vert.Value {
 	transformResult.Code = string(result.Output)
 	transformResult.Map = createSourceMapString(source, result, transformOptions)
 	return vert.ValueOf(transformResult)
 }
 
 func createInlineSourceMap(source string, transformResult *TransformResult, result printer.PrintResult, transformOptions transform.TransformOptions) vert.Value {
 	sourcemapString := createSourceMapString(source, result, transformOptions)
 	inlineSourcemap := `//# sourceMappingURL=data:application/json;charset=utf-8;base64,` + base64.StdEncoding.EncodeToString([]byte(sourcemapString))
 	transformResult.Code = string(result.Output) + "\n" + inlineSourcemap
@@ -526,7 +532,7 @@
 	return vert.ValueOf(transformResult)
 }
 
 func createBothSourceMap(source string, transformResult *TransformResult, result printer.PrintResult, transformOptions transform.TransformOptions) vert.Value {
 	sourcemapString := createSourceMapString(source, result, transformOptions)
 	inlineSourcemap := `//# sourceMappingURL=data:application/json;charset=utf-8;base64,` + base64.StdEncoding.EncodeToString([]byte(sourcemapString))
 	transformResult.Code = string(result.Output) + "\n" + inlineSourcemap

internal/doctype.go

@@ -154,3 +154,53 @@ var quirkyIDs = []string{
 	"-//webtechs//dtd mozilla html 2.0//",
 	"-//webtechs//dtd mozilla html//",
 }
+
+func parseDoctypeExact(s string) (n *Node) {
+	n = &Node{Type: DoctypeNode}
+
+	// Find the name.
+	space := strings.IndexAny(s, whitespace)
+	if space == -1 {
+		space = len(s)
+	}
+	n.Data = s[:space]
+	n.Data = strings.ToLower(n.Data)
+	s = strings.TrimLeft(s[space:], whitespace)
+
+	if len(s) < 6 {
+		// It can't start with "PUBLIC" or "SYSTEM".
+		// Ignore the rest of the string.
+		return n
+	}
+
+	key := strings.ToLower(s[:6])
+	s = s[6:]
+	for key == "public" || key == "system" {
+		s = strings.TrimLeft(s, whitespace)
+		if s == "" {
+			break
+		}
+		quote := s[0]
+		if quote != '"' && quote != '\'' {
+			break
+		}
+		s = s[1:]
+		q := strings.IndexRune(s, rune(quote))
+		var id string
+		if q == -1 {
+			id = s
+			s = ""
+		} else {
+			id = s[:q]
+			s = s[q+1:]
+		}
+		n.Attr = append(n.Attr, Attribute{Key: key, Val: id})
+		if key == "public" {
+			key = "system"
+		} else {
+			key = ""
+		}
+	}
+
+	return n
+}

internal/parser.go

@@ -393,7 +393,6 @@ func (p *parser) addExpression() {
 		Loc:           p.generateLoc(),
 		Namespace:     p.top().Namespace,
 	})
-
 }
 
 func isFragment(data string) bool {
@@ -725,6 +724,38 @@ func beforeHTMLIM(p *parser) bool {
 	return false
 }
 
+func initialIMExact(p *parser) bool {
+	switch p.tok.Type {
+	case FrontmatterFenceToken:
+		p.setOriginalIM()
+		p.im = frontmatterIM
+		return false
+	case TextToken:
+		p.tok.Data = strings.TrimLeft(p.tok.Data, whitespace)
+		if len(p.tok.Data) == 0 {
+			// It was all whitespace, so ignore it.
+			return true
+		}
+	case CommentToken:
+		p.doc.AppendChild(&Node{
+			Type: CommentNode,
+			Data: p.tok.Data,
+			Loc:  p.generateLoc(),
+		})
+		return true
+	case DoctypeToken:
+		n := parseDoctypeExact(p.tok.Data)
+		p.doc.AppendChild(n)
+		p.im = inLiteralIMExact
+		return true
+	}
+	if p.frontmatterState == FrontmatterInitial {
+		p.addFrontmatter(true)
+	}
+	p.im = inLiteralIMExact
+	return false
+}
+
 // Section 12.2.6.4.3.
 func beforeHeadIM(p *parser) bool {
 	switch p.tok.Type {
@@ -1776,6 +1807,14 @@ func textIM(p *parser) bool {
 	return p.tok.Type == EndTagToken
 }
 
+func parseText(p *parser) {
+	d := p.tok.Data
+	if d == "" {
+		return
+	}
+	p.addText(d)
+}
+
 // Section 12.2.6.4.9.
 func inTableIM(p *parser) bool {
 	switch p.tok.Type {
@@ -2805,6 +2844,42 @@ func inExpressionIM(p *parser) bool {
 	return p.tok.Type == EndTagToken
 }
 
+func inLiteralIMExact(p *parser) bool {
+	switch p.tok.Type {
+	case DoctypeToken:
+		n := parseDoctypeExact(p.tok.Data)
+		p.doc.AppendChild(n)
+	case StartTagToken:
+		p.addElement()
+		if p.hasSelfClosingToken {
+			p.addLoc()
+			p.oe.pop()
+			p.acknowledgeSelfClosingTag()
+		}
+	case StartExpressionToken:
+		p.addExpression()
+		p.setOriginalIM()
+		p.im = inExpressionIMExact
+	case ErrorToken:
+		// ignore the token
+	case TextToken:
+		parseText(p)
+	case CommentToken:
+		p.addChild(&Node{
+			Type: CommentNode,
+			Data: p.tok.Data,
+			Loc:  p.generateLoc(),
+		})
+	case EndTagToken:
+		p.addLoc()
+		p.oe.pop()
+	case EndExpressionToken:
+		p.addLoc()
+		p.oe.pop()
+	}
+	return true
+}
+
 func ignoreTheRemainingTokens(p *parser) bool {
 	return true
 }
@@ -2817,6 +2892,34 @@ func getExitLiteralFunc(p *parser) func() bool {
 	}
 }
 
+func inExpressionIMExact(p *parser) bool {
+	switch p.tok.Type {
+	case ErrorToken:
+		p.oe.pop()
+	case TextToken:
+		parseText(p)
+		return true
+	case StartTagToken, EndTagToken:
+		return inLiteralIMExact(p)
+	case EndExpressionToken:
+		p.addLoc()
+		p.oe.pop()
+		p.im = p.originalIM
+		p.originalIM = nil
+		return true
+	case CommentToken:
+		p.addChild(&Node{
+			Type: CommentNode,
+			Data: p.tok.Data,
+			Loc:  p.generateLoc(),
+		})
+		return true
+	}
+	p.im = p.originalIM
+	p.originalIM = nil
+	return p.tok.Type == EndTagToken
+}
+
 const whitespaceOrNUL = whitespace + "\x00"
 
 // Section 12.2.6.5
@@ -3052,6 +3155,14 @@ func ParseOptionEnableLiteral(enable bool) ParseOption {
 	}
 }
 
+func ParseOptionExperimentalBetterLiteralThingy(enable bool) ParseOption {
+	return func(p *parser) {
+		if enable {
+			p.im = initialIMExact
+		}
+	}
+}
+
 // ParseWithOptions is like Parse, with options.
 func ParseWithOptions(r io.Reader, opts ...ParseOption) (*Node, error) {
 	p := &parser{

internal/printer/__printer_js__/Preserve_namespaces_in_expressions_-_with_exact_parsing.snap

@@ -0,0 +1,41 @@
+
+[TestPrinter/Preserve_namespaces_in_expressions_-_with_exact_parsing - 1]
+## Input
+
+```
+<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink"><rect xlink:href={`#${iconId}`}></svg>
+```
+
+## Output
+
+```js
+import {
+  Fragment,
+  render as $$render,
+  createAstro as $$createAstro,
+  createComponent as $$createComponent,
+  renderComponent as $$renderComponent,
+  renderHead as $$renderHead,
+  maybeRenderHead as $$maybeRenderHead,
+  unescapeHTML as $$unescapeHTML,
+  renderSlot as $$renderSlot,
+  mergeSlots as $$mergeSlots,
+  addAttribute as $$addAttribute,
+  spreadAttributes as $$spreadAttributes,
+  defineStyleVars as $$defineStyleVars,
+  defineScriptVars as $$defineScriptVars,
+  renderTransition as $$renderTransition,
+  createTransitionScope as $$createTransitionScope,
+  renderScript as $$renderScript,
+  createMetadata as $$createMetadata
+} from "http://localhost:3000/";
+
+export const $$metadata = $$createMetadata(import.meta.url, { modules: [], hydratedComponents: [], clientOnlyComponents: [], hydrationDirectives: new Set([]), hoisted: [] });
+
+const $$Component = $$createComponent(($$result, $$props, $$slots) => {
+
+return $$render`${$$maybeRenderHead($$result)}<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink"><rect${$$addAttribute(`#${iconId}`, "xlink:href")}></rect></svg>`;
+}, undefined, undefined);
+export default $$Component;
+```
+---