Use the v2 scanner as the default and remove the old one (#144)

* Parse tests passing

* Fix a header edge case

* Fix bad resolver test case

* All working!

* Remove token.URL and ast.KindURL

* Fix pathological parser recovery fuzz test

* oops lint

* Add todo

Author: FollowTheProcess

internal/syntax/ast/ast_test.go

@@ -36,14 +36,14 @@ func TestNode(t *testing.T) {
 		},
 		{
 			name: "url literal",
-			node: ast.URL{
+			node: ast.TextLiteral{
 				Value: "https://example.com",
-				Token: token.Token{Kind: token.URL, Start: 0, End: 19},
-				Type:  ast.KindURL,
+				Token: token.Token{Kind: token.Text, Start: 0, End: 19},
+				Type:  ast.KindTextLiteral,
 			},
-			start: token.Token{Kind: token.URL, Start: 0, End: 19},
-			end:   token.Token{Kind: token.URL, Start: 0, End: 19},
-			kind:  ast.KindURL,
+			start: token.Token{Kind: token.Text, Start: 0, End: 19},
+			end:   token.Token{Kind: token.Text, Start: 0, End: 19},
+			kind:  ast.KindTextLiteral,
 		},
 		{
 			name: "ident",
@@ -240,7 +240,7 @@ func TestNode(t *testing.T) {
 				},
 				URL: ast.TextLiteral{
 					Value: "https://example.com",
-					Token: token.Token{Kind: token.URL, Start: 9, End: 28},
+					Token: token.Token{Kind: token.Text, Start: 9, End: 28},
 					Type:  ast.KindTextLiteral,
 				},
 				Method: ast.Method{
@@ -260,7 +260,7 @@ func TestNode(t *testing.T) {
 				Body: nil,
 				URL: ast.TextLiteral{
 					Value: "https://example.com",
-					Token: token.Token{Kind: token.URL, Start: 9, End: 28},
+					Token: token.Token{Kind: token.Text, Start: 9, End: 28},
 					Type:  ast.KindTextLiteral,
 				},
 				Method: ast.Method{
@@ -271,7 +271,7 @@ func TestNode(t *testing.T) {
 				Type: ast.KindRequest,
 			},
 			start: token.Token{Kind: token.Separator, Start: 0, End: 3},
-			end:   token.Token{Kind: token.URL, Start: 9, End: 28},
+			end:   token.Token{Kind: token.Text, Start: 9, End: 28},
 			kind:  ast.KindRequest,
 		},
 		{
@@ -308,7 +308,7 @@ func TestNode(t *testing.T) {
 				},
 				URL: ast.TextLiteral{
 					Value: "https://example.com",
-					Token: token.Token{Kind: token.URL, Start: 9, End: 28},
+					Token: token.Token{Kind: token.Text, Start: 9, End: 28},
 					Type:  ast.KindTextLiteral,
 				},
 				Method: ast.Method{
@@ -340,7 +340,7 @@ func TestNode(t *testing.T) {
 				},
 				URL: ast.TextLiteral{
 					Value: "https://example.com",
-					Token: token.Token{Kind: token.URL, Start: 9, End: 28},
+					Token: token.Token{Kind: token.Text, Start: 9, End: 28},
 					Type:  ast.KindTextLiteral,
 				},
 				Method: ast.Method{
@@ -375,10 +375,10 @@ func TestNode(t *testing.T) {
 			// |------ left ------|-- interp --|- right -|
 			name: "full interp",
 			node: ast.InterpolatedExpression{
-				Left: ast.URL{
+				Left: ast.TextLiteral{
 					Value: "https://example.com/",
-					Token: token.Token{Kind: token.URL, Start: 0, End: 20},
-					Type:  ast.KindURL,
+					Token: token.Token{Kind: token.Text, Start: 0, End: 20},
+					Type:  ast.KindTextLiteral,
 				},
 				Interp: ast.Interp{
 					Expr: ast.Ident{
@@ -390,15 +390,15 @@ func TestNode(t *testing.T) {
 					Close: token.Token{Kind: token.CloseInterp, Start: 31, End: 33},
 					Type:  ast.KindInterp,
 				},
-				Right: ast.URL{
+				Right: ast.TextLiteral{
 					Value: "/items/123",
-					Token: token.Token{Kind: token.URL, Start: 33, End: 43},
-					Type:  ast.KindURL,
+					Token: token.Token{Kind: token.Text, Start: 33, End: 43},
+					Type:  ast.KindTextLiteral,
 				},
 				Type: ast.KindInterpolatedExpression,
 			},
-			start: token.Token{Kind: token.URL, Start: 0, End: 20},
-			end:   token.Token{Kind: token.URL, Start: 33, End: 43},
+			start: token.Token{Kind: token.Text, Start: 0, End: 20},
+			end:   token.Token{Kind: token.Text, Start: 33, End: 43},
 			kind:  ast.KindInterpolatedExpression,
 		},
 		{
@@ -417,26 +417,26 @@ func TestNode(t *testing.T) {
 					Close: token.Token{Kind: token.CloseInterp, Start: 11, End: 13},
 					Type:  ast.KindInterp,
 				},
-				Right: ast.URL{
+				Right: ast.TextLiteral{
 					Value: "/items/3",
-					Token: token.Token{Kind: token.URL, Start: 13, End: 21},
-					Type:  ast.KindURL,
+					Token: token.Token{Kind: token.Text, Start: 13, End: 21},
+					Type:  ast.KindTextLiteral,
 				},
 				Type: ast.KindInterpolatedExpression,
 			},
 			start: token.Token{Kind: token.OpenInterp, Start: 0, End: 2},
-			end:   token.Token{Kind: token.URL, Start: 13, End: 21},
+			end:   token.Token{Kind: token.Text, Start: 13, End: 21},
 			kind:  ast.KindInterpolatedExpression,
 		},
 		{
 			// https://example/com/{{ endpoint }}
 			// |------ left ------|-- interp ---|
 			name: "interp no right",
 			node: ast.InterpolatedExpression{
-				Left: ast.URL{
+				Left: ast.TextLiteral{
 					Value: "https://example.com/",
-					Token: token.Token{Kind: token.URL, Start: 0, End: 20},
-					Type:  ast.KindURL,
+					Token: token.Token{Kind: token.Text, Start: 0, End: 20},
+					Type:  ast.KindTextLiteral,
 				},
 				Interp: ast.Interp{
 					Expr: ast.Ident{
@@ -451,7 +451,7 @@ func TestNode(t *testing.T) {
 				Right: nil,
 				Type:  ast.KindInterpolatedExpression,
 			},
-			start: token.Token{Kind: token.URL, Start: 0, End: 20},
+			start: token.Token{Kind: token.Text, Start: 0, End: 20},
 			end:   token.Token{Kind: token.CloseInterp, Start: 32, End: 34},
 			kind:  ast.KindInterpolatedExpression,
 		},

internal/syntax/ast/expression.go

@@ -72,38 +72,6 @@ func (t TextLiteral) Kind() Kind {
 // expressionNode marks a [TextLiteral] as an [ast.Expression].
 func (t TextLiteral) expressionNode() {}
 
-// URL is a literal URL expression.
-type URL struct {
-	// The url value (unquoted)
-	Value string `yaml:"value"`
-
-	// The [token.URL] token.
-	Token token.Token `yaml:"token"`
-
-	// Type is [KindURL].
-	Type Kind `yaml:"type"`
-}
-
-// Start returns the first token of the URL, which is
-// obviously just the [token.URL].
-func (u URL) Start() token.Token {
-	return u.Token
-}
-
-// End returns the last token in the URL, which is also
-// the [token.URL].
-func (u URL) End() token.Token {
-	return u.Token
-}
-
-// Kind returns [KindURL].
-func (u URL) Kind() Kind {
-	return u.Type
-}
-
-// expressionNode marks a [URL] as an [ast.Expression].
-func (u URL) expressionNode() {}
-
 // Interp is a text interpolation expression.
 type Interp struct {
 	// Expr is the expression inside the interpolation.

internal/syntax/ast/kind.go

@@ -12,7 +12,6 @@ const (
 	KindVarStatement                       // VarStatement
 	KindIdent                              // Ident
 	KindTextLiteral                        // TextLiteral
-	KindURL                                // URL
 	KindInterp                             // Interp
 	KindPrompt                             // Prompt
 	KindRequest                            // Request

internal/syntax/ast/kind_string.go

@@ -23,9 +23,15 @@ import (
 	"go.followtheprocess.codes/zap/internal/syntax/token"
 )
 
-// ErrParse is a generic parsing error, details on the error are passed
-// to the parser's [syntax.ErrorHandler] at the moment it occurs.
-var ErrParse = errors.New("parse error")
+var (
+	// ErrParse is a generic parsing error, details on the error are passed
+	// to the parser's [syntax.ErrorHandler] at the moment it occurs.
+	ErrParse = errors.New("parse error")
+
+	// ErrAbort is a fatal parsing error, this means the parser encountered
+	// pathological syntax it was not able to recover from and has aborted.
+	ErrAbort = errors.New("fatal parse error, aborting")
+)
 
 // Parser is the http file parser.
 type Parser struct {
@@ -72,14 +78,20 @@ func (p *Parser) Parse() (ast.File, error) {
 	for !p.current.Is(token.EOF) {
 		if p.current.Is(token.Error) {
 			p.error("Error token from scanner")
-			p.synchronise()
+
+			if abortErr := p.synchronise(); abortErr != nil {
+				return file, fmt.Errorf("%w: %w", ErrAbort, abortErr)
+			}
 
 			continue
 		}
 
 		statement, err := p.parseStatement()
 		if err != nil {
-			p.synchronise()
+			if abortErr := p.synchronise(); abortErr != nil {
+				return file, fmt.Errorf("%w: %w", ErrAbort, abortErr)
+			}
+
 			continue
 		}
 
@@ -226,14 +238,34 @@ func (p *Parser) bytes() []byte {
 //
 // synchronise discards tokens until it sees the next Separator, EOF after which
 // point the parser should be back in sync and can continue normally.
-func (p *Parser) synchronise() {
+//
+// It does this up to a maximum limit, which if reached will cause synchronise
+// to return a non-nil error, indicating that no progress has been made while
+// synchronising and the parser should abort.
+func (p *Parser) synchronise() error {
+	p.hadErrors = true
+
+	// We try a max of 5 times to synchronise, if we haven't found
+	// something good by then, bail out.
+	const maxAttempts = 5
+
+	attempt := 0
+
 	for {
 		p.advance()
 
-		if p.current.Is(token.Separator, token.EOF) {
+		attempt++
+
+		if p.current.Is(token.Separator, token.EOF) || attempt >= maxAttempts {
 			break
 		}
 	}
+
+	if attempt >= maxAttempts {
+		return fmt.Errorf("%w: could not synchronise parser after %d attempts", ErrAbort, maxAttempts)
+	}
+
+	return nil
 }
 
 // parseStatement parses a statement.
@@ -438,7 +470,7 @@ func (p *Parser) parseRequest() (ast.Request, error) {
 
 	result.Method = method
 
-	if err = p.expect(token.URL, token.OpenInterp); err != nil {
+	if err = p.expect(token.Text, token.OpenInterp); err != nil {
 		return result, err
 	}
 
@@ -491,6 +523,9 @@ func (p *Parser) parseRequest() (ast.Request, error) {
 		}
 
 		result.Body = bodyFile
+	case token.Error:
+		p.error("parse error while parsing request body")
+		return result, ErrParse
 	default:
 		// Nothing, not all requests have a body
 	}
@@ -548,8 +583,6 @@ func (p *Parser) parseExpression(precedence int) (ast.Expression, error) {
 		expr, err = p.parseInterpolatedExpression(nil)
 	case token.Ident:
 		expr = p.parseIdent()
-	case token.URL:
-		expr = p.parseURL()
 	case token.Body:
 		expr, err = p.parseBody()
 	default:
@@ -619,7 +652,7 @@ func (p *Parser) parseInterpolatedExpression(left ast.Expression) (ast.Interpola
 
 	precedence := p.current.Precedence()
 
-	if p.next.Is(token.Text, token.OpenInterp, token.Ident, token.URL, token.Body) && p.shouldParseRHS(left) {
+	if p.next.Is(token.Text, token.OpenInterp, token.Body) && p.shouldParseRHS(left) {
 		p.advance()
 
 		right, err := p.parseExpression(precedence)
@@ -663,8 +696,6 @@ func (p *Parser) shouldParseRHS(left ast.Expression) bool {
 		return p.next.Is(token.Text)
 	case ast.KindIdent:
 		return p.next.Is(token.Ident)
-	case ast.KindURL:
-		return p.next.Is(token.URL)
 	case ast.KindBody:
 		return p.next.Is(token.Body)
 	default:
@@ -683,17 +714,6 @@ func (p *Parser) parseTextLiteral() ast.TextLiteral {
 	return text
 }
 
-// parseURL parses a URL literal.
-func (p *Parser) parseURL() ast.URL {
-	result := ast.URL{
-		Value: p.text(),
-		Token: p.current,
-		Type:  ast.KindURL,
-	}
-
-	return result
-}
-
 // parseHeader parses a Header statement.
 func (p *Parser) parseHeader() (ast.Header, error) {
 	result := ast.Header{

internal/syntax/parser/parser.go

@@ -0,0 +1,2 @@
+go test fuzz v1
+[]byte("### Body\nPOST https://api.somewhere.com/items/1\n\n{\n  \"somethi\xfeS\xe7C\xb3\x8f?ng\": \"here\"\n}\n\n<> response.json\n")

internal/syntax/parser/testdata/fuzz/FuzzParser/4e13f80736f1e3d9

@@ -1,8 +1,10 @@
+# TODO: The positions are off by 1 or 2 in some places
+
 -- src.http --
 ### Bad
 GET https://blowup.com
 
 < £$%^&
 -- want.txt --
 bad-body-file.txtar:4:1-2: Error token from scanner
-bad-body-file.txtar:4:5: unrecognised character: '£'
+bad-body-file.txtar:4:4: unexpected character: '£'

internal/syntax/parser/testdata/invalid/bad-body-file.txtar

@@ -6,4 +6,4 @@ GET https://api.something.com/v1
 Content-Type application/json
 -- want.txt --
 bad-headers.txtar:3:1-13: Error token from scanner
-bad-headers.txtar:3:13: expected ':', got ' '
+bad-headers.txtar:3:12: invalid header, expected ':', got ' '