Scan HTTP headers (#18)

* Scan simple headers

* WIP: interpolated headers

* Fix silly bug with headers

* Add a scanner fuzz test

Author: FollowTheProcess

Taskfile.yml

@@ -69,6 +69,8 @@ tasks:
 
   lint:
     desc: Run linting
+    deps:
+      - fmt
     sources:
       - "**/*.go"
       - .golangci.yml

internal/syntax/scanner/scanner.go

@@ -429,7 +429,7 @@ func scanEq(s *Scanner) scanFn {
 	return scanStart
 }
 
-// scanInterp scans an opening '{{' token.
+// scanInterp scans an entire interpolation of {{ <contents> }}.
 func scanInterp(s *Scanner) scanFn {
 	// Absorb no more than 2 '{'
 	count := 0
@@ -537,10 +537,99 @@ func scanURL(s *Scanner) scanFn {
 
 	s.emit(token.URL)
 
+	// Is the next thing headers?
+	s.skip(unicode.IsSpace)
+
+	if isAlpha(s.peek()) {
+		return scanHeaders
+	}
+
 	// TODO(@FollowTheProcess): Handle HTTP version, headers, body etc.
 	return scanStart
 }
 
+// scanHeaders scans a series of HTTP headers, one per line, emitting
+// tokens as it goes.
+//
+// It stops when it sees the next character is not a valid ident character
+// and so could not be another header.
+func scanHeaders(s *Scanner) scanFn {
+	s.takeWhile(isIdent)
+
+	// Header without a colon or value e.g. 'Content-Type'
+	// this is unfinished so is an error, like an unterminated interpolation.
+	if s.peek() == eof {
+		s.error("unexpected eof")
+		return nil
+	}
+
+	s.emit(token.Header)
+
+	if s.peek() != ':' {
+		s.errorf("expected ':', got %q", s.peek())
+		return nil
+	}
+
+	// Consume the ':' now we know it exists, and skip over any whitespace
+	// on the same line until we get to the header value
+	s.next()
+	s.emit(token.Colon)
+	s.skip(isLineSpace)
+
+	// for next := s.next(); next != '\n' && next != eof; next = s.next() {
+	// 	if s.restHasPrefix("{{") {
+	// 		// Emit whatever we've captured at this point as text (if there is anything)
+	// 		// and go scan the interpolation
+	// 		if s.start != s.pos {
+	// 			// We have absorbed stuff
+	// 			s.emit(token.Text)
+	// 		}
+	// 		scanInterp(s)
+	// 	}
+	// }
+
+	// Handle interpolation somewhere inside the header value
+	// e.g. Authorization: Bearer {{ token }}
+	for {
+		if s.restHasPrefix("{{") {
+			// Emit what we have captured up to this point (if there is anything) as Text and then
+			// switch to scanning the interpolation
+			if s.start != s.pos {
+				// We have absorbed stuff, emit it
+				s.emit(token.Text)
+			}
+
+			scanInterp(s)
+		}
+
+		// Scan any text on the same line
+		next := s.peek()
+		if next == '\n' || next == eof {
+			break
+		}
+
+		s.next()
+	}
+
+	// If we absorbed any text, emit it.
+	//
+	// This could be empty because the entire header value could have just been an interp
+	// e.g. X-Api-Key: {{ key }}
+	if s.start != s.pos {
+		s.emit(token.Text)
+	}
+
+	// Now for the fun bit, call itself if there are more headers
+	s.skip(unicode.IsSpace)
+
+	if isAlpha(s.peek()) {
+		return scanHeaders
+	}
+
+	// TODO(@FollowTheProcess): Handle request body
+	return scanStart
+}
+
 // isAlpha reports whether r is an alpha character.
 func isAlpha(r rune) bool {
 	return (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z')

internal/syntax/scanner/scanner_test.go

@@ -373,6 +373,53 @@ func TestValid(t *testing.T) {
 	}
 }
 
+func FuzzScanner(f *testing.F) {
+	// Get all the .http source from testdata for the corpus
+	pattern := filepath.Join("testdata", "valid", "*.txtar")
+	files, err := filepath.Glob(pattern)
+	test.Ok(f, err)
+
+	for _, file := range files {
+		archive, err := txtar.ParseFile(file)
+		test.Ok(f, err)
+
+		src, ok := archive.Read("src.http")
+		test.True(f, ok, test.Context("file %s does not contain 'src.http'", file))
+
+		f.Add(src)
+	}
+
+	// Property: The scanner never panics or loops indefinitely, fuzz
+	// by default will catch both of these
+	f.Fuzz(func(t *testing.T, src string) {
+		// Note: no ErrorHandler installed, because if we let the scanner report syntax
+		// errors it would kill the fuzz test straight away e.g. on the first invalid
+		// utf-8 char
+		scanner := scanner.New("fuzz", []byte(src), nil)
+
+		for {
+			tok := scanner.Scan()
+			if tok.Is(token.EOF, token.Error) {
+				break
+			}
+
+			// Property: Positions must be positive integers
+			test.True(t, tok.Start >= 0, test.Context("token start position (%d) was negative", tok.Start))
+			test.True(t, tok.End >= 0, test.Context("token end position (%d) was negative", tok.End))
+
+			// Property: The kind must be one of the known kinds
+			test.True(
+				t,
+				(tok.Kind >= token.EOF) && (tok.Kind <= token.MethodTrace),
+				test.Context("token %s was not one of the pre-defined kinds", tok),
+			)
+
+			// Property: End must be >= Start
+			test.True(t, tok.End >= tok.Start, test.Context("token %s had invalid start and end positions", tok))
+		}
+	})
+}
+
 // testFailHandler returns a [syntax.ErrorHandler] that handles scanning errors by failing
 // the enclosing test.
 func testFailHandler(tb testing.TB) syntax.ErrorHandler {

internal/syntax/scanner/testdata/valid/headers.txtar

@@ -0,0 +1,21 @@
+-- src.http --
+### Test
+GET https://api.somewhere.com/items/1
+Content-Type: application/json
+Accept: application/json
+X-Something-Else: yes
+-- tokens.txt --
+<Token::Separator start=0, end=3>
+<Token::Comment start=4, end=8>
+<Token::MethodGet start=9, end=12>
+<Token::URL start=13, end=46>
+<Token::Header start=47, end=59>
+<Token::Colon start=59, end=60>
+<Token::Text start=61, end=77>
+<Token::Header start=78, end=84>
+<Token::Colon start=84, end=85>
+<Token::Text start=86, end=102>
+<Token::Header start=103, end=119>
+<Token::Colon start=119, end=120>
+<Token::Text start=121, end=124>
+<Token::EOF start=125, end=125>

internal/syntax/scanner/testdata/valid/interpolated-headers.txtar

@@ -0,0 +1,39 @@
+-- src.http --
+@token = shhh
+@json = application/json
+
+### Test
+GET https://api.somewhere.com/items/1
+Content-Type: {{ json }}
+Authorization: Bearer {{ token }}
+Accept: {{ json }}
+-- tokens.txt --
+<Token::At start=0, end=1>
+<Token::Ident start=1, end=6>
+<Token::Eq start=7, end=8>
+<Token::Text start=9, end=13>
+<Token::At start=14, end=15>
+<Token::Ident start=15, end=19>
+<Token::Eq start=20, end=21>
+<Token::Text start=22, end=38>
+<Token::Separator start=40, end=43>
+<Token::Comment start=44, end=48>
+<Token::MethodGet start=49, end=52>
+<Token::URL start=53, end=86>
+<Token::Header start=87, end=99>
+<Token::Colon start=99, end=100>
+<Token::OpenInterp start=101, end=103>
+<Token::Ident start=104, end=108>
+<Token::CloseInterp start=109, end=111>
+<Token::Header start=112, end=125>
+<Token::Colon start=125, end=126>
+<Token::Text start=127, end=134>
+<Token::OpenInterp start=134, end=136>
+<Token::Ident start=137, end=142>
+<Token::CloseInterp start=143, end=145>
+<Token::Header start=146, end=152>
+<Token::Colon start=152, end=153>
+<Token::OpenInterp start=154, end=156>
+<Token::Ident start=157, end=161>
+<Token::CloseInterp start=162, end=164>
+<Token::EOF start=165, end=165>

internal/syntax/token/kind_string.go

@@ -18,8 +18,10 @@ const (
 	At                            // At
 	Ident                         // Ident
 	Eq                            // Eq
+	Colon                         // Colon
 	Text                          // Text
 	URL                           // URL
+	Header                        // Header
 	OpenInterp                    // OpenInterp
 	CloseInterp                   // CloseInterp
 	Name                          // Name