Scan request headers

Author: FollowTheProcess

internal/syntax/scanner/v2/scanner.go

@@ -412,6 +412,9 @@ func scanText(s *Scanner) stateFn {
 // scanRequest scans inside a HTTP request definition.
 //
 // The opening '###' and any request comment has already been consumed.
+//
+// The only thing allowed top level in a request is '#', '//' and
+// a request method.
 func scanRequest(s *Scanner) stateFn {
 	s.skip(unicode.IsSpace)
 
@@ -428,7 +431,6 @@ func scanRequest(s *Scanner) stateFn {
 		return scanRequestSlash
 	default:
 		if isUpperAlpha(char) {
-			// A request method
 			return scanMethod
 		}
 
@@ -528,8 +530,68 @@ func scanURL(s *Scanner) stateFn {
 	s.takeWhile(isURL)
 	s.emit(token.Text)
 
-	// TODO(@FollowTheProcess): Should move the state to scanHTTPVersion or scanHeaders
-	return scanStart
+	s.skip(isLineSpace)
+
+	// Is there a HTTP version?
+	if s.restHasPrefix("HTTP/") {
+		return scanHTTPVersion
+	}
+
+	s.skip(unicode.IsSpace)
+
+	if isIdent(s.peek()) {
+		return scanHeaders
+	}
+
+	// TODO(@FollowTheProcess): Probably should be scan body
+	return scanRequest
+}
+
+// scanHTTPVersion scans a literal 'HTTP/<version>' declaration.
+func scanHTTPVersion(s *Scanner) stateFn {
+	s.takeExact("HTTP/")
+
+	s.takeWhile(isDigit)
+
+	if s.take(".") {
+		// It's e.g 1.2
+		s.takeWhile(isDigit)
+	}
+
+	s.emit(token.HTTPVersion)
+
+	return scanHeaders
+}
+
+// scanHeaders scans a request header.
+func scanHeaders(s *Scanner) stateFn {
+	s.takeWhile(isIdent)
+	s.emit(token.Ident)
+
+	if s.peek() != ':' {
+		s.errorf("invalid header, expected ':', got %q", s.peek())
+		return nil
+	}
+
+	s.take(":")
+	s.emit(token.Colon)
+
+	s.skip(isLineSpace)
+
+	if isText(s.peek()) {
+		s.takeWhile(isText)
+		s.emit(token.Text)
+	}
+
+	s.skip(unicode.IsSpace)
+
+	// If there are more headers, call this function again!
+	if isIdent(s.peek()) {
+		return scanHeaders
+	}
+
+	// TODO(@FollowTheProcess): Scan body
+	return scanRequest
 }
 
 // isLineSpace reports whether r is a non line terminating whitespace character,

internal/syntax/scanner/v2/testdata/invalid/single-slash-global.txtar

@@ -0,0 +1,8 @@
+-- src.http --
+# I'm a normal comment
+/ <- This is wrong, should be two '//'
+-- tokens.txt --
+<Token::Comment start=2, end=22>
+<Token::Error start=23, end=24>
+-- errors.txt --
+single-slash-global.txtar:2:1: invalid use of '/', two '//' mark a comment start, got '/'

internal/syntax/scanner/v2/testdata/invalid/single-slash-request.txtar

@@ -0,0 +1,11 @@
+-- src.http --
+###
+# I'm a normal comment
+/ <- This is wrong, should be two '//'
+GET https://example.com
+-- tokens.txt --
+<Token::Separator start=0, end=3>
+<Token::Comment start=6, end=26>
+<Token::Error start=27, end=28>
+-- errors.txt --
+single-slash-request.txtar:3:1: invalid use of '/', two '//' mark a comment start, got '/'

internal/syntax/scanner/v2/testdata/valid/headers.txtar

@@ -0,0 +1,21 @@
+-- src.http --
+### Test
+GET https://api.somewhere.com/items/1
+Content-Type: application/json
+Accept: application/json
+X-Something-Else: yes
+-- tokens.txt --
+<Token::Separator start=0, end=3>
+<Token::Comment start=4, end=8>
+<Token::MethodGet start=9, end=12>
+<Token::Text start=13, end=46>
+<Token::Ident start=47, end=59>
+<Token::Colon start=59, end=60>
+<Token::Text start=61, end=77>
+<Token::Ident start=78, end=84>
+<Token::Colon start=84, end=85>
+<Token::Text start=86, end=102>
+<Token::Ident start=103, end=119>
+<Token::Colon start=119, end=120>
+<Token::Text start=121, end=124>
+<Token::EOF start=125, end=125>

internal/syntax/scanner/v2/testdata/valid/http-version.txtar

@@ -0,0 +1,9 @@
+-- src.http --
+###
+GET https://api.somewhere.com/items/1 HTTP/1.1
+-- tokens.txt --
+<Token::Separator start=0, end=3>
+<Token::MethodGet start=4, end=7>
+<Token::Text start=8, end=41>
+<Token::HTTPVersion start=42, end=50>
+<Token::EOF start=51, end=51>