We actually need the header token (#142)

Author: FollowTheProcess

internal/syntax/scanner/v2/scanner.go

@@ -45,7 +45,6 @@ type stateFn func(*Scanner) stateFn
 // Scanner is the http file scanner.
 type Scanner struct {
 	tokens      chan token.Token    // Channel on which to emit scanned tokens.
-	state       stateFn             // The scanner's current state
 	name        string              // Name of the file
 	diagnostics []syntax.Diagnostic // Diagnostics gathered during scanning
 	src         []byte              // Raw source text
@@ -71,7 +70,6 @@ func New(name string, src []byte) *Scanner {
 		stack:  make([]stateFn, 0, stackSize),
 		name:   name,
 		src:    src,
-		state:  scanStart,
 		line:   1,
 	}
 
@@ -780,7 +778,7 @@ func scanHeader(s *Scanner) stateFn {
 	s.takeWhile(isIdent)
 
 	if s.pos > s.start {
-		s.emit(token.Ident)
+		s.emit(token.Header)
 	}
 
 	if s.peek() != ':' {

internal/syntax/scanner/v2/testdata/invalid/header-no-colon.txtar

@@ -6,7 +6,7 @@ Authorization Bearer secret
 <Token::Separator start=0, end=3>
 <Token::MethodGet start=4, end=7>
 <Token::Text start=8, end=27>
-<Token::Ident start=28, end=41>
+<Token::Header start=28, end=41>
 <Token::Error start=41, end=41>
 -- errors.txt --
 header-no-colon.txtar:3:14: invalid header, expected ':', got ' '

internal/syntax/scanner/v2/testdata/valid/full.txtar

@@ -87,16 +87,16 @@ Authorization: Bearer somesecret token
 <Token::MethodPut start=138, end=141>
 <Token::Text start=142, end=175>
 <Token::HTTPVersion start=176, end=182>
-<Token::Ident start=183, end=195>
+<Token::Header start=183, end=195>
 <Token::Colon start=195, end=196>
 <Token::Text start=197, end=213>
-<Token::Ident start=214, end=220>
+<Token::Header start=214, end=220>
 <Token::Colon start=220, end=221>
 <Token::Text start=222, end=238>
-<Token::Ident start=239, end=255>
+<Token::Header start=239, end=255>
 <Token::Colon start=255, end=256>
 <Token::Text start=257, end=260>
-<Token::Ident start=261, end=274>
+<Token::Header start=261, end=274>
 <Token::Colon start=274, end=275>
 <Token::Text start=276, end=283>
 <Token::OpenInterp start=283, end=285>
@@ -138,7 +138,7 @@ Authorization: Bearer somesecret token
 <Token::Comment start=693, end=724>
 <Token::MethodPost start=725, end=729>
 <Token::Text start=730, end=763>
-<Token::Ident start=764, end=776>
+<Token::Header start=764, end=776>
 <Token::Colon start=776, end=777>
 <Token::Text start=778, end=794>
 <Token::LeftAngle start=796, end=797>
@@ -152,16 +152,16 @@ Authorization: Bearer somesecret token
 <Token::Separator start=838, end=841>
 <Token::MethodGet start=842, end=845>
 <Token::Text start=846, end=865>
-<Token::Ident start=866, end=872>
+<Token::Header start=866, end=872>
 <Token::Colon start=872, end=873>
 <Token::Text start=874, end=877>
-<Token::Ident start=878, end=890>
+<Token::Header start=878, end=890>
 <Token::Colon start=890, end=891>
 <Token::Text start=892, end=908>
-<Token::Ident start=909, end=918>
+<Token::Header start=909, end=918>
 <Token::Colon start=918, end=919>
 <Token::Text start=920, end=929>
-<Token::Ident start=930, end=943>
+<Token::Header start=930, end=943>
 <Token::Colon start=943, end=944>
 <Token::Text start=945, end=968>
 <Token::EOF start=969, end=969>

internal/syntax/scanner/v2/testdata/valid/headers.txtar

@@ -9,13 +9,13 @@ X-Something-Else: yes
 <Token::Comment start=4, end=8>
 <Token::MethodGet start=9, end=12>
 <Token::Text start=13, end=46>
-<Token::Ident start=47, end=59>
+<Token::Header start=47, end=59>
 <Token::Colon start=59, end=60>
 <Token::Text start=61, end=77>
-<Token::Ident start=78, end=84>
+<Token::Header start=78, end=84>
 <Token::Colon start=84, end=85>
 <Token::Text start=86, end=102>
-<Token::Ident start=103, end=119>
+<Token::Header start=103, end=119>
 <Token::Colon start=119, end=120>
 <Token::Text start=121, end=124>
 <Token::EOF start=125, end=125>

internal/syntax/scanner/v2/testdata/valid/interp-before-body.txtar

@@ -16,7 +16,7 @@ Authorization: Bearer {{ token }}
 <Token::Separator start=15, end=18>
 <Token::MethodPost start=19, end=23>
 <Token::Text start=24, end=43>
-<Token::Ident start=44, end=57>
+<Token::Header start=44, end=57>
 <Token::Colon start=57, end=58>
 <Token::Text start=59, end=66>
 <Token::OpenInterp start=66, end=68>

internal/syntax/scanner/v2/testdata/valid/interpolated-headers.txtar

@@ -20,18 +20,18 @@ Accept: {{ json }}
 <Token::Comment start=44, end=48>
 <Token::MethodGet start=49, end=52>
 <Token::Text start=53, end=86>
-<Token::Ident start=87, end=99>
+<Token::Header start=87, end=99>
 <Token::Colon start=99, end=100>
 <Token::OpenInterp start=101, end=103>
 <Token::Ident start=104, end=108>
 <Token::CloseInterp start=109, end=111>
-<Token::Ident start=112, end=125>
+<Token::Header start=112, end=125>
 <Token::Colon start=125, end=126>
 <Token::Text start=127, end=134>
 <Token::OpenInterp start=134, end=136>
 <Token::Ident start=137, end=142>
 <Token::CloseInterp start=143, end=145>
-<Token::Ident start=146, end=152>
+<Token::Header start=146, end=152>
 <Token::Colon start=152, end=153>
 <Token::OpenInterp start=154, end=156>
 <Token::Ident start=157, end=161>