Skip to content

Commit 05a8d87

Browse files
shmsrantonmedv
authored andcommitted
Fix handling of space(s) in between operator
1 parent ddd630d commit 05a8d87

File tree

4 files changed

+28
-24
lines changed

4 files changed

+28
-24
lines changed

parser/lexer/lexer.go

Lines changed: 11 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@ func Lex(source *file.Source) ([]Token, error) {
1414
tokens: make([]Token, 0),
1515
}
1616

17-
l.loc = file.Location{1, 0}
17+
l.loc = file.Location{Line: 1, Column: 0}
1818
l.prev = l.loc
1919
l.startLoc = l.loc
2020

@@ -31,10 +31,9 @@ func Lex(source *file.Source) ([]Token, error) {
3131

3232
type lexer struct {
3333
input string
34-
state stateFn
3534
tokens []Token
3635
start, end int // current position in input
37-
width int // last rune with
36+
width int // last rune width
3837
startLoc file.Location // start location
3938
prev, loc file.Location // prev location of end location, end location
4039
err *file.Error
@@ -120,14 +119,17 @@ func (l *lexer) acceptRun(valid string) {
120119
}
121120

122121
func (l *lexer) acceptWord(word string) bool {
123-
pos := l.end
124-
loc := l.loc
125-
prev := l.prev
122+
pos, loc, prev := l.end, l.loc, l.prev
123+
124+
// Skip spaces (U+0020) if any
125+
r := l.peek()
126+
for ; r == ' '; r = l.peek() {
127+
l.next()
128+
}
129+
126130
for _, ch := range word {
127131
if l.next() != ch {
128-
l.end = pos
129-
l.loc = loc
130-
l.prev = prev
132+
l.end, l.loc, l.prev = pos, loc, prev
131133
return false
132134
}
133135
}

parser/lexer/lexer_test.go

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -2,11 +2,12 @@ package lexer_test
22

33
import (
44
"fmt"
5-
"github.com/stretchr/testify/assert"
6-
"github.com/stretchr/testify/require"
75
"strings"
86
"testing"
97

8+
"github.com/stretchr/testify/assert"
9+
"github.com/stretchr/testify/require"
10+
1011
"github.com/antonmedv/expr/file"
1112
. "github.com/antonmedv/expr/parser/lexer"
1213
)
@@ -63,7 +64,7 @@ var lexTests = []lexTest{
6364
},
6465
},
6566
{
66-
`not in not abc not i not(false) not in`,
67+
`not in not abc not i not(false) not in not in`,
6768
[]Token{
6869
{Kind: Operator, Value: "not in"},
6970
{Kind: Operator, Value: "not"},
@@ -74,8 +75,8 @@ var lexTests = []lexTest{
7475
{Kind: Bracket, Value: "("},
7576
{Kind: Identifier, Value: "false"},
7677
{Kind: Bracket, Value: ")"},
77-
{Kind: Operator, Value: "not"},
78-
{Kind: Operator, Value: "in"},
78+
{Kind: Operator, Value: "not in"},
79+
{Kind: Operator, Value: "not in"},
7980
{Kind: EOF},
8081
},
8182
},
@@ -162,7 +163,6 @@ func TestLex_error(t *testing.T) {
162163
tests := strings.Split(strings.Trim(errorTests, "\n"), "\n\n")
163164

164165
for _, test := range tests {
165-
166166
input := strings.SplitN(test, "\n", 2)
167167
if len(input) != 2 {
168168
t.Errorf("syntax error in test: %q", test)

parser/lexer/state.go

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -125,10 +125,12 @@ loop:
125125
}
126126

127127
func not(l *lexer) stateFn {
128-
if l.acceptWord(" in") {
129-
l.emit(Operator)
130-
} else {
131-
l.emit(Operator)
128+
switch l.acceptWord("in") {
129+
case true:
130+
l.emitValue(Operator, "not in")
131+
case false:
132+
l.emitValue(Operator, "not")
132133
}
134+
133135
return root
134136
}

parser/lexer/token.go

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -10,11 +10,11 @@ type Kind string
1010

1111
const (
1212
Identifier Kind = "Identifier"
13-
Number = "Number"
14-
String = "String"
15-
Operator = "Operator"
16-
Bracket = "Bracket"
17-
EOF = "EOF"
13+
Number Kind = "Number"
14+
String Kind = "String"
15+
Operator Kind = "Operator"
16+
Bracket Kind = "Bracket"
17+
EOF Kind = "EOF"
1818
)
1919

2020
type Token struct {

0 commit comments

Comments
 (0)