Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
33 changes: 33 additions & 0 deletions src/main/java/org/codelibs/sai/internal/parser/AbstractParser.java
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@

import java.util.HashMap;
import java.util.Map;
import java.util.function.BooleanSupplier;

import org.codelibs.sai.internal.ir.IdentNode;
import org.codelibs.sai.internal.ir.LiteralNode;
Expand Down Expand Up @@ -149,6 +150,38 @@ protected final TokenType T(final int i) {
return Token.descType(getToken(i));
}

/**
* Run a speculative token-type lookahead that leaves no trace on the token stream.
*
* The lexer stops scanning right after an ambiguous token - a "/", or a "<" in
* scripting mode - so that {@link Lexer#scanLiteral} can still reinterpret it as a
* regular expression or a here string. Plain {@link #T(int)} lookahead defeats
* that: it resumes the lexer past the ambiguous token, and scanLiteral then refuses
* to rescan a token the stream has moved beyond, so the literal is lost for good.
*
* This runs the probe and rewinds both the lexer and the token stream to where they
* were, leaving the ambiguity for the real parse to resolve with the grammar
* context it actually has.
*
* @param probe the lookahead to run.
* @return whatever the probe returned.
*/
protected final boolean lookahead(final BooleanSupplier probe) {
final Lexer.State state = lexer.saveState();
final boolean pause = lexer.isPauseOnNextLeftBrace();
final int last = stream.last();

try {
return probe.getAsBoolean();
} finally {
while (stream.last() > last) {
stream.removeLast();
}
lexer.restoreState(state);
lexer.setPauseOnNextLeftBrace(pause);
}
}

/**
* Seek next token that is not an EOL or comment.
*
Expand Down
19 changes: 19 additions & 0 deletions src/main/java/org/codelibs/sai/internal/parser/Lexer.java
Original file line number Diff line number Diff line change
Expand Up @@ -283,6 +283,25 @@ void restoreState(final State state) {
last = state.last;
}

/**
* Whether the next left brace opens a function body the lexer has to pause on.
* Saved and restored around a speculative lookahead, which has to leave no trace.
*
* @return the current value of the flag.
*/
boolean isPauseOnNextLeftBrace() {
return pauseOnNextLeftBrace;
}

/**
* Restore the flag read by {@link #isPauseOnNextLeftBrace()}.
*
* @param pause the value to restore.
*/
void setPauseOnNextLeftBrace(final boolean pause) {
this.pauseOnNextLeftBrace = pause;
}

/**
* Add a new token to the stream.
*
Expand Down
8 changes: 8 additions & 0 deletions src/main/java/org/codelibs/sai/internal/parser/Parser.java
Original file line number Diff line number Diff line change
Expand Up @@ -3483,6 +3483,10 @@ private Expression functionExpression(final boolean isStatement, final boolean t
* @return true if an arrow function starts at the current token.
*/
private boolean isArrowFunction() {
return lookahead(this::isArrowFunctionAhead);
}

private boolean isArrowFunctionAhead() {
if (type == IDENT || isNonStrictModeIdent()) {
return T(k + 1) == ARROW;
}
Expand Down Expand Up @@ -4420,6 +4424,10 @@ private static IdentNode referenceTo(final IdentNode ident) {
* @return true if a destructuring assignment starts at the current token
*/
private boolean isDestructuringAssignment() {
return lookahead(this::isDestructuringAssignmentAhead);
}

private boolean isDestructuringAssignmentAhead() {
if (type != LBRACKET && type != LBRACE) {
return false;
}
Expand Down
65 changes: 65 additions & 0 deletions test/script/basic/es6/regex-literals.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,65 @@
/*
* Copyright (c) 2026, CodeLibs Project and/or its affiliates. All rights reserved.
*
* This code is free software; you can redistribute it and/or modify it
* under the terms of the GNU General Public License version 2 only, as
* published by the Free Software Foundation.
*
* This code is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
* version 2 for more details (a copy is included in the LICENSE file that
* accompanied this code).
*
* You should have received a copy of the GNU General Public License version
* 2 along with this work; if not, write to the Free Software Foundation,
* Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
*/

/**
* A regular expression literal in the positions the ES6 lookahead passes over.
*
* A "/" is ambiguous until the grammar resolves it, so the lexer stops right
* after one and lets the parser rescan it. The arrow and destructuring-assignment
* lookaheads run before that happens, so each of these positions is one the
* lookahead has to leave untouched.
*
* @test
* @run
* @option --language=es6
*/

// A parenthesised expression - where the arrow lookahead starts.
print((/a+/).source);
print((/ab+c/.test("abbc")));

// An array literal and an object literal - where the destructuring-assignment
// lookahead starts.
print([/b+/][0].source);
print({ re: /c+/ }.re.source);
print([1, /d+/][1].source);

// Not the first token in the brackets, so the lookahead has already scanned past
// something else by the time it reaches the slash.
print({ a: 1, re: /e+/ }.re.source);

// Inside a conditional, which the arrow lookahead scans through.
var cond = true;
print((cond ? /f+/ : 1).source);

// Nested one function deeper, so the enclosing function is re-parsed on demand.
print([1].map(function () { return (/g+/).source; })[0]);

// A default value in arrow parameters. Division there has always worked; a regular
// expression is the same position and now works too.
var half = (a = 1 / 2) => a;
print(half());

var re = (a = /h+/) => a;
print(re().source);

var sum = (a, b = 4 / 2) => a + b;
print(sum(1));

// Ordinary division is still division, not the start of a literal.
print(10 / 2 / 1);
12 changes: 12 additions & 0 deletions test/script/basic/es6/regex-literals.js.EXPECTED
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
a+
true
b+
c+
d+
e+
f+
g+
0.5
h+
3
5