Skip to content

Commit 962360b

Browse files
committed
fix token delimiter replacement for all occurences
1 parent d41550e commit 962360b

File tree

1 file changed

+2
-1
lines changed

1 file changed

+2
-1
lines changed

tokenizers/javascript/tokenizer.js

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -51,8 +51,9 @@ const tokenizer = function(code, parentId, blockId) {
5151
// TODO: refactor these
5252
const tokens = immutable.List(tokensRaw).flatMap((token) => {
5353
if (token.value.indexOf(TOKEN_DELIMITER) != -1)
54+
tokenDelimiters = new RegExep(TOKEN_DELIMITER, 'g')
5455
token.value =
55-
token.value.replace(TOKEN_DELIMITER, TOKEN_DELIMITER_REPLACEMENT)
56+
token.value.replace(tokenDelimiters, TOKEN_DELIMITER_REPLACEMENT)
5657

5758
// NOTE: get rid of all whitespaces, dey sak
5859
if (token.value.search(WHITESPACES) != -1)

0 commit comments

Comments
 (0)