diff --git a/src/kinds.jl b/src/kinds.jl index 6de2f26a..4b24d98b 100644 --- a/src/kinds.jl +++ b/src/kinds.jl @@ -112,8 +112,11 @@ const _kind_names = "BEGIN_ASSIGNMENTS" "=" "+=" + "+%=" "-=" # Also used for "−=" + "-%=" "*=" + "*%=" "/=" "//=" "|=" @@ -121,8 +124,11 @@ const _kind_names = "÷=" "%=" "<<=" + "<<%=" ">>=" + ">>%=" ">>>=" + ">>>%=" "\\=" "&=" ":=" @@ -623,7 +629,9 @@ const _kind_names = "BEGIN_PLUS" "\$" "+" + "+%" "-" # also used for "−" + "-%" "++" "⊕" "⊖" @@ -684,6 +692,7 @@ const _kind_names = # Level 10 "BEGIN_TIMES" "*" + "*%" "/" "÷" "%" @@ -773,6 +782,9 @@ const _kind_names = "<<" ">>" ">>>" + "<<%" + ">>%" + ">>>%" "END_BITSHIFTS" # Level 13 diff --git a/src/tokenize.jl b/src/tokenize.jl index 9c19c040..0085908a 100644 --- a/src/tokenize.jl +++ b/src/tokenize.jl @@ -796,11 +796,23 @@ function lex_greater(l::Lexer) if accept(l, '>') if accept(l, '=') return emit(l, K">>>=") + elseif accept(l, '%') + if accept(l, '=') + return emit(l, K">>>%=") + else + return emit(l, K">>>%") + end else # >>>?, ? not a = return emit(l, K">>>") end elseif accept(l, '=') return emit(l, K">>=") + elseif accept(l, '%') + if accept(l, "=") + return emit(l, K">>%=") + else + return emit(l, K">>%") + end else return emit(l, K">>") end @@ -818,6 +830,12 @@ function lex_less(l::Lexer) if accept(l, '<') if accept(l, '=') return emit(l, K"<<=") + elseif accept(l, '%') + if accept(l, '=') + return emit(l, K"<<%=") + else + return emit(l, K"<<%") + end else # '<") elseif accept(l, '=') return emit(l, K"-=") + elseif accept(l, '%') + if accept(l, '=') + return emit(l, K"-%=") + else + return emit(l, K"-%") + end end return emit(l, K"-") end @@ -933,6 +963,12 @@ function lex_star(l::Lexer) return emit(l, K"Error**") # "**" is an invalid operator use ^ elseif accept(l, '=') return emit(l, K"*=") + elseif accept(l, '%') + if accept(l, '=') + return emit(l, K"*%=") + else + return emit(l, K"*%") + end end return emit(l, K"*") end diff --git a/test/fuzz_test.jl b/test/fuzz_test.jl index 15bfa79d..aefe305e 100644 --- a/test/fuzz_test.jl +++ b/test/fuzz_test.jl @@ -77,9 +77,12 @@ const all_tokens = [ "=" "+=" + "+%=" "-=" # Also used for "−=" + "-%=" "−=" "*=" + "*%=" "/=" "//=" "|=" @@ -87,8 +90,11 @@ const all_tokens = [ "÷=" "%=" "<<=" + "<<%=" ">>=" + ">>%=" ">>>=" + ">>>%=" "\\=" "&=" ":=" @@ -560,7 +566,9 @@ const all_tokens = [ "\$" "+" + "+%" "-" # also used for "−" + "-%" "−" "++" "⊕" @@ -618,6 +626,7 @@ const all_tokens = [ "¦" "*" + "*%" "/" "÷" "%" @@ -701,8 +710,11 @@ const all_tokens = [ "//" "<<" + "<<%" ">>" + ">>%" ">>>" + ">>>%" "^" "↑" diff --git a/test/tokenize.jl b/test/tokenize.jl index 26ab044a..c71a6139 100644 --- a/test/tokenize.jl +++ b/test/tokenize.jl @@ -166,6 +166,9 @@ end @testset "test added operators" begin @test tok("1+=2", 2).kind == K"+=" + @test tok("1+%=2", 2).kind == K"+%=" + @test tok("1-%=2", 2).kind == K"-%=" + @test tok("1*%=2", 2).kind == K"*%=" @test tok("1-=2", 2).kind == K"-=" @test tok("1:=2", 2).kind == K":=" @test tok("1*=2", 2).kind == K"*=" @@ -876,7 +879,7 @@ end @testset "new ops" begin ops = [ - raw"= += -= *= /= //= \= ^= ÷= %= <<= >>= >>>= |= &= ⊻= ≔ ⩴ ≕ ~ := $=" + raw"= += +%= -= -%= *= *%= /= //= \= ^= ÷= %= <<= <<%= >>= >>%= >>>= >>>%= |= &= ⊻= ≔ ⩴ ≕ ~ := $=" raw"=>" raw"?" raw"← → ↔ ↚ ↛ ↞ ↠ ↢ ↣ ↦ ↤ ↮ ⇎ ⇍ ⇏ ⇐ ⇒ ⇔ ⇴ ⇶ ⇷ ⇸ ⇹ ⇺ ⇻ ⇼ ⇽ ⇾ ⇿ ⟵ ⟶ ⟷ ⟹ ⟺ ⟻ ⟼ ⟽ ⟾ ⟿ ⤀ ⤁ ⤂ ⤃ ⤄ ⤅ ⤆ ⤇ ⤌ ⤍ ⤎ ⤏ ⤐ ⤑ ⤔ ⤕ ⤖ ⤗ ⤘ ⤝ ⤞ ⤟ ⤠ ⥄ ⥅ ⥆ ⥇ ⥈ ⥊ ⥋ ⥎ ⥐ ⥒ ⥓ ⥖ ⥗ ⥚ ⥛ ⥞ ⥟ ⥢ ⥤ ⥦ ⥧ ⥨ ⥩ ⥪ ⥫ ⥬ ⥭ ⥰ ⧴ ⬱ ⬰ ⬲ ⬳ ⬴ ⬵ ⬶ ⬷ ⬸ ⬹ ⬺ ⬻ ⬼ ⬽ ⬾ ⬿ ⭀ ⭁ ⭂ ⭃ ⭄ ⭇ ⭈ ⭉ ⭊ ⭋ ⭌ ← → ⇜ ⇝ ↜ ↝ ↩ ↪ ↫ ↬ ↼ ↽ ⇀ ⇁ ⇄ ⇆ ⇇ ⇉ ⇋ ⇌ ⇚ ⇛ ⇠ ⇢ ↷ ↶ ↺ ↻ -->" @@ -886,10 +889,10 @@ end raw"<|" raw"|>" raw": .. … ⁝ ⋮ ⋱ ⋰ ⋯" - raw"$ + - ¦ | ⊕ ⊖ ⊞ ⊟ ++ ∪ ∨ ⊔ ± ∓ ∔ ∸ ≏ ⊎ ⊻ ⊽ ⋎ ⋓ ⧺ ⧻ ⨈ ⨢ ⨣ ⨤ ⨥ ⨦ ⨧ ⨨ ⨩ ⨪ ⨫ ⨬ ⨭ ⨮ ⨹ ⨺ ⩁ ⩂ ⩅ ⩊ ⩌ ⩏ ⩐ ⩒ ⩔ ⩖ ⩗ ⩛ ⩝ ⩡ ⩢ ⩣" - raw"* / ⌿ ÷ % & ⋅ ∘ × \ ∩ ∧ ⊗ ⊘ ⊙ ⊚ ⊛ ⊠ ⊡ ⊓ ∗ ∙ ∤ ⅋ ≀ ⊼ ⋄ ⋆ ⋇ ⋉ ⋊ ⋋ ⋌ ⋏ ⋒ ⟑ ⦸ ⦼ ⦾ ⦿ ⧶ ⧷ ⨇ ⨰ ⨱ ⨲ ⨳ ⨴ ⨵ ⨶ ⨷ ⨸ ⨻ ⨼ ⨽ ⩀ ⩃ ⩄ ⩋ ⩍ ⩎ ⩑ ⩓ ⩕ ⩘ ⩚ ⩜ ⩞ ⩟ ⩠ ⫛ ⊍ ▷ ⨝ ⟕ ⟖ ⟗" + raw"$ + +% - -% ¦ | ⊕ ⊖ ⊞ ⊟ ++ ∪ ∨ ⊔ ± ∓ ∔ ∸ ≏ ⊎ ⊻ ⊽ ⋎ ⋓ ⧺ ⧻ ⨈ ⨢ ⨣ ⨤ ⨥ ⨦ ⨧ ⨨ ⨩ ⨪ ⨫ ⨬ ⨭ ⨮ ⨹ ⨺ ⩁ ⩂ ⩅ ⩊ ⩌ ⩏ ⩐ ⩒ ⩔ ⩖ ⩗ ⩛ ⩝ ⩡ ⩢ ⩣" + raw"* *% / ⌿ ÷ % & ⋅ ∘ × \ ∩ ∧ ⊗ ⊘ ⊙ ⊚ ⊛ ⊠ ⊡ ⊓ ∗ ∙ ∤ ⅋ ≀ ⊼ ⋄ ⋆ ⋇ ⋉ ⋊ ⋋ ⋌ ⋏ ⋒ ⟑ ⦸ ⦼ ⦾ ⦿ ⧶ ⧷ ⨇ ⨰ ⨱ ⨲ ⨳ ⨴ ⨵ ⨶ ⨷ ⨸ ⨻ ⨼ ⨽ ⩀ ⩃ ⩄ ⩋ ⩍ ⩎ ⩑ ⩓ ⩕ ⩘ ⩚ ⩜ ⩞ ⩟ ⩠ ⫛ ⊍ ▷ ⨝ ⟕ ⟖ ⟗" raw"//" - raw"<< >> >>>" + raw"<< <<% >> >>% >>> >>>%" raw"^ ↑ ↓ ⇵ ⟰ ⟱ ⤈ ⤉ ⤊ ⤋ ⤒ ⤓ ⥉ ⥌ ⥍ ⥏ ⥑ ⥔ ⥕ ⥘ ⥙ ⥜ ⥝ ⥠ ⥡ ⥣ ⥥ ⥮ ⥯ ↑ ↓" raw"::" raw"."