From b59b98b6b4d5f55b0851e1cb3ed4d991f1bc0d3a Mon Sep 17 00:00:00 2001 From: Em Chu Date: Tue, 11 Feb 2025 13:42:27 -0800 Subject: [PATCH 1/3] Parse arr[end] differently from arr[var"end"] --- src/expr.jl | 4 ++++ src/kinds.jl | 4 ++++ src/parser.jl | 10 +++++++++- 3 files changed, 17 insertions(+), 1 deletion(-) diff --git a/src/expr.jl b/src/expr.jl index 638e0b75..3fc071cb 100644 --- a/src/expr.jl +++ b/src/expr.jl @@ -76,6 +76,10 @@ function _leaf_to_Expr(source, txtbuf, txtbuf_offset, head, srcrange, node) return k == K"error" ? Expr(:error) : Expr(:error, "$(_token_error_descriptions[k]): `$(source[srcrange])`") + elseif k == K"RefBegin" + return Expr(:begin) + elseif k == K"RefEnd" + return Expr(:end) else val = isnothing(node) ? parse_julia_literal(txtbuf, head, srcrange .+ txtbuf_offset) : diff --git a/src/kinds.jl b/src/kinds.jl index c7d27e35..e946790e 100644 --- a/src/kinds.jl +++ b/src/kinds.jl @@ -198,6 +198,10 @@ register_kinds!(JuliaSyntax, 0, [ # macro name may not appear as characters in the source: The `@` may be # detached from the macro name as in `@A.x` (ugh!!), or have a _str or _cmd # suffix appended. + "BEGIN_REF_IDENTIFIERS" + "RefBegin" + "RefEnd" + "BEGIN_REF_IDENTIFIERS" "BEGIN_MACRO_NAMES" "MacroName" "StringMacroName" diff --git a/src/parser.jl b/src/parser.jl index d1a91478..a00835fe 100644 --- a/src/parser.jl +++ b/src/parser.jl @@ -15,7 +15,7 @@ struct ParseState space_sensitive::Bool # Seeing `for` stops parsing macro arguments and makes a generator for_generator::Bool - # Treat 'end' like a normal symbol instead of a reserved word + # Treat begin/end like special symbols instead of reserved words end_symbol::Bool # Treat newline like ordinary whitespace instead of as a potential separator whitespace_newline::Bool @@ -3627,6 +3627,14 @@ function parse_atom(ps::ParseState, check_identifiers=true, has_unary_prefix=fal elseif check_identifiers && is_closing_token(ps, leading_kind) # :(end) ==> (quote-: (error end)) bump(ps, error="invalid identifier") + elseif ps.end_symbol && leading_kind in KSet"begin end" + # https://github.com/JuliaLang/julia/issues/57269 + # Parse a[begin] differently from a[var"begin"] + if leading_kind == K"begin" + bump(ps, remap_kind=K"RefBegin") + elseif leading_kind == K"end" + bump(ps, remap_kind=K"RefEnd") + end else # Remap keywords to identifiers. # :end ==> (quote-: end) From beede64dd37e8732eb4a7e544452c83b56741489 Mon Sep 17 00:00:00 2001 From: Em Chu Date: Wed, 12 Feb 2025 09:41:22 -0800 Subject: [PATCH 2/3] Remove unique ref kinds, specify version TODO: Tests need fixing --- src/expr.jl | 4 ++-- src/kinds.jl | 4 ---- src/parser.jl | 8 ++------ test/parser.jl | 6 ++++++ 4 files changed, 10 insertions(+), 12 deletions(-) diff --git a/src/expr.jl b/src/expr.jl index 3fc071cb..8bbebaa2 100644 --- a/src/expr.jl +++ b/src/expr.jl @@ -76,9 +76,9 @@ function _leaf_to_Expr(source, txtbuf, txtbuf_offset, head, srcrange, node) return k == K"error" ? Expr(:error) : Expr(:error, "$(_token_error_descriptions[k]): `$(source[srcrange])`") - elseif k == K"RefBegin" + elseif k == K"begin" # begin/end as firstindex/lastindex return Expr(:begin) - elseif k == K"RefEnd" + elseif k == K"end" return Expr(:end) else val = isnothing(node) ? diff --git a/src/kinds.jl b/src/kinds.jl index e946790e..c7d27e35 100644 --- a/src/kinds.jl +++ b/src/kinds.jl @@ -198,10 +198,6 @@ register_kinds!(JuliaSyntax, 0, [ # macro name may not appear as characters in the source: The `@` may be # detached from the macro name as in `@A.x` (ugh!!), or have a _str or _cmd # suffix appended. - "BEGIN_REF_IDENTIFIERS" - "RefBegin" - "RefEnd" - "BEGIN_REF_IDENTIFIERS" "BEGIN_MACRO_NAMES" "MacroName" "StringMacroName" diff --git a/src/parser.jl b/src/parser.jl index a00835fe..95ac8d9c 100644 --- a/src/parser.jl +++ b/src/parser.jl @@ -3627,14 +3627,10 @@ function parse_atom(ps::ParseState, check_identifiers=true, has_unary_prefix=fal elseif check_identifiers && is_closing_token(ps, leading_kind) # :(end) ==> (quote-: (error end)) bump(ps, error="invalid identifier") - elseif ps.end_symbol && leading_kind in KSet"begin end" + elseif ps.end_symbol && leading_kind in KSet"begin end" && ps.stream.version >= (1, 12) # https://github.com/JuliaLang/julia/issues/57269 # Parse a[begin] differently from a[var"begin"] - if leading_kind == K"begin" - bump(ps, remap_kind=K"RefBegin") - elseif leading_kind == K"end" - bump(ps, remap_kind=K"RefEnd") - end + bump(ps) else # Remap keywords to identifiers. # :end ==> (quote-: end) diff --git a/test/parser.jl b/test/parser.jl index f208e24c..b43f8a8c 100644 --- a/test/parser.jl +++ b/test/parser.jl @@ -410,6 +410,9 @@ tests = [ "(a=1)[]" => "(ref (parens (= a 1)))" "a[end]" => "(ref a end)" "a[begin]" => "(ref a begin)" + "a[var\"end\"]" => "(ref a (var end))" + "a[var\"begin\"]" => "(ref a (var begin))" + "a[var\"test\"]" => "(ref a (var test))" "a[:(end)]" => "(typed_hcat a (quote-: (parens (error-t))) (error-t))" "T[x y]" => "(typed_hcat T x y)" "T[x ; y]" => "(typed_vcat T x y)" @@ -1153,6 +1156,9 @@ parsestmt_with_kind_tests = [ "a >>= b" => "(op= a::Identifier >>::Identifier b::Identifier)" ":+=" => "(quote-: +=::op=)" ":.+=" => "(quote-: (. +=::op=))" + # TODO: specify version 1.12 + # "a[begin]" => "(ref a::Identifier begin::begin)" + # "a[end]" => "(ref a::Identifier end::end)" ] @testset "parser `Kind` remapping" begin From 17be4e33b713d4e0f41f5ae3c5abcaef68bd9c90 Mon Sep 17 00:00:00 2001 From: Em Chu Date: Wed, 12 Feb 2025 17:11:55 -0800 Subject: [PATCH 3/3] Fix version (1.13) and tests --- src/parser.jl | 2 +- test/parser.jl | 13 +++++++++---- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/src/parser.jl b/src/parser.jl index 95ac8d9c..3d246677 100644 --- a/src/parser.jl +++ b/src/parser.jl @@ -3627,7 +3627,7 @@ function parse_atom(ps::ParseState, check_identifiers=true, has_unary_prefix=fal elseif check_identifiers && is_closing_token(ps, leading_kind) # :(end) ==> (quote-: (error end)) bump(ps, error="invalid identifier") - elseif ps.end_symbol && leading_kind in KSet"begin end" && ps.stream.version >= (1, 12) + elseif ps.end_symbol && leading_kind in KSet"begin end" && ps.stream.version >= (1, 13) # https://github.com/JuliaLang/julia/issues/57269 # Parse a[begin] differently from a[var"begin"] bump(ps) diff --git a/test/parser.jl b/test/parser.jl index b43f8a8c..37338335 100644 --- a/test/parser.jl +++ b/test/parser.jl @@ -1156,14 +1156,19 @@ parsestmt_with_kind_tests = [ "a >>= b" => "(op= a::Identifier >>::Identifier b::Identifier)" ":+=" => "(quote-: +=::op=)" ":.+=" => "(quote-: (. +=::op=))" - # TODO: specify version 1.12 - # "a[begin]" => "(ref a::Identifier begin::begin)" - # "a[end]" => "(ref a::Identifier end::end)" + ((v=v"1.13",), "a[begin]") => "(ref a::Identifier begin::begin)" + ((v=v"1.13",), "a[end]") => "(ref a::Identifier end::end)" ] @testset "parser `Kind` remapping" begin @testset "$(repr(input))" for (input, output) in parsestmt_with_kind_tests - input = ((show_kind=true,), input) + if !(input isa AbstractString) + opts, input_s = input + else + opts = NamedTuple() + input_s = input + end + input = ((show_kind=true, opts...), input_s) test_parse(JuliaSyntax.parse_stmts, input, output) end end