Provide mechanism for Julia syntax evolution

Compiler/test/inference.jl

@@ -6463,6 +6463,7 @@ end == TypeError
 # Issue #58257 - Hang in inference during BindingPartition resolution
 module A58257
     module B58257
+        const age = Base.get_world_counter()
         using ..A58257
         # World age here is N
     end
@@ -6474,7 +6475,7 @@ end
 ## The sequence of events is critical here.
 A58257.get!      # Creates binding partition in A, N+1:∞
 A58257.B58257.get!    # Creates binding partition in A.B, N+1:∞
-Base.invoke_in_world(UInt(38678), getglobal, A58257, :get!) # Expands binding partition in A through <N
+Base.invoke_in_world(A58257.B58257.age, getglobal, A58257, :get!) # Expands binding partition in A through <N
 @test Base.infer_return_type(A58257.f) == typeof(Base.get!) # Attempt to lookup A.B in world age N hangs
 
 function tt57873(a::Vector{String}, pref)

JuliaLowering/src/ast.jl

@@ -175,6 +175,7 @@ function makeleaf(ctx, srcref, k::Kind, value; kws...)
               k == K"Char"    ? convert(Char,    value) :
               k == K"Value"   ? value                   :
               k == K"Bool"    ? value                   :
+              k == K"VERSION" ? value                   :
               error("Unexpected leaf kind `$k`")
         makeleaf(graph, srcref, k; value=val, kws...)
     end

JuliaLowering/src/compat.jl

@@ -211,6 +211,9 @@ function _insert_convert_expr(@nospecialize(e), graph::SyntaxGraph, src::SourceA
         id_inner = _insert_tree_node(graph, K"String", src; value=e)
         setchildren!(graph, st_id, [id_inner])
         return st_id, src
+    elseif e isa VersionNumber
+        st_id = _insert_tree_node(graph, K"VERSION", src, JS.set_numeric_flags(e.minor*10); value=e)
+        return st_id, src
     elseif !(e isa Expr)
         # There are other kinds we could potentially back-convert (e.g. Float),
         # but Value should work fine.
@@ -398,11 +401,14 @@ function _insert_convert_expr(@nospecialize(e), graph::SyntaxGraph, src::SourceA
             child_exprs[2] = maybe_unwrap_arg(e.args[2])
         end
     elseif e.head === :module
-        @assert nargs === 3
-        if !e.args[1]
+        @assert nargs in (3, 4)
+        has_version = !isa(e.args[1], Bool)
+        if !e.args[1+has_version]
             st_flags |= JS.BARE_MODULE_FLAG
         end
-        child_exprs = Any[e.args[2], e.args[3]]
+        child_exprs = has_version ?
+            Any[e.args[1], e.args[2+has_version], e.args[3+has_version]] :
+            Any[e.args[2+has_version], e.args[3+has_version]]
     elseif e.head === :do
         # Expr:
         # (do (call f args...) (-> (tuple lam_args...) (block ...)))

JuliaLowering/src/eval.jl

@@ -78,19 +78,26 @@ function lower_step(iter, push_mod=nothing)
         push!(iter.todo, (ex, false, 1))
         return lower_step(iter)
     elseif k == K"module"
-        name = ex[1]
+        name_or_version = ex[1]
+        version = nothing
+        if kind(name_or_version) == K"VERSION"
+            version = name_or_version.value
+            name = ex[2]
+        else
+            name = name_or_version
+        end
         if kind(name) != K"Identifier"
             throw(LoweringError(name, "Expected module name"))
         end
         newmod_name = Symbol(name.name_val)
-        body = ex[2]
+        body = ex[end]
         if kind(body) != K"block"
             throw(LoweringError(body, "Expected block in module body"))
         end
         std_defs = !has_flags(ex, JuliaSyntax.BARE_MODULE_FLAG)
         loc = source_location(LineNumberNode, ex)
         push!(iter.todo, (body, true, 1))
-        return Core.svec(:begin_module, newmod_name, std_defs, loc)
+        return Core.svec(:begin_module, version, newmod_name, std_defs, loc)
     else
         # Non macro expansion parts of lowering
         ctx2, ex2 = expand_forms_2(iter.ctx, ex)
@@ -480,9 +487,9 @@ function _eval(mod, iter)
             break
         elseif type == :begin_module
             push!(modules, mod)
-            filename = something(thunk[4].file, :none)
-            mod = @ccall jl_begin_new_module(mod::Any, thunk[2]::Symbol, thunk[3]::Cint,
-                                             filename::Cstring, thunk[4].line::Cint)::Module
+            filename = something(thunk[5].file, :none)
+            mod = @ccall jl_begin_new_module(mod::Any, thunk[3]::Symbol, thunk[2]::Any, thunk[4]::Cint,
+                                             filename::Cstring, thunk[5].line::Cint)::Module
             new_mod = mod
         elseif type == :end_module
             @ccall jl_end_new_module(mod::Module)::Cvoid
@@ -510,10 +517,11 @@ function _eval(mod, iter, new_mod=nothing)
             @assert !in_new_mod
             break
         elseif type == :begin_module
-            name = thunk[2]::Symbol
-            std_defs = thunk[3]
+            version = thunk[2]
+            name = thunk[3]::Symbol
+            std_defs = thunk[4]
             result = Core.eval(mod,
-                Expr(:module, std_defs, name,
+                Expr(:module, (version === nothing ? () : (version,))..., std_defs, name,
                      Expr(:block, thunk[4], Expr(:call, m->_eval(m, iter, m), name)))
             )
         elseif type == :end_module

JuliaLowering/test/macros.jl

@@ -431,6 +431,7 @@ end
         )
     end
     """) ≈ @ast_ [K"module"
+        v"1.14.0"::K"VERSION"
         "AA"::K"Identifier"
         [K"block"
         ]

JuliaSyntax/src/integration/expr.jl

@@ -199,7 +199,7 @@ end
 
 function parseargs!(retexpr::Expr, loc::LineNumberNode, cursor, source, txtbuf::Vector{UInt8}, txtbuf_offset::UInt32)
     args = retexpr.args
-    firstchildhead = head(cursor)
+    firstchildhead = secondchildhead = head(cursor)
     firstchildrange::UnitRange{UInt32} = byte_range(cursor)
     itr = reverse_nontrivia_children(cursor)
     r = iterate(itr)
@@ -208,11 +208,12 @@ function parseargs!(retexpr::Expr, loc::LineNumberNode, cursor, source, txtbuf::
         r = iterate(itr, state)
         expr = node_to_expr(child, source, txtbuf, txtbuf_offset)
         @assert expr !== nothing
+        secondchildhead = firstchildhead
         firstchildhead = head(child)
         firstchildrange = byte_range(child)
         pushfirst!(args, fixup_Expr_child(head(cursor), expr, r === nothing))
     end
-    return (firstchildhead, firstchildrange)
+    return (firstchildhead, secondchildhead, firstchildrange)
 end
 
 _expr_leaf_val(node::SyntaxNode, _...) = node.val
@@ -235,6 +236,9 @@ function node_to_expr(cursor, source, txtbuf::Vector{UInt8}, txtbuf_offset::UInt
             return k == K"error" ?
                 Expr(:error) :
                 Expr(:error, "$(_token_error_descriptions[k]): `$(source[srcrange])`")
+        elseif k == K"VERSION"
+            nv = numeric_flags(flags(nodehead))
+            return VersionNumber(1, nv ÷ 10, nv % 10)
         else
             scoped_val = _expr_leaf_val(cursor, txtbuf, txtbuf_offset)
             val = @isexpr(scoped_val, :scope_layer) ? scoped_val.args[1] : scoped_val
@@ -292,10 +296,11 @@ function node_to_expr(cursor, source, txtbuf::Vector{UInt8}, txtbuf_offset::UInt
     end
 
     # Now recurse to parse all arguments
-    (firstchildhead, firstchildrange) = parseargs!(retexpr, loc, cursor, source, txtbuf, txtbuf_offset)
+    (firstchildhead, secondchildhead, firstchildrange) =
+        parseargs!(retexpr, loc, cursor, source, txtbuf, txtbuf_offset)
 
     return _node_to_expr(retexpr, loc, srcrange,
-                         firstchildhead, firstchildrange,
+                         firstchildhead, secondchildhead, firstchildrange,
                          nodehead, source)
 end
 
@@ -318,7 +323,7 @@ end
 # tree types.
 @noinline function _node_to_expr(retexpr::Expr, loc::LineNumberNode,
                                  srcrange::UnitRange{UInt32},
-                                 firstchildhead::SyntaxHead,
+                                 firstchildhead::SyntaxHead, secondchildhead::SyntaxHead,
                                  firstchildrange::UnitRange{UInt32},
                                  nodehead::SyntaxHead,
                                  source)
@@ -355,6 +360,11 @@ end
                 # Fix up for custom cmd macros like foo`x`
                 args[2] = a2.args[3]
             end
+            if kind(secondchildhead) == K"VERSION"
+                # Encode the syntax version into `loc` so that the argument order
+                # matches what ordinary macros expect.
+                loc = Core.MacroSource(loc, popat!(args, 2))
+            end
         end
         do_lambda = _extract_do_lambda!(args)
         _reorder_parameters!(args, 2)
@@ -554,8 +564,8 @@ end
             pushfirst!((args[2]::Expr).args, loc)
         end
     elseif k == K"module"
-        pushfirst!(args, !has_flags(nodehead, BARE_MODULE_FLAG))
-        pushfirst!((args[3]::Expr).args, loc)
+        insert!(args, kind(firstchildhead) == K"VERSION" ? 2 : 1, !has_flags(nodehead, BARE_MODULE_FLAG))
+        pushfirst!((args[end]::Expr).args, loc)
     elseif k == K"quote"
         if length(args) == 1
             a1 = only(args)

JuliaSyntax/src/integration/hooks.jl

@@ -162,7 +162,7 @@ end
 # Debug log file for dumping parsed code
 const _debug_log = Ref{Union{Nothing,IO}}(nothing)
 
-function core_parser_hook(code, filename::String, lineno::Int, offset::Int, options::Symbol)
+function core_parser_hook(code, filename::String, lineno::Int, offset::Int, options::Symbol; syntax_version = v"1.13")
     try
         # TODO: Check that we do all this input wrangling without copying the
         # code buffer
@@ -184,7 +184,7 @@ function core_parser_hook(code, filename::String, lineno::Int, offset::Int, opti
             write(_debug_log[], code)
         end
 
-        stream = ParseStream(code, offset+1)
+        stream = ParseStream(code, offset+1; version = syntax_version)
         if options === :statement || options === :atom
             # To copy the flisp parser driver:
             # * Parsing atoms      consumes leading trivia

JuliaSyntax/src/julia/kinds.jl

@@ -247,6 +247,7 @@ register_kinds!(JuliaSyntax, 0, [
             "public"
             "type"
             "var"
+            "VERSION"
         "END_CONTEXTUAL_KEYWORDS"
     "END_KEYWORDS"
 

JuliaSyntax/src/julia/literal_parsing.jl

@@ -408,6 +408,9 @@ function parse_julia_literal(txtbuf::Vector{UInt8}, head::SyntaxHead, srcrange)
         return had_error ? ErrorVal() : String(take!(io))
     elseif k == K"Bool"
         return txtbuf[first(srcrange)] == u8"t"
+    elseif k == K"VERSION"
+        nv = numeric_flags(head)
+        return VersionNumber(1, nv ÷ 10, nv % 10)
     end
 
     # TODO: Avoid allocating temporary String here

JuliaSyntax/src/julia/parser.jl

@@ -1488,13 +1488,23 @@ function parse_unary_prefix(ps::ParseState, has_unary_prefix=false)
     end
 end
 
-function maybe_parsed_macro_name(ps, processing_macro_name, mark)
+function maybe_parsed_macro_name(ps, processing_macro_name, last_identifier_orig_kind, mark)
     if processing_macro_name
         emit(ps, mark, K"macro_name")
+        maybe_parsed_special_macro(ps, last_identifier_orig_kind)
     end
     return false
 end
 
+function maybe_parsed_special_macro(ps, last_identifier_orig_kind)
+    is_syntax_version_macro = last_identifier_orig_kind == K"VERSION"
+    if is_syntax_version_macro && ps.stream.version >= (1, 14)
+        # Encode the current parser version into an invisible token
+        bump_invisible(ps, K"VERSION",
+            set_numeric_flags(ps.stream.version[2] * 10))
+    end
+end
+
 # Parses a chain of suffixes at function call precedence, leftmost binding
 # tightest. This handles
 #  * Bracketed calls like a() b[] c{}
@@ -1543,7 +1553,7 @@ function parse_call_chain(ps::ParseState, mark, is_macrocall=false)
             # @+x y       ==> (macrocall (macro_name +) x y)
             # [email protected]       ==> (macrocall (. A (macro_name .)) x)
             processing_macro_name = maybe_parsed_macro_name(
-                ps, processing_macro_name, mark)
+                ps, processing_macro_name, last_identifier_orig_kind, mark)
             let ps = with_space_sensitive(ps)
                 # Space separated macro arguments
                 # A.@foo a b    ==> (macrocall (. A (macro_name foo)) a b)
@@ -1577,7 +1587,7 @@ function parse_call_chain(ps::ParseState, mark, is_macrocall=false)
             # (a=1)()  ==>  (call (parens (= a 1)))
             # f (a)    ==>  (call f (error-t) a)
             processing_macro_name = maybe_parsed_macro_name(
-                ps, processing_macro_name, mark)
+                ps, processing_macro_name, last_identifier_orig_kind, mark)
             bump_disallowed_space(ps)
             bump(ps, TRIVIA_FLAG)
             opts = parse_call_arglist(ps, K")")
@@ -1598,7 +1608,7 @@ function parse_call_chain(ps::ParseState, mark, is_macrocall=false)
             end
         elseif k == K"["
             processing_macro_name = maybe_parsed_macro_name(
-                ps, processing_macro_name, mark)
+                ps, processing_macro_name, last_identifier_orig_kind, mark)
             m = position(ps)
             # a [i]  ==>  (ref a (error-t) i)
             bump_disallowed_space(ps)
@@ -1666,7 +1676,7 @@ function parse_call_chain(ps::ParseState, mark, is_macrocall=false)
                 if is_macrocall
                     # Recover by pretending we do have the syntax
                     processing_macro_name = maybe_parsed_macro_name(
-                        ps, processing_macro_name, mark)
+                        ps, processing_macro_name, last_identifier_orig_kind, mark)
                     # @M.(x)  ==> (macrocall (dotcall (macro_name M) (error-t) x))
                     bump_invisible(ps, K"error", TRIVIA_FLAG)
                     emit_diagnostic(ps, mark,
@@ -1720,6 +1730,7 @@ function parse_call_chain(ps::ParseState, mark, is_macrocall=false)
                 macro_atname_range = (m, position(ps))
                 is_macrocall = true
                 emit(ps, mark, K".")
+                maybe_parsed_special_macro(ps, last_identifier_orig_kind)
             elseif k == K"'"
                 # f.'  =>  (dotcall-post f (error '))
                 bump(ps, remap_kind=K"Identifier")  # bump '
@@ -1760,7 +1771,7 @@ function parse_call_chain(ps::ParseState, mark, is_macrocall=false)
             emit(ps, mark, K"call", POSTFIX_OP_FLAG)
         elseif k == K"{"
             processing_macro_name = maybe_parsed_macro_name(
-                ps, processing_macro_name, mark)
+                ps, processing_macro_name, last_identifier_orig_kind, mark)
             # Type parameter curlies and macro calls
             m = position(ps)
             # S {a} ==> (curly S (error-t) a)
@@ -2065,6 +2076,13 @@ function parse_resword(ps::ParseState)
             # module do \n end  ==>  (module (error do) (block))
             bump(ps, error="Invalid module name")
         else
+            if ps.stream.version >= (1, 14)
+                # Encode the parser version that parsed this module - the runtime
+                # will use this to set the same parser version for runtime `include`
+                # etc into this module.
+                bump_invisible(ps, K"VERSION",
+                    set_numeric_flags(ps.stream.version[2] * 10))
+            end
             # module $A end  ==>  (module ($ A) (block))
             parse_unary_prefix(ps)
         end

JuliaSyntax/src/julia/tokenize.jl

@@ -1245,12 +1245,12 @@ function lex_identifier(l::Lexer, c)
     end
 end
 
-# This creates a hash for chars in [a-z] using 5 bit per char.
+# This creates a hash for chars in [A-z] using 6 bit per char.
 # Requires an additional input-length check somewhere, because
-# this only works up to ~12 chars.
+# this only works up to ~10 chars.
 @inline function simple_hash(c::Char, h::UInt64)
-    bytehash = (clamp(c - 'a' + 1, -1, 30) % UInt8) & 0x1f
-    h << 5 + bytehash
+    bytehash = (clamp(c - 'A' + 1, -1, 60) % UInt8) & 0x3f
+    h << 6 + bytehash
 end
 
 function simple_hash(str)
@@ -1305,10 +1305,11 @@ K"outer",
 K"primitive",
 K"type",
 K"var",
+K"VERSION"
 ]
 
 const _true_hash = simple_hash("true")
 const _false_hash = simple_hash("false")
-const _kw_hash = Dict(simple_hash(lowercase(string(kw))) => kw for kw in kws)
+const _kw_hash = Dict(simple_hash(string(kw)) => kw for kw in kws)
 
 end # module

JuliaSyntax/test/expr.jl

@@ -55,7 +55,7 @@
             @test parsestmt("a;b") ==
                 Expr(:toplevel, :a, :b)
 
-            @test parsestmt("module A\n\nbody\nend") ==
+            @test parsestmt("module A\n\nbody\nend"; version=v"1.13") ==
                 Expr(:module,
                      true,
                      :A,
@@ -798,9 +798,11 @@
     end
 
     @testset "module" begin
-        @test parsestmt("module A end") ==
+        @test parsestmt("module A end"; version=v"1.13") ==
             Expr(:module, true,  :A, Expr(:block, LineNumberNode(1), LineNumberNode(1)))
-        @test parsestmt("baremodule A end") ==
+        @test parsestmt("module A end"; version=v"1.14") ==
+            Expr(:module, v"1.14", true,  :A, Expr(:block, LineNumberNode(1), LineNumberNode(1)))
+        @test parsestmt("baremodule A end"; version=v"1.13") ==
             Expr(:module, false, :A, Expr(:block, LineNumberNode(1), LineNumberNode(1)))
     end
 

JuliaSyntax/test/parser.jl

@@ -377,6 +377,14 @@ tests = [
         "@doc x\n\ny"  =>  "(macrocall (macro_name doc) x)"
         "@doc x\nend"  =>  "(macrocall (macro_name doc) x)"
 
+        # Special 1.14 @VERSION parsing rules
+        ((v=v"1.13",), "@VERSION")        =>  "(macrocall (macro_name VERSION))"
+        ((v=v"1.13",), "@A.B.VERSION")    =>  "(macrocall (macro_name (. (. A B) VERSION)))"
+        ((v=v"1.13",), "A.B.@VERSION")     =>  "(macrocall (. (. A B) (macro_name VERSION)))"
+        ((v=v"1.14",), "@VERSION")        =>  "(macrocall (macro_name VERSION) v\"1.14.0\")"
+        ((v=v"1.14",), "@A.B.VERSION")    =>  "(macrocall (macro_name (. (. A B) VERSION)) v\"1.14.0\")"
+        ((v=v"1.14",), "A.B.@VERSION")     =>  "(macrocall (. (. A B) (macro_name VERSION)) v\"1.14.0\")"
+
         # calls with brackets
         "f(a,b)"  => "(call f a b)"
         "f(a,)"   => "(call-, f a)"