From 64bcec643528e84cf1738c0a39e900c1773e6627 Mon Sep 17 00:00:00 2001 From: Lukasz Samson Date: Fri, 7 Feb 2025 14:40:46 +0100 Subject: [PATCH 1/7] fix: Eat semicolon after eol --- lib/spitfire.ex | 5 ++++- test/spitfire_test.exs | 12 ++++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/lib/spitfire.ex b/lib/spitfire.ex index 19b1c8a..91a9c88 100644 --- a/lib/spitfire.ex +++ b/lib/spitfire.ex @@ -2326,7 +2326,10 @@ defmodule Spitfire do end defp eat_eol(parser) do - eat(%{:eol => true, :";" => true}, parser) + case eat(%{:eol => true, :";" => true}, parser) do + %{current_token: {token, _}} = parser when token in [:eol, :";"] -> eat_eol(parser) + parser -> parser + end end defp eat_eol_at(parser, idx) do diff --git a/test/spitfire_test.exs b/test/spitfire_test.exs index afee842..009d5bf 100644 --- a/test/spitfire_test.exs +++ b/test/spitfire_test.exs @@ -42,6 +42,18 @@ defmodule SpitfireTest do # assert Spitfire.parse(code) == s2q(code) end + test "semicolon in block" do + code = """ + defmodule MyModule do + import List + + ;(__cursor__()) + end + """ + + assert Spitfire.parse(code) == s2q(code) + end + test "parses valid elixir" do code = """ defmodule Foo do From 9a18f1c0175eaf73d83faf0fb323b2ccdd7cdb0a Mon Sep 17 00:00:00 2001 From: Lukasz Samson Date: Fri, 7 Feb 2025 14:41:54 +0100 Subject: [PATCH 2/7] fix: Handle semicolon in peek_token_eat_eol --- lib/spitfire.ex | 2 +- test/spitfire_test.exs | 8 ++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/lib/spitfire.ex b/lib/spitfire.ex index 91a9c88..5edd861 100644 --- a/lib/spitfire.ex +++ b/lib/spitfire.ex @@ -2392,7 +2392,7 @@ defmodule Spitfire do :eof end - defp peek_token_eat_eol(%{peek_token: {:eol, _token}} = parser) do + defp peek_token_eat_eol(%{peek_token: {token, _token}} = parser) when token in [:eol, :";"] do peek_token_eat_eol(next_token(parser)) end diff --git a/test/spitfire_test.exs b/test/spitfire_test.exs index 009d5bf..d4c5185 100644 --- a/test/spitfire_test.exs +++ b/test/spitfire_test.exs @@ -1737,6 +1737,14 @@ defmodule SpitfireTest do assert Spitfire.parse(code) == s2q(code) end + test "default args semicolons" do + code = ~S''' + def foo(arg \\ :value) do; :ok; end + ''' + + assert Spitfire.parse(code) == s2q(code) + end + test "literal encoder" do codes = [ ~S''' From a14b6ba87b7719d6b76b1664cbd71d10edb03756 Mon Sep 17 00:00:00 2001 From: Lukasz Samson Date: Fri, 7 Feb 2025 15:06:48 +0100 Subject: [PATCH 3/7] docs: Add comment --- lib/spitfire.ex | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/spitfire.ex b/lib/spitfire.ex index 5edd861..dcaaeb3 100644 --- a/lib/spitfire.ex +++ b/lib/spitfire.ex @@ -2604,6 +2604,7 @@ defmodule Spitfire do nil end + # NOTE no need to handle ; here as this function is used only for building metadata and counting newlines there defp peek_newlines(%{peek_token: {:eol, {_line, _col, newlines}}}) when is_integer(newlines) do newlines end From ea645ba79520474281ff510067937a6949779c64 Mon Sep 17 00:00:00 2001 From: Lukasz Samson Date: Fri, 7 Feb 2025 15:10:24 +0100 Subject: [PATCH 4/7] fix: Handle semicolon on start --- lib/spitfire.ex | 2 +- test/spitfire_test.exs | 9 +++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/lib/spitfire.ex b/lib/spitfire.ex index dcaaeb3..7be4948 100644 --- a/lib/spitfire.ex +++ b/lib/spitfire.ex @@ -133,7 +133,7 @@ defmodule Spitfire do # eat all the beginning eol tokens in case the file starts with a comment parser = - while current_token(parser) == :eol <- parser do + while current_token(parser) in [:eol, :";"] <- parser do next_token(parser) end diff --git a/test/spitfire_test.exs b/test/spitfire_test.exs index d4c5185..ec1a3e0 100644 --- a/test/spitfire_test.exs +++ b/test/spitfire_test.exs @@ -1727,6 +1727,15 @@ defmodule SpitfireTest do assert Spitfire.parse(code) == s2q(code) end + test "starts with a semicolon" do + code = """ + ; + some_code = :foo + """ + + assert Spitfire.parse(code) == s2q(code) + end + test "default args" do code = ~S''' def foo(arg \\ :value) do From 0a1ef194bc14e417660ee79b72224e1d98302fef Mon Sep 17 00:00:00 2001 From: Lukasz Samson Date: Fri, 7 Feb 2025 15:22:19 +0100 Subject: [PATCH 5/7] docs: Add comments --- lib/spitfire.ex | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/lib/spitfire.ex b/lib/spitfire.ex index 7be4948..d014fc3 100644 --- a/lib/spitfire.ex +++ b/lib/spitfire.ex @@ -230,6 +230,8 @@ defmodule Spitfire do precedence < power end + # TODO Is anything needed here? Removing all of these tokens does not break any tests + # TODO Should we add :";" to the list? @terminals MapSet.new([:eol, :eof, :"}", :")", :"]", :">>"]) @terminals_with_comma MapSet.put(@terminals, :",") defp(parse_expression(parser, assoc \\ @lowest, is_list \\ false, is_map \\ false, is_top \\ false, is_stab \\ false)) @@ -308,9 +310,10 @@ defmodule Spitfire do else @terminals_with_comma end - + {parser, is_valid} = validate_peek(parser, current_token_type(parser)) - + # TODO should we handle ; here? + # TODO removing peek_token(parser) != :eol does not break any tests if is_valid do while (is_nil(Map.get(parser, :stab_state)) and not MapSet.member?(terminals, peek_token(parser))) && (current_token(parser) != :do and peek_token(parser) != :eol) && From 485c503759a509fb127b1787dd02c3261e34eedb Mon Sep 17 00:00:00 2001 From: Lukasz Samson Date: Fri, 7 Feb 2025 15:36:13 +0100 Subject: [PATCH 6/7] docs: Add comment --- lib/spitfire.ex | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/spitfire.ex b/lib/spitfire.ex index d014fc3..d5d512a 100644 --- a/lib/spitfire.ex +++ b/lib/spitfire.ex @@ -1670,6 +1670,9 @@ defmodule Spitfire do else {left, parser} = prefix.(parser) + # TODO should we add ; here? + # TODO is anything needed? Removing all tokens does not break any tests + terminals = [:eol, :eof, :"}", :")", :"]", :">>"] terminals = [:eol, :eof, :"}", :")", :"]", :">>"] {parser, is_valid} = validate_peek(parser, current_token_type(parser)) From 17d4b8f03fbd42ba410f93297cfe1bea836762ef Mon Sep 17 00:00:00 2001 From: Lukasz Samson Date: Fri, 7 Feb 2025 15:41:55 +0100 Subject: [PATCH 7/7] fix: Handle semicolon in parse_grouped_expression Add comment --- lib/spitfire.ex | 12 ++++++------ test/spitfire_test.exs | 18 ++++++++++++++++++ 2 files changed, 24 insertions(+), 6 deletions(-) diff --git a/lib/spitfire.ex b/lib/spitfire.ex index d5d512a..f12b374 100644 --- a/lib/spitfire.ex +++ b/lib/spitfire.ex @@ -310,7 +310,7 @@ defmodule Spitfire do else @terminals_with_comma end - + {parser, is_valid} = validate_peek(parser, current_token_type(parser)) # TODO should we handle ; here? # TODO removing peek_token(parser) != :eol does not break any tests @@ -399,7 +399,7 @@ defmodule Spitfire do cond do # if the next token is the closing paren or if the next token is a newline and the next next token is the closing paren - peek_token(parser) == :")" || (peek_token(parser) == :eol && peek_token(next_token(parser)) == :")") -> + peek_token(parser) == :")" || (peek_token(parser) in [:eol, :";"] && peek_token(next_token(parser)) == :")") -> parser = parser |> Map.put(:nesting, old_nesting) @@ -431,11 +431,11 @@ defmodule Spitfire do {ast, parser} # if the next token is a new line, but the next next token is not the closing paren (implied from previous clause) - peek_token(parser) == :eol or current_token(parser) == :-> -> + peek_token(parser) in [:eol, :";"] or current_token(parser) == :-> -> # second conditon checks of the next next token is a closing paren or another expression {exprs, parser} = while2 current_token(parser) == :-> || - (peek_token(parser) == :eol && parser |> next_token() |> peek_token() != :")") <- parser do + (peek_token(parser) in [:eol, :";"] && parser |> next_token() |> peek_token() != :")") <- parser do {ast, parser} = case Map.get(parser, :stab_state) do %{ast: lhs} -> @@ -482,7 +482,7 @@ defmodule Spitfire do # handles if the closing paren is on a new line or the same line parser = - if peek_token(parser) == :eol do + if peek_token(parser) in [:eol, :";"] do next_token(parser) else parser @@ -1673,7 +1673,6 @@ defmodule Spitfire do # TODO should we add ; here? # TODO is anything needed? Removing all tokens does not break any tests terminals = [:eol, :eof, :"}", :")", :"]", :">>"] - terminals = [:eol, :eof, :"}", :")", :"]", :">>"] {parser, is_valid} = validate_peek(parser, current_token_type(parser)) @@ -2331,6 +2330,7 @@ defmodule Spitfire do parser end + # TODO this may be too greedy. Probably the better option would be to have distinct eat_eol/1 and eat_eol_or_semicolon/1 defp eat_eol(parser) do case eat(%{:eol => true, :";" => true}, parser) do %{current_token: {token, _}} = parser when token in [:eol, :";"] -> eat_eol(parser) diff --git a/test/spitfire_test.exs b/test/spitfire_test.exs index ec1a3e0..6d62862 100644 --- a/test/spitfire_test.exs +++ b/test/spitfire_test.exs @@ -427,6 +427,9 @@ defmodule SpitfireTest do !false ) ''', + ~S''' + (; !false;) + ''', "(not false)", ~S''' a do @@ -1007,6 +1010,21 @@ defmodule SpitfireTest do assert Spitfire.parse(code) == s2q(code) + code = ~S''' + ( + min_line = line(meta); max_line = closing_line(meta); Enum.any?(comments, fn %{line: line} -> line > min_line and line < max_line end) + ) + ''' + + assert Spitfire.parse(code) == s2q(code) + + code = ~S''' + ( + min_line = line(meta); max_line = closing_line(meta); Enum.any?(comments, fn %{line: line} -> line > min_line and line < max_line end); ) + ''' + + assert Spitfire.parse(code) == s2q(code) + code = ~S''' (min_line = line(meta) max_line = closing_line(meta)