fix(pglt_statement_splitter): properly split for windows (#216)

* chore(pglt_statement_splitter): add test for comments

* debug

* debug

* debug

* debug

* maybe fix?

* debug

* debug

* fix?

* fix?

* cleanup

* fix: weorkflow

* fix: weorkflow

* chore: add test for consecutive newlines

* fix: regex

Author: psteinroe

.github/workflows/publish.dispatch.yml

@@ -39,7 +39,7 @@ jobs:
                 return;
               }
 
-              if (releases.length < 100) { 
+              if (releases.length < 100) {
                 exhausted = true;
               } else if (page >= 10) {
                 throw new Error("We iterated over 10 pages. Does the script work?");
@@ -55,8 +55,8 @@ jobs:
       - name: Abort
         if: steps.validate-release.outputs.has-release != 'true'
         run: |
-          { 
-            echo "Tag ${{ github.event.inputs.release_tag }} not found."
+          {
+            echo "Tag ${{ github.event.inputs.release-tag }} not found."
             exit 1
           }
 

.github/workflows/publish.reusable.yml

@@ -9,7 +9,6 @@ on:
       is-prerelease:
         type: string
         required: true
-        default: "false"
 
 jobs:
   publish:
@@ -52,17 +51,17 @@ jobs:
       - name: Publish npm packages as nightly
         if: inputs.is-prerelease == 'true'
         run: |
-          for package in packages/@pglt/*; do 
-            npm publish $package --tag nightly --access public --provenance
+          for package in packages/@pglt/*; do
+            npm publish "$package" --tag nightly --access public --provenance
           done
         env:
           NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} #
 
       - name: Publish npm packages as latest
         if: inputs.is-prerelease != 'true'
         run: |
-          for package in packages/@pglt/*; do 
-            npm publish $package --tag latest --access public --provenance
+          for package in packages/@pglt/*; do
+            npm publish "$package" --tag latest --access public --provenance
           done
         env:
           NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}

crates/pglt_lexer/src/lib.rs

@@ -61,8 +61,18 @@ pub static WHITESPACE_TOKENS: &[SyntaxKind] = &[
     SyntaxKind::SqlComment,
 ];
 
-static PATTERN_LEXER: LazyLock<Regex> =
-    LazyLock::new(|| Regex::new(r"(?P<whitespace> +)|(?P<newline>\r?\n+)|(?P<tab>\t+)").unwrap());
+static PATTERN_LEXER: LazyLock<Regex> = LazyLock::new(|| {
+    #[cfg(windows)]
+    {
+        // On Windows, treat \r\n as a single newline token
+        Regex::new(r"(?P<whitespace> +)|(?P<newline>(\r\n|\n)+)|(?P<tab>\t+)").unwrap()
+    }
+    #[cfg(not(windows))]
+    {
+        // On other platforms, just check for \n
+        Regex::new(r"(?P<whitespace> +)|(?P<newline>\n+)|(?P<tab>\t+)").unwrap()
+    }
+});
 
 fn whitespace_tokens(input: &str) -> VecDeque<Token> {
     let mut tokens = VecDeque::new();
@@ -202,6 +212,22 @@ mod tests {
         assert_eq!(tokens[1].kind, SyntaxKind::Newline);
     }
 
+    #[test]
+    fn test_consecutive_newlines() {
+        // Test with multiple consecutive newlines
+        #[cfg(windows)]
+        let input = "select\r\n\r\n1";
+        #[cfg(not(windows))]
+        let input = "select\n\n1";
+
+        let tokens = lex(input).unwrap();
+
+        // Check that we have exactly one newline token between "select" and "1"
+        assert_eq!(tokens[0].kind, SyntaxKind::Select);
+        assert_eq!(tokens[1].kind, SyntaxKind::Newline);
+        assert_eq!(tokens[2].kind, SyntaxKind::Iconst);
+    }
+
     #[test]
     fn test_whitespace_tokens() {
         let input = "select 1";

crates/pglt_statement_splitter/src/lib.rs

@@ -114,6 +114,12 @@ mod tests {
         ]);
     }
 
+    #[test]
+    fn single_newlines() {
+        Tester::from("select 1\nfrom contact\n\nselect 3")
+            .expect_statements(vec!["select 1\nfrom contact", "select 3"]);
+    }
+
     #[test]
     fn alter_column() {
         Tester::from("alter table users alter column email drop not null;")

crates/pglt_statement_splitter/src/parser.rs

@@ -183,6 +183,17 @@ impl Parser {
     }
 }
 
+#[cfg(windows)]
+/// Returns true if the token is relevant for the paring process
+///
+/// On windows, a newline is represented by `\r\n` which is two characters.
+fn is_irrelevant_token(t: &Token) -> bool {
+    WHITESPACE_TOKENS.contains(&t.kind)
+        && (t.kind != SyntaxKind::Newline || t.text == "\r\n" || t.text.chars().count() == 1)
+}
+
+#[cfg(not(windows))]
+/// Returns true if the token is relevant for the paring process
 fn is_irrelevant_token(t: &Token) -> bool {
     WHITESPACE_TOKENS.contains(&t.kind)
         && (t.kind != SyntaxKind::Newline || t.text.chars().count() == 1)

crates/pglt_statement_splitter/tests/data/with_comments__4.sql

@@ -0,0 +1,13 @@
+-- test
+select id, name, test1231234123, unknown from co;
+
+-- in between two statements
+
+select 14433313331333 -- after a statement
+
+alter table --within a statement
+test drop column id;
+
+select lower('test');
+--after a statement
+