TASK: Fix Tokenizer not tokenizing all tokens

for example ExpressionNode::fromString('(true) ? 42 : "foo"')
would previously only tokenize `(true)` due to the brace

this is not an issue, when used in the return statement

Author: mhsdesign

src/Parser/Ast/ExpressionNode.php

@@ -126,7 +126,7 @@ public static function fromTokens(\Iterator &$tokens, Precedence $precedence = P
             );
         }
 
-        while ($tokens->valid()) {
+        while (!Scanner::isEnd($tokens)) {
             Scanner::skipSpaceAndComments($tokens);
 
             switch (Scanner::type($tokens)) {

src/Parser/Tokenizer/Scanner.php

@@ -169,8 +169,14 @@ public static function isEnd(\Iterator $tokens): bool
     /**
      * @param \Iterator<mixed,Token> $tokens
      */
-    public static function debugPrint(\Iterator $tokens): string
+    public static function debugPrint(\Iterator &$tokens): string
     {
+        $tokens = (function(): \Generator {
+            throw new \Exception('Once debugged, $tokens is empty.');
+            // @phpstan-ignore-next-line
+            yield;
+        })();
+
         $tokensAsArray = [];
         while ($tokens->valid()) {
             $tokensAsArray[] = [

src/Parser/Tokenizer/Tokenizer.php

@@ -44,7 +44,10 @@ public static function fromSource(Source $source): Tokenizer
      */
     public function getIterator(): \Iterator
     {
-        yield from self::block($this->source->getIterator());
+        $fragments = $this->source->getIterator();
+        while ($fragments->valid()) {
+            yield from self::block($fragments);
+        }
     }
 
     /**

test/Unit/Target/Php/Transpiler/TernaryOperation/TernaryOperationTranspilerTest.php

@@ -37,6 +37,7 @@ public function ternaryOperationExamples(): array
     {
         return [
             'true ? 42 : "foo"' => ['true ? 42 : "foo"', '(true ? 42 : \'foo\')'],
+            '(true) ? 42 : "foo"' => ['(true) ? 42 : "foo"', '(true ? 42 : \'foo\')'],
             'a ? 42 : "foo"' => ['a ? 42 : "foo"', '($this->a ? 42 : \'foo\')'],
             'true ? b : "foo"' => ['true ? b : "foo"', '(true ? $this->b : \'foo\')'],
             'true ? 42 : c' => ['true ? 42 : c', '(true ? 42 : $this->c)'],
@@ -70,4 +71,4 @@ public function transpilesTernaryOperationNodes(string $ternaryOperationAsString
             $actualTranspilationResult
         );
     }
-}
+}