fromjcl 0.4.0: parser bug fixes, Rich syntax highlighting, code quality

Bug fixes:
- Scanner: false-positive ContinueComment state on lines where content
  reaches column 72 (long template JOB cards) caused the next statement
  to fail as an invalid continued comment; state is now reset and the
  record re-dispatched.
- Scanner: lowercase characters in jobnames (template placeholders like
  TKTxxx1) were rejected; name-char validation now accepts any ASCII
  letter.
- Scanner: multi-element JOB account fields like (B004273,BIN#,BLDG#)
  were truncated at the first comma; paren-nesting is now tracked in the
  keyword parse context.
- CLI: piping --to json on JCL with instream data produced invalid JSON
  because Rich's Pygments lexer expanded \n escape sequences to literal
  newlines; syntax highlighting now activates only for TTY stdout.

Added:
- Rich dependency; colored syntax-highlighted output for json/yaml/jcl/
  zoau/mvscmd targets when stdout is a terminal.

Code quality:
- 27 refurb modernizations across 10+ files (FURB102/108/113/115/123/
  124/138/142/143/173).
- Voice audit: removed 10 comments that restated code or referenced
  prior state.

Author: msradam

CHANGELOG.md

@@ -1,5 +1,32 @@
 # Changelog
 
+## 0.4.0 (2026-05-23)
+
+### Added
+
+- Syntax-highlighted terminal output via Rich. `--to json`, `--to yaml`,
+  `--to jcl`, `--to zoau`, and `--to mvscmd` colorize output when stdout
+  is a TTY (monokai theme); piped output is plain text, byte-for-byte
+  identical to file output (`-o`).
+
+### Fixed
+
+- **Scanner: false-positive continuation state** on lines where content
+  reaches column 72 (e.g., template JOB cards with long `MSGCLASS=`
+  values). The scanner incorrectly set `ContinueComment` state, causing
+  the next statement to fail with `Invalid continued comment record`.
+- **Scanner: lowercase jobname rejection.** Jobnames with lowercase
+  characters (common in template JCL, e.g., `TKTxxx1`) were rejected as
+  invalid. Name-character validation now accepts any ASCII letter.
+- **Scanner: multi-element JOB account truncation.** Account fields of
+  the form `(B004273,BIN#,BLDG#,DEPT#)` were split at the first comma
+  inside the parentheses. Paren-nesting is now tracked in the keyword
+  context so the full group is preserved.
+- **CLI: invalid JSON when piping `--to json`** on JCL with instream
+  data. Rich's Pygments JSON lexer was expanding `\n` escape sequences
+  to literal newlines inside string values. Syntax highlighting now
+  activates only for interactive terminals.
+
 ## 0.3.1 (2026-05-18)
 
 ### Fixed

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "fromjcl"
-version = "0.3.1"
+version = "0.4.0"
 description = "Parse IBM z/OS JCL and serialize to JSON, YAML, CSV, or roundtrip JCL."
 readme = "README.md"
 license = "Apache-2.0"
@@ -27,6 +27,7 @@ classifiers = [
 # z/OS as well as Linux/macOS/Windows.
 dependencies = [
     "pyyaml>=6.0",
+    "rich>=13.0",
     "typer>=0.12",
 ]
 
@@ -66,6 +67,7 @@ dev = [
     "bandit>=1.7",
     "pip-audit>=2.7",
     "twine>=5.0",
+    "refurb>=2.3.1",
 ]
 
 [build-system]

src/fromjcl/_scanner.py

@@ -157,9 +157,9 @@ def to_dict(self) -> dict[str, Any]:
             "name": self.name,
             "lines": self.lines,
             "keyword_col": self.keyword_col,
-            "record_lens": list(self.record_lens),
-            "instream_records": list(self.instream_records),
-            "parameters": list(self.kvps),
+            "record_lens": self.record_lens.copy(),
+            "instream_records": self.instream_records.copy(),
+            "parameters": self.kvps.copy(),
             "scanned_lines": [
                 {
                     "parm_text": sl.parm_text,
@@ -201,15 +201,15 @@ def _skip_blanks(text: str, start: int, end: int) -> tuple[int, int]:
 
 
 def _is_name_char(c: str) -> bool:
-    return c.isupper() or c.isdigit() or c in NATIONAL
+    return (c.isascii() and c.isalpha()) or c.isdigit() or c in NATIONAL
 
 
 def _is_valid_name(buf: str) -> tuple[bool, int]:
     """buf is text AFTER the // prefix. Return (is_valid, name_len)."""
     if not buf:
         return False, 0
     c0 = buf[0]
-    if not (c0.isupper() or c0 in NATIONAL):
+    if not ((c0.isascii() and c0.isalpha()) or c0 in NATIONAL):
         return False, 0
 
     dot = 0
@@ -301,8 +301,7 @@ def _add_scanned_line(
         )
 
     def _append_scanned_comment(self, text: str, start: int, end: int) -> None:
-        """C appendScannedComment: strip blanks, append to last scan line's
-        comment_text, creating an empty comment string if needed."""
+        """C appendScannedComment."""
         start, end = _skip_blanks(text, start, end)
         chunk = text[start:end]
         tail = self._cur.scan_lines[-1]
@@ -393,8 +392,7 @@ def _scan_parameters(self, text: str, column: int) -> None:
                 if (
                     dlm_orig is not None
                     and len(dlm_orig) == DELIM_LEN + 2
-                    and dlm_orig[0] == "'"
-                    and dlm_orig[-1] == "'"
+                    and dlm_orig[0] == dlm_orig[-1] == "'"
                 ):
                     dlm = dlm_orig[1:3]
                 else:
@@ -424,10 +422,14 @@ def _scan_sub_parameters(self) -> None:
             while i < end:
                 c = text[i]
                 if context == ParmContext.InKeyword:
-                    if c == "=":
+                    if c == "(":
+                        paren_nest += 1
+                    elif c == ")" and paren_nest > 0:
+                        paren_nest -= 1
+                    elif c == "=" and paren_nest == 0:
                         context = ParmContext.InValue
                         cur_value = i + 1
-                    elif c == ",":
+                    elif c == "," and paren_nest == 0:
                         if i + 1 == end:
                             comment = cur_line_comment
                             hn = True
@@ -584,7 +586,7 @@ def _scan_conditional(self, text: str, column: int) -> None:
             self.state = ScanState.ContinueConditional
 
     def _scan_comment_stmt(self, text: str) -> None:
-        """//* comment. Comment text is cropped to the original line length."""
+        """Comment text is cropped to the original line length."""
         self._add_stmt("//*", None)
         raw_len = len(self._current_raw)
         content_end = max(PREFIX_LEN + 1, min(raw_len, JCL_RECLEN))
@@ -757,7 +759,13 @@ def _process_inline_record(self, text: str) -> None:
 
     def _process_continued_comment(self, text: str) -> None:
         if text[0] != "/" or text[1] != "/" or text[2] != " ":
-            raise ValueError("Invalid continued comment record")
+            # A new JCL statement arrived while continuation was expected
+            # (e.g. a parameter line whose content reached col 72 set the
+            # flag spuriously). Treat this record as the start of a fresh
+            # statement rather than a hard error.
+            self.state = ScanState.NotContinued
+            self._process_jcl_record(text)
+            return
         self._add_to_stmt()
         self._scan_parameters(text, PREFIX_LEN + 1)
 
@@ -781,24 +789,22 @@ def _process_continued_conditional(self, text: str) -> None:
     def _process_continued_parameter(self, text: str) -> None:
         # //* inside continued params: append to the current scan line's
         # comment_text with a leading newline. No addToStatement.
-        if text[0] == "/" and text[1] == "/" and text[2] == "*":
+        if text[0] == text[1] == "/" and text[2] == "*":
             tail = self._cur.scan_lines[-1]
             existing = tail.comment_text or ""
             tail.comment_text = existing + "\n"
             self._append_scanned_comment(text, PREFIX_LEN + 1, JCL_TXTLEN)
             return
         if text[0] != "/" or text[1] != "/" or text[2] != " ":
-            raise ValueError("Invalid continued parameter record")
+            # Same lenient recovery as _process_continued_comment.
+            self.state = ScanState.NotContinued
+            self._process_jcl_record(text)
+            return
         self._add_to_stmt()
         self._scan_parameters(text, PREFIX_LEN + 1)
 
     def _process_jes3_continued_dataset(self, text: str) -> None:
-        if (
-            text[0] == "/"
-            and text[1] == "/"
-            and text[2] == "*"
-            and _word_starts(text[3:], "ENDDATASET")
-        ):
+        if text[0] == text[1] == "/" and text[2] == "*" and _word_starts(text[3:], "ENDDATASET"):
             # C: addStatement(0, JES3_KEYWORD); opens a new //* stmt.
             self._add_stmt("//*", None)
             self.state = ScanState.NotContinued
@@ -827,7 +833,6 @@ def process_record(self, text: str, raw: str | None = None) -> None:
         else:
             raise RuntimeError(f"Unreachable scan state: {current}")
 
-        # Inline data lines go to instream_records; JCL records to record_lens.
         if len(self.stmts) > before:
             self._cur.record_lens.append(len(self._current_raw))
         elif self.stmts and was_inline:

src/fromjcl/_zoau_flags.py

@@ -1,7 +1,6 @@
 # SPDX-License-Identifier: Apache-2.0
 """Frozen snapshot of ZOAU 1.x command flags (55 verbs, 493 flags).
 
-Originally extracted from manpages; the generator has been removed.
 To refresh against a newer ZOAU release, run `man <verb>` on each verb
 in FLAGS_BY_VERB below and reshape the synopsis lines into the existing
 list structure.

src/fromjcl/cli.py

@@ -9,6 +9,8 @@
 from typing import Annotated
 
 import typer
+from rich.console import Console
+from rich.syntax import Syntax
 
 from fromjcl.models import Job
 from fromjcl.parser import parse, parse_bytes
@@ -41,6 +43,16 @@ class InputFormat(StrEnum):
 
 _ZOAU_FORMATS = {OutputFormat.mvscmd, OutputFormat.zoau}
 
+_RICH_LEXERS: dict[OutputFormat, str] = {
+    OutputFormat.json: "json",
+    OutputFormat.yaml: "yaml",
+    OutputFormat.jcl: "jcl",
+    OutputFormat.zoau: "bash",
+    OutputFormat.mvscmd: "bash",
+}
+
+_console = Console()
+
 
 def _require_extra(extra: str, marker_module: str) -> None:
     """Exit with help if pip install fromjcl[<extra>] hasn't been run."""
@@ -55,12 +67,19 @@ def _require_extra(extra: str, marker_module: str) -> None:
         raise typer.Exit(code=2) from None
 
 
-def _write_output(output: str, dest: str | None) -> None:
+def _write_output(output: str, dest: str | None, fmt: OutputFormat | None = None) -> None:
     """Write output to a file or stdout. Both paths ensure exactly one
-    trailing newline so piping `--to jcl` matches the `-o file` form."""
+    trailing newline so piping `--to jcl` matches the `-o file` form.
+    When writing to a terminal, syntax-highlights via Rich."""
     text = output if output.endswith("\n") else output + "\n"
     if dest:
         Path(dest).write_text(text)
+        return
+    lexer = _RICH_LEXERS.get(fmt) if fmt else None
+    if lexer and _console.is_terminal:
+        _console.print(
+            Syntax(text.rstrip("\n"), lexer, theme="monokai", background_color="default")
+        )
     else:
         sys.stdout.write(text)
 
@@ -121,7 +140,7 @@ def convert(
         except (ValueError, KeyError) as e:
             typer.echo(f"Error: {e}", err=True)
             raise typer.Exit(code=1) from e
-        _write_output(result, output)
+        _write_output(result, output, OutputFormat.jcl)
         return
 
     try:
@@ -159,7 +178,7 @@ def convert(
     else:  # pragma: no cover - Enum exhaustiveness; defensive default.
         result = ""
 
-    _write_output(result, output)
+    _write_output(result, output, to)
 
     if warnings:
         typer.echo(f"fromjcl: validation failed ({len(warnings)} issue(s)):", err=True)

src/fromjcl/converters/_conditions.py

@@ -150,6 +150,5 @@ def referenced_step_names(items: list[Any]) -> set[str]:
             text = getattr(item, attr, None)
             if not text:
                 continue
-            for m in pattern.finditer(text):
-                refs.add(m.group(1).split(".")[0].lower())
+            refs.update(m.group(1).split(".")[0].lower() for m in pattern.finditer(text))
     return refs

src/fromjcl/converters/classify.py

@@ -227,7 +227,7 @@ def _classify_iefbr14(step: Step) -> DatasetOps:
             # NEW+CATLG/KEEP allocates; OLD/MOD/SHR + DELETE deletes.
             if disp.status == "NEW" and disp.normal in ("CATLG", "KEEP"):
                 creates.append((name, ds))
-            elif (disp.normal == "DELETE" or disp.abnormal == "DELETE") and disp.status in (
+            elif "DELETE" in (disp.normal, disp.abnormal) and disp.status in (
                 "OLD",
                 "MOD",
                 "SHR",
@@ -277,7 +277,7 @@ def _gather_iebgener_info(step: Step) -> _IEBGenerInfo:
     return info
 
 
-# Matcher order matters: specific patterns first, then more general.
+# Matcher order matters.
 
 
 def _match_path_to_dsn(info: _IEBGenerInfo) -> CopyDataset | None:
@@ -466,8 +466,7 @@ def _classify_sort(step: Step) -> TextReplace | Fallback:
     find = joined[findrep.start(1) : findrep.end(1)]
     repl = joined[findrep.start(2) : findrep.end(2)]
 
-    dd_map = build_dd_map(step)
-    sortin = dd_map.get("SORTIN")
+    sortin = build_dd_map(step).get("SORTIN")
     if not (sortin and sortin.datasets):
         return Fallback(reason="SORT: Missing SORTIN dataset")
 
@@ -697,7 +696,7 @@ def _join_continuations(sysin_data: str) -> list[str]:
                 current = ""
             continue
 
-        ends_continued = line.endswith(",") or line.endswith("-")
+        ends_continued = line.endswith((",", "-"))
         if current:
             # Trailing comma joins flush (parsers want the comma); trailing
             # dash inserts a space (IDCAMS treats line-break as whitespace).

src/fromjcl/converters/common.py

@@ -143,8 +143,12 @@ def build_mvscmd_command(step: Step, force_auth: bool | None = None) -> list[str
     if step.program:
         parts.append(f"--pgm={step.program}")
     elif step.proc:
-        result.append(f"# WARNING: PROC={step.proc} cannot be executed by {exe}.")
-        result.append("# Expand the PROC or find the program it calls.")
+        result.extend(
+            (
+                f"# WARNING: PROC={step.proc} cannot be executed by {exe}.",
+                "# Expand the PROC or find the program it calls.",
+            )
+        )
         parts.append("--pgm=UNKNOWN")
 
     if step.parm:

src/fromjcl/converters/shell/_scaffold.py

@@ -33,8 +33,7 @@ def emit(job: Job, render_step: Callable[[Step], list[str]], header_tag: str) ->
         lines.append("# WARNING: This job uses symbolic parameters:")
         for sym, val in job.symbols.items():
             lines.append(f"#   {sym}={val}")
-        lines.append("# You may need to substitute these values manually.")
-        lines.append("")
+        lines.extend(("# You may need to substitute these values manually.", ""))
 
     referenced = _conditions.referenced_step_names(job.steps)
     for cond, steps in _conditions.group_consecutive_by_condition(job.steps):
@@ -67,8 +66,12 @@ def _emit_conditional(
     warn = _conditions._approx_warning(cond)
     if warn:
         out.append(f"# WARNING: {warn}")
-    out.append(f"# Steps: {', '.join(s.name for s in steps)}")
-    out.append(f"if (( {_conditions.to_shell(cond)} )); then")
+    out.extend(
+        (
+            f"# Steps: {', '.join(s.name for s in steps)}",
+            f"if (( {_conditions.to_shell(cond)} )); then",
+        )
+    )
     body_has_command = False
     for step in steps:
         out.append(f"    # Step: {step.name}")
@@ -85,6 +88,5 @@ def _emit_conditional(
     if not body_has_command:
         # bash if ... fi requires at least one command between then/fi.
         out.append("    :")
-    out.append("fi")
-    out.append("")
+    out.extend(("fi", ""))
     return out

src/fromjcl/converters/shell/zoau.py

@@ -84,12 +84,10 @@ def _render_dataset_ops(intent: DatasetOps, sym: dict[str, str]) -> list[str]:
     result: list[str] = []
     for dd_name, dataset in intent.creates:
         dsn = resolve_symbols(dataset.dsn, sym)
-        result.append(f"# DD {dd_name}: Allocate dataset")
-        result.append(_build_dtouch(dataset, dsn))
+        result.extend((f"# DD {dd_name}: Allocate dataset", _build_dtouch(dataset, dsn)))
     for raw_dsn in intent.deletes:
         dsn = resolve_symbols(raw_dsn, sym)
-        result.append("# Delete dataset")
-        result.append(f'drm "{dsn}"')
+        result.extend(("# Delete dataset", f'drm "{dsn}"'))
     if not result:
         result.append("# IEFBR14 with no actionable DDs")
     return result
@@ -228,9 +226,6 @@ def _render_backup(intent: BackupRestore, sym: dict[str, str]) -> list[str]:
     return ["# ADRDSSU: Unknown operation"]
 
 
-# ZOAU-only converters (no Ansible module equivalent)
-
-
 def _convert_isrsupc(step: Step) -> list[str]:
     newdd = olddd = None
     sysin_data = get_sysin(step)
@@ -257,12 +252,10 @@ def _convert_isrsupc(step: Step) -> list[str]:
 
 
 def _render_iehlist(intent: IEHListOps) -> list[str]:
-    result: list[str] = []
-    for dsn in intent.pds_dsns:
-        result.append(f'mls "{dsn}"')
-    for vol in intent.vtoc_volumes:
-        result.append(f"vtocls {vol}")
-    return result
+    return [
+        *(f'mls "{dsn}"' for dsn in intent.pds_dsns),
+        *(f"vtocls {vol}" for vol in intent.vtoc_volumes),
+    ]
 
 
 def _render_iehprogm(intent: IEHPROGMOps) -> list[str]: