fix(lexer): compute span ends by character count, not byte length (#72)

## Summary

The lexer scans the source as a `Vec<char>`, so token positions and AST
`Span`s are **character indices**. But span *ends* were computed as `pos
+ value.len()` — adding the token value's **byte** length to a
**character** position. For any token containing multibyte UTF-8, the
end overshoots by the byte/char delta, so `Node::source_text` returns
the wrong slice for every **non-final** token.

The existing `source_text_multibyte_utf8` test never caught this because
its multibyte word was the *last* token, where the overshoot clamps
harmlessly to the source end.

### Example (before)

```
echo "✓ valid" || echo "✗ bad"
  Command 0..16  source_text = `echo "✓ valid" |`   ← wrong (bleeds into the `||`)
```

### After

```
  Command 0..14  source_text = `echo "✓ valid"`     ← correct
```

## The fix

Use `value.chars().count()` at the four sites that derive an end offset
from a start position:

- `lexer/mod.rs` — `last_token_end` (the source of most node ends via
`parser/mod.rs:134`)
- `parser/simple_command.rs` — word-node span
- `parser/case_select.rs` — pattern word span
- `token.rs` — `Token::adjacent_to`

…and correct the `Span` doc comment to state offsets are character
indices.

## Downstream impact

Surfaced as [tokf #383](mpecan/tokf#383):
compound shell commands mixing an emoji/`✓`/`✗` `echo` with a rewritten
pipe were spliced into invalid shell (`| ||`, dropped `; `). tokf
consumes these spans to split compound commands.

## Tests

Adds regression tests for multibyte in **non-final** words, compound
segments, pipelines, and astral-plane emoji (`🎉`). Full suite: **262
passing**, clippy + fmt clean. No ASCII spans change (char count == byte
len for ASCII), so no existing test moved.

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

Author: mpecan

src/ast.rs

@@ -1,6 +1,11 @@
 use crate::lexer::word_builder::{QuotingContext, WordSpanKind};
 
-/// Source span representing a byte range in the original input.
+/// Source span representing a **character** range in the original input.
+///
+/// Offsets are character indices (the lexer scans the source as a `Vec<char>`),
+/// not byte offsets — they diverge for multibyte UTF-8. Use
+/// [`Node::source_text`] to recover the source slice; it converts these
+/// character indices to byte offsets before slicing.
 #[derive(Debug, Clone, PartialEq, Eq)]
 pub struct Span {
     pub start: usize,

src/lexer/mod.rs

@@ -317,7 +317,11 @@ impl Lexer {
         } else {
             self.read_token()?
         };
-        self.last_token_end = tok.pos + tok.value.len();
+        // `pos` is a character index (the lexer scans a `Vec<char>`), so the
+        // span end must advance by the token's character count, not its byte
+        // length. Using `value.len()` (bytes) corrupts spans for any token
+        // containing multibyte UTF-8 (tokf #383).
+        self.last_token_end = tok.pos + tok.value.chars().count();
         Ok(tok)
     }
 

src/parser/case_select.rs

@@ -19,7 +19,8 @@ impl Parser {
                 value: word_tok.value.clone(),
                 spans: word_tok.spans,
             },
-            Span::new(word_tok.pos, word_tok.pos + word_tok.value.len()),
+            // Char count, not byte length: `pos` is a character index (#383).
+            Span::new(word_tok.pos, word_tok.pos + word_tok.value.chars().count()),
         ));
 
         self.lexer.set_command_start();

src/parser/simple_command.rs

@@ -146,7 +146,9 @@ impl Parser {
 /// which uses `Node::empty` (no span) for AST constructors where positional
 /// information is not available.
 fn build_word_node(tok: Token) -> Node {
-    let word_span = Span::new(tok.pos, tok.pos + tok.value.len());
+    // `pos` is a character index; advance the span end by the token's char
+    // count, not its byte length, so multibyte words span correctly (#383).
+    let word_span = Span::new(tok.pos, tok.pos + tok.value.chars().count());
     let parts = word_parts::decompose_word_with_spans(&tok.value, &tok.spans);
     Node::new(
         NodeKind::Word {

src/parser/tests.rs

@@ -8,6 +8,30 @@ fn parse(source: &str) -> Vec<Node> {
     parser.parse_all().unwrap()
 }
 
+/// Flatten a (possibly nested) `List`/`Pipeline` into its leaf command nodes,
+/// in source order. Test helper for span-boundary assertions.
+fn collect_leaf_commands<'a>(node: &'a Node, out: &mut Vec<&'a Node>) {
+    match &node.kind {
+        NodeKind::List { items } => {
+            for item in items {
+                collect_leaf_commands(&item.command, out);
+            }
+        }
+        NodeKind::Pipeline { commands, .. } => {
+            for cmd in commands {
+                collect_leaf_commands(cmd, out);
+            }
+        }
+        _ => out.push(node),
+    }
+}
+
+/// Slice `source` by **character** offsets, mirroring how `Node::source_text`
+/// resolves spans. Test helper for asserting inter-span gaps.
+fn char_slice(source: &str, start: usize, end: usize) -> String {
+    source.chars().skip(start).take(end - start).collect()
+}
+
 #[test]
 fn simple_command() {
     let nodes = parse("echo hello");
@@ -401,6 +425,67 @@ fn source_text_multibyte_utf8() {
     assert_eq!(words[1].source_text(source), "café");
 }
 
+#[test]
+fn source_text_multibyte_non_last_word() {
+    // Regression (tokf #383): a multibyte word that is NOT the last token.
+    // Before the char-count span fix, the word's span end was computed as
+    // `pos + byte_len`, overshooting into the following token — here `café`
+    // would bleed into ` bar`. The previous `source_text_multibyte_utf8`
+    // test missed this because its multibyte word was last, so the
+    // overshooting span clamped harmlessly to the source end.
+    let source = "echo café bar";
+    let nodes = parse(source);
+    assert_eq!(nodes[0].source_text(source), "echo café bar");
+    let NodeKind::Command { words, .. } = &nodes[0].kind else {
+        unreachable!("expected Command");
+    };
+    assert_eq!(words[0].source_text(source), "echo");
+    assert_eq!(words[1].source_text(source), "café");
+    assert_eq!(words[2].source_text(source), "bar");
+}
+
+#[test]
+fn source_text_multibyte_compound_segments() {
+    // Regression (tokf #383): multibyte in non-final segments of a compound
+    // list. Each leaf command's source_text must be exact, and the spans
+    // must leave the operator gaps intact for a consumer that slices
+    // `source[prev.span.end .. next.span.start]` as the separator.
+    let source = "echo \"✓ ok\" || echo \"✗ no\"; ls";
+    let nodes = parse(source);
+    let mut leaves = Vec::new();
+    collect_leaf_commands(&nodes[0], &mut leaves);
+    let texts: Vec<&str> = leaves.iter().map(|n| n.source_text(source)).collect();
+    assert_eq!(texts, vec!["echo \"✓ ok\"", "echo \"✗ no\"", "ls"]);
+    // The gap between the first two leaves must be exactly the `||` operator
+    // (converting the character spans to byte offsets, as source_text does).
+    let gap = char_slice(source, leaves[0].span.end, leaves[1].span.start);
+    assert_eq!(gap, " || ");
+}
+
+#[test]
+fn source_text_multibyte_pipeline() {
+    // Regression (tokf #383): multibyte in the left command of a pipeline.
+    let source = "echo \"✓\" | grep x";
+    let nodes = parse(source);
+    let NodeKind::Pipeline { commands, .. } = &nodes[0].kind else {
+        unreachable!("expected Pipeline");
+    };
+    assert_eq!(commands[0].source_text(source), "echo \"✓\"");
+    assert_eq!(commands[1].source_text(source), "grep x");
+}
+
+#[test]
+fn source_text_astral_plane_emoji() {
+    // Regression (tokf #383): a 4-byte (astral-plane) char before a newline
+    // separator. Previously `echo 🎉` overshot its span and the second
+    // command's slice underflowed, panicking on a non-char-boundary index.
+    let source = "echo 🎉\nls";
+    let nodes = parse(source);
+    assert_eq!(nodes.len(), 2);
+    assert_eq!(nodes[0].source_text(source), "echo 🎉");
+    assert_eq!(nodes[1].source_text(source), "ls");
+}
+
 #[test]
 fn source_text_if_compound() {
     let source = "if true; then echo yes; fi";

src/token.rs

@@ -183,8 +183,12 @@ impl Token {
     }
 
     /// Returns true if this token is immediately adjacent to `other` (no whitespace).
-    pub const fn adjacent_to(&self, other: &Self) -> bool {
-        self.pos + self.value.len() == other.pos
+    ///
+    /// `pos` is a character index, so adjacency is measured in characters: the
+    /// token's char count, not its byte length (which diverges for multibyte
+    /// UTF-8 — tokf #383).
+    pub fn adjacent_to(&self, other: &Self) -> bool {
+        self.pos + self.value.chars().count() == other.pos
     }
 
     pub const fn eof(pos: usize, line: usize) -> Self {