refactor(html)!: html module rewrite

.devcontainer/Dockerfile

@@ -1,18 +1,19 @@
-# See here for image contents: https://github.com/microsoft/vscode-dev-containers/tree/v0.187.0/containers/rust/.devcontainer/base.Dockerfile
+# See here for image contents: https://github.com/devcontainers/images/tree/main/src/rust
 
-FROM mcr.microsoft.com/devcontainers/rust:1-1-bullseye
+FROM mcr.microsoft.com/devcontainers/rust:1-1-bookworm
 
-# [Optional] Uncomment this section to install additional packages.
+# Install Python and uv for lxml reference tests
 RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \
-    && apt-get -y install --no-install-recommends python3-lxml
+    && apt-get -y install --no-install-recommends python3-full
+
+COPY --from=ghcr.io/astral-sh/uv:latest /uv /usr/local/bin/uv
+
+COPY postCreateCommand.sh /tmp/postCreateCommand.sh
+RUN chmod +x /tmp/postCreateCommand.sh
 
 USER vscode
 
 # Install nightly rust
 RUN rustup toolchain install nightly
 
-# Install pip
 ENV PATH="${PATH}:/home/vscode/.local/bin"
-RUN curl https://bootstrap.pypa.io/get-pip.py -o /tmp/get-pip.py \
-    && python3 /tmp/get-pip.py \
-    && rm /tmp/get-pip.py

.devcontainer/devcontainer.json

@@ -23,7 +23,7 @@
 			// Add the IDs of extensions you want installed when the container is created.
 			"extensions": [
 				"matklad.rust-analyzer",
-				"bungcip.better-toml",
+				"tamasfe.even-better-toml",
 				"vadimcn.vscode-lldb",
 				"mutantdino.resourcemonitor",
 				"ms-azuretools.vscode-docker"
@@ -33,7 +33,7 @@
 	// Use 'forwardPorts' to make a list of ports inside the container available locally.
 	// "forwardPorts": [],
 	// Use 'postCreateCommand' to run commands after the container is created.
-	"postCreateCommand": "pip install -r tests/lxml_tests/requirements.txt",
+	"postCreateCommand": "/tmp/postCreateCommand.sh",
 	// Comment out connect as root instead. More info: https://aka.ms/vscode-remote/containers/non-root.
 	"remoteUser": "vscode"
 }

.devcontainer/postCreateCommand.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+# Set up the Python venv for lxml reference tests using uv.
+# The Rust lxml tests expect the venv at tests/lxml_tests/.venv/bin/python.
+cd /workspaces/Skyscraper/tests/lxml_tests && uv sync

.gitignore

@@ -1,3 +1,7 @@
 /target
 Cargo.lock
 test.html
+.env/
+.venv/
+log.txt
+.context/

.vscode/settings.json

@@ -1,5 +1,5 @@
 {
     "rust-analyzer.linkedProjects": [
         "./Cargo.toml"
-    ]
+    ],
 }

CLAUDE.md

@@ -0,0 +1,96 @@
+# CLAUDE.md
+
+This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository.
+
+## Project Overview
+
+Skyscraper is a Rust library for scraping HTML documents with XPath expressions. It has its own HTML parser (following the WHATWG spec) and XPath parser/evaluator. Published on crates.io as `skyscraper` (v0.7.0-beta.2, MIT license). Many XPath features still have `todo!()` stubs.
+
+## Build & Test Commands
+
+```sh
+cargo build                  # Build
+cargo test                   # Run all tests
+cargo test <test_name>       # Run a single test by name
+cargo test --test <file>     # Run a specific test file (e.g. --test html_tests)
+cargo bench                  # Run all benchmarks (criterion)
+```
+
+Stack overflow regression tests (Windows-specific, run in CI on windows-latest):
+```sh
+cargo test --test run_stack_overflow_tests -- --include-ignored --test-threads=1
+```
+
+Reference testing against Python lxml (managed via uv):
+```sh
+cd tests/lxml_tests && uv sync          # Install Python deps (first time / after changes)
+cat tests/samples/James-LG_Skyscraper.html | uv run --directory tests/lxml_tests python xpath.py "//div"
+```
+The Rust lxml tests (`tests/xpath_tests/lxml_tests.rs`) automatically use the venv at `tests/lxml_tests/.venv/bin/python` when available.
+
+Feature flag: `debug_prints` enables debug output during parsing.
+
+## Architecture
+
+The crate has two core modules (`src/html/` and `src/xpath/`) plus a small `vecpointer` utility. The `#![warn(missing_docs)]` lint is enabled at the crate root.
+
+### HTML Module (`src/html/`)
+
+Parses HTML text into an arena-based tree (`indextree` crate).
+
+- **`HtmlDocument`** — owns the arena tree; has a `root_node: DocumentNode`
+- **`DocumentNode`** — wrapper around `indextree::NodeId`; provides `children()`, `parent()`, `get_text()`, `get_all_text()`
+- **`HtmlNode`** — enum: `Tag(HtmlTag)` | `Text(HtmlText)`. Text nodes are separate to preserve ordering in mixed content.
+- **Grammar subsystem** (`src/html/grammar/`):
+  - **Tokenizer** (`tokenizer/`) — state-machine HTML tokenizer per WHATWG spec, including named character references
+  - **Parser** (`mod.rs`) — main parsing logic with WHATWG insertion modes
+  - **Document builder** (`document_builder.rs`) — constructs the tree from token stream
+  - **Insertion modes** (`insertion_mode_impls/`) — WHATWG tree construction insertion modes (in_body is the largest)
+- Key constants: `VOID_TAGS`, `SPECIAL_ELEMENTS`, `GENERATE_IMPLIED_END_TAG_TYPES`, `ELEMENT_IN_SCOPE_TYPES` (defined via `once_cell::Lazy`)
+
+### XPath Module (`src/xpath/`)
+
+Parses XPath strings and evaluates them against an `XpathItemTree` (converted from `HtmlDocument`).
+
+- **`Xpath`** — compiled XPath expression; call `.apply(&XpathItemTree)` to evaluate
+- **`XpathItemTree`** — tree structure mirroring the HTML document for XPath evaluation; created via `XpathItemTree::from(&document)`
+- **Grammar subsystem** (`src/xpath/grammar/`, ~70 files):
+  - `expressions/` — all expression types (path, logical, comparison, arithmetic, conditional, for/let, quantified, postfix, etc.)
+  - `expressions/path_expressions/steps/axes/` — forward axes (child, descendant, attribute, descendant-or-self) and reverse axes (parent)
+  - `data_model/` — XPath node types (document, element, text, attribute, comment, PI)
+  - `types/` — XPath type system (sequence types, element/attribute tests)
+  - `terminal_symbols.rs` — token definitions
+  - `recipes.rs`, `whitespace_recipes.rs` — nom parser combinator helpers
+  - `xml_names.rs` — XML name validation
+
+### Parsing Approach
+
+Both modules use **nom** parser combinators extensively. The `recipes.rs` and `whitespace_recipes.rs` modules provide reusable combinator patterns. The HTML tokenizer uses a state machine pattern, while XPath parsing is a recursive descent grammar built from nom combinators.
+
+## Test Structure
+
+- `tests/html_tests/` — HTML parsing correctness
+- `tests/xpath_tests/` — XPath parsing and evaluation (many sub-files: predicates, axes, functions, type matching, etc.)
+- `tests/test_framework/` — shared test utilities for document comparison
+- `tests/samples/` — sample HTML files for integration tests
+- `tests/lxml_tests/` — Python reference tests comparing against lxml
+- `tests/stack_overflow_tests/` — separate Cargo project for Windows stack overflow regression
+
+Tests use `indoc` for readable multi-line HTML strings and `proptest` for property-based testing.
+
+## Reference Specifications (`.context/`)
+
+The `.context/` directory (gitignored) contains downloaded W3C specifications for offline reference. New resources can be added here as needed to avoid refetching.
+
+Current contents:
+- `xpath-31-spec.html` — [XPath 3.1](https://www.w3.org/TR/2017/REC-xpath-31-20170321/) (W3C Recommendation)
+- `xpath-datamodel-31-spec.html` — [XPath Data Model 3.1](https://www.w3.org/TR/2017/REC-xpath-datamodel-31-20170321/) (W3C Recommendation)
+- `xpath-functions-31-spec.html` — [XPath Functions and Operators 3.1](https://www.w3.org/TR/2017/REC-xpath-functions-31-20170321/) (W3C Recommendation)
+
+## Features Backlog
+
+[`docs/features-backlog.md`](docs/features-backlog.md) tracks spec gaps, known limitations, and design decisions for both the HTML parser and XPath evaluator. Check it before implementing new features to avoid duplicating known issues.
+
+## CI
+
+GitHub Actions (`.github/workflows/rust.yml`): runs `cargo test` in a devcontainer on ubuntu-latest, plus stack overflow tests on windows-latest with Rust nightly.

Cargo.toml

@@ -19,10 +19,10 @@ indexmap = "2.0.0"
 log = "0.4.19"
 nom = "7.1.3"
 ordered-float = "4.2.0"
-once_cell = "1.19.0"
 enum-extract-macro = "0.1.1"
 enum-extract-error = "0.1.1"
 regex = "1.10.3"
+unicode-normalization = "0.1.25"
 
 [dev-dependencies]
 criterion = "0.5.1"
@@ -32,7 +32,11 @@ proptest = "1.3.1"
 regex = "1.10.3"
 serde_json = "1.0.113"
 serde = "1.0.196"
+itertools = "0.13.0"
 
 [[bench]]
 name = "benchmarks"
 harness = false
+
+[features]
+debug_prints = []

README.md

@@ -7,9 +7,7 @@
 
 Rust library to scrape HTML documents with XPath expressions.
 
-> This library is major-version 0 because there are still `todo!` calls for many xpath features.
->If you encounter one that you feel should be prioritized, open an issue on [GitHub](https://github.com/James-LG/Skyscraper/issues).
->
+> This library is major-version 0 as the API is still evolving.
 > See the [Supported XPath Features](#supported-xpath-features) section for details.
 
 ## HTML Parsing
@@ -51,6 +49,10 @@ assert_eq!(parent_node, parent_of_child0);
 assert_eq!(parent_node, parent_of_child1);
 ```
 
+### WHATWG Compliance Note
+
+Skyscraper's HTML parser follows the [WHATWG parsing specification](https://html.spec.whatwg.org/multipage/parsing.html). One notable consequence is **implicit `<tbody>` insertion**: when `<tr>`, `<td>`, or `<th>` elements appear as direct children of `<table>`, the parser automatically wraps them in a `<tbody>` element (per [WHATWG §13.2.6.4.9](https://html.spec.whatwg.org/multipage/parsing.html#parsing-main-intable)). This matches browser behavior but differs from parsers like Python's lxml, which does not insert `<tbody>`. As a result, XPath expressions like `//table/*` or `//table//*` may return different results than lxml for the same input HTML. To avoid this discrepancy, use explicit `<tbody>` tags in your HTML or account for the implicit element in your XPath expressions.
+
 ## XPath Expressions
 
 Skyscraper is capable of parsing XPath strings and applying them to HTML documents.
@@ -106,20 +108,24 @@ Below is a non-exhaustive list of all the features that are currently supported.
 1. Predicates:
     1. Attributes: `//div[@class='hi']`
     1. Indexing: `//div[1]`
-1. Functions:
-    1. `fn:root()`
-    1. `contains(haystack, needle)`
-1. Forward axes:
-    1. Child: `child::*`
-    1. Descendant: `descendant::*`
-    1. Attribute: `attribute::*`
-    1. DescendentOrSelf: `descendant-or-self::*`
-    1. (more coming soon)
-1. Reverse axes:
-    1. Parent:  `parent::*`
-    1. (more coming soon)
-1. Treat expressions: `/html treat as node()`
-
-This should cover most XPath use-cases.
+    1. Arbitrary expressions: `//div[contains(@class, 'hi')]`
+1. Forward axes: `child::`, `descendant::`, `attribute::`, `self::`, `descendant-or-self::`, `following-sibling::`, `following::`, `namespace::`
+1. Reverse axes: `parent::`, `ancestor::`, `preceding-sibling::`, `preceding::`, `ancestor-or-self::`
+1. Operators:
+    1. Logical: `and`, `or`
+    1. Comparison: `=`, `!=`, `<`, `>`, `<=`, `>=`, `eq`, `ne`, `lt`, `gt`, `le`, `ge`
+    1. Arithmetic: `+`, `-`, `*`, `div`, `idiv`, `mod`
+    1. String concatenation: `||`
+    1. Sequence: `union`/`|`, `intersect`, `except`, `to`
+    1. Simple map: `!`
+    1. Arrow: `=>`
+    1. Node comparison: `is`, `<<`, `>>`
+1. Expressions: `if`/`then`/`else`, `for`, `let`, `some`/`every` (quantified)
+1. Type expressions: `instance of`, `cast as`, `castable as`, `treat as`
+1. Functions (100+): string (`contains`, `starts-with`, `ends-with`, `substring`, `concat`, `normalize-space`, `upper-case`, `lower-case`, `translate`, `matches`, `replace`, `tokenize`, ...), numeric (`round`, `floor`, `ceiling`, `abs`, `sum`, `avg`, `min`, `max`, ...), boolean (`not`, `true`, `false`, `boolean`), sequence (`count`, `empty`, `exists`, `reverse`, `sort`, `distinct-values`, `head`, `tail`, `subsequence`, ...), node (`name`, `local-name`, `root`, `path`, `has-children`, `data`, ...), higher-order (`for-each`, `filter`, `fold-left`, `fold-right`, ...), and more
+1. Maps and arrays construction and access
+
 If your use case requires an unimplemented feature,
 please open an issue on [GitHub](https://github.com/James-LG/Skyscraper/issues).
+
+See [`docs/features-backlog.md`](docs/features-backlog.md) for a detailed list of spec gaps, known limitations, and design decisions.

benches/benchmarks.rs

@@ -2,9 +2,20 @@ use criterion::{criterion_group, criterion_main};
 
 mod html_benchmark;
 mod xpath_benchmark;
+mod xpath_eval_benchmark;
 
 use crate::html_benchmark::*;
 use crate::xpath_benchmark::*;
+use crate::xpath_eval_benchmark::*;
 
-criterion_group!(benches, benchmark_html_parse, benchmark_xpath_parse);
+criterion_group!(
+    benches,
+    benchmark_html_parse,
+    benchmark_xpath_parse,
+    benchmark_axes,
+    benchmark_predicates,
+    benchmark_functions,
+    benchmark_expressions,
+    benchmark_real_world,
+);
 criterion_main!(benches);