Skip to content

Commit 7ecae0c

Browse files
committed
refactor: decide the listing exemption without a ratio
Ask whether every link stands alone in its own paragraph instead of counting how many do, which removes the majority threshold. Measured identical on the reporter's page, on tests/comparison_small.py over all 960 EVAL_PAGES, and on the suite; the added test pins the boundary the simpler form draws.
1 parent 49e8201 commit 7ecae0c

2 files changed

Lines changed: 17 additions & 5 deletions

File tree

tests/unit_tests.py

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1528,6 +1528,19 @@ def test_link_density_links_sharing_a_paragraph_pruned():
15281528
assert trafilatura.htmlprocessing.link_density_test(element, text)[0] is True
15291529

15301530

1531+
def test_link_density_listing_with_one_shared_paragraph_pruned():
1532+
"the exemption is all-or-nothing on purpose: one paragraph holding two links makes the \
1533+
container a farm again, which is what lets the check work without a ratio or a threshold."
1534+
from trafilatura.utils import trim
1535+
1536+
items = "".join(f'<p><ref target="/d{i}.pdf">Instruktionsbok MC 258 part {i}</ref></p>' for i in range(14))
1537+
shared = '<p><ref target="/a">Also see this page</ref> <ref target="/b">and this other one</ref></p>'
1538+
element = html.fromstring(f"<body><div>{items}{shared}</div><p>real article sibling here</p></body>")[0]
1539+
text = trim(element.text_content())
1540+
assert len(text) > 300
1541+
assert trafilatura.htmlprocessing.link_density_test(element, text)[0] is True
1542+
1543+
15311544
def test_overall_discard_legacy_tokens():
15321545
"regression on the legacy single-PR discard tokens, each decided by a full-WMB single-token A/B \
15331546
(see xpaths.py audit note): 'yin' STAYS (net-positive despite English '-ying'/'y+Info' \

trafilatura/htmlprocessing.py

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -124,13 +124,12 @@ def collect_link_info(
124124

125125

126126
def is_paragraph_listing(links_xpath: list[HtmlElement]) -> bool:
127-
"Tell a document listing (each link its own paragraph) from a farm (links running together)"
128-
own_paragraph = 0
127+
"Tell a document listing (every link alone in its paragraph) from a farm (links running together)"
129128
for link in links_xpath:
130129
parent = link.getparent()
131-
if parent is not None and parent.tag == "p" and len(parent.findall(".//ref")) == 1:
132-
own_paragraph += 1
133-
return own_paragraph * 2 > len(links_xpath)
130+
if parent is None or parent.tag != "p" or len(parent.findall(".//ref")) > 1:
131+
return False
132+
return True
134133

135134

136135
def link_density_test(element: HtmlElement, text: str, favor_precision: bool = False) -> tuple[bool, list[str]]:

0 commit comments

Comments
 (0)