Resolve sitemap locations by XML namespace

Author: be-student

docs/tutorial-corpus.rst

@@ -28,6 +28,8 @@ In order to gather web documents it can be useful to download the portions of a
 
 A comprehensive overview of the available documents can be obtained faster and more efficiently using sitemaps and feeds than by systematically crawling. These formats are machine-readable and can reveal content that may not be reachable through the browsable interface. However, link inspection and filtering prior to download is recommended to avoid undesired content — see `link filtering`_ below.
 
+XML sitemap locations are resolved by namespace, so default and prefixed sitemap namespaces are supported. Location tags in extension namespaces (such as image sitemaps) are ignored. Legacy XML without a namespace is also supported.
+
 In addition, Trafilatura supports multilingual and multinational sitemaps, for example when a site targets different languages through paths like ``/en/…`` and ``/de/…``.
 
 .. hint::

tests/sitemaps_tests.py

@@ -220,3 +220,63 @@ def test_whole():
     trafilatura.settings.MAX_SITEMAPS_SEEN = 1
     results = sitemaps.sitemap_search("https://www.sitemaps.org", target_lang="de")
     assert len(results) == 8
+
+
+@pytest.mark.parametrize("prefix", ["", "sm:", "site-map:"])
+@pytest.mark.parametrize("declaration", ["", '<?xml version="1.0" encoding="UTF-8"?>'])
+@pytest.mark.parametrize("index", [False, True])
+def test_sitemap_namespaces(prefix, declaration, index):
+    """Namespace prefixes do not change page or nested sitemap discovery."""
+    root, child = ("sitemapindex", "sitemap") if index else ("urlset", "url")
+    namespace = f'xmlns{":" + prefix[:-1] if prefix else ""}="http://www.sitemaps.org/schemas/sitemap/0.9"'
+    url = "https://example.org/nested.xml" if index else "https://example.org/page"
+    sitemap = sitemaps.SitemapObject("https://example.org", "example.org", [])
+    sitemap.current_url = "https://example.org/sitemap.xml"
+    sitemap.content = (
+        f"{declaration}<{prefix}{root} {namespace}>"
+        f"<{prefix}{child}><{prefix}loc><![CDATA[{url}]]></{prefix}loc></{prefix}{child}>"
+        f"</{prefix}{root}>"
+    )
+    sitemap.process()
+    assert (sitemap.sitemap_urls, sitemap.urls) == (([url], []) if index else ([], [url]))
+
+
+def test_sitemap_namespace_scope():
+    """Ignore extension locations even when a prefix is rebound locally."""
+    sitemap = sitemaps.SitemapObject("https://example.org", "example.org", [])
+    sitemap.current_url = "https://example.org/sitemap.xml"
+    sitemap.content = (
+        '<s:urlset xmlns:s="http://www.sitemaps.org/schemas/sitemap/0.9" xmlns:image="urn:image">'
+        "<s:url><s:loc>https://example.org/page</s:loc>"
+        "<image:loc>https://example.org/image</image:loc>"
+        '<s:loc xmlns:s="urn:other">https://example.org/other</s:loc>'
+        "</s:url></s:urlset>"
+    )
+    sitemap.process()
+    assert sitemap.urls == ["https://example.org/page"]
+
+
+@pytest.mark.parametrize("location", ["&external;", "https://example.org/&external;", "https://example.org/<nested/>"])
+def test_sitemap_locations_do_not_expand_entities(location):
+    """Location extraction does not resolve entities or concatenate child markup."""
+    sitemap = sitemaps.SitemapObject("https://example.org", "example.org", [])
+    sitemap.content = (
+        '<!DOCTYPE urlset [<!ENTITY external SYSTEM "file:///not-a-sitemap-resource">]>'
+        '<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">'
+        f"<url><loc>{location}</loc></url></urlset>"
+    )
+    sitemap.extract_sitemap_links()
+    assert not sitemap.urls
+
+
+def test_sitemap_namespace_link_limit(monkeypatch):
+    """Namespaced extraction keeps the existing maximum-location bound."""
+    monkeypatch.setattr(sitemaps, "MAX_LINKS", 1)
+    sitemap = sitemaps.SitemapObject("https://example.org", "example.org", [])
+    sitemap.content = (
+        '<s:urlset xmlns:s="http://www.sitemaps.org/schemas/sitemap/0.9">'
+        "<s:url><s:loc>https://example.org/first?a=1&amp;b=2</s:loc></s:url>"
+        "<s:url><s:loc>https://example.org/second</s:loc></s:url></s:urlset>"
+    )
+    sitemap.extract_sitemap_links()
+    assert sitemap.urls == ["https://example.org/first?a=1&b=2"]

trafilatura/sitemaps.py

@@ -17,21 +17,22 @@
     get_hostinfo,
     lang_filter,
 )
+from lxml import etree
 
 from .deduplication import is_similar_domain
 from .downloads import fetch_url, is_live_page
 from .settings import MAX_LINKS, MAX_SITEMAPS_SEEN
 
 LOGGER = logging.getLogger(__name__)
 
-LINK_REGEX = re.compile(r"<loc>(?:<!\[CDATA\[)?(http.+?)(?:\]\]>)?</loc>")
+SITEMAP_NAMESPACE = "http://www.sitemaps.org/schemas/sitemap/0.9"
 XHTML_REGEX = re.compile(r"<xhtml:link.+?>", re.DOTALL)
 HREFLANG_REGEX = re.compile(r'href=["\'](.+?)["\']')
 WHITELISTED_PLATFORMS = re.compile(
     r"(?:blogger|blogpost|ghost|hubspot|livejournal|medium|typepad|squarespace|tumblr|weebly|wix|wordpress)\."
 )
 
-SITEMAP_FORMAT = re.compile(r"^.{0,5}<\?xml|<sitemap|<urlset")
+SITEMAP_FORMAT = re.compile(r"^.{0,5}<\?xml|<(?:[\w.-]+:)?(?:sitemapindex|sitemap|urlset)\b")
 DETECT_SITEMAP_LINK = re.compile(r"\.xml(\..{2,4})?$|\.xml[?#]")
 DETECT_LINKS = re.compile(r'https?://[^\s<"]+')
 SCRUB_REGEX = re.compile(r"\?.*$|#.*$")
@@ -141,8 +142,18 @@ def handle_lang_link(attrs: str) -> None:
         self.extract_links(XHTML_REGEX, 0, handle_lang_link)
 
     def extract_sitemap_links(self) -> None:
-        "Extract sitemap links and web page links from a sitemap file."
-        self.extract_links(LINK_REGEX, 1, self.handle_link)  # process middle part of the match tuple
+        "Extract locations in the sitemap namespace (or legacy unnamespaced XML)."
+        parser = etree.XMLParser(encoding="utf-8", resolve_entities=False, no_network=True, recover=True)
+        try:
+            tree = etree.fromstring(self.content.encode("utf-8"), parser)
+        except etree.XMLSyntaxError:
+            return
+        if tree is None:
+            return
+        for element in islice(tree.iter("loc", f"{{{SITEMAP_NAMESPACE}}}loc"), MAX_LINKS):
+            # Entity references and nested markup are not part of a location URL.
+            if len(element) == 0 and element.text and element.text.startswith("http"):
+                self.handle_link(element.text)
 
     def process(self) -> None:
         "Download a sitemap and extract the links it contains."