Synced webpages from sitemap to Document model for search.

Author: sarahboyce

docs/management/commands/update_docs.py

@@ -132,6 +132,8 @@ def build_doc_release(self, release, force=False, interactive=False):
         if self.verbosity >= 1:
             self.stdout.write(f"Starting update for {release} at {datetime.now()}...")
 
+        release.sync_from_sitemap()
+
         # checkout_dir is shared for all languages.
         checkout_dir = settings.DOCS_BUILD_ROOT.joinpath("sources", release.version)
         parent_build_dir = settings.DOCS_BUILD_ROOT.joinpath(

docs/models.py

@@ -5,6 +5,7 @@
 from functools import partial, reduce
 from pathlib import Path
 
+import requests
 from django.conf import settings
 from django.contrib.postgres.indexes import GinIndex
 from django.contrib.postgres.search import (
@@ -34,8 +35,16 @@
     START_SEL,
     STOP_SEL,
     TSEARCH_CONFIG_LANGUAGES,
+    DocumentationCategory,
+    fetch_html,
     get_document_search_vector,
 )
+from .utils import extract_inner_html
+
+
+def get_search_config(lang):
+    """Determine the PostgreSQL search language"""
+    return TSEARCH_CONFIG_LANGUAGES.get(lang[:2], DEFAULT_TEXT_SEARCH_CONFIG)
 
 
 class DocumentReleaseQuerySet(models.QuerySet):
@@ -175,7 +184,7 @@ def sync_to_db(self, decoded_documents):
         the database. Deletes all the release's documents first then
         reinserts them as needed.
         """
-        self.documents.all().delete()
+        self.documents.exclude(metadata__parents=DocumentationCategory.WEBSITE).delete()
 
         # Read excluded paths from robots.docs.txt.
         robots_path = settings.BASE_DIR.joinpath(
@@ -206,16 +215,54 @@ def sync_to_db(self, decoded_documents):
                 path=document_path,
                 title=html.unescape(strip_tags(document["title"])),
                 metadata=document,
-                config=TSEARCH_CONFIG_LANGUAGES.get(
-                    self.lang[:2], DEFAULT_TEXT_SEARCH_CONFIG
-                ),
+                config=get_search_config(self.lang),
             )
-        for document in self.documents.all():
+        for document in self.documents.exclude(
+            metadata__parents=DocumentationCategory.WEBSITE
+        ):
             document.metadata["breadcrumbs"] = list(
                 Document.objects.breadcrumbs(document).values("title", "path")
             )
             document.save(update_fields=("metadata",))
 
+    def sync_from_sitemap(self):
+        from djangoproject.urls.www import sitemaps
+
+        if self.lang != "en" or self.release:
+            return
+
+        for sitemap in sitemaps.values():
+            for url in sitemap().get_urls():
+                try:
+                    page_html = fetch_html(url["location"])
+                except requests.RequestException:
+                    continue
+                try:
+                    main_html = extract_inner_html(page_html, tag="main")
+                    title = extract_inner_html(page_html, tag="h1")
+                except ValueError:
+                    continue
+                Document.objects.update_or_create(
+                    release=self,
+                    path=url["location"],
+                    defaults={
+                        "title": title,
+                        "metadata": {
+                            "body": main_html,
+                            "breadcrumbs": [
+                                {
+                                    "path": DocumentationCategory.WEBSITE,
+                                    "title": "Website",
+                                },
+                            ],
+                            "parents": DocumentationCategory.WEBSITE,
+                            "title": title,
+                            "toc": "",
+                        },
+                        "config": get_search_config(self.lang),
+                    },
+                )
+
 
 def _clean_document_path(path):
     # We have to be a bit careful to reverse-engineer the correct
@@ -228,7 +275,9 @@ def _clean_document_path(path):
 
 
 def document_url(doc):
-    if doc.path:
+    if doc.metadata.get("parents") == DocumentationCategory.WEBSITE:
+        return doc.path
+    elif doc.path:
         kwargs = {
             "lang": doc.release.lang,
             "version": doc.release.version,
@@ -273,6 +322,14 @@ def search(self, query_text, release, document_category=None):
                 config=models.F("config"),
             )
             base_filter = Q(release_id=release.id)
+            if release.lang == "en" and release.version != "dev":
+                dev_release = DocumentRelease.objects.get_by_version_and_lang(
+                    "dev", "en"
+                )
+                base_filter |= Q(
+                    release_id=dev_release.id,
+                    metadata__parents=DocumentationCategory.WEBSITE,
+                )
             if document_category:
                 base_filter &= Q(metadata__parents__startswith=document_category)
             base_qs = (

docs/search.py

@@ -1,3 +1,4 @@
+import requests
 from django.contrib.postgres.search import SearchVector
 from django.db.models import TextChoices
 from django.db.models.fields.json import KeyTextTransform
@@ -67,10 +68,41 @@ class DocumentationCategory(TextChoices):
     TOPICS = "topics", _("Using Django")
     HOWTO = "howto", _("How-to guides")
     RELEASE_NOTES = "releases", _("Release notes")
+    WEBSITE = "website", _("Django Website")
 
     @classmethod
     def parse(cls, value, default=None):
         try:
             return cls(value)
         except ValueError:
             return None
+
+
+def fetch_html(url, timeout=10):
+    """
+    Fetch the HTML of a page if status code is 200.
+    Simulates a human browser and accepts only text/html.
+    """
+
+    headers = {
+        "User-Agent": (
+            "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
+            "AppleWebKit/537.36 (KHTML, like Gecko) "
+            "Chrome/123.0.0.0 Safari/537.36 Edg/123.0.0.0"
+        ),
+        "Accept": "text/html",
+        "Accept-Language": "en-US,en;q=0.9",
+    }
+
+    response = requests.get(url, headers=headers, timeout=timeout)
+
+    if response.status_code == 200:
+        content_type = response.headers.get("Content-Type", "")
+        if "text/html" in content_type:
+            return response.text
+        else:
+            raise requests.RequestException(f"Unexpected Content-Type: {content_type}")
+    else:
+        raise requests.RequestException(
+            f"Failed to fetch {url}, status code: {response.status_code}"
+        )

docs/templates/docs/search_results.html

@@ -43,11 +43,11 @@ <h2>{% translate "No search query given" %}</h2>
         {% for result in page.object_list %}
           <dt>
             <h2 class="result-title">
-              <a href="{% url 'document-detail' lang=result.release.lang version=result.release.version url=result.path host 'docs' %}{% if not start_sel in result.headline %}{{ result.highlight|fragment }}{% endif %}">{{ result.headline|safe }}</a>
+              <a href="{{ result.get_absolute_url }}{% if not start_sel in result.headline %}{{ result.highlight|fragment }}{% endif %}">{{ result.headline|safe }}</a>
             </h2>
             <span class="meta breadcrumbs">
               {% for breadcrumb in result.breadcrumbs %}
-                <a href="{% url 'document-detail' lang=result.release.lang version=result.release.version url=breadcrumb.path host 'docs' %}">{{ breadcrumb.title }}</a>{% if not forloop.last %} <span class="arrow">»</span>{% endif %}
+                <a href="{{ result.get_absolute_url }}">{{ breadcrumb.title }}</a>{% if not forloop.last %} <span class="arrow">»</span>{% endif %}
               {% endfor %}
             </span>
           </dt>
@@ -60,7 +60,7 @@ <h2 class="result-title">
               <ul class="code-links">
                 {% for name, value in result_code_links.items %}
                   <li>
-                    <a href="{% url 'document-detail' lang=result.release.lang version=result.release.version url=result.path host 'docs' %}#{{ value.full_path }}">
+                    <a href="{{ result.get_absolute_url }}#{{ value.full_path }}">
                       <div>
                         <code>{{ name }}</code>
                         {% if value.module_path %}<div class="meta">{{ value.module_path }}</div>{% endif %}

docs/tests/test_models.py

@@ -173,6 +173,7 @@ def test_get_available_languages_by_version(self):
 class DocumentManagerTest(TestCase):
     @classmethod
     def setUpTestData(cls):
+        DocumentRelease.objects.create(lang="en")
         cls.release = DocumentRelease.objects.create(
             release=Release.objects.create(version="1.2.3"),
         )

docs/tests/test_utils.py

@@ -3,7 +3,7 @@
 
 from django.test import SimpleTestCase
 
-from ..utils import get_doc_path, sanitize_for_trigram
+from ..utils import extract_inner_html, get_doc_path, sanitize_for_trigram
 
 
 class TestUtils(SimpleTestCase):
@@ -38,3 +38,38 @@ def test_sanitize_for_trigram(self):
         ]:
             with self.subTest(query=query):
                 self.assertEqual(sanitize_for_trigram(query), sanitized_query)
+
+    def test_extract_inner_html(self):
+        for html, expected_output in [
+            ("<main><p>Hello</p></main>", "<p>Hello</p>"),
+            (
+                '<header>Test</header><main id="app" class="container">'
+                "<h1>Title</h1></main>",
+                "<h1>Title</h1>",
+            ),
+            ("<main>&amp; &lt; &gt; &#169;</main>", "& < > ©"),
+            ("<main></main>", ""),
+            ("<main>Hello world</main>", "Hello world"),
+            ("<main><h1>Hi</h1>Text<p>Bye</p></main>", "<h1>Hi</h1>Text<p>Bye</p>"),
+        ]:
+            self.assertEqual(extract_inner_html(html, tag="main"), expected_output)
+
+    def test_extract_inner_html_multiple_same_tags_raises(self):
+        with self.assertRaisesMessage(
+            ValueError, "<main> occurs more than once in HTML."
+        ):
+            extract_inner_html(
+                "<main>One main</main><main id='dupe'>Two main</main>", tag="main"
+            )
+
+    def test_extract_inner_html_multiple_same_tags_nested_raises(self):
+        with self.assertRaisesMessage(
+            ValueError, "Nested <main> tags are not allowed."
+        ):
+            extract_inner_html(
+                "<main>One main<main id='dupe'>Two main</main></main>", tag="main"
+            )
+
+    def test_extract_inner_html_tag_not_found_raises(self):
+        with self.assertRaisesMessage(ValueError, "<main> not found in HTML."):
+            extract_inner_html("<p>Test</p>", tag="main")

docs/tests/test_views.py

@@ -91,7 +91,7 @@ def test_search_type_filter_all(self):
         )
         self.assertEqual(response.status_code, 200)
         self.assertContains(
-            response, "4 results for <em>generic</em> in version 5.1", html=True
+            response, "5 results for <em>generic</em> in version 5.1", html=True
         )
         self.assertContains(response, self.active_filter, count=1)
         self.assertContains(response, f"{self.active_filter}All</a>", html=True)
@@ -122,7 +122,7 @@ def test_search_category_filter_invalid_doc_categories(self):
         )
         self.assertEqual(response.status_code, 200)
         self.assertContains(
-            response, "4 results for <em>generic</em> in version 5.1", html=True
+            response, "5 results for <em>generic</em> in version 5.1", html=True
         )
         self.assertContains(response, self.active_filter, count=1)
         self.assertContains(response, f"{self.active_filter}All</a>", html=True)

docs/utils.py

@@ -1,5 +1,6 @@
 import re
 import unicodedata
+from html.parser import HTMLParser
 
 from django.conf import settings
 from django.http import Http404
@@ -92,3 +93,59 @@ def get_module_path(name, full_path):
     if full_path.endswith(name_suffix):
         return full_path.removesuffix(name_suffix)
     return None
+
+
+class SingleTagInnerHTMLExtractor(HTMLParser):
+    def __init__(self, target_tag):
+        super().__init__()
+        self.target_tag = target_tag.lower()
+        self.capturing = False
+        self.inner_html = []
+        self.tag_count = 0
+
+    def handle_starttag(self, tag, attrs):
+        tag = tag.lower()
+        if tag == self.target_tag:
+            self.tag_count += 1
+            if self.capturing:
+                # Nested target tag not allowed.
+                raise ValueError(f"Nested <{self.target_tag}> tags are not allowed.")
+            self.capturing = True
+        elif self.capturing:
+            self.inner_html.append(self.get_starttag_text())
+
+    def handle_endtag(self, tag):
+        tag = tag.lower()
+        if self.capturing:
+            if tag == self.target_tag:
+                self.capturing = False
+            else:
+                self.inner_html.append(f"</{tag}>")
+
+    def handle_data(self, data):
+        if self.capturing:
+            self.inner_html.append(data)
+
+    def handle_entityref(self, name):
+        if self.capturing:
+            self.inner_html.append(f"&{name};")
+
+    def handle_charref(self, name):
+        if self.capturing:
+            self.inner_html.append(f"&#{name};")
+
+
+def extract_inner_html(html, tag):
+    """
+    Extracts the inner HTML of a tag that appears exactly once.
+    """
+    parser = SingleTagInnerHTMLExtractor(tag)
+    parser.feed(html)
+    parser.close()
+
+    if parser.tag_count == 0:
+        raise ValueError(f"<{tag}> not found in HTML.")
+    if parser.tag_count > 1:
+        raise ValueError(f"<{tag}> occurs more than once in HTML.")
+
+    return "".join(parser.inner_html)