Added AI summarization of news items (#1897)

daveoconnor · daveoconnor · commit 61bdba30a127 · 2025-09-25T17:23:22.000-07:00
diff --git a/.github/workflows/actions-gcp.yaml b/.github/workflows/actions-gcp.yaml
@@ -71,6 +71,7 @@ jobs:
           SECRET_KEY: "for-testing-only"
           REDIS_HOST: "localhost"
           CI: "true"
+          OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }}
         run: |
           python -m pytest
 
diff --git a/.github/workflows/actions.yml b/.github/workflows/actions.yml
@@ -61,6 +61,7 @@ jobs:
           SECRET_KEY: "for-testing-only"
           REDIS_HOST: "localhost"
           CI: "true"
+          OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }}
         run: |
           python -m pytest
 
diff --git a/config/settings.py b/config/settings.py
@@ -581,3 +581,5 @@
     MIDDLEWARE.append("debug_toolbar.middleware.DebugToolbarMiddleware")
 
 BOOST_BRANCHES = ["master", "develop"]
+OPENROUTER_URL = "https://openrouter.ai/api/v1"
+OPENROUTER_API_KEY = env("OPENROUTER_API_KEY")
diff --git a/core/templatetags/text_helpers.py b/core/templatetags/text_helpers.py
@@ -34,8 +34,6 @@ def replace_match(match):
         else:
             word = word_or_link
 
-        print(word_or_link)
-
         if link_inner_match:
             if len(word) > ln + 10:
                 start = word[: ((ln + 10) // 2)]
diff --git a/env.template b/env.template
@@ -78,3 +78,5 @@ PROD_LISTS_CORE_DB_DUMP_URL=gs://boostbackups/db1/daily/
 PROD_LISTS_CORE_DB_DUMP_FILE_WILDCARD=lists_production_core.db1*
 PROD_LISTS_WEB_DB_DUMP_URL=gs://boostbackups/db1/daily/
 PROD_LISTS_WEB_DB_DUMP_FILE_WILDCARD=lists_production_web.db1*
+
+OPENROUTER_API_KEY=
diff --git a/news/constants.py b/news/constants.py
@@ -1,2 +1,3 @@
 NEWS_APPROVAL_SALT = "news-approval"
 MAGIC_LINK_EXPIRATION = 3600 * 24  # 24h
+CONTENT_SUMMARIZATION_THRESHOLD = 1000  # characters
diff --git a/news/helpers.py b/news/helpers.py
@@ -1,18 +1,13 @@
-# import requests
-#
-# from django.conf import settings
-#
-#
-# def get_link_preview_data(link):
-#     """gets the link preview json response from LinkPreview api"""
-#     api_url = "https://api.linkpreview.net"
-#     api_key = settings.LINK_PREVIEW_API_KEY
-#     target = link
-#
-#     # TODO: Add additional field `image_size` to help validate image https://docs.linkpreview.net/#image-processing-and-validation
-#     response = requests.get(
-#         api_url,
-#         headers={'X-Linkpreview-Api-Key': api_key},
-#         params={'q': target},
-#     )
-#     return response.json()
+from bs4 import BeautifulSoup
+
+
+def extract_content(html: str) -> str:
+    soup = BeautifulSoup(html, "html.parser")
+    non_visible_tags = ["style", "script", "head", "meta", "[document]"]
+    for script_or_style in soup(non_visible_tags):
+        script_or_style.decompose()
+    text = soup.get_text(separator="\n")
+    lines = (line.strip() for line in text.splitlines())
+    # drop blank lines
+    minimized = [line for line in lines if line]
+    return "\n".join(minimized)
diff --git a/news/management/__init__.py b/news/management/__init__.py
diff --git a/news/management/commands/__init__.py b/news/management/commands/__init__.py
diff --git a/news/management/commands/backpopulate_summaries.py b/news/management/commands/backpopulate_summaries.py
@@ -0,0 +1,36 @@
+import djclick as click
+from news.models import Entry
+from news.tasks import summary_dispatcher
+
+
+@click.command()
+@click.option(
+    "--dry-run",
+    is_flag=True,
+    help="Show which entries would be processed without actually dispatching tasks",
+)
+def command(dry_run):
+    """Backpopulate summary field for news entries where summary is not set."""
+
+    entries_without_summary = Entry.objects.filter(summary="")
+    count = entries_without_summary.count()
+
+    if count == 0:
+        click.echo("No entries found without summaries.")
+        return
+
+    if dry_run:
+        click.echo(f"Would process {count} entries:")
+        for entry in entries_without_summary[:10]:
+            click.echo(f"  - {entry.pk}: {entry.title}")
+        if count > 10:
+            click.echo(f"  ... and {count - 10} more")
+        return
+
+    click.echo(f"Processing {count} entries without summaries...")
+
+    for entry in entries_without_summary:
+        click.echo(f"Dispatching summary task for entry {entry.pk}: {entry.title}")
+        summary_dispatcher.delay(entry.pk)
+
+    click.echo(f"Dispatched summary tasks for {count} entries.")
diff --git a/news/migrations/0011_entry_summary.py b/news/migrations/0011_entry_summary.py
@@ -0,0 +1,22 @@
+# Generated by Django 4.2.16 on 2025-09-02 21:21
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("news", "0010_news_attachment"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="entry",
+            name="summary",
+            field=models.TextField(
+                blank=True,
+                default="",
+                help_text="AI generated summary. Delete to regenerate.",
+            ),
+        ),
+    ]
diff --git a/news/models.py b/news/models.py
@@ -1,5 +1,6 @@
 from pathlib import Path
 
+from structlog import get_logger
 from django.contrib.auth import get_user_model
 from django.db import models
 from django.db.models import Case, Value, When
@@ -17,8 +18,11 @@
 )
 
 from . import acl
+from .constants import CONTENT_SUMMARIZATION_THRESHOLD
+from .tasks import summary_dispatcher
 
 User = get_user_model()
+logger = get_logger(__name__)
 
 
 class EntryManager(models.Manager):
@@ -86,6 +90,9 @@ class AlreadyApprovedError(Exception):
     approved_at = models.DateTimeField(null=True, blank=True)
     modified_at = models.DateTimeField(auto_now=True)
     publish_at = models.DateTimeField(default=now)
+    summary = models.TextField(
+        blank=True, default="", help_text="AI generated summary. Delete to regenerate."
+    )
 
     objects = EntryManager()
 
@@ -154,6 +161,21 @@ def is_video(self):
             result = False
         return result
 
+    @cached_property
+    def determined_news_type(self):
+        if self.is_blogpost:
+            return "blogpost"
+        elif self.is_link:
+            return "link"
+        elif self.is_news:
+            return "news"
+        elif self.is_poll:
+            return "poll"
+        elif self.is_video:
+            return "video"
+        else:
+            return None
+
     def approve(self, user, commit=True):
         """Mark this entry as approved by the given `user`."""
         if self.is_approved:
@@ -163,10 +185,28 @@ def approve(self, user, commit=True):
         if commit:
             self.save(update_fields=["moderator", "approved_at", "modified_at"])
 
+    @cached_property
+    def use_summary(self):
+        return self.summary and (
+            not self.content or len(self.content) > CONTENT_SUMMARIZATION_THRESHOLD
+        )
+
+    @cached_property
+    def visible_content(self):
+        if self.use_summary:
+            return self.summary
+        return self.content
+
     def save(self, *args, **kwargs):
         if not self.slug:
             self.slug = slugify(self.title)
-        return super().save(*args, **kwargs)
+        result = super().save(*args, **kwargs)
+
+        if not self.summary:
+            logger.info(f"Passing {self.pk=} to dispatcher")
+            summary_dispatcher.delay(self.pk)
+
+        return result
 
     def get_absolute_url(self):
         return reverse("news-detail", args=[self.slug])
diff --git a/news/tasks.py b/news/tasks.py
@@ -0,0 +1,142 @@
+from textwrap import dedent
+from openai import OpenAI, OpenAIError
+import requests
+import structlog
+
+from config.celery import app
+from config.settings import OPENROUTER_API_KEY, OPENROUTER_URL
+from news.constants import CONTENT_SUMMARIZATION_THRESHOLD
+from news.helpers import extract_content
+
+logger = structlog.get_logger(__name__)
+
+
+@app.task(bind=True, max_retries=3, autoretry_for=(OpenAIError,))
+def summarize_content(self, content: str, title: str, model: str) -> str:
+    """Summarize content using an LLM model."""
+    if not content:
+        logger.warning("No content provided to summarize, skipping.")
+        raise ValueError("No content provided to summarize.")
+    logger.info(f"Summarizing {content[:100]=}... with {model=}")
+    max_length = 256
+    system_prompt = dedent(
+        f"""
+        You are an experienced technical writer tasked with summarizing content. Provide
+        a brief description of what the content after the "----" is discussing.
+        The title is also provided and may be in the content, repeating it in the
+        summary would be redundant so should be avoided.
+        Your summary should be concise, clear, and capture the main points of the
+        content. It should be less than {max_length} characters, with a single paragraph
+        of text, without going into detail. Before returning your response, check if
+        it's less than {max_length} characters, if not, shorten it until it is.
+        Write summaries in an impersonal, passive voice, never attributing actions to
+        'the author' or similar.
+        If no content is provided, do not return anything at all.
+        Don't format with markdown, html, or any other markup, just plain text.
+        Avoid adding any personal opinions or extraneous information.
+        Do not allow any NSFW content such as profanity, sexual content, or violence to
+        be returned in the summary, work around it.
+        Do not allow any security vulnerabilities to be returned in the summary, work
+        around them.
+        """
+    )
+    user_prompt = dedent(
+        f"""
+        Please provide a summary of the following content:
+        ----
+        Title: {title}
+        Content: {content}
+        """
+    )
+    messages = [
+        {"role": "system", "content": system_prompt},
+        {"role": "user", "content": user_prompt},
+    ]
+    logger.debug(f"{messages=}")
+    content = None
+    try:
+        client = OpenAI(base_url=OPENROUTER_URL, api_key=OPENROUTER_API_KEY)
+        response = client.chat.completions.create(model=model, messages=messages)
+        content = response.choices[0].message.content
+        logger.info(
+            f"Received summarized content for {content[:100]=}: {len(content)=}..."
+        )
+    except (AttributeError, IndexError) as e:
+        logger.error(f"Error getting summarized content: {e=}")
+    return content
+
+
+@app.task
+def save_entry_summary_value(summary: str, pk: int):
+    from news.models import Entry
+
+    entry = Entry.objects.get(pk=pk)
+    entry.summary = summary
+    entry.save()
+
+
+@app.task
+def summary_dispatcher(pk: int):
+    from news.models import Entry
+
+    entry = Entry.objects.get(pk=pk)
+    logger.info(f"Dispatching {pk=} with {entry.news_type=}")
+    handler = {
+        "news": set_summary_for_event_entry,
+        "blogpost": set_summary_for_event_entry,
+        "link": set_summary_for_link_entry,
+        "video": set_summary_for_video_entry,
+        "poll": set_summary_for_poll_entry,
+    }[entry.determined_news_type]
+    logger.info(f"Dispatching summary task for {pk=} to {handler.__name__=}")
+    handler.delay(pk)
+
+
+@app.task
+def set_summary_for_event_entry(pk: int):
+    from news.models import Entry
+
+    entry = Entry.objects.get(pk=pk)
+    logger.info(f"dispatching summarize task for {pk=} with {entry.content[:40]=}...")
+    if entry.content and len(entry.content) < CONTENT_SUMMARIZATION_THRESHOLD:
+        logger.warning(f"Content too short to summarize for {pk=}, skipping.")
+        return
+    logger.info(f"handing off {pk=} to summarize_content task")
+    summarize_content.apply_async(
+        (entry.content, entry.title, "gpt-oss-120b"),
+        link=save_entry_summary_value.s(pk),
+    )
+
+
+@app.task
+def set_summary_for_link_entry(pk: int):
+    logger.info(f"Setting summary for link entry {pk=}")
+    from news.models import Entry
+
+    entry = Entry.objects.get(pk=pk)
+    try:
+        logger.info(f"Fetching content from {entry.external_url=} for entry.{pk=}")
+        response = requests.get(entry.external_url, timeout=10)
+        response.raise_for_status()
+        markup = response.text
+        logger.debug(f"Fetched {len(markup)=} for entry.{pk=}...")
+        content = extract_content(markup)
+        logger.info(f"extracted content from {entry.external_url=}, {markup[:100]=}")
+    except requests.RequestException as e:
+        logger.error(f"Error fetching content from {entry.external_url=}: {e=}")
+        return
+
+    logger.info(f"dispatching summarize task for {pk=} with {content[:40]=}...")
+    summarize_content.apply_async(
+        (content, entry.title, "gpt-oss-120b"), link=save_entry_summary_value.s(pk)
+    )
+
+
+@app.task
+def set_summary_for_video_entry(pk: int):
+    logger.info("Summarization not implemented")
+
+
+@app.task
+def set_summary_for_poll_entry(pk: int):
+    logger.info("Summarization not implemented")
diff --git a/requirements.in b/requirements.in
@@ -34,6 +34,7 @@ jsoncomment
 unidecode
 wordcloud
 lxml
+openai
 
 # Logging
 django-tracer
diff --git a/requirements.txt b/requirements.txt
diff --git a/templates/news/detail.html b/templates/news/detail.html
diff --git a/templates/news/list.html b/templates/news/list.html

Original file line number	Diff line number	Diff line change
`@@ -1,2 +1,3 @@`
`1`	`1`	`NEWS_APPROVAL_SALT = "news-approval"`
`2`	`2`	`MAGIC_LINK_EXPIRATION = 3600 * 24 # 24h`
	`3`	`+CONTENT_SUMMARIZATION_THRESHOLD = 1000 # characters`