fix: request xai release notes as html (#72)

Author: parrisdigital

convex/ingest.ts

@@ -69,9 +69,28 @@ function canUseConditionalCache(source: SourceFetchTarget) {
   return sourceLooksLikeDirectFeed(source) || !source.parserKey || !FEED_PARSER_KEYS.has(source.parserKey);
 }
 
-function buildFetchHeaders(userAgent: string, source?: SourceFetchTarget) {
+function sourceNeedsHtmlFirstAccept(source?: SourceFetchTarget) {
+  if (!source) {
+    return false;
+  }
+
+  if (source.parserKey === "xai:docs_page") {
+    return true;
+  }
+
+  try {
+    const url = new URL(source.url);
+    return url.hostname === "docs.x.ai" && url.pathname.replace(/\/$/, "") === "/developers/release-notes";
+  } catch {
+    return /docs\.x\.ai\/developers\/release-notes\/?$/i.test(source.url);
+  }
+}
+
+export function buildFetchHeaders(userAgent: string, source?: SourceFetchTarget) {
   const accept = sourceLooksLikeDirectFeed(source ?? { url: "" })
     ? "application/rss+xml,application/atom+xml,application/xml;q=0.9,text/xml;q=0.8,*/*;q=0.7"
+    : sourceNeedsHtmlFirstAccept(source)
+      ? "text/html,application/xhtml+xml,application/xml;q=0.9,text/plain;q=0.8,*/*;q=0.7"
     : "text/markdown,text/html,application/xhtml+xml,application/xml;q=0.9,text/plain;q=0.8,*/*;q=0.7";
 
   const headers: Record<string, string> = {

src/lib/ingestion/ingest-state.test.ts

@@ -5,6 +5,7 @@ import {
   classifyThrownError,
   SourceIngestionError,
 } from "../../../convex/ingestionErrors";
+import { buildFetchHeaders } from "../../../convex/ingest";
 import {
   findSameSourceCandidateByTitle,
   getFailureBackoffUntil,
@@ -21,6 +22,30 @@ import {
   shouldPollLifecycleState,
 } from "../../../convex/sourceLifecycle";
 
+describe("buildFetchHeaders", () => {
+  it("requests xAI release notes as HTML instead of the stripped Markdown view", () => {
+    const headers = buildFetchHeaders("VersionWatchBot/1.0", {
+      url: "https://docs.x.ai/developers/release-notes",
+      parserKey: "xai:docs_page",
+      sourceType: "docs_page",
+    });
+
+    expect(headers.Accept).toContain("text/html");
+    expect(headers.Accept).not.toContain("text/markdown");
+  });
+
+  it("keeps direct feeds on RSS/XML-first accept headers", () => {
+    const headers = buildFetchHeaders("VersionWatchBot/1.0", {
+      url: "https://resend.com/changelog/index.xml",
+      parserKey: "resend:rss",
+      sourceType: "rss",
+    });
+
+    expect(headers.Accept).toContain("application/rss+xml");
+    expect(headers.Accept).not.toContain("text/html");
+  });
+});
+
 describe("hasMeaningfulTitle", () => {
   it("allows short semver release titles for GitHub release sources", () => {
     expect(hasMeaningfulTitle("v1.14.21", "https://github.com/anomalyco/opencode/releases")).toBe(true);