No dupe citations (#286)

Author: mbertrand

ai_chatbots/chatbots_test.py

@@ -348,16 +348,15 @@ async def test_syllabus_bot_tool(
             }
             for resource in raw_results
         ],
-        "citation_sources": [
-            {
-                "id": resource.get("resource_point_id"),
+        "citation_sources": {
+            resource.get("resource_point_id"): {
                 "citation_title": resource.get("title")
                 or resource.get("content_title"),
                 "citation_url": resource.get("url"),
             }
             for resource in raw_results
             if resource.get("url")
-        ],
+        },
         "metadata": {},
     }
 

ai_chatbots/tools.py

@@ -249,7 +249,7 @@ def _content_file_search(url, params, *, exclude_canvas=True):
         raw_results = response.json().get("results", [])
         # Simplify the response to only include the main properties
         simplified_results = []
-        citations = []
+        citations = {}
         for result in raw_results:
             platform = result.get("platform", {}).get("code")
             # Currently, canvas contentfiles have blank platform values,
@@ -262,15 +262,12 @@ def _content_file_search(url, params, *, exclude_canvas=True):
                 "run_title": result.get("run_title"),
             }
             simplified_results.append(simplified_result)
-            if result.get("url"):
-                citations.append(
-                    {
-                        "id": result["resource_point_id"],
-                        "citation_url": result.get("url"),
-                        "citation_title": result.get("title")
-                        or result.get("content_title"),
-                    }
-                )
+            if result.get("url") and not citations.get(result["resource_point_id"]):
+                citations[result["resource_point_id"]] = {
+                    "citation_url": result.get("url"),
+                    "citation_title": result.get("title")
+                    or result.get("content_title"),
+                }
         full_output = {
             "results": simplified_results,
             "citation_sources": citations,

ai_chatbots/tools_test.py

@@ -155,15 +155,20 @@ def test_search_content_files(  # noqa: PLR0913
     )
     assert len(results["results"]) == len(content_chunk_results["results"])
     assert len(results["citation_sources"]) == len(
-        [result for result in content_chunk_results["results"] if result["url"]]
+        {
+            result["resource_point_id"]
+            for result in content_chunk_results["results"]
+            if result["url"]
+        }
     )
     for idx, result in enumerate(content_chunk_results["results"]):
         if content_chunk_results["results"][idx]["url"]:
-            assert {
-                "id": result["resource_point_id"],
+            assert results["citation_sources"][
+                content_chunk_results["results"][idx]["resource_point_id"]
+            ] == {
                 "citation_url": result.get("url"),
                 "citation_title": (result.get("title") or result["content_title"]),
-            } in results["citation_sources"]
+            }
 
 
 @pytest.mark.parametrize("exclude_canvas", [True, False])
@@ -190,7 +195,13 @@ def test_search_canvas_content_files(
         len(content_chunk_results["results"]) if not exclude_canvas else 0
     )
     assert len(results["citation_sources"]) == (
-        len([result for result in content_chunk_results["results"] if result["url"]])
+        len(
+            {
+                result["resource_point_id"]
+                for result in content_chunk_results["results"]
+                if result["url"]
+            }
+        )
         if not exclude_canvas
         else 0
     )

test_json/content_file_chunks.json

@@ -34,7 +34,7 @@
       "course_number": ["MITx+6.00.1x"],
       "description": null,
       "key": "2cab2fd265631bc527845acf2027686c",
-      "resource_point_id": "3c77d1af-8a99-58ea-bc3a-67691b5c6ca6",
+      "resource_point_id": "4c77d1af-8a99-58ea-bc3a-67691b5c6ca6",
       "url": "http://ocw.mit.edu/fake/url/2/",
       "file_extension": ".xml",
       "resource_readable_id": "MITx+6.00.1x",
@@ -61,7 +61,7 @@
       "course_number": ["MITx+6.00.1x"],
       "description": null,
       "key": "fb9fa09c795cd17544d2e5c917db2dca",
-      "resource_point_id": "3c77d1af-8a99-58ea-bc3a-67691b5c6ca6",
+      "resource_point_id": "5c77d1af-8a99-58ea-bc3a-67691b5c6ca6",
       "url": "http://ocw.mit.edu/fake/url/3/",
       "file_extension": ".xml",
       "resource_readable_id": "MITx+6.00.1x",
@@ -88,8 +88,8 @@
       "course_number": ["MITx+6.00.1x"],
       "description": null,
       "key": "d5dab5756147a6466f96284a68df85b8",
-      "resource_point_id": "3c77d1af-8a99-58ea-bc3a-67691b5c6ca6",
-      "url": null,
+      "resource_point_id": "5c77d1af-8a99-58ea-bc3a-67691b5c6ca6",
+      "url": "http://ocw.mit.edu/fake/url/3/",
       "file_extension": ".xml",
       "resource_readable_id": "MITx+6.00.1x",
       "run_title": "Introduction to Computer Science and Programming Using Python",