Add separate field for ocw topics, use best field to assign related topics (#1600)

Author: mbertrand

frontends/api/src/generated/v1/api.ts

@@ -71,6 +71,40 @@ def parse_delivery(course_data: dict) -> list[str]:
     return delivery
 
 
+def parse_learn_topics(course_data: dict) -> list[dict]:
+    """
+    Parse topics. Use the "mit_learn_topics" field if it exists and isn't empty,
+    otherwise use and transform the "topics" field values.
+
+    Args:
+        course_data (dict): The course data
+
+    Returns:
+        list[dict]: The topics
+    """
+    mitlearn_topics = course_data.get("mit_learn_topics") or []
+    ocw_topics = course_data.get("topics") or []
+    if mitlearn_topics:
+        # Should already be in the correct format
+        return [
+            {"name": topic_name}
+            for topic_name in sorted(
+                {topic for topics in mitlearn_topics for topic in topics}
+            )
+        ]
+    else:
+        # Topics need to be transformed
+        return transform_topics(
+            [
+                {"name": topic_name}
+                for topic_name in sorted(
+                    {topic for topics in ocw_topics for topic in topics}
+                )
+            ],
+            OFFERED_BY["code"],
+        )
+
+
 def transform_content_files(
     s3_resource: boto3.resource,
     course_prefix: str,
@@ -329,10 +363,6 @@ def transform_course(course_data: dict) -> dict:
     readable_term = f"+{slugify(term)}" if term else ""
     readable_year = f"_{course_data.get('year')}" if year else ""
     readable_id = f"{course_data[PRIMARY_COURSE_ID]}{readable_term}{readable_year}"
-    topics = transform_topics(
-        [{"name": topic} for topics in course_data.get("topics") for topic in topics],
-        OFFERED_BY["code"],
-    )
     image_src = course_data.get("image_src")
 
     return {
@@ -365,7 +395,14 @@ def transform_course(course_data: dict) -> dict:
                 is_ocw=True,
             ),
         },
-        "topics": topics,
+        "topics": parse_learn_topics(course_data),
+        "ocw_topics": sorted(
+            {
+                topic_name
+                for topic_sublist in course_data.get("topics", [])
+                for topic_name in topic_sublist
+            }
+        ),
         "runs": [transform_run(course_data)],
         "resource_type": LearningResourceType.course.name,
         "unique_field": UNIQUE_FIELD,

learning_resources/etl/ocw.py

@@ -18,11 +18,15 @@
 )
 from learning_resources.etl.constants import CourseNumberType, ETLSource
 from learning_resources.etl.ocw import (
+    parse_learn_topics,
     transform_content_files,
     transform_contentfile,
     transform_course,
 )
-from learning_resources.factories import ContentFileFactory
+from learning_resources.factories import (
+    ContentFileFactory,
+    LearningResourceTopicFactory,
+)
 from learning_resources.models import ContentFile
 from learning_resources.utils import (
     get_s3_object_and_read,
@@ -239,6 +243,14 @@ def test_transform_course(  # noqa: PLR0913
     )
     transformed_json = transform_course(extracted_json)
     if expected_uid:
+        assert transformed_json["ocw_topics"] == [
+            "Anthropology",
+            "Ethnography",
+            "Humanities",
+            "Philosophy",
+            "Political Philosophy",
+            "Social Science",
+        ]
         assert transformed_json["readable_id"] == expected_id
         assert transformed_json["etl_source"] == ETLSource.ocw.name
         assert transformed_json["delivery"] == expected_delivery
@@ -295,3 +307,45 @@ def test_transform_course(  # noqa: PLR0913
         )
     else:
         assert transformed_json is None
+
+
+@pytest.mark.parametrize("has_learn_topics", [True, False])
+def test_parse_topics(mocker, has_learn_topics):
+    """Topics should be assigned correctly based on mitlearn topics if present, ocw topics if not"""
+    ocw_topics = [
+        ["Social Science", "Anthropology", "Ethnography"],
+        ["Social Science", "Political Science", "International Relations"],
+    ]
+    mit_learn_topics = (
+        [["Social Sciences", "Anthropology"], ["Social Sciences", "Political Science"]]
+        if has_learn_topics
+        else []
+    )
+    course_data = {
+        "topics": ocw_topics,
+        "mit_learn_topics": mit_learn_topics,
+    }
+    mocker.patch(
+        "learning_resources.etl.utils.load_offeror_topic_map",
+        return_value={
+            "Political Philosophy": ["Philosophy"],
+            "Ethnography": ["Anthropology"],
+            "International Relations": ["Political Science"],
+        },
+    )
+    for topic in ("Social Sciences", "Anthropology", "Political Science"):
+        LearningResourceTopicFactory.create(name=topic)
+    topics_dict = parse_learn_topics(course_data)
+    if has_learn_topics:
+        assert topics_dict == [
+            {"name": "Anthropology"},
+            {"name": "Political Science"},
+            {"name": "Social Sciences"},
+        ]
+    else:
+        assert topics_dict == [
+            {"name": "Anthropology"},
+            {"name": "Anthropology"},
+            {"name": "Political Science"},
+            {"name": "Political Science"},
+        ]

learning_resources/etl/ocw_test.py

@@ -0,0 +1,23 @@
+# Generated by Django 4.2.16 on 2024-09-23 18:04
+
+import django.contrib.postgres.fields
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("learning_resources", "0068_learningresource_format_pace"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="learningresource",
+            name="ocw_topics",
+            field=django.contrib.postgres.fields.ArrayField(
+                base_field=models.CharField(max_length=128),
+                blank=True,
+                default=list,
+                size=None,
+            ),
+        ),
+    ]

learning_resources/migrations/0069_learningresource_ocw_topics.py

@@ -411,6 +411,7 @@ class LearningResource(TimestampedModel):
         choices=((member.name, member.value) for member in LearningResourceType),
     )
     topics = models.ManyToManyField(LearningResourceTopic)
+    ocw_topics = ArrayField(models.CharField(max_length=128), default=list, blank=True)
     offered_by = models.ForeignKey(
         LearningResourceOfferor, null=True, on_delete=models.SET_NULL
     )

learning_resources/models.py

@@ -254,6 +254,7 @@ def test_learning_resource_serializer(  # noqa: PLR0913
             serializers.LearningResourceTopicSerializer(topic).data
             for topic in resource.topics.all()
         ],
+        "ocw_topics": sorted(resource.ocw_topics),
         "runs": [
             serializers.LearningResourceRunSerializer(instance=run).data
             for run in resource.runs.all()

learning_resources/serializers_test.py

@@ -71,6 +71,7 @@ class FilterConfig:
     "run_id": FilterConfig("run_id", case_sensitive=True),
     "resource_id": FilterConfig("resource_id", case_sensitive=True),
     "topic": FilterConfig("topics.name"),
+    "ocw_topic": FilterConfig("ocw_topics"),
     "level": FilterConfig("runs.level.code"),
     "department": FilterConfig("departments.department_id"),
     "platform": FilterConfig("platform.code"),
@@ -184,6 +185,7 @@ class FilterConfig:
             "channel_url": {"type": "keyword"},
         },
     },
+    "ocw_topics": {"type": "keyword"},
     "offered_by": {
         "type": "nested",
         "properties": {

learning_resources_search/constants.py

@@ -276,6 +276,11 @@ class SearchRequestSerializer(serializers.Serializer):
         child=serializers.CharField(),
         help_text="The topic name. To see a list of options go to api/v1/topics/",
     )
+    ocw_topic = serializers.ListField(
+        required=False,
+        child=serializers.CharField(),
+        help_text="The ocw topic name.",
+    )
     dev_mode = serializers.BooleanField(
         required=False,
         allow_null=True,