Merge pull request #360 from GenomicDataInfrastructure/improved-multilangual

Improved multilangual

Author: amercader

.gitignore

@@ -16,3 +16,4 @@ build/*
 tmp/*
 package/DEBIAN/control
 *.swp
+.idea

ckanext/dcat/profiles/base.py

@@ -533,10 +533,36 @@ def _agents_details(self, subject, predicate):
         """
 
         agents = []
+        default_locale = config.get("ckan.locale_default", "") or ""
+        default_lang = default_locale.split("_")[0] if default_locale else None
+
         for agent in self.g.objects(subject, predicate):
             agent_details = {}
             agent_details["uri"] = str(agent) if isinstance(agent, term.URIRef) else ""
-            agent_details["name"] = self._object_value(agent, FOAF.name)
+
+            names = list(self.g.objects(agent, FOAF.name))
+            translations = {}
+            fallback_name = ""
+            for name_literal in names:
+                if isinstance(name_literal, Literal):
+                    value = str(name_literal)
+                    lang = name_literal.language
+                    if lang:
+                        translations[lang] = value
+                    elif not fallback_name:
+                        fallback_name = value
+                elif not fallback_name:
+                    fallback_name = str(name_literal)
+
+            if translations:
+                agent_details["name_translated"] = translations
+                if default_lang and translations.get(default_lang):
+                    agent_details["name"] = translations[default_lang]
+                else:
+                    agent_details["name"] = fallback_name or next(iter(translations.values()))
+            else:
+                agent_details["name"] = fallback_name
+
             agent_details["email"] = self._without_mailto(
                 self._object_value(agent, FOAF.mbox)
             )
@@ -839,8 +865,25 @@ def _add_agent_to_graph(self, subject_ref, predicate, agent_dict):
         self.g.add((agent_ref, RDF.type, FOAF.Organization))
         self.g.add((agent_ref, RDF.type, FOAF.Agent))
 
+        name_translated = agent_dict.get("name_translated")
+        translated_values = set()
+        if isinstance(name_translated, dict):
+            for lang, values in name_translated.items():
+                if not values:
+                    continue
+                if isinstance(values, (list, tuple)):
+                    iterable = values
+                else:
+                    iterable = [values]
+                for value in iterable:
+                    if value:
+                        self.g.add((agent_ref, FOAF.name, Literal(value, lang=lang)))
+                        translated_values.add((lang, value))
+
         if agent_dict.get("name"):
-            self.g.add((agent_ref, FOAF.name, Literal(agent_dict["name"])))
+            name_value = agent_dict["name"]
+            if not translated_values or all(val != name_value for _, val in translated_values):
+                self.g.add((agent_ref, FOAF.name, Literal(name_value)))
         if agent_dict.get("email"):
             email = agent_dict["email"]
             if not email.startswith("mailto:"):
@@ -856,11 +899,26 @@ def _add_agent_to_graph(self, subject_ref, predicate, agent_dict):
             self.g.add((agent_ref, DCT.identifier, Literal(agent_dict["identifier"])))
 
         for sub_org in agent_dict.get("actedOnBehalfOf", []):
-            if sub_org.get("name"):
+            if sub_org.get("name") or sub_org.get("name_translated"):
                 org_ref = BNode()
                 self.g.add((agent_ref, PROV.actedOnBehalfOf, org_ref))
                 self.g.add((org_ref, RDF.type, PROV.Organization))
-                self.g.add((org_ref, FOAF.name, Literal(sub_org["name"])))
+
+                sub_translations = sub_org.get("name_translated", {}) or {}
+                if isinstance(sub_translations, dict):
+                    for lang, values in sub_translations.items():
+                        if not values:
+                            continue
+                        if isinstance(values, (list, tuple)):
+                            iterable = values
+                        else:
+                            iterable = [values]
+                        for value in iterable:
+                            if value:
+                                self.g.add((org_ref, FOAF.name, Literal(value, lang=lang)))
+
+                if sub_org.get("name"):
+                    self.g.add((org_ref, FOAF.name, Literal(sub_org["name"])))
 
         return agent_ref
 

ckanext/dcat/profiles/euro_dcat_ap_scheming.py

@@ -70,6 +70,29 @@ def _parse_list_value(data_dict, field_name):
                     except ValueError:
                         pass
 
+        def _supports_agent_translations(field_name):
+            schema_field = self._schema_field(field_name)
+            if schema_field and "repeating_subfields" in schema_field:
+                return any(
+                    subfield.get("field_name") == "name_translated"
+                    for subfield in schema_field["repeating_subfields"]
+                )
+            return False
+
+        def _prune_agent_translations(agent_list):
+            pruned = []
+            for agent_entry in agent_list:
+                if isinstance(agent_entry, dict):
+                    agent_entry = dict(agent_entry)
+                    agent_entry.pop("name_translated", None)
+                    acted_lists = agent_entry.get("actedOnBehalfOf")
+                    if isinstance(acted_lists, list):
+                        agent_entry["actedOnBehalfOf"] = _prune_agent_translations(acted_lists)
+                    pruned.append(agent_entry)
+                else:
+                    pruned.append(agent_entry)
+            return pruned
+
         for field_name in dataset_dict.keys():
             _parse_list_value(dataset_dict, field_name)
 
@@ -117,6 +140,8 @@ def _parse_list_value(data_dict, field_name):
             key, predicate = item
             agents = self._agents_details(dataset_ref, predicate)
             if agents:
+                if not _supports_agent_translations(key):
+                    agents = _prune_agent_translations(agents)
                 dataset_dict[key] = agents
 
         # Add any qualifiedRelations
@@ -239,7 +264,25 @@ def _add_agents(
                 self.g.add((agent_ref, RDF.type, FOAF.Agent))
                 self.g.add((dataset_ref, rdf_predicate, agent_ref))
 
-                self._add_triple_from_dict(agent, agent_ref, FOAF.name, "name")
+                name_translated = agent.get("name_translated")
+                translated_values = set()
+                if isinstance(name_translated, dict):
+                    for lang, values in name_translated.items():
+                        if not values:
+                            continue
+                        if isinstance(values, (list, tuple)):
+                            iterable = values
+                        else:
+                            iterable = [values]
+                        for value in iterable:
+                            if value:
+                                self.g.add((agent_ref, FOAF.name, Literal(value, lang=lang)))
+                                translated_values.add((lang, value))
+
+                if agent.get("name"):
+                    name_value = agent["name"]
+                    if not translated_values or all(val != name_value for _, val in translated_values):
+                        self.g.add((agent_ref, FOAF.name, Literal(name_value)))
                 self._add_triple_from_dict(
                     agent, agent_ref, FOAF.homepage, "url", _type=URIRef
                 )

ckanext/dcat/profiles/euro_health_dcat_ap.py

@@ -23,6 +23,12 @@
     "dpv": DPV,
 }
 
+# HealthDCAT-AP fields that can contain language-tagged literals
+MULTILINGUAL_LITERAL_FIELDS = {
+    "population_coverage": HEALTHDCATAP.populationCoverage,
+    "publisher_note": HEALTHDCATAP.publisherNote,
+}
+
 
 class EuropeanHealthDCATAPProfile(EuropeanDCATAP3Profile):
     """
@@ -42,7 +48,11 @@ def parse_dataset(self, dataset_dict, dataset_ref):
         return dataset_dict
 
     def _parse_health_fields(self, dataset_dict, dataset_ref):
-        self.__parse_healthdcat_stringvalues(dataset_dict, dataset_ref)
+        multilingual_fields = set(self._multilingual_dataset_fields())
+
+        self.__parse_healthdcat_stringvalues(
+            dataset_dict, dataset_ref, multilingual_fields
+        )
         self.__parse_healthdcat_booleanvalues(dataset_dict, dataset_ref)
         self.__parse_healthdcat_intvalues(dataset_dict, dataset_ref)
 
@@ -78,7 +88,9 @@ def __parse_healthdcat_intvalues(self, dataset_dict, dataset_ref):
             if value is not None:
                 dataset_dict[key] = value
 
-    def __parse_healthdcat_stringvalues(self, dataset_dict, dataset_ref):
+    def __parse_healthdcat_stringvalues(
+        self, dataset_dict, dataset_ref, multilingual_fields
+    ):
         for (key, predicate,) in (
             ("analytics", HEALTHDCATAP.analytics),
             ("code_values", HEALTHDCATAP.hasCodeValues),
@@ -92,9 +104,18 @@ def __parse_healthdcat_stringvalues(self, dataset_dict, dataset_ref):
             ("publisher_type", HEALTHDCATAP.publisherType),
             ("purpose", DPV.hasPurpose),
         ):
-            values = self._object_value_list(dataset_ref, predicate)
-            if values:
-                dataset_dict[key] = values
+            if (
+                key in MULTILINGUAL_LITERAL_FIELDS
+                and key in multilingual_fields
+            ):
+                value = self._object_value(
+                    dataset_ref, predicate, multilingual=True
+                )
+            else:
+                value = self._object_value_list(dataset_ref, predicate)
+
+            if value:
+                dataset_dict[key] = value
 
     def __parse_healthdcat_booleanvalues(self, dataset_dict, dataset_ref):
         for key, predicate in (
@@ -169,25 +190,45 @@ def graph_from_dataset(self, dataset_dict, dataset_ref):
             self.g.bind(prefix, namespace)
 
         # key, predicate, fallbacks, _type, _class
-        items = [
+        list_items = [
             ("analytics", HEALTHDCATAP.analytics, None, URIRefOrLiteral),
             ("code_values", HEALTHDCATAP.hasCodeValues, None, URIRefOrLiteral),
             ("coding_system", HEALTHDCATAP.hasCodingSystem, None, URIRefOrLiteral),
             ("health_category", HEALTHDCATAP.healthCategory, None, URIRefOrLiteral),
             ("health_theme", HEALTHDCATAP.healthCategory, None, URIRefOrLiteral),
             ("legal_basis", DPV.hasLegalBasis, None, URIRefOrLiteral),
-            (
-                "population_coverage",
-                HEALTHDCATAP.populationCoverage,
-                None,
-                URIRefOrLiteral,
-            ),
             ("personal_data", DPV.hasPersonalData, None, URIRef),
-            ("publisher_note", HEALTHDCATAP.publisherNote, None, URIRefOrLiteral),
             ("publisher_type", HEALTHDCATAP.publisherType, None, URIRefOrLiteral),
             ("purpose", DPV.hasPurpose, None, URIRefOrLiteral),
         ]
-        self._add_list_triples_from_dict(dataset_dict, dataset_ref, items)
+        self._add_list_triples_from_dict(dataset_dict, dataset_ref, list_items)
+
+        multilingual_fields = set(self._multilingual_dataset_fields())
+        for key, predicate in MULTILINGUAL_LITERAL_FIELDS.items():
+            value = self._get_dataset_value(dataset_dict, key)
+            if not value:
+                continue
+
+            if key in multilingual_fields and isinstance(value, dict):
+                for lang, translated_value in value.items():
+                    if translated_value:
+                        self.g.add(
+                            (
+                                dataset_ref,
+                                predicate,
+                                Literal(translated_value, lang=lang),
+                            )
+                        )
+                continue
+
+            self._add_triple_from_dict(
+                dataset_dict,
+                dataset_ref,
+                predicate,
+                key,
+                list_value=True,
+                _type=URIRefOrLiteral,
+            )
 
         if "trusted_data_holder" in dataset_dict:
             self.g.add(