[3.3.x][Fixes #8689] Extend the ResourceBase metadata model with an opaque JSONField (#8727)

* [Fixes #8689] Extend the ResourceBase metadata model with an opaque JSONField

* [Fixes #8689] Fix missing resource_type for new form instances

* [Fixes #8689] Add test and UI fix for doc, maps and geoapps

* [Fixes #8689] Fix flakee8 formatting

* [Fixes #8689] Extra metadata json saved with format

* [Fixes #8689] Refactor validation def, start defining endpoint for API

* [Fixes #8689] Definition of extra-metadata endpoints for resources

* [Fixes #8689] Converting metadata from jsonfield to manytomany relation

* [Fixes #8689] Fix views with new relation and prettify json on UI

* [Fixes #8689] Fix serializer

* [Fixes #8689] Fix custom metadata endpoint, update metadata schema

* [Fixes #8689] Fix flake8 issues

* [Fixes #8689] Remove endpoint from each resorce, keep it only on base resource

* [Fixes #8689] Fix broken tests

* [Fixes #8689] Add metadata filtering in API v1

* [Fixes #8689] Add test for metadata filtering in API v1

* [Fixes #8689] Fix some of broken tests

* [Fixes #8689] fix flake8

* [Fixes #8689] fix tests

* [Fixes #8689] removed typo on settings.py

* [Fixes #8689] fix broken build

* [Fixes #8689] fix minor error on filter convertion

* [Fixes #8689] fix flake8

* [Fixes #8689] Update default schema structure

Author: mattiagiupponi

geonode/api/resourcebase_api.py

@@ -95,7 +95,8 @@ class CommonMetaApi:
         'date': ALL,
         'purpose': ALL,
         'uuid': ALL_WITH_RELATIONS,
-        'abstract': ALL
+        'abstract': ALL,
+        'metadata': ALL_WITH_RELATIONS
     }
     ordering = ['date', 'title', 'popular_count']
     max_limit = None
@@ -168,6 +169,9 @@ def build_filters(self, filters=None, ignore_bad_filters=False, **kwargs):
             orm_filters.update({'polymorphic_ctype__model__in': [filt.lower() for filt in filters.getlist('app_type__in')]})
         if 'extent' in filters:
             orm_filters.update({'extent': filters['extent']})
+        _metadata = {f"metadata__{_k}": _v for _k, _v in filters.items() if _k.startswith('metadata__')}
+        if _metadata:
+            orm_filters.update({"metadata_filters": _metadata})
         orm_filters['f_method'] = filters['f_method'] if 'f_method' in filters else 'and'
         if not settings.SEARCH_RESOURCES_EXTENDED:
             return self._remove_additional_filters(orm_filters)
@@ -186,6 +190,9 @@ def apply_filters(self, request, applicable_filters):
         metadata_only = applicable_filters.pop('metadata_only', False)
         filtering_method = applicable_filters.pop('f_method', 'and')
         polyphormic_model = applicable_filters.pop('polymorphic_ctype__model__in', None)
+
+        metadata_filters = applicable_filters.pop('metadata_filters', None)
+
         if filtering_method == 'or':
             filters = Q()
             for f in applicable_filters.items():
@@ -234,6 +241,9 @@ def apply_filters(self, request, applicable_filters):
         if keywords:
             filtered = self.filter_h_keywords(filtered, keywords)
 
+        if metadata_filters:
+            filtered = filtered.filter(**metadata_filters)
+
         # return filtered
         return get_visible_resources(
             filtered,
@@ -589,6 +599,9 @@ def format_objects(self, objects):
                 except Exception as e:
                     logger.exception(e)
 
+            if formatted_obj.get('metadata', None):
+                formatted_obj['metadata'] = [model_to_dict(_m) for _m in formatted_obj['metadata']]
+
             formatted_objects.append(formatted_obj)
 
         return formatted_objects

geonode/api/tests.py

@@ -30,6 +30,7 @@
 from guardian.shortcuts import get_anonymous_user
 
 from geonode import geoserver
+from geonode.base.models import ExtraMetadata
 from geonode.maps.models import Map
 from geonode.layers.models import Layer
 from geonode.utils import check_ogc_backend
@@ -360,6 +361,35 @@ def test_category_filters(self):
         self.assertValidJSONResponse(resp)
         self.assertEqual(len(self.deserialize(resp)['objects']), 5)
 
+    def test_metadata_filters(self):
+        """Test category filtering"""
+        _r = Layer.objects.first()
+        _m = ExtraMetadata.objects.create(
+            resource=_r,
+            metadata={
+                "name": "metadata-updated",
+                "slug": "metadata-slug-updated",
+                "help_text": "this is the help text-updated",
+                "field_type": "str-updated",
+                "value": "my value-updated",
+                "category": "category"
+            }
+        )
+        _r.metadata.add(_m)
+        # check we get the correct layers number returnered filtering on one
+        # and then two different categories
+        filter_url = f"{self.list_url}?metadata__category=category"
+
+        resp = self.api_client.get(filter_url)
+        self.assertValidJSONResponse(resp)
+        self.assertEqual(len(self.deserialize(resp)['objects']), 1)
+
+        filter_url = f"{self.list_url}?metadata__category=not-existing-category"
+
+        resp = self.api_client.get(filter_url)
+        self.assertValidJSONResponse(resp)
+        self.assertEqual(len(self.deserialize(resp)['objects']), 0)
+
     def test_tag_filters(self):
         """Test keywords filtering"""
 

geonode/base/api/serializers.py

@@ -16,6 +16,7 @@
 # along with this program. If not, see <http://www.gnu.org/licenses/>.
 #
 #########################################################################
+from django.db.models.query import QuerySet
 from slugify import slugify
 from urllib.parse import urljoin
 
@@ -35,15 +36,16 @@
 
 from geonode.favorite.models import Favorite
 from geonode.base.models import (
-    ResourceBase,
+    ExtraMetadata,
     HierarchicalKeyword,
+    License,
     Region,
+    ResourceBase,
     RestrictionCodeType,
-    License,
-    TopicCategory,
     SpatialRepresentationType,
     ThesaurusKeyword,
-    ThesaurusKeywordLabel
+    ThesaurusKeywordLabel,
+    TopicCategory,
 )
 from geonode.groups.models import (
     GroupCategory,
@@ -263,6 +265,24 @@ def get_attribute(self, instance):
         return build_absolute_uri(instance.detail_url)
 
 
+class ExtraMetadataSerializer(DynamicModelSerializer):
+    class Meta:
+        model = ExtraMetadata
+        name = 'ExtraMetadata'
+        fields = ('pk', 'metadata')
+
+    def to_representation(self, obj):
+
+        if isinstance(obj, QuerySet):
+            out = []
+            for el in obj:
+                out.append({**{"id": el.id}, **el.metadata})
+            return out
+        elif isinstance(obj, list):
+            return obj
+        return {**{"id": obj.id}, **obj.metadata}
+
+
 class ThumbnailUrlField(DynamicComputedField):
 
     def __init__(self, **kwargs):
@@ -381,6 +401,8 @@ def __init__(self, *args, **kwargs):
         self.fields['spatial_representation_type'] = DynamicRelationField(
             SpatialRepresentationTypeSerializer, embed=True, many=False)
 
+    metadata = DynamicRelationField(ExtraMetadataSerializer, embed=False, many=True, deferred=True)
+
     class Meta:
         model = ResourceBase
         name = 'resource'
@@ -397,7 +419,7 @@ class Meta:
             'popular_count', 'share_count', 'rating', 'featured', 'is_published', 'is_approved',
             'detail_url', 'embed_url', 'created', 'last_updated',
             'raw_abstract', 'raw_purpose', 'raw_constraints_other',
-            'raw_supplemental_information', 'raw_data_quality_statement', 'metadata_only', 'processed'
+            'raw_supplemental_information', 'raw_data_quality_statement', 'metadata_only', 'processed', "metadata"
             # TODO
             # csw_typename, csw_schema, csw_mdsource, csw_insert_date, csw_type, csw_anytext, csw_wkt_geometry,
             # metadata_uploaded, metadata_uploaded_preserve, metadata_xml,

geonode/base/api/tests.py

@@ -36,16 +36,18 @@
 
 from geonode import geoserver
 from geonode.layers.models import Layer
+from geonode.tests.base import GeoNodeBaseTestSupport
 from geonode.utils import check_ogc_backend, set_resource_default_links
 from geonode.favorite.models import Favorite
 from geonode.documents.models import Document
 from geonode.base.utils import build_absolute_uri
 from geonode.thumbs.exceptions import ThumbnailError
-from geonode.base.populate_test_data import create_models
+from geonode.base.populate_test_data import create_models, create_single_layer
 from geonode.security.utils import get_resources_with_perms
 
 from geonode.base.models import (
     CuratedThumbnail,
+    ExtraMetadata,
     HierarchicalKeyword,
     Region,
     ResourceBase,
@@ -1021,3 +1023,64 @@ def test_set_thumbnail_from_bbox_from_logged_user_for_existing_dataset_raise_exp
         }
         self.assertEqual(response.status_code, 500)
         self.assertEqual(expected, response.json())
+
+
+class TestExtraMetadataBaseApi(GeoNodeBaseTestSupport):
+    def setUp(self):
+        self.layer = create_single_layer('single_layer')
+        self.metadata = {
+            "filter_header": "Foo Filter header",
+            "field_name": "metadata-name",
+            "field_label": "this is the help text",
+            "field_value": "foo"
+        }
+        m = ExtraMetadata.objects.create(
+            resource=self.layer,
+            metadata=self.metadata
+        )
+        self.layer.metadata.add(m)
+        self.mdata = ExtraMetadata.objects.first()
+
+    def test_get_will_return_the_list_of_extra_metadata(self):
+        self.client.login(username="admin", password="admin")
+        url = reverse('base-resources-extra-metadata', args=[self.layer.id])
+        response = self.client.get(url, content_type='application/json')
+        self.assertTrue(200, response.status_code)
+        expected = [
+            {**{"id": self.mdata.id}, **self.metadata}
+        ]
+        self.assertEqual(expected, response.json())
+
+    def test_put_will_update_the_whole_metadata(self):
+        self.client.login(username="admin", password="admin")
+        url = reverse('base-resources-extra-metadata', args=[self.layer.id])
+        input_metadata = {
+            "id": self.mdata.id,
+            "filter_header": "Foo Filter header",
+            "field_name": "metadata-updated",
+            "field_label": "this is the help text",
+            "field_value": "foo"
+        }
+        response = self.client.put(url, data=[input_metadata], content_type='application/json')
+        self.assertTrue(200, response.status_code)
+        self.assertEqual([input_metadata], response.json())
+
+    def test_post_will_add_new_metadata(self):
+        self.client.login(username="admin", password="admin")
+        url = reverse('base-resources-extra-metadata', args=[self.layer.id])
+        input_metadata = {
+            "filter_header": "Foo Filter header",
+            "field_name": "metadata-updated",
+            "field_label": "this is the help text",
+            "field_value": "foo"
+        }
+        response = self.client.post(url, data=[input_metadata], content_type='application/json')
+        self.assertTrue(201, response.status_code)
+        self.assertEqual(2, len(response.json()))
+
+    def test_delete_will_delete_single_metadata(self):
+        self.client.login(username="admin", password="admin")
+        url = reverse('base-resources-extra-metadata', args=[self.layer.id])
+        response = self.client.delete(url, data=[self.mdata.id], content_type='application/json')
+        self.assertTrue(200, response.status_code)
+        self.assertEqual([], response.json())

geonode/base/api/views.py

@@ -38,7 +38,7 @@
 from geonode.favorite.models import Favorite
 from geonode.thumbs.exceptions import ThumbnailError
 from geonode.thumbs.thumbnails import create_thumbnail
-from geonode.base.models import HierarchicalKeyword, Region, ResourceBase, TopicCategory, ThesaurusKeyword
+from geonode.base.models import ExtraMetadata, HierarchicalKeyword, Region, ResourceBase, TopicCategory, ThesaurusKeyword
 from geonode.base.api.filters import DynamicSearchFilter, ExtentFilter, FavoriteFilter
 from geonode.groups.models import GroupProfile, GroupMember
 from geonode.layers.models import Layer
@@ -70,6 +70,8 @@
     ThesaurusKeywordSerializer,
 )
 from .pagination import GeoNodeApiPagination
+from geonode.base.api.serializers import ExtraMetadataSerializer
+from geonode.base.utils import validate_extra_metadata
 
 import logging
 
@@ -466,3 +468,84 @@ def set_thumbnail_from_bbox(self, request, resource_id):
             traceback.print_exc()
             logger.error(e)
             return Response(data={"message": e.args[0], "success": False}, status=500, exception=True)
+
+    @extend_schema(
+        methods=["get", "put", "delete", "post"], description="Get/Update/Delete/Add extra metadata for resource"
+    )
+    @action(
+        detail=True,
+        methods=["get", "put", "delete", "post"],
+        permission_classes=[
+            IsOwnerOrAdmin,
+        ],
+        url_path=r"extra_metadata",  # noqa
+        url_name="extra-metadata",
+    )
+    def extra_metadata(self, request, pk=None):
+        _obj = self.get_object()
+        if request.method == "GET":
+            # get list of available metadata
+            queryset = _obj.metadata.all()
+            _filters = [{f"metadata__{key}": value} for key, value in request.query_params.items()]
+            if _filters:
+                queryset = queryset.filter(**_filters[0])
+            return Response(ExtraMetadataSerializer().to_representation(queryset))
+        if not request.method == "DELETE":
+            try:
+                extra_metadata = validate_extra_metadata(request.data, _obj)
+            except Exception as e:
+                return Response(status=500, data=e.args[0])
+
+        if request.method == "PUT":
+            '''
+            update specific metadata. The ID of the metadata is required to perform the update
+            [
+                {
+                        "id": 1,
+                        "name": "foo_name",
+                        "slug": "foo_sug",
+                        "help_text": "object",
+                        "field_type": "int",
+                        "value": "object",
+                        "category": "object"
+                }
+            ]
+            '''
+            for _m in extra_metadata:
+                _id = _m.pop('id')
+                ResourceBase.objects.filter(id=_obj.id).first().metadata.filter(id=_id).update(metadata=_m)
+            logger.info("metadata updated for the selected resource")
+            _obj.refresh_from_db()
+            return Response(ExtraMetadataSerializer().to_representation(_obj.metadata.all()))
+        elif request.method == "DELETE":
+            # delete single metadata
+            '''
+            Expect a payload with the IDs of the metadata that should be deleted. Payload be like:
+            [4, 3]
+            '''
+            ResourceBase.objects.filter(id=_obj.id).first().metadata.filter(id__in=request.data).delete()
+            _obj.refresh_from_db()
+            return Response(ExtraMetadataSerializer().to_representation(_obj.metadata.all()))
+        elif request.method == "POST":
+            # add new metadata
+            '''
+            [
+                {
+                        "name": "foo_name",
+                        "slug": "foo_sug",
+                        "help_text": "object",
+                        "field_type": "int",
+                        "value": "object",
+                        "category": "object"
+                }
+            ]
+            '''
+            for _m in extra_metadata:
+                new_m = ExtraMetadata.objects.create(
+                    resource=_obj,
+                    metadata=_m
+                )
+                new_m.save()
+                _obj.metadata.add(new_m)
+            _obj.refresh_from_db()
+            return Response(ExtraMetadataSerializer().to_representation(_obj.metadata.all()), status=201)