fix: Adjust the pagination logic to use a limit+1 strategy for determining if more results are available

- Deleted the `.gitmodules` file as it is no longer needed.
- Updated the `get_paginated_inverse_references` function to simplify the return structure by removing the total count from the response, focusing on pagination with `has_more`.
- Adjusted the pagination logic to use a limit+1 strategy for determining if more results are available.
- Enhanced error handling in the `find_similar_resources` function to validate limit and offset parameters, ensuring they are positive integers.
- Improved test cases for linked resources and similar resources to reflect the updated response structure and pagination logic.

Author: arcangelo7

.gitmodules

@@ -12,53 +12,28 @@
 
 linked_resources_bp = Blueprint("linked_resources", __name__, url_prefix="/api/linked-resources")
 
-def get_paginated_inverse_references(subject_uri: str, limit: int, offset: int) -> tuple[list[dict], int, bool]:
+def get_paginated_inverse_references(subject_uri: str, limit: int, offset: int) -> tuple[list[dict], bool]:
     """
-    Get paginated entities that reference this entity.
+    Get paginated entities that reference this entity using the limit+1 strategy.
 
     Args:
         subject_uri: URI of the entity to find references to.
-        limit: Maximum number of references to return.
+        limit: Maximum number of references to return per page.
         offset: Number of references to skip.
 
     Returns:
         A tuple containing:
-            - List of dictionaries containing reference information.
-            - Total count of references.
+            - List of dictionaries containing reference information (max 'limit' items).
             - Boolean indicating if there are more references.
     """
     sparql = get_sparql()
     custom_filter = get_custom_filter()
-    total_count = 0
     references = []
+    # Fetch limit + 1 to check if there are more results
+    query_limit = limit + 1
 
     try:
-        # Count Query
-        count_query_parts = [
-            "SELECT (COUNT(DISTINCT ?s) as ?count) WHERE {",
-        ]
-        if is_virtuoso:
-            count_query_parts.append("    GRAPH ?g { ?s ?p ?o . }")
-            count_query_parts.append(f"    FILTER(?g NOT IN (<{'>, <'.join(VIRTUOSO_EXCLUDED_GRAPHS)}>))")
-        else:
-            count_query_parts.append("    ?s ?p ?o .")
-
-        count_query_parts.extend([
-            f"    FILTER(?o = <{subject_uri}>)",
-            "    FILTER(?p != <http://www.w3.org/1999/02/22-rdf-syntax-ns#type>)",
-            "}"
-        ])
-        count_query = "\n".join(count_query_parts)
-
-        sparql.setQuery(count_query)
-        sparql.setReturnFormat(JSON)
-        count_results = sparql.query().convert()
-        if count_results.get("results", {}).get("bindings", []):
-            count_binding = count_results["results"]["bindings"][0]
-            if "count" in count_binding:
-                total_count = int(count_binding["count"]["value"])
-
-        # Main Query with pagination
+        # Main Query with pagination (limit + 1)
         query_parts = [
             "SELECT DISTINCT ?s ?p WHERE {",
         ]
@@ -71,7 +46,7 @@ def get_paginated_inverse_references(subject_uri: str, limit: int, offset: int)
         query_parts.extend([
             f"    FILTER(?o = <{subject_uri}>)",
             "    FILTER(?p != <http://www.w3.org/1999/02/22-rdf-syntax-ns#type>)",
-            f"}} ORDER BY ?s OFFSET {offset} LIMIT {limit}"
+            f"}} ORDER BY ?s OFFSET {offset} LIMIT {query_limit}" # Use query_limit
         ])
         main_query = "\n".join(query_parts)
 
@@ -80,7 +55,14 @@ def get_paginated_inverse_references(subject_uri: str, limit: int, offset: int)
         results = sparql.query().convert()
 
         bindings = results.get("results", {}).get("bindings", [])
-        for result in bindings:
+
+        # Determine if there are more results
+        has_more = len(bindings) > limit
+
+        # Process only up to 'limit' results
+        results_to_process = bindings[:limit]
+
+        for result in results_to_process:
             subject = result["s"]["value"]
             predicate = result["p"]["value"]
 
@@ -100,13 +82,12 @@ def get_paginated_inverse_references(subject_uri: str, limit: int, offset: int)
                 "label": label
             })
 
-        has_more = offset + limit < total_count
-        return references, total_count, has_more
+        return references, has_more
 
     except Exception as e:
         tb_str = traceback.format_exc()
         current_app.logger.error(f"Error fetching inverse references for {subject_uri}: {e}\n{tb_str}")
-        return [], 0, False
+        return [], False
 
 @linked_resources_bp.route("/", methods=["GET"])
 @login_required
@@ -125,11 +106,10 @@ def get_linked_resources_api():
     if limit <= 0 or offset < 0:
          return jsonify({"status": "error", "message": gettext("Limit must be positive and offset non-negative")}), 400
 
-    references, total_count, has_more = get_paginated_inverse_references(subject_uri, limit, offset)
+    references, has_more = get_paginated_inverse_references(subject_uri, limit, offset)
 
     return jsonify({
         "status": "success",
         "results": references,
-        "total_count": total_count,
         "has_more": has_more
     })

heritrace/routes/linked_resources.py

@@ -213,12 +213,18 @@ def find_similar_resources():
     """Find resources potentially similar to a given subject based on shared literal properties."""
     subject_uri = request.args.get("subject_uri")
     entity_type = request.args.get("entity_type") # Primary entity type
-    limit = int(request.args.get("limit", 5))
-    offset = int(request.args.get("offset", 0))
+    try:
+        limit = int(request.args.get("limit", 5))
+        offset = int(request.args.get("offset", 0))
+    except ValueError:
+        return jsonify({"status": "error", "message": gettext("Invalid limit or offset parameter")}), 400
 
     if not subject_uri or not entity_type:
         return jsonify({"status": "error", "message": gettext("Missing required parameters (subject_uri, entity_type)")}), 400
 
+    if limit <= 0 or offset < 0:
+        return jsonify({"status": "error", "message": gettext("Limit must be positive and offset non-negative")}), 400
+
     try:
         sparql = get_sparql()
         custom_filter = get_custom_filter()
@@ -272,46 +278,33 @@ def find_similar_resources():
         if not similarity_conditions:
             return jsonify({"status": "success", "results": []})
 
-        # Get total count query (for pagination)
-        count_query_parts = [
-            "SELECT (COUNT(DISTINCT ?similar) as ?count) WHERE {",
-            f"  ?similar a <{entity_type}> .",
-            f"  FILTER(?similar != <{subject_uri}>)",
-            "  {",
-            "    " + " \n    UNION\n    ".join(similarity_conditions),
-            "  }",
-            "}"
-        ]
-        count_query = "\n".join(count_query_parts)
-
-        sparql.setQuery(count_query)
-        sparql.setReturnFormat(JSON)
-        count_results = sparql.query().convert()
-        total_count = 0
-        if count_results.get("results", {}).get("bindings", []):
-            count_binding = count_results["results"]["bindings"][0]
-            if "count" in count_binding:
-                total_count = int(count_binding["count"]["value"])
-
-        # Main query with pagination
+        # Main query with pagination (fetch limit + 1)
+        query_limit = limit + 1
         query_parts = [
             "SELECT DISTINCT ?similar WHERE {",
             f"  ?similar a <{entity_type}> .",
             f"  FILTER(?similar != <{subject_uri}>)",
             "  {",
             "    " + " \n    UNION\n    ".join(similarity_conditions),
             "  }",
-            f"}} OFFSET {offset} LIMIT {limit}"
+            f"}} ORDER BY ?similar OFFSET {offset} LIMIT {query_limit}" # Use query_limit
         ]
         final_query = "\n".join(query_parts)
 
         sparql.setQuery(final_query)
         sparql.setReturnFormat(JSON)
         results = sparql.query().convert()
 
-        candidate_uris = [item["similar"]["value"] for item in results.get("results", {}).get("bindings", [])]
+        bindings = results.get("results", {}).get("bindings", [])
+        candidate_uris = [item["similar"]["value"] for item in bindings]
+
+        # Determine if there are more results
+        has_more = len(candidate_uris) > limit
+
+        # Process only up to 'limit' results
+        results_to_process = candidate_uris[:limit]
         transformed_results = []
-        for uri in candidate_uris:
+        for uri in results_to_process:
             readable_label = custom_filter.human_readable_entity(uri, [entity_type]) 
             sim_types = get_entity_types(uri)
             type_labels = [custom_filter.human_readable_predicate(type_uri, sim_types) for type_uri in sim_types]
@@ -322,12 +315,10 @@ def find_similar_resources():
                 "type_labels": type_labels
             })
 
-        has_more = offset + limit < total_count
         return jsonify({
             "status": "success", 
             "results": transformed_results, 
             "has_more": has_more,
-            "total_count": total_count
         })
 
     except Exception as e:

heritrace/routes/merge.py

@@ -0,0 +1,127 @@
+/**
+ * Reusable function to load resources via AJAX with pagination.
+ *
+ * @param {object} options - Configuration options.
+ * @param {string} options.containerSelector - jQuery selector for the list container.
+ * @param {string} options.noResultsSelector - jQuery selector for the "no results" message element.
+ * @param {string} options.loadMoreSelector - jQuery selector for the "load more" button.
+ * @param {string} options.apiUrl - The URL for the AJAX request.
+ * @param {object} options.ajaxData - An object containing static data to send with the AJAX request (e.g., { subject_uri: '...' }).
+ * @param {function} options.renderItemCallback - A function that takes a single result item and returns the HTML string for it.
+ * @param {number} [options.resultsPerPage=5] - Number of items per page.
+ * @param {string} [options.loadingHtml='<div class="text-center my-3"><div class="spinner-border spinner-border-sm" role="status"><span class="visually-hidden">Loading...</span></div></div>'] - HTML for the loading indicator.
+ * @param {string} [options.errorText='Error loading resources.'] - Text to display on AJAX error.
+ */
+function loadResources(options) {
+    const {
+        containerSelector,
+        noResultsSelector,
+        loadMoreSelector,
+        apiUrl,
+        ajaxData,
+        renderItemCallback,
+        resultsPerPage = 5,
+        loadingHtml = '<div class="text-center my-3 resource-loading-indicator"><div class="spinner-border spinner-border-sm" role="status"><span class="visually-hidden">Loading...</span></div></div>',
+        errorText = 'Error loading resources.' // Default error text
+    } = options;
+
+    const $container = $(containerSelector);
+    const $noResults = $(noResultsSelector);
+    const $loadMoreButton = $(loadMoreSelector);
+    const $parent = $container.parent(); // Get parent to append loading indicator
+
+    let currentPage = 1;
+    let totalLoaded = 0;
+    let hasMore = true;
+    let $loadingIndicator = null;
+
+    function fetchData(page) {
+        if (!hasMore) return;
+
+        // Remove existing indicator and add new one
+        if ($loadingIndicator) {
+            $loadingIndicator.remove();
+        }
+        $loadingIndicator = $(loadingHtml);
+        $parent.append($loadingIndicator);
+        $loadMoreButton.hide();
+
+        const requestData = {
+            ...ajaxData, // Include static data
+            limit: resultsPerPage,
+            offset: (page - 1) * resultsPerPage
+        };
+
+        $.ajax({
+            url: apiUrl,
+            method: 'GET',
+            data: requestData,
+            success: function(response) {
+                if ($loadingIndicator) $loadingIndicator.remove();
+
+                if (response.status === 'success' && response.results && response.results.length > 0) {
+                    if (page === 1) {
+                        $container.empty(); // Clear only on first load
+                    }
+                    $noResults.hide();
+
+                    response.results.forEach(function(item) {
+                        const itemHtml = renderItemCallback(item);
+                        if (itemHtml) {
+                            $container.append(itemHtml);
+                        }
+                    });
+
+                    totalLoaded += response.results.length;
+                    // Use response.has_more if available, otherwise estimate based on resultsPerPage
+                    hasMore = response.has_more !== undefined ? response.has_more : (response.results.length === resultsPerPage);
+
+
+                    $container.show();
+                    if (hasMore) {
+                        $loadMoreButton.show();
+                    } else {
+                        $loadMoreButton.hide();
+                    }
+
+                } else {
+                    hasMore = false; // Stop loading if error or no results
+                    if (page === 1) { // Only show "no results" if it's the first page and no results came back
+                         if (response.status === 'success' && (!response.results || response.results.length === 0)) {
+                            // Keep the original "no results" text if the API call was successful but empty
+                         } else {
+                            // Otherwise, show a generic error or the specific API message
+                            $noResults.text(response.message || errorText);
+                         }
+                        $noResults.show();
+                        $container.hide(); // Hide container if empty on first load
+                    }
+                    $loadMoreButton.hide(); // Hide button if error or no results on subsequent pages
+
+                    if (response.status !== 'success') {
+                        console.error(`Error fetching resources from ${apiUrl}:`, response.message);
+                    }
+                }
+            },
+            error: function(jqXHR) {
+                if ($loadingIndicator) $loadingIndicator.remove();
+                hasMore = false; // Stop loading on error
+                 if (page === 1) {
+                    $container.hide();
+                    $noResults.text(errorText).show(); // Show generic error on first page AJAX fail
+                }
+                $loadMoreButton.hide(); // Hide button on error
+                console.error(`AJAX error fetching resources from ${apiUrl}:`, jqXHR.statusText);
+            }
+        });
+    }
+
+    // Initial load
+    fetchData(currentPage);
+
+    // Load more on button click
+    $loadMoreButton.on('click', function() {
+        currentPage++;
+        fetchData(currentPage);
+    });
+}