Handle missing files & placeholder docs in the PackageTableView

src/css/metacatui-common.css

@@ -7937,7 +7937,8 @@ ul.side-nav-items {
 }
 
 .data-package-item td.error > [contenteditable] {
-  border: 2px solid red;
+  border: 1.5px solid red;
+  border-radius: 6px;
 }
 /* Editor package table column 1 */
 .data-package-item td:first-of-type {

src/js/collections/DataPackage.js

@@ -433,6 +433,43 @@ define([
         }
       },
 
+      /**
+       * Fetches the system metadata for all member models that are marked as
+       * placeholder documents (i.e., isPlaceHolder_b === true). This property
+       * is retrieved from Solr and indicates that the file ID is referenced in
+       * a resource map, but has either not yet been indexed or is missing from
+       * the repository.
+       * @returns {Promise} A promise that resolves when all placeholder member
+       * models have been fetched.
+       * @since 0.0.0
+       */
+      async fetchSysMetaForPlaceholders() {
+        const placeholder_prop = "isPlaceHolder_b";
+        const placeholder_models = this.filter(
+          (model) => model.get(placeholder_prop) === true,
+        );
+        await this.fetchMemberModels(placeholder_models);
+      },
+
+      /**
+       * Checks whether any member models are missing files
+       * @returns {boolean} True if any member models are missing files
+       * @since 0.0.0
+       */
+      missingFilesDetected() {
+        return this.some((model) => model.fileDoesNotExist());
+      },
+
+      /**
+       * Returns an array of member models that are missing files
+       * @returns {Backbone.Model[]} An array of member models that are missing
+       * files
+       * @since 0.0.0
+       */
+      getMissingFileModels() {
+        return this.filter((model) => model.fileDoesNotExist());
+      },
+
       /**
        * Fetches member models in batches to avoid fetching all members
        * simultaneously.
@@ -555,8 +592,13 @@ define([
           // Wait for whichever finishes first
           return await Promise.race([fetchPromise, timerPromise]);
         } catch (err) {
-          // Retry if we still have attempts left
-          if (attempt >= maxRetries - 1) {
+          // Retry if we still have attempts left and the type of error makes
+          // sense to retry
+          const dontRetryErrors = [401, 403, 404, 410];
+          if (
+            attempt >= maxRetries - 1 ||
+            dontRetryErrors.includes(memberModel.get("errorStatus"))
+          ) {
             throw err;
           }
           // Recursively call ourselves with an incremented attempt count
@@ -681,8 +723,12 @@ define([
                 resolve(model);
               });
             },
-            error: (m, response) => {
-              reject(new Error(response?.statusText || "Model fetch failed"));
+            error: (_m, response) => {
+              model.set("errorStatus", response.status);
+              model.set("errorMessage", response.statusText);
+              reject(new Error(response.statusText || "Fetch failed"), {
+                cause: response,
+              });
             },
           });
         });

src/js/common/QueryService.js

@@ -24,7 +24,9 @@ define(["jquery"], ($) => {
    * URL.
    * @property {boolean} [usePost] Force POST / GET (overrides auto-choice).
    * @property {boolean} [useAuth=true] Inject MetacatUI auth headers?
-   * @property {boolean} [archived] Include archived items? Default `false`.
+   * @property {boolean} [archived] Include archived resources in the results. A
+   * special filter query is applied to include all docs that have any value for
+   * the `archived` field.
    * @property {boolean} [group] Use Solr grouping (group=true)?
    * @property {string} [groupField] Field to group by (if `group` is true).
    * @property {number} [groupLimit] Limit of groups to return (if `group` is
@@ -483,7 +485,6 @@ define(["jquery"], ($) => {
         });
       }
 
-      // TODO - are there other values possible for the archived param?
       if (archived) {
         params["archived"] = "archived:*";
       }

src/js/models/DataONEObject.js

@@ -392,7 +392,10 @@ define([
        */
       parse(response) {
         // If the response is XML
-        if (typeof response === "string" && response.indexOf("<") == 0) {
+        if (
+          typeof response === "string" &&
+          response.trim().indexOf("<") === 0
+        ) {
           const responseDoc = $.parseHTML(response);
           let systemMetadata;
 
@@ -1890,6 +1893,17 @@ define([
         );
       },
 
+      /**
+       * Checks for a 404 error in the error status or message
+       * @returns {boolean} True if a 404 error is found
+       * @since 0.0.0
+       */
+      fileDoesNotExist() {
+        if (this.get("errorStatus") === 404) return true;
+        if (this.get("errorMessage")?.includes("404")) return true;
+        return false;
+      },
+
       /**
        * A utility function that will format an XML string or XML nodes by camel-casing the node names, as necessary
        * @param {string|Element} xml - The XML to format

src/js/models/PackageModel.js

@@ -196,7 +196,7 @@ define([
           fields:
             `resourceMap,fileName,obsoletes,obsoletedBy,size,formatType,formatId,` +
             `id,datasource,rightsHolder,dateUploaded,archived,title,origin,` +
-            `prov_instanceOfClass,isDocumentedBy,isPublic`,
+            `prov_instanceOfClass,isDocumentedBy,isPublic,isPlaceHolder_b`,
           rows: 1000,
           archived: this.get("getArchivedMembers") ? true : false,
         })
@@ -235,7 +235,7 @@ define([
         return this;
       },
 
-      /*
+      /**
        * Send custom options to the Backbone.Model.fetch() function
        */
       fetch(options = {}) {
@@ -291,7 +291,7 @@ define([
         });
       },
 
-      /*
+      /**
        * Deserialize a Package from OAI-ORE RDF XML
        */
       parse(response) {

src/js/views/DataItemView.js

@@ -114,6 +114,14 @@ define([
       render() {
         // Prevent duplicate listeners
         this.stopListening();
+        // listen for changes to rerender the view
+        this.listenTo(
+          this.model,
+          "change:fileName change:title change:id change:formatType " +
+            "change:formatId change:type change:resourceMap change:documents change:isDocumentedBy " +
+            "change:size change:nodeLevel change:uploadStatus change:errorMessage change:errorStatus",
+          this.render,
+        );
 
         let itemPathParts = [];
 
@@ -483,15 +491,6 @@ define([
               this.toggleSaving,
             );
 
-            // listen for changes to rerender the view
-            this.listenTo(
-              this.model,
-              "change:fileName change:title change:id change:formatType " +
-                "change:formatId change:type change:resourceMap change:documents change:isDocumentedBy " +
-                "change:size change:nodeLevel change:uploadStatus change:errorMessage",
-              this.render,
-            ); // render changes to the item
-
             const view = this;
             this.listenTo(this.model, "replace", (newModel) => {
               view.model = newModel;
@@ -642,6 +641,10 @@ define([
               .attr("data-trigger", "hover")
               .attr("data-delay", "300")
               .attr("data-title", objectTitleTooltip);
+
+            // Check for errors indicating that the file does not exist and
+            // show im UI
+            this.handleNonExistentFile();
           }
         }
 
@@ -1462,13 +1465,18 @@ define([
        * @since 2.32.1
        */
       showError(message) {
+        let messageNormalized = message;
+        if (messageNormalized === "404") {
+          messageNormalized =
+            "This file does not exist in the repository. Please remove or replace it.";
+        }
         this.$el.removeClass("loading");
         const nameColumn = this.$(".name");
         nameColumn.addClass("error");
         // Append an error message
         this.errorEl = $(document.createElement("div"))
           .addClass("error-message")
-          .text(`There was an error: ${message}`);
+          .text(`Error: ${messageNormalized}`);
         nameColumn.append(this.errorEl);
         const icon = this.$(".type-icon");
         icon.addClass("error");
@@ -1578,8 +1586,22 @@ define([
         return null;
       },
 
+      /**
+       * Handle the case where the file does not exist on the server
+       * @since 0.0.0
+       */
+      handleNonExistentFile() {
+        if (!this.model.fileDoesNotExist()) return;
+        this.$el.addClass("non-existent-file");
+        if (this.downloadButtonView) {
+          this.downloadButtonView.inactivate(
+            "This file does not exist on the server, but is referenced in the metadata. Please contact the author for assistance.",
+          );
+        }
+      },
+
       downloadFile(e) {
-        this.downloadButtonView.download(e);
+        this.downloadButtonView?.download(e);
       },
 
       // Member row metrics for the package table

src/js/views/DataPackageView.js

@@ -1,4 +1,4 @@
-"use strict";
+"use strict";
 
 define([
   "jquery",
@@ -161,8 +161,8 @@ define([
 
         if (this.edit) {
           // Listen for  add events because models are being merged
-          this.listenTo(this.dataPackage, "add", this.addOne);
-          this.listenTo(this.dataPackage, "fileAdded", this.addOne);
+          this.stopListening(this.dataPackage, "add fileAdded", this.addOne);
+          this.listenTo(this.dataPackage, "add fileAdded", this.addOne);
         }
 
         // Render the current set of models in the DataPackage
@@ -204,6 +204,7 @@ define([
 
         // Don't add duplicate rows
         if (this.$(`.data-package-item[data-id='${item.id}']`).length) return;
+        if (_.contains(Object.keys(this.subviews), item.id)) return;
 
         // Don't add data package
         if (
@@ -217,10 +218,6 @@ define([
         let parentRow;
         let delayedModels;
 
-        if (_.contains(Object.keys(this.subviews), item.id)) {
-          return; // Don't double render
-        }
-
         let itemPath = null;
         const view = this;
         if (!_.isEmpty(this.atLocationObj)) {
@@ -503,10 +500,7 @@ define([
             this.sortedFilePathObj = sortedFilePathObj;
 
             this.addFilesAndFolders(sortedFilePathObj);
-          } else {
-            this.dataPackage.each(this.addOne, this, this.dataPackage);
           }
-
           this.dataPackage.each(this.addOne, this, this.dataPackage);
         } else {
           this.dataPackage.each(this.addOne, this);

src/js/views/DownloadButtonView.js

@@ -112,17 +112,9 @@ define([
             // then we can assume the resource map object is private. So disable
             // the download button.
             if (!this.model.get("indexDoc")) {
-              this.$el
-                .attr("disabled", "disabled")
-                .addClass("disabled")
-                .attr("href", "")
-                .tooltip({
-                  trigger: "hover",
-                  placement: "top",
-                  delay: 500,
-                  title:
-                    "This dataset may contain private data, so each data file should be downloaded individually.",
-                });
+              this.inactivate(
+                "This dataset may contain private data, so each data file should be downloaded individually.",
+              );
             }
           }
           // For individual DataONEObjects
@@ -142,27 +134,34 @@ define([
           this.model.type === "Package" &&
           this.model.getTotalSize() > MetacatUI.appModel.get("maxDownloadSize")
         ) {
-          this.$el
-            .addClass("tooltip-this")
-            .attr("disabled", "disabled")
-            .attr(
-              "data-title",
-              "This dataset is too large to download as a package. Please download the files individually or contact us for alternate data access.",
-            )
-            .attr("data-placement", "top")
-            .attr("data-trigger", "hover")
-            .attr("data-container", "body");
-
-          // Removing the `href` attribute while disabling the download button.
-          this.$el.removeAttr("href");
-
-          // Removing pointer as cursor and setting to default
-          this.$el.css("cursor", "default");
+          this.inactivate(
+            `This dataset is too large to download all at once. Please download individual files separately."`,
+          );
         }
 
         return this;
       },
 
+      /**
+       * Prevents the download button from being clickable and adds a tooltip
+       * with a message explaining why.
+       * @param {string} [message] - The message to display in the tooltip.
+       * @since 0.0.0
+       */
+      inactivate(message = "This file is not available for download.") {
+        this.$el.addClass("disabled").attr("disabled", "disabled");
+        this.$el.css("cursor", "default");
+        this.$el.removeAttr("href");
+
+        this.$el.tooltip({
+          trigger: "hover",
+          placement: "top",
+          delay: 100,
+          title: message,
+          container: "body",
+        });
+      },
+
       /**
        * Handles the download event when the button is clicked. Checks for
        * conditions like authentication, public/private access, and download size