Add export and import of offline documentation

The offline page can now save installed documentations to a JSON file and
restore them later or on another computer, without downloading them again.
Single documentations are exported from the action column, all of them at
once with the new Export all button; Import restores either kind of file.

The file holds each doc's pages, the index file cached in localStorage,
and the mtime the doc was installed with, so that a restored doc that has
been updated since shows up as outdated instead of up-to-date. Importing
enables the docs that aren't enabled yet, which is also what creates their
object stores, and reloads the app so their indexes get loaded.

Closes #336

Author: simon04

assets/javascripts/app/offline_backup.js

@@ -0,0 +1,185 @@
+// Exports the offline data (the pages stored in IndexedDB and the index files
+// cached in localStorage) to a JSON file, and imports it back — either to
+// restore a backup after the browser evicted the data, or to move the
+// documentations to another computer without downloading them again.
+app.OfflineBackup = class OfflineBackup {
+  static TYPE = "devdocs-offline";
+  static VERSION = 1;
+  static MIME_TYPE = "application/json";
+
+  filename(docs) {
+    const date = new Date().toISOString().slice(0, 10);
+    const name = docs.length === 1 ? docs[0].slug : "offline";
+    return `devdocs-${name}-${date}.json`;
+  }
+
+  // Calls back with a Blob containing every installed doc among `docs`, and
+  // the number of docs it holds. Docs that aren't installed are skipped.
+  export(docs, onProgress, onSuccess, onError) {
+    const chunks = [
+      `{"type":"${OfflineBackup.TYPE}","version":${
+        OfflineBackup.VERSION
+      },"date":"${new Date().toISOString()}","docs":[`,
+    ];
+    let count = 0;
+    let i = 0;
+
+    var next = () => {
+      const doc = docs[i++];
+
+      if (!doc) {
+        if (count === 0) {
+          onError("empty");
+          return;
+        }
+        chunks.push("]}");
+        onSuccess(new Blob(chunks, { type: OfflineBackup.MIME_TYPE }), count);
+        return;
+      }
+
+      onProgress(doc, i, docs.length);
+      app.db.dump(doc, (result) => {
+        if (result) {
+          // Serialize each doc on its own instead of building one big object,
+          // to avoid holding the whole backup in memory twice.
+          chunks.push(
+            (count++ === 0 ? "" : ",") +
+              JSON.stringify(this.serializeDoc(doc, result)),
+          );
+        }
+        setTimeout(next, 0);
+      });
+    };
+
+    next();
+  }
+
+  serializeDoc(doc, result) {
+    const entry = { slug: doc.slug, mtime: result.mtime, db: result.data };
+    const index = app.localStorage.get(doc.slug);
+    // Ship the index file too, so that the doc can be used on a computer that
+    // never downloaded it (the app falls back to the network otherwise).
+    if (index && index[0] === result.mtime) {
+      entry.index = index[1];
+    }
+    return entry;
+  }
+
+  import(file, onProgress, onSuccess, onError) {
+    if (!file || (file.type && file.type !== OfflineBackup.MIME_TYPE)) {
+      onError("invalid");
+      return;
+    }
+
+    const reader = new FileReader();
+    reader.onloadend = () => {
+      const data = (() => {
+        try {
+          return JSON.parse(reader.result);
+        } catch (error) {}
+      })();
+
+      if (!data || data.type !== OfflineBackup.TYPE || !Array.isArray(data.docs)) {
+        onError("invalid");
+        return;
+      }
+      if (data.version > OfflineBackup.VERSION) {
+        onError("version");
+        return;
+      }
+
+      this.importDocs(data.docs, onProgress, onSuccess, onError);
+    };
+    reader.onerror = () => onError("invalid");
+    reader.readAsText(file);
+  }
+
+  importDocs(entries, onProgress, onSuccess, onError) {
+    const queue = [];
+    const skipped = [];
+
+    for (var entry of entries) {
+      var doc = entry?.db && this.findDoc(entry.slug);
+      if (doc) {
+        queue.push([doc, entry]);
+      } else {
+        skipped.push(entry?.slug || "?");
+      }
+    }
+
+    if (queue.length === 0) {
+      onError("unknown", skipped);
+      return;
+    }
+
+    const enabled = this.enableDocs(queue.map(([doc]) => doc));
+    const total = queue.length;
+    const imported = [];
+    const failed = [];
+    let i = 0;
+
+    var next = () => {
+      const item = queue[i++];
+
+      if (!item) {
+        onSuccess({ docs: imported, skipped, failed, enabled });
+        return;
+      }
+
+      const [doc, entry] = item;
+      const mtime = entry.mtime || doc.mtime;
+      onProgress(doc, i, total);
+
+      if (entry.index) {
+        // Keyed by the backup's mtime so that Doc#_getCache discards it when
+        // the documentation has been updated since the backup was made.
+        app.localStorage.set(doc.slug, [mtime, entry.index]);
+      }
+
+      app.db.store(
+        doc,
+        entry.db,
+        mtime,
+        () => {
+          imported.push(doc);
+          setTimeout(next, 0);
+        },
+        () => {
+          failed.push(doc.slug);
+          setTimeout(next, 0);
+        },
+      );
+    };
+
+    next();
+  }
+
+  findDoc(slug) {
+    return (
+      app.docs.findBy("slug", slug) || app.disabledDocs.findBy("slug", slug)
+    );
+  }
+
+  // Enabling the docs up-front is what makes their object stores exist: the
+  // schema bump triggers DB#onUpgradeNeeded, which only creates stores for the
+  // enabled docs. Returns the number of docs that weren't enabled before.
+  enableDocs(docs) {
+    let enabled = 0;
+
+    for (var doc of docs) {
+      if (app.docs.contains(doc)) {
+        continue;
+      }
+      app.disabledDocs.remove(doc);
+      app.docs.add(doc);
+      enabled += 1;
+    }
+
+    if (enabled > 0) {
+      app.docs.sort();
+      app.saveDocs();
+    }
+
+    return enabled;
+  }
+};

assets/javascripts/templates/pages/offline_tmpl.js

@@ -8,7 +8,7 @@ app.templates.offlinePage = (docs, hasPersistence, isPersistent) => `\
     }>Install updates automatically
   </label>
   <div class="_docs-links">
-    <button type="button" class="_btn-link" data-action-all="install">Install all</button><button type="button" class="_btn-link" data-action-all="update"><strong>Update all</strong></button><button type="button" class="_btn-link" data-action-all="uninstall">Uninstall all</button>
+    <button type="button" class="_btn-link" data-action-all="install" title="Download every enabled documentation for offline use">Install all</button><button type="button" class="_btn-link" data-action-all="update" title="Download the current version of every outdated documentation"><strong>Update all</strong></button><button type="button" class="_btn-link" data-action-all="uninstall" title="Delete the offline data of every installed documentation">Uninstall all</button><button type="button" class="_btn-link _show" data-export-docs title="Save the installed documentations to a file, to restore them later or on another computer">Export all</button><label class="_btn-link _file-btn _show" title="Restore documentations from a previously exported file">Import<input type="file" name="importDocs" accept="application/json,.json"></label>
   </div>
 </div>
 
@@ -23,6 +23,7 @@ app.templates.offlinePage = (docs, hasPersistence, isPersistent) => `\
     ${docs}
   </table>
 </div>
+<div id="_offline-backup-status"></div>
 <div id="_offline-persistence-note">
   ${offlinePersistenceNote(hasPersistence, isPersistent)}
 </div>
@@ -33,6 +34,8 @@ app.templates.offlinePage = (docs, hasPersistence, isPersistent) => `\
       The app also uses <a href="https://devdocs.io/dom/service_worker_api/using_service_workers">Service Workers</a> and <a href="https://devdocs.io/dom/web_storage_api">localStorage</a> to cache the assets and index files.
   <dt>Can I close the tab/browser?
   <dd>${canICloseTheTab()}
+  <dt>How do I move the documentations to another computer?
+  <dd>Export them to a file using the buttons above, copy it over, and import it there. The other computer still needs to load DevDocs once while online for the app itself to be cached.
   <dt>What if I don't update a documentation?
   <dd>You'll see outdated content and some pages will be missing or broken, because the rest of the app (including data for the search and sidebar) uses a different caching mechanism that's updated automatically.
   <dt>I found a bug, where do I report it?
@@ -44,6 +47,46 @@ app.templates.offlinePage = (docs, hasPersistence, isPersistent) => `\
 </dl>\
 `;
 
+app.templates.backupProgress = (action, doc, i, total) =>
+  `${action} ${doc.fullName}\u2026 (${i}/${total})`;
+
+app.templates.backupExported = (count) =>
+  `Exported ${count} ${pluralizeDocs(count)}.`;
+
+app.templates.backupImported = function (result) {
+  let html = `<strong>Imported ${result.docs.length} ${pluralizeDocs(
+    result.docs.length
+  )}.</strong>`;
+
+  if (result.failed.length > 0) {
+    html += ` Couldn't be stored: ${result.failed.join(", ")}.`;
+  }
+  if (result.skipped.length > 0) {
+    html += ` Not available anymore: ${result.skipped.join(", ")}.`;
+  }
+  if (result.enabled > 0) {
+    html += " Reloading\u2026";
+  }
+
+  return html;
+};
+
+app.templates.backupError = function (reason) {
+  switch (reason) {
+    case "empty":
+      return "<strong>No documentation is installed.</strong> Install one before exporting.";
+    case "unknown":
+      return "<strong>Nothing to import.</strong> This file doesn't contain any documentation that DevDocs still offers.";
+    case "version":
+      return "<strong>This file was exported by a newer version of DevDocs.</strong> Reload the app and try again.";
+    default:
+      return "<strong>The file you selected is invalid.</strong> Only files exported from this page can be imported.";
+  }
+};
+
+var pluralizeDocs = (count) =>
+  count === 1 ? "documentation" : "documentations";
+
 app.templates.persistenceError = function (exception) {
   const reason = exception
     ? `<code class="_label">${exception.name}: ${exception.message}</code>`
@@ -106,11 +149,11 @@ app.templates.offlineDoc = function (doc, status) {
     : outdated
       ? `\
 <td><strong>Outdated</strong></td>
-<td><button type="button" class="_btn-link _bold" data-action="update">Update</button> - <button type="button" class="_btn-link" data-action="uninstall">Uninstall</button></td>\
+<td><button type="button" class="_btn-link _bold" data-action="update">Update</button> &bull; <button type="button" class="_btn-link" data-action="uninstall">Uninstall</button> &bull; <button type="button" class="_btn-link" data-action="export">Export</button></td>\
 `
       : `\
 <td>Up&#8209;to&#8209;date</td>
-<td><button type="button" class="_btn-link" data-action="uninstall">Uninstall</button></td>\
+<td><button type="button" class="_btn-link" data-action="uninstall">Uninstall</button> &bull; <button type="button" class="_btn-link" data-action="export">Export</button></td>\
 `;
 
   return html + "</tr>";

assets/javascripts/views/content/offline_page.js

@@ -77,7 +77,9 @@ app.views.OfflinePage = class OfflinePage extends app.View {
   onClick(event) {
     let el = $.eventTarget(event);
     let action = el.getAttribute("data-action");
-    if (action) {
+    if (action === "export") {
+      this.exportDoc(this.docByEl(el), el);
+    } else if (action) {
       const doc = this.docByEl(el);
       if (action === "update") {
         action = "install";
@@ -102,6 +104,8 @@ app.views.OfflinePage = class OfflinePage extends app.View {
       }
     } else if (el.hasAttribute("data-enable-persistence")) {
       this.requestPersistence();
+    } else if (el.hasAttribute("data-export-docs")) {
+      this.exportDocs(app.docs.all());
     }
   }
 
@@ -149,6 +153,104 @@ app.views.OfflinePage = class OfflinePage extends app.View {
   onChange(event) {
     if (event.target.name === "autoUpdate") {
       app.settings.set("manualUpdate", !event.target.checked);
+    } else if (event.target.name === "importDocs") {
+      this.importDocs(event.target);
+    }
+  }
+
+  backup() {
+    return this._backup || (this._backup = new app.OfflineBackup());
+  }
+
+  // Exports `docs` into a single file. Returns false when another backup is
+  // already running, in which case `onDone` is never called.
+  exportDocs(docs, onDone) {
+    if (this.backingUp) {
+      return false;
+    }
+    this.backingUp = true;
+    const backup = this.backup();
+
+    const done = (html, isError, success) => {
+      this.backingUp = false;
+      if (!this.activated) {
+        return;
+      }
+      this.setBackupStatus(html, isError);
+      if (onDone) {
+        onDone(success);
+      }
+    };
+
+    backup.export(
+      docs,
+      (doc, i, total) =>
+        this.setBackupStatus(
+          this.tmpl("backupProgress", "Exporting", doc, i, total),
+        ),
+      (blob, count) => {
+        $.download(blob, backup.filename(docs));
+        done(this.tmpl("backupExported", count), false, true);
+      },
+      () => done(this.tmpl("backupError", "empty"), true, false),
+    );
+
+    return true;
+  }
+
+  exportDoc(doc, el) {
+    const started = this.exportDocs([doc], (success) =>
+      success ? this.onInstallSuccess(doc) : this.onInstallError(doc),
+    );
+    if (started) {
+      el.parentNode.innerHTML = "Exporting\u2026";
+    }
+  }
+
+  importDocs(input) {
+    if (this.backingUp) {
+      return;
+    }
+    this.backingUp = true;
+    const file = input.files[0];
+    input.value = ""; // so that picking the same file again fires a change event
+
+    this.backup().import(
+      file,
+      (doc, i, total) =>
+        this.setBackupStatus(
+          this.tmpl("backupProgress", "Importing", doc, i, total),
+        ),
+      (result) => {
+        this.backingUp = false;
+        if (!this.activated) {
+          return;
+        }
+        this.setBackupStatus(
+          this.tmpl("backupImported", result),
+          result.failed.length > 0,
+        );
+        // Newly enabled docs have no index in memory; reboot to load them.
+        // Otherwise just refresh the rows that changed, to keep the message.
+        if (result.enabled > 0) {
+          this.delay(() => app.reboot(), 2000);
+        } else {
+          for (var doc of result.docs) {
+            this.onInstallSuccess(doc);
+          }
+        }
+      },
+      (reason) => {
+        this.backingUp = false;
+        this.setBackupStatus(this.tmpl("backupError", reason), true);
+      },
+    );
+  }
+
+  setBackupStatus(html, isError) {
+    const el = this.find("#_offline-backup-status");
+    if (el) {
+      el.innerHTML = `<p class="_note${isError ? " _note-red" : ""}">${html}`;
     }
   }