Merge pull request #68 from man-group/cleaner-index-page

Cleaner index page

Author: jonbannister

README.md

@@ -16,7 +16,10 @@ Productionise and schedule your Jupyter Notebooks, just as interactively as you
 ![Screenshot of some notebook results](https://raw.githubusercontent.com/man-group/notebooker/master/docs/images/nbkr_results.png)
 
 ## All results are accessible from the home page
-![Screenshot of Executing a notebook](https://raw.githubusercontent.com/man-group/notebooker/master/docs/images/nbkr_homepage.png)
+![Screenshot of the Notebooker homepage](https://raw.githubusercontent.com/man-group/notebooker/master/docs/images/nbkr_homepage.png)
+
+## Drill down into each template's results
+![Screenshot of result listings](https://raw.githubusercontent.com/man-group/notebooker/master/docs/images/nbkr_results_listing.png)
 
 
 ## Getting started

docs/images/nbkr_homepage.png

@@ -2,22 +2,29 @@ The Notebooker webapp
 =====================
 
 Notebooker's primary interface is a simple webapp written to allow users to view and
-run Notebooker reports. It displays all results in a handy grid, and allows for rerunning
+run Notebooker reports. It first displays all unique template names which have ever run, and a drill-down
+view lists all results for that notebook template in a handy grid, allowing for rerunning
 and parameter tweaking.
 The entrypoint used to run Notebooks via the webapp is the
-same as the external API, so as long as you are using the same environment (e.g. within
+same as the external API; as long as you are using the same environment (e.g. within
 a docker image) you will get consistent results.
 
 
 Report dashboard
 ----------------
 The home page of the Notebooker webapp displays an overview of all reports which have recently run.
+
+.. image:: /images/nbkr_homepage.png
+   :width: 400
+   :alt: Screenshot of Notebooker webapp homepage
+
+Clicking on one of these elements will bring up an overview of all reports which have recently run.
 It is possible to view each full report by clicking "Result". It's also possible to rerun, delete, and
 copy parameters of each report in the grid.
 
-.. image:: /images/notebooker_homepage.png
+.. image:: /images/nbkr_results_listing.png
    :width: 400
-   :alt: Screenshot of Notebooker webapp homepage
+   :alt: Screenshot of Notebooker results listing
 
 
 Running a report
@@ -40,7 +47,7 @@ Running a report
 .. warning::
     In order to prevent users having to write JSON, the Override parameters box actually takes raw python statements
     and converts them into JSON. Therefore, it is strongly recommended that you run Notebooker in an environment
-    where you either completely trust all of the user base,  or within a docker container
+    where you either completely trust all of the user base, or within a docker container
     where executing variable assignments will not have any negative side-effects.
 
 Customisable elements:
@@ -49,6 +56,7 @@ Customisable elements:
 * Override parameters - the values which will override the parameters in the report (in python). Can be left blank.
 * Email to - upon completion of the report, who should it be emailed to? Can be left blank.
 * Generate PDF output - whether to generate PDFs or not. Requires xelatex to be installed - see :ref:`export to pdf`
+* Hide code from email and PDF output - whether to display the notebook code when producing output emails and PDFs.
 
 Viewing results
 ---------------
@@ -67,6 +75,7 @@ If the job fails, the stack trace will be presented to allow for easier debuggin
 
 
 | If the job succeeds, the .ipynb will have been converted into HTML for viewing on this page.
+| **Please note** for user convenience, all notebook code is hidden by default.
 | You can also get to this view by clicking the blue "Result" button on the homepage.
 | If you are using a framework such as seaborn or matplotlib, the images will be available and served by the webapp.
 | If you are using plotly, you can use offline mode to store the required javascript within the HTML render,

docs/images/nbkr_results_listing.png

@@ -8,6 +8,7 @@
 
 SUBMISSION_TIMEOUT = 3
 RUNNING_TIMEOUT = 60
+DEFAULT_RESULT_LIMIT = 100
 CANCEL_MESSAGE = "The webapp shut down while this job was running. Please resubmit with the same parameters."
 TEMPLATE_DIR_SEPARATOR = "^"
 DEFAULT_SERIALIZER = "PyMongoResultSerializer"

docs/images/notebooker_homepage.png

@@ -1,5 +1,7 @@
 import datetime
 import json
+from collections import Counter, defaultdict
+
 from abc import ABC
 from logging import getLogger
 from typing import Any, AnyStr, Dict, List, Optional, Tuple, Union, Iterator
@@ -300,20 +302,45 @@ def get_check_result(
         result = self.library.find_one({"job_id": job_id}, {"_id": 0})
         return self._convert_result(result)
 
+    def _get_raw_results(self, base_filter, projection, limit):
+        if "status" in base_filter:
+            base_filter["status"].update({"$ne": JobStatus.DELETED.value})
+        else:
+            base_filter["status"] = {"$ne": JobStatus.DELETED.value}
+        return self.library.find(base_filter, projection).sort("update_time", -1).limit(limit)
+
+    def get_count_and_latest_time_per_report(self):
+        reports = list(
+            self._get_raw_results(
+                base_filter={},
+                projection={"report_name": 1, "job_start_time": 1, "scheduler_job_id": 1, "_id": 0},
+                limit=0,
+            )
+        )
+        jobs_by_name = defaultdict(list)
+        for r in reports:
+            jobs_by_name[r["report_name"]].append(r)
+        output = {}
+        for report, all_runs in jobs_by_name.items():
+            latest_start_time = max(r["job_start_time"] for r in all_runs)
+            scheduled_runs = len([x for x in all_runs if x.get("scheduler_job_id")])
+            output[report] = {"count": len(all_runs), "latest_run": latest_start_time, "scheduler_runs": scheduled_runs}
+        return output
+
     def get_all_results(
         self,
         since: Optional[datetime.datetime] = None,
         limit: Optional[int] = 100,
         mongo_filter: Optional[Dict] = None,
         load_payload: bool = True,
     ) -> Iterator[Union[NotebookResultComplete, NotebookResultError, NotebookResultPending]]:
-        base_filter = {"status": {"$ne": JobStatus.DELETED.value}}
+        base_filter = {}
         if mongo_filter:
             base_filter.update(mongo_filter)
         if since:
             base_filter.update({"update_time": {"$gt": since}})
         projection = REMOVE_ID_PROJECTION if load_payload else REMOVE_PAYLOAD_FIELDS_AND_ID_PROJECTION
-        results = self.library.find(base_filter, projection).sort("update_time", -1).limit(limit)
+        results = self._get_raw_results(base_filter, projection, limit)
         for res in results:
             if res:
                 converted_result = self._convert_result(res, load_payload=load_payload)
@@ -404,8 +431,8 @@ def get_latest_successful_job_ids_for_name_all_params(self, report_name: str) ->
 
         return [result["job_id"] for result in results]
 
-    def n_all_results(self):
-        return self.library.find({"status": {"$ne": JobStatus.DELETED.value}}).count()
+    def n_all_results_for_report_name(self, report_name: str) -> int:
+        return self._get_raw_results({"report_name": report_name}, {}, 0).count()
 
     def delete_result(self, job_id: AnyStr) -> None:
         self.update_check_status(job_id, JobStatus.DELETED)

docs/webapp/webapp.rst

@@ -1,7 +1,11 @@
+import datetime
+from collections import defaultdict
 from datetime import datetime as dt
 from logging import getLogger
 from typing import Callable, Dict, Iterator, List, Mapping, Optional, Tuple
 
+import babel.dates
+import inflection
 from flask import url_for
 
 from notebooker import constants
@@ -106,9 +110,10 @@ def get_all_result_keys(
     return all_keys
 
 
-def get_all_available_results_json(serializer: MongoResultSerializer, limit: int) -> List[constants.NotebookResultBase]:
+def get_all_available_results_json(serializer: MongoResultSerializer, limit: int, report_name: str = None) -> List[constants.NotebookResultBase]:
     json_output = []
-    for result in serializer.get_all_results(limit=limit, load_payload=False):
+    mongo_filter = {"report_name": report_name} if report_name is not None else {}
+    for result in serializer.get_all_results(mongo_filter=mongo_filter, limit=limit, load_payload=False):
         output = result.saveable_output()
         output["result_url"] = url_for(
             "serve_results_bp.task_results", job_id=output["job_id"], report_name=output["report_name"]
@@ -126,6 +131,16 @@ def get_all_available_results_json(serializer: MongoResultSerializer, limit: int
     return json_output
 
 
+def get_count_and_latest_time_per_report(serializer: MongoResultSerializer):
+    reports = serializer.get_count_and_latest_time_per_report()
+    output = {}
+    for report_name, metadata in sorted(reports.items(), key=lambda x: x[1]["latest_run"], reverse=True):
+        metadata["report_name"] = report_name
+        metadata["time_diff"] = babel.dates.format_timedelta(datetime.datetime.utcnow() - metadata["latest_run"])
+        output[inflection.titleize(report_name)] = metadata
+    return output
+
+
 def get_latest_successful_job_results_all_params(
     report_name: str,
     serializer: MongoResultSerializer,

notebooker/constants.py

@@ -121,7 +121,10 @@ def setup_app(flask_app: Flask, web_config: WebappConfig):
     logging.basicConfig(level=logging.getLevelName(web_config.LOGGING_LEVEL))
     flask_app.config.from_object(web_config)
     flask_app.config.update(
-        TEMPLATES_AUTO_RELOAD=web_config.DEBUG, EXPLAIN_TEMPLATE_LOADING=True, DEBUG=web_config.DEBUG
+        TEMPLATES_AUTO_RELOAD=web_config.DEBUG,
+        EXPLAIN_TEMPLATE_LOADING=True,
+        DEBUG=web_config.DEBUG,
+        TESTING=web_config.DEBUG,
     )
     flask_app = setup_scheduler(flask_app, web_config)
     return flask_app

notebooker/serialization/mongo.py

@@ -39,6 +39,7 @@ def _report_hunter(webapp_config: WebappConfig, run_once: bool = False, timeout:
                 JobStatus.SUBMITTED: now - datetime.timedelta(minutes=SUBMISSION_TIMEOUT),
                 JobStatus.PENDING: now - datetime.timedelta(minutes=RUNNING_TIMEOUT),
             }
+            cutoff.update({k.value: v for (k, v) in cutoff.items()})  # Add value to dict for backwards compat
             for result in all_pending:
                 this_cutoff = cutoff.get(result.status)
                 if result.job_start_time <= this_cutoff: