Returns reporting state (#136)

* Adds empty reporting data json file to zip contents

* Moves CheckResult into models.py to prevent circular imports

* Started implementing generate_json_reporting_data

* WIP generation of reporting data

* WIP

* Adds ReadingType and PackedReadings

* Adds serializable Site class

* Adds sites (and site der) to reporting data

* Adds timeline to reporting data

* Fixes dataframe serialization bug

* Revert runner back to original reporting code

* Replace multiple asserts with assert_class_instance_equality

* Adds version number to reporting data

* Put version into reporting data filename

* Fix linter issues

* Fix linter issues

* Updates cactus-schema dependency to v0.0.15

---------

Co-authored-by: LachlanJW <watsonjlachlan@gmail.com>

Author: mikejturner

pyproject.toml

@@ -22,7 +22,7 @@ classifiers = [
 dependencies = [
   "aiohttp>=3.11.12,<4",
   "cactus-test-definitions>=1.9.2,<2",
-  "cactus-schema>=0.0.14,<1",
+  "cactus-schema>=0.0.15,<1",
   "envoy @ git+https://github.com/bsgip/envoy.git@v1.3.2",
   "psycopg[binary]>=3.2.5,<4",
   "asyncpg>=0.30.0,<1",

src/cactus_runner/app/check.py

@@ -1,11 +1,10 @@
 import http
 import logging
 import re
-from dataclasses import dataclass
 from datetime import datetime, timedelta
 from itertools import chain
 from typing import Annotated, Any, Iterable, Optional, Sequence
-from cactus_runner.app.uri import does_endpoint_match
+
 import pydantic
 import pydantic.alias_generators
 import pydantic.fields
@@ -41,7 +40,13 @@
     ResolvedParam,
     resolve_variable_expressions_from_parameters,
 )
-from cactus_runner.models import ActiveTestProcedure, ClientCertificateType, RequestEntry
+from cactus_runner.app.uri import does_endpoint_match
+from cactus_runner.models import (
+    ActiveTestProcedure,
+    CheckResult,
+    ClientCertificateType,
+    RequestEntry,
+)
 
 logger = logging.getLogger(__name__)
 
@@ -153,14 +158,6 @@ class ParamsDERCapabilityContents(pydantic.BaseModel):
     ] = None
 
 
-@dataclass
-class CheckResult:
-    """Represents the results of a running a single check"""
-
-    passed: bool  # True if the check is considered passed or successful. False otherwise
-    description: Optional[str]  # Human readable description of what the check "considered" or wants to elaborate about
-
-
 class SoftChecker:
     """Collects all failed results suppressing them until finalized"""
 

src/cactus_runner/app/finalize.py

@@ -9,6 +9,7 @@
 from pathlib import Path
 from typing import cast
 
+import pandas as pd
 from envoy.server.model.archive.site import ArchiveSiteDERSetting
 from envoy.server.model.site import SiteDERSetting
 from sqlalchemy import select
@@ -35,10 +36,22 @@
 )
 from cactus_runner.app.requests_archive import copy_request_response_files_to_archive
 from cactus_runner.app.status import get_active_runner_status
-from cactus_runner.models import RunnerState
+from cactus_runner.models import (
+    CheckResult,
+    PackedReadings,
+    ReadingType,
+    ReportingData,
+    RunnerState,
+    Site,
+)
+
+# Cactus runner supports returning different versions of the reporting data
+# Define the currently preferred reporting data version
+CURRENT_REPORTING_DATA_VERSION: int = 1
 
 GENERATION_ERRORS_FILE_NAME = "generation-errors.txt"
 
+
 logger = logging.getLogger(__name__)
 
 
@@ -66,10 +79,12 @@ def get_file_name_no_extension(file_path: str) -> str:
 
 def get_zip_contents(
     json_status_summary: str | None,
+    json_reporting_data: str | None,
     log_file_paths: list[str],
     pdf_data: bytes | None,
     errors: list[str],
     filename_infix: str = "",
+    reporting_data_filename_prefix: str | None = "ReportingData",
 ) -> bytes:
     """Returns the contents of the zipped test procedures artifacts in bytes."""
 
@@ -89,6 +104,12 @@ def get_zip_contents(
             with open(file_path, "w") as f:
                 f.write(json_status_summary)
 
+        # Create reporting data json file
+        if json_reporting_data is not None and reporting_data_filename_prefix is not None:
+            file_path = archive_dir / f"{reporting_data_filename_prefix}{filename_infix}.json"
+            with open(file_path, "w") as f:
+                f.write(json_reporting_data)
+
         # Copy all log files into the archive - preserving the names
         for log_file_path in log_file_paths:
             log_file_name = get_file_name_no_extension(log_file_path)
@@ -221,6 +242,44 @@ async def generate_pdf(
     return pdf_data
 
 
+async def generate_json_reporting_data(
+    runner_state: RunnerState,
+    check_results: dict[str, CheckResult],
+    readings: dict[ReadingType, pd.DataFrame],
+    reading_counts: dict[ReadingType, int],
+    sites: list[Site],
+    timeline: timeline.Timeline | None,
+    errors,
+    version: int = 1,
+    set_max_w_varied: bool = False,
+) -> str | None:
+    created_at = datetime.now(timezone.utc)
+
+    try:
+        # Repack readings into something serializable
+        packed_readings = [
+            PackedReadings(reading_type=k, readings_as_json=readings[k].to_json(), reading_counts=v)
+            for k, v in reading_counts.items()
+        ]
+
+        reporting_data = ReportingData.v(version)(
+            created_at=created_at,
+            runner_state=runner_state,
+            check_results=check_results,
+            readings=packed_readings,
+            sites=sites,
+            timeline=timeline,
+            set_max_w_varied=set_max_w_varied,
+        )
+        json_reporting_data = reporting_data.to_json()
+    except Exception as exc:
+        logger.error("Error generating reporting data. Omitting reporting data from final zip.", exc_info=exc)
+        errors.append(f"Error generating reporting data: {exc}")
+        json_reporting_data = None
+
+    return json_reporting_data
+
+
 async def finish_active_test(runner_state: RunnerState, session: AsyncSession) -> bytes:
     """For the specified RunnerState - move the active test into a "Finished" state by calculating the final ZIP
     contents. Raises NoActiveTestProcedure if there isn't an active test procedure for the specified RunnerState
@@ -277,11 +336,11 @@ async def finish_active_test(runner_state: RunnerState, session: AsyncSession) -
     # Add a "virtual" check covering XSD errors in incoming requests
     xsd_error_counts = [len(rh.body_xml_errors) for rh in runner_state.request_history if rh.body_xml_errors]
     if xsd_error_counts:
-        xsd_check = check.CheckResult(
+        xsd_check = CheckResult(
             False, f"Detected {sum(xsd_error_counts)} xsd errors over {len(xsd_error_counts)} request(s)."
         )
     else:
-        xsd_check = check.CheckResult(True, "No XSD errors detected in any requests.")
+        xsd_check = CheckResult(True, "No XSD errors detected in any requests.")
     check_results["all-requests-xsd-valid"] = xsd_check
 
     # Figure out the testing timeline
@@ -299,7 +358,7 @@ async def finish_active_test(runner_state: RunnerState, session: AsyncSession) -
             errors.append(f"Failed to generate test timeline: {exc}")
             test_timeline = None
 
-    # Fetch raw DB data and create PDF
+        # Fetch raw DB data and create PDF
     try:
         sites = await get_sites(session)
         readings = await get_readings(session, reading_specifiers=MANDATORY_READING_SPECIFIERS)
@@ -329,15 +388,38 @@ async def finish_active_test(runner_state: RunnerState, session: AsyncSession) -
             errors=errors,
             set_max_w_varied=set_max_w_varied,
         )
+
+        # Convert to serialisable types
+        serializable_readings = {ReadingType.from_site_reading_type(k): v for k, v in readings.items()}
+        serializable_reading_counts = {ReadingType.from_site_reading_type(k): v for k, v in reading_counts.items()}
+        serializable_sites = [Site.from_site(s) for s in sites]
+
+        # Collect reporting state into json object
+        reporting_data_version = CURRENT_REPORTING_DATA_VERSION
+        json_reporting_data = await generate_json_reporting_data(
+            runner_state=runner_state,
+            check_results=check_results,
+            readings=serializable_readings,
+            reading_counts=serializable_reading_counts,
+            sites=serializable_sites,
+            timeline=test_timeline,
+            errors=errors,
+            version=reporting_data_version,
+            set_max_w_varied=set_max_w_varied,
+        )
+        reporting_data_filename_prefix = f"ReportingData_v{reporting_data_version}"
     except Exception as exc:
         logger.error("Failed to generate PDF report", exc_info=exc)
         errors.append(f"Failed to generate PDF report: {exc}")
         pdf_data = None
+        json_reporting_data = None
+        reporting_data_filename_prefix = None
 
     generation_timestamp = now.replace(microsecond=0)
 
     active_test_procedure.finished_zip_data = get_zip_contents(
         json_status_summary=json_status_summary,
+        json_reporting_data=json_reporting_data,
         log_file_paths=[
             LOG_FILE_ENVOY_SERVER,
             LOG_FILE_ENVOY_ADMIN,
@@ -346,6 +428,7 @@ async def finish_active_test(runner_state: RunnerState, session: AsyncSession) -
         ],
         pdf_data=pdf_data,
         filename_infix=f"_{int(generation_timestamp.timestamp())}_{active_test_procedure.name}",
+        reporting_data_filename_prefix=reporting_data_filename_prefix,
         errors=errors,
     )
     return active_test_procedure.finished_zip_data

src/cactus_runner/app/readings.py

@@ -1,6 +1,6 @@
 from dataclasses import dataclass
 from decimal import Decimal
-from typing import Sequence
+from typing import Any, Sequence
 
 import pandas as pd
 from envoy.server.model.site_reading import SiteReading, SiteReadingType
@@ -217,8 +217,16 @@ def scale_readings(reading_type: SiteReadingType, readings: Sequence[SiteReading
     if not readings:
         raise ValueError("Expected at least 1 entry in readings. Got 0/None")
 
+    def filter_attributes(attributes: dict[str, Any]):
+        """Removes attributes that start with _.
+
+        This is mainly about targeting and removing sqlalchemy attributes
+        that are prefixed by `_sa_`.
+        """
+        return {k: v for k, v in attributes.items() if not k.startswith("_")}
+
     # Convert list of readings into a dataframe
-    df = pd.DataFrame([reading.__dict__ for reading in readings])
+    df = pd.DataFrame([filter_attributes(reading.__dict__) for reading in readings])
 
     # Calculate value with proper scaling applied (power_10)
     scale_factor = Decimal(10**reading_type.power_of_ten_multiplier)

src/cactus_runner/app/reporting.py

@@ -53,10 +53,10 @@
 )
 
 from cactus_runner import __version__ as cactus_runner_version
-from cactus_runner.app.check import CheckResult
 from cactus_runner.app.envoy_common import ReadingLocation
 from cactus_runner.app.timeline import Timeline, duration_to_label
 from cactus_runner.models import (
+    CheckResult,
     ClientCertificateType,
     ClientInteraction,
     ClientInteractionType,

src/cactus_runner/app/timeline.py

@@ -4,6 +4,7 @@
 from itertools import chain
 from typing import Any, Callable, Sequence, cast
 
+from dataclass_wizard import JSONWizard
 from envoy.server.model.archive import ArchiveBase
 from envoy.server.model.archive.doe import (
     ArchiveDynamicOperatingEnvelope,
@@ -25,7 +26,7 @@
 
 
 @dataclass
-class TimelineDataStream:
+class TimelineDataStream(JSONWizard):
 
     label: str  # Descriptive label of this data stream
     offset_watt_values: list[
@@ -36,7 +37,7 @@ class TimelineDataStream:
 
 
 @dataclass
-class Timeline:
+class Timeline(JSONWizard):
     """Represents a series of regular "power" observations aligned on interval_seconds offsets relative to start"""
 
     start: datetime  # The basis time