diff --git a/README.md b/README.md index 3b974825..51f1a60b 100644 --- a/README.md +++ b/README.md @@ -73,8 +73,6 @@ Typically settings are set by setting an environment variable with the same name | **Setting** | **Type** | **Purpose** | | ----------- | -------- | ----------- | | `cert_header` | `string` | The name of the HTTP header that API endpoints will look for to validate a client. This should be set by the TLS termination point and can contain either a full client certificate in PEM format or the sha256 fingerprint of that certificate. defaults to "x-forwarded-client-cert" | -| `default_doe_import_active_watts` | `float` | If set - the DefaultDERControl endpoint will be activated with the DOE extensions for import being set to this value (requires `default_doe_export_active_watts`)| -| `default_doe_export_active_watts` | `float` | If set - the DefaultDERControl endpoint will be activated with the DOE extensions for export being set to this value (requires `default_doe_import_active_watts`)| | `allow_device_registration` | `bool` | If True - the registration workflows that enable unrecognised certs to generate/manage a single EndDevice (tied to that cert) will be enabled. Otherwise any cert will need to be registered out of band and assigned to an aggregator before connections can be made. Defaults to False| | `static_registration_pin` | `int` | If set - all new EndDevice registrations will have their Registration PIN set to this value (use 5 digit form). Uses a random number generator otherwise. | | `nmi_validation_enabled` | `bool` | If `true` - all updates of `ConnectionPoint` resource will trigger validation on `ConnectionPoint.id` against on AEMO's NMI Allocation List (Version 13 – November 2022). Defaults to `false`. | diff --git a/postman/envoy-admin-server.postman_collection.json b/postman/envoy-admin-server.postman_collection.json index 7148790b..89646e22 100644 --- a/postman/envoy-admin-server.postman_collection.json +++ b/postman/envoy-admin-server.postman_collection.json @@ -407,65 +407,6 @@ }, "response": [] }, - { - "name": "POST Update site control default", - "request": { - "method": "POST", - "header": [], - "body": { - "mode": "raw", - "raw": "{\n \"import_limit_watts\": {\"value\": 500},\n \"export_limit_watts\": {\"value\": 500},\n \"generation_limit_watts\": {\"value\": 500},\n \"load_limit_watts\": {\"value\": 500},\n \"ramp_rate_percent_per_second\": {\"value\": 500}\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": { - "raw": "{{HOST}}/site/{{SITE_ID}}/control_default", - "host": [ - "{{HOST}}" - ], - "path": [ - "site", - "{{SITE_ID}}", - "control_default" - ] - } - }, - "response": [] - }, - { - "name": "GET site control default", - "protocolProfileBehavior": { - "disableBodyPruning": true - }, - "request": { - "method": "GET", - "header": [], - "body": { - "mode": "raw", - "raw": "{\n \"import_limit_watts\" 500,\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": { - "raw": "{{HOST}}/site/{{SITE_ID}}/control_default", - "host": [ - "{{HOST}}" - ], - "path": [ - "site", - "{{SITE_ID}}", - "control_default" - ] - } - }, - "response": [] - }, { "name": "POST Update server runtime config", "request": { @@ -791,6 +732,65 @@ }, "response": [] }, + { + "name": "POST Update site control group default", + "request": { + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"import_limit_watts\": {\"value\": 500},\n \"export_limit_watts\": {\"value\": 500},\n \"generation_limit_watts\": {\"value\": 500},\n \"load_limit_watts\": {\"value\": 500},\n \"ramp_rate_percent_per_second\": {\"value\": 500}\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{HOST}}/site_control_group/{{GROUP_ID}}/default", + "host": [ + "{{HOST}}" + ], + "path": [ + "site_control_group", + "{{GROUP_ID}}", + "default" + ] + } + }, + "response": [] + }, + { + "name": "GET site control group default", + "protocolProfileBehavior": { + "disableBodyPruning": true + }, + "request": { + "method": "GET", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"import_limit_watts\" 500,\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{HOST}}/site_control_group/{{GROUP_ID}}/default", + "host": [ + "{{HOST}}" + ], + "path": [ + "site_control_group", + "{{GROUP_ID}}", + "default" + ] + } + }, + "response": [] + }, { "name": "POST Site controls", "request": { diff --git a/pyproject.toml b/pyproject.toml index 429db827..82612d2f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,7 +9,6 @@ addopts = "--strict-markers" markers = [ "cert_header: marks tests to use a custom value for cert_header instead of the default", "href_prefix: marks tests to use a custom value for href_prefix instead of the default of None", - "no_default_doe: marks tests to disable the default DOE config values (disables default DERControl endpoint)", "azure_ad_auth: marks tests to enable the azure active directory auth dependency", "azure_ad_db: marks tests to enable the azure active directory dynamic db creds dependency (requires azure_ad_auth)", "azure_ad_db_refresh_secs: marks tests to set the config value azure_ad_db_refresh_secs (requires azure_ad_db)", @@ -50,7 +49,7 @@ name = "envoy" dynamic = ["version", "readme"] requires-python = ">=3.9,<4.0" dependencies = [ - "envoy_schema==0.30.0", + "envoy_schema==0.31.0", "fastapi>=0.94.1", "sqlalchemy>=2.0.0", "alembic", diff --git a/src/envoy/admin/api/config.py b/src/envoy/admin/api/config.py index 735c10f1..342d219e 100644 --- a/src/envoy/admin/api/config.py +++ b/src/envoy/admin/api/config.py @@ -1,18 +1,12 @@ import logging from http import HTTPStatus -from envoy_schema.admin.schema.config import ( - ControlDefaultRequest, - ControlDefaultResponse, - RuntimeServerConfigRequest, - RuntimeServerConfigResponse, -) -from envoy_schema.admin.schema.uri import ServerConfigRuntimeUri, SiteControlDefaultConfigUri +from envoy_schema.admin.schema.config import RuntimeServerConfigRequest, RuntimeServerConfigResponse +from envoy_schema.admin.schema.uri import ServerConfigRuntimeUri from fastapi import APIRouter from fastapi_async_sqlalchemy import db from envoy.admin.manager.config import ConfigManager -from envoy.server.api.error_handler import LoggedHttpException logger = logging.getLogger(__name__) @@ -40,31 +34,3 @@ async def get_runtime_config() -> RuntimeServerConfigResponse: RuntimeServerConfigResponse """ return await ConfigManager.fetch_config_response(db.session) - - -@router.post(SiteControlDefaultConfigUri, status_code=HTTPStatus.NO_CONTENT, response_model=None) -async def update_site_control_default(site_id: int, body: ControlDefaultRequest) -> None: - """Updates the control default config for the specified site. Any missing values will NOT be updated. - - Body: - single ControlDefaultRequest object. - - Returns: - None - """ - result = await ConfigManager.update_site_control_default(db.session, site_id, body) - if not result: - raise LoggedHttpException(logger, None, HTTPStatus.NOT_FOUND, f"site_id {site_id} not found") - - -@router.get(SiteControlDefaultConfigUri, status_code=HTTPStatus.OK, response_model=ControlDefaultResponse) -async def get_site_control_default(site_id: int) -> ControlDefaultResponse: - """Gets the control default config for the specified site. - - Returns: - ControlDefaultResponse or 404 - """ - result = await ConfigManager.fetch_site_control_default_response(db.session, site_id) - if not result: - raise LoggedHttpException(logger, None, HTTPStatus.NOT_FOUND, f"site_id {site_id} not found") - return result diff --git a/src/envoy/admin/api/site_control.py b/src/envoy/admin/api/site_control.py index e8418c35..42f4155a 100644 --- a/src/envoy/admin/api/site_control.py +++ b/src/envoy/admin/api/site_control.py @@ -5,6 +5,8 @@ from asyncpg.exceptions import CardinalityViolationError # type: ignore from envoy_schema.admin.schema.site_control import ( + SiteControlGroupDefaultRequest, + SiteControlGroupDefaultResponse, SiteControlGroupPageResponse, SiteControlGroupRequest, SiteControlGroupResponse, @@ -12,6 +14,7 @@ SiteControlRequest, ) from envoy_schema.admin.schema.uri import ( + SiteControlGroupDefaultUri, SiteControlGroupListUri, SiteControlGroupUri, SiteControlRangeUri, @@ -145,3 +148,31 @@ async def delete_site_controls_in_range(group_id: int, period_start: datetime, p await SiteControlListManager.delete_site_controls_in_range( db.session, site_control_group_id=group_id, site_id=None, period_start=period_start, period_end=period_end ) + + +@router.post(SiteControlGroupDefaultUri, status_code=HTTPStatus.NO_CONTENT, response_model=None) +async def update_site_control_default(group_id: int, body: SiteControlGroupDefaultRequest) -> None: + """Updates the control default config for the specified SiteControlGroup. Any missing values will NOT be updated. + + Body: + single SiteControlGroupDefaultRequest object. + + Returns: + None + """ + result = await SiteControlGroupManager.update_site_control_default(db.session, group_id, body) + if not result: + raise LoggedHttpException(logger, None, HTTPStatus.NOT_FOUND, f"group_id {group_id} not found") + + +@router.get(SiteControlGroupDefaultUri, status_code=HTTPStatus.OK, response_model=SiteControlGroupDefaultResponse) +async def get_site_control_default(group_id: int) -> SiteControlGroupDefaultResponse: + """Gets the control default config for the specified SiteControlGroup + + Returns: + SiteControlGroupDefaultResponse or 404 + """ + result = await SiteControlGroupManager.fetch_site_control_default_response(db.session, group_id) + if not result: + raise LoggedHttpException(logger, None, HTTPStatus.NOT_FOUND, f"group_id {group_id} not found") + return result diff --git a/src/envoy/admin/crud/site.py b/src/envoy/admin/crud/site.py index 86d2ea4b..9ec5a855 100644 --- a/src/envoy/admin/crud/site.py +++ b/src/envoy/admin/crud/site.py @@ -119,7 +119,6 @@ async def select_single_site_no_scoping( site_id: int, include_groups: bool = False, include_der: bool = False, - include_site_default: bool = False, ) -> Optional[Site]: """Admin selecting of a single site - no filtering on aggregator is made.""" @@ -136,8 +135,5 @@ async def select_single_site_no_scoping( selectinload(Site.site_ders).selectinload(SiteDER.site_der_status), ) - if include_site_default: - stmt = stmt.options(selectinload(Site.default_site_control)) - resp = await session.execute(stmt) return resp.scalar_one_or_none() diff --git a/src/envoy/admin/manager/config.py b/src/envoy/admin/manager/config.py index 2a2cae8f..a1b33ab0 100644 --- a/src/envoy/admin/manager/config.py +++ b/src/envoy/admin/manager/config.py @@ -1,22 +1,13 @@ from datetime import datetime, timezone -from decimal import Decimal -from typing import Optional - -from envoy_schema.admin.schema.config import ( - ControlDefaultRequest, - ControlDefaultResponse, - RuntimeServerConfigRequest, - RuntimeServerConfigResponse, -) + +from envoy_schema.admin.schema.config import RuntimeServerConfigRequest, RuntimeServerConfigResponse from sqlalchemy.ext.asyncio import AsyncSession -from envoy.admin.crud.site import select_single_site_no_scoping from envoy.notification.manager.notification import NotificationManager from envoy.server.crud.server import select_server_config from envoy.server.manager.server import _map_server_config from envoy.server.manager.time import utc_now from envoy.server.model.server import RuntimeServerConfig as ConfigEntity -from envoy.server.model.site import DefaultSiteControl from envoy.server.model.subscription import SubscriptionResource @@ -100,72 +91,3 @@ async def fetch_config_response(session: AsyncSession) -> RuntimeServerConfigRes changed_time=changed_time, created_time=created_time, ) - - @staticmethod - async def update_site_control_default(session: AsyncSession, site_id: int, request: ControlDefaultRequest) -> bool: - now = utc_now() - - site = await select_single_site_no_scoping(session, site_id, include_site_default=True) - if site is None: - return False - - if site.default_site_control is None: - site.default_site_control = DefaultSiteControl(changed_time=now, site_id=site.site_id, version=0) - else: - site.default_site_control.changed_time = now - - if request.import_limit_watts is not None: - site.default_site_control.import_limit_active_watts = request.import_limit_watts.value - - if request.export_limit_watts is not None: - site.default_site_control.export_limit_active_watts = request.export_limit_watts.value - - if request.generation_limit_watts is not None: - site.default_site_control.generation_limit_active_watts = request.generation_limit_watts.value - - if request.load_limit_watts is not None: - site.default_site_control.load_limit_active_watts = request.load_limit_watts.value - - if request.ramp_rate_percent_per_second is not None: - ramp_rate_value = ( - int(request.ramp_rate_percent_per_second.value) - if request.ramp_rate_percent_per_second.value is not None - else None - ) - site.default_site_control.ramp_rate_percent_per_second = ramp_rate_value - - site.default_site_control.version = site.default_site_control.version + 1 - await session.commit() - - await NotificationManager.notify_changed_deleted_entities(SubscriptionResource.DEFAULT_SITE_CONTROL, now) - - return True - - @staticmethod - async def fetch_site_control_default_response( - session: AsyncSession, site_id: int - ) -> Optional[ControlDefaultResponse]: - """Fetches the current site control default values as a ControlDefaultResponse for external communication""" - site = await select_single_site_no_scoping(session, site_id, include_site_default=True) - if not site: - return None - if site.default_site_control: - default_config = site.default_site_control - else: - default_config = DefaultSiteControl( - changed_time=site.changed_time, created_time=site.created_time, site_id=site.site_id - ) - - return ControlDefaultResponse( - ramp_rate_percent_per_second=( - Decimal(default_config.ramp_rate_percent_per_second) - if default_config.ramp_rate_percent_per_second is not None - else None - ), - server_default_import_limit_watts=default_config.import_limit_active_watts, - server_default_export_limit_watts=default_config.export_limit_active_watts, - server_default_generation_limit_watts=default_config.generation_limit_active_watts, - server_default_load_limit_watts=default_config.load_limit_active_watts, - changed_time=default_config.changed_time, - created_time=default_config.created_time, - ) diff --git a/src/envoy/admin/manager/site_control.py b/src/envoy/admin/manager/site_control.py index 8f1b2d15..c01be892 100644 --- a/src/envoy/admin/manager/site_control.py +++ b/src/envoy/admin/manager/site_control.py @@ -1,7 +1,10 @@ from datetime import datetime +from decimal import Decimal from typing import Optional from envoy_schema.admin.schema.site_control import ( + SiteControlGroupDefaultRequest, + SiteControlGroupDefaultResponse, SiteControlGroupPageResponse, SiteControlGroupRequest, SiteControlGroupResponse, @@ -20,8 +23,11 @@ ) from envoy.admin.mapper.site_control import SiteControlGroupListMapper, SiteControlListMapper from envoy.notification.manager.notification import NotificationManager +from envoy.server.crud.archive import copy_rows_into_archive from envoy.server.crud.doe import select_site_control_group_by_id, select_site_control_group_fsa_ids from envoy.server.manager.time import utc_now +from envoy.server.model.archive.doe import ArchiveSiteControlGroupDefault +from envoy.server.model.doe import SiteControlGroupDefault from envoy.server.model.subscription import SubscriptionResource @@ -71,6 +77,82 @@ async def get_site_control_group_by_id( return SiteControlGroupListMapper.map_to_response(scg) + @staticmethod + async def update_site_control_default( + session: AsyncSession, group_id: int, request: SiteControlGroupDefaultRequest + ) -> bool: + now = utc_now() + + scg = await select_site_control_group_by_id(session, group_id, include_default=True) + if scg is None: + return False + + if scg.site_control_group_default is None: + scg.site_control_group_default = SiteControlGroupDefault( + changed_time=now, version=0, site_control_group_id=group_id + ) + else: + # If there is an existing record - lets archive it BEFORE we update it + await copy_rows_into_archive( + session, + SiteControlGroupDefault, + ArchiveSiteControlGroupDefault, + lambda q: q.where(SiteControlGroupDefault.site_control_group_id == group_id), + ) + scg.site_control_group_default.changed_time = now + + if request.import_limit_watts is not None: + scg.site_control_group_default.import_limit_active_watts = request.import_limit_watts.value + + if request.export_limit_watts is not None: + scg.site_control_group_default.export_limit_active_watts = request.export_limit_watts.value + + if request.generation_limit_watts is not None: + scg.site_control_group_default.generation_limit_active_watts = request.generation_limit_watts.value + + if request.load_limit_watts is not None: + scg.site_control_group_default.load_limit_active_watts = request.load_limit_watts.value + + if request.ramp_rate_percent_per_second is not None: + ramp_rate_value = ( + int(request.ramp_rate_percent_per_second.value) + if request.ramp_rate_percent_per_second.value is not None + else None + ) + scg.site_control_group_default.ramp_rate_percent_per_second = ramp_rate_value + + scg.site_control_group_default.version = scg.site_control_group_default.version + 1 + + await session.commit() + + await NotificationManager.notify_changed_deleted_entities(SubscriptionResource.DEFAULT_SITE_CONTROL, now) + + return True + + @staticmethod + async def fetch_site_control_default_response( + session: AsyncSession, group_id: int + ) -> Optional[SiteControlGroupDefaultResponse]: + """Fetches the current site control group default values as a SiteControlGroupDefaultResponse for external + communication""" + scg = await select_site_control_group_by_id(session, group_id, include_default=True) + if not scg or not scg.site_control_group_default: + return None + + return SiteControlGroupDefaultResponse( + ramp_rate_percent_per_second=( + Decimal(scg.site_control_group_default.ramp_rate_percent_per_second) + if scg.site_control_group_default.ramp_rate_percent_per_second is not None + else None + ), + server_default_import_limit_watts=scg.site_control_group_default.import_limit_active_watts, + server_default_export_limit_watts=scg.site_control_group_default.export_limit_active_watts, + server_default_generation_limit_watts=scg.site_control_group_default.generation_limit_active_watts, + server_default_load_limit_watts=scg.site_control_group_default.load_limit_active_watts, + changed_time=scg.site_control_group_default.changed_time, + created_time=scg.site_control_group_default.created_time, + ) + class SiteControlListManager: @staticmethod diff --git a/src/envoy/notification/crud/batch.py b/src/envoy/notification/crud/batch.py index 7a28618a..e73cab96 100644 --- a/src/envoy/notification/crud/batch.py +++ b/src/envoy/notification/crud/batch.py @@ -13,12 +13,12 @@ orm_relationship_map_parent_entities, ) from envoy.notification.crud.common import ( - ArchiveControlGroupScopedDefaultSiteControl, ArchiveSiteScopedFunctionSetAssignment, ArchiveSiteScopedSiteControlGroup, - ControlGroupScopedDefaultSiteControl, + ArchiveSiteScopedSiteControlGroupDefault, SiteScopedFunctionSetAssignment, SiteScopedSiteControlGroup, + SiteScopedSiteControlGroupDefault, TArchiveResourceModel, TResourceModel, ) @@ -27,9 +27,12 @@ from envoy.server.crud.server import select_server_config from envoy.server.manager.der_constants import PUBLIC_SITE_DER_ID from envoy.server.model.aggregator import Aggregator -from envoy.server.model.archive.doe import ArchiveDynamicOperatingEnvelope, ArchiveSiteControlGroup +from envoy.server.model.archive.doe import ( + ArchiveDynamicOperatingEnvelope, + ArchiveSiteControlGroup, + ArchiveSiteControlGroupDefault, +) from envoy.server.model.archive.site import ( - ArchiveDefaultSiteControl, ArchiveSite, ArchiveSiteDER, ArchiveSiteDERAvailability, @@ -39,16 +42,8 @@ ) from envoy.server.model.archive.site_reading import ArchiveSiteReading, ArchiveSiteReadingType from envoy.server.model.archive.tariff import ArchiveTariffGeneratedRate -from envoy.server.model.doe import DynamicOperatingEnvelope, SiteControlGroup -from envoy.server.model.site import ( - DefaultSiteControl, - Site, - SiteDER, - SiteDERAvailability, - SiteDERRating, - SiteDERSetting, - SiteDERStatus, -) +from envoy.server.model.doe import DynamicOperatingEnvelope, SiteControlGroup, SiteControlGroupDefault +from envoy.server.model.site import Site, SiteDER, SiteDERAvailability, SiteDERRating, SiteDERSetting, SiteDERStatus from envoy.server.model.site_reading import SiteReading, SiteReadingType from envoy.server.model.subscription import Subscription, SubscriptionResource from envoy.server.model.tariff import TariffGeneratedRate @@ -160,10 +155,10 @@ def get_batch_key(resource: SubscriptionResource, entity: TResourceModel) -> tup status = cast(SiteDERStatus, entity) # type: ignore # Pretty sure this is a mypy quirk return (status.site_der.site.aggregator_id, status.site_der.site_id, PUBLIC_SITE_DER_ID) elif resource == SubscriptionResource.DEFAULT_SITE_CONTROL: - default_control = cast(ControlGroupScopedDefaultSiteControl, entity) # type: ignore + default_control = cast(SiteScopedSiteControlGroupDefault, entity) # type: ignore return ( - default_control.original.site.aggregator_id, - default_control.original.site_id, + default_control.aggregator_id, + default_control.site_id, default_control.site_control_group_id, ) elif resource == SubscriptionResource.FUNCTION_SET_ASSIGNMENTS: @@ -207,7 +202,7 @@ def get_subscription_filter_id(resource: SubscriptionResource, entity: TResource elif resource == SubscriptionResource.FUNCTION_SET_ASSIGNMENTS: return -1 # There are no subscriptions to a single FSA elif resource == SubscriptionResource.DEFAULT_SITE_CONTROL: - return cast(ControlGroupScopedDefaultSiteControl, entity).site_control_group_id # type: ignore + return cast(SiteScopedSiteControlGroupDefault, entity).site_control_group_id # type: ignore elif resource == SubscriptionResource.SITE_CONTROL_GROUP: return cast(SiteScopedSiteControlGroup, entity).original.fsa_id # type: ignore else: @@ -233,7 +228,7 @@ def get_site_id(resource: SubscriptionResource, entity: TResourceModel) -> int: elif resource == SubscriptionResource.SITE_DER_STATUS: return cast(SiteDERStatus, entity).site_der.site_id # type: ignore # Pretty sure this is a mypy quirk elif resource == SubscriptionResource.DEFAULT_SITE_CONTROL: - return cast(ControlGroupScopedDefaultSiteControl, entity).original.site_id # type: ignore + return cast(SiteScopedSiteControlGroupDefault, entity).site_id # type: ignore elif resource == SubscriptionResource.FUNCTION_SET_ASSIGNMENTS: return cast(SiteScopedFunctionSetAssignment, entity).site_id # type: ignore elif resource == SubscriptionResource.SITE_CONTROL_GROUP: @@ -669,50 +664,32 @@ async def fetch_der_status_by_changed_at( async def fetch_default_site_controls_by_changed_at( session: AsyncSession, timestamp: datetime -) -> AggregatorBatchedEntities[ControlGroupScopedDefaultSiteControl, ArchiveControlGroupScopedDefaultSiteControl]: # type: ignore # noqa: E501 +) -> AggregatorBatchedEntities[SiteScopedSiteControlGroupDefault, ArchiveSiteScopedSiteControlGroupDefault]: # type: ignore # noqa: E501 """Fetches all DefaultSiteControl instances matching the specified changed_at and returns them keyed by their aggregator/site id Also fetches any site from the archive that was deleted at the specified timestamp""" active_defaults, deleted_defaults = await fetch_entities_with_archive_by_datetime( - session, DefaultSiteControl, ArchiveDefaultSiteControl, timestamp - ) - - # Now fetch all site's so we can populate the site_relationship - referenced_site_ids = { - e.site_id - for e in cast( - Iterable[Union[DefaultSiteControl, ArchiveDefaultSiteControl]], - chain(active_defaults, deleted_defaults), - ) - } - active_sites, deleted_sites = await fetch_entities_with_archive_by_id( - session, Site, ArchiveSite, referenced_site_ids + session, SiteControlGroupDefault, ArchiveSiteControlGroupDefault, timestamp ) - # Map the "site" relationship - orm_relationship_map_parent_entities( - cast(Iterable[Union[DefaultSiteControl, ArchiveDefaultSiteControl]], chain(active_defaults, deleted_defaults)), - lambda e: e.site_id, - {e.site_id: e for e in cast(Iterable[Union[Site, ArchiveSite]], chain(active_sites, deleted_sites))}, - "site", - ) - - # The defaults will need to vary per SiteControlGroup so we generate an instance per site_control_group_id - site_control_group_ids = (await session.execute(select(SiteControlGroup.site_control_group_id))).scalars().all() + # We need to generate a notification per site ID - so fetch all of those + aggregator_site_ids = (await session.execute(select(Site.aggregator_id, Site.site_id).order_by(Site.site_id))).all() - site_control_scoped_actives = [ - ControlGroupScopedDefaultSiteControl(scg_id, ad) for ad in active_defaults for scg_id in site_control_group_ids + scoped_actives = [ + SiteScopedSiteControlGroupDefault(agg_id, site_id, ad.site_control_group_id, ad) + for ad in active_defaults + for agg_id, site_id in aggregator_site_ids ] - site_control_scoped_deleted = [ - ArchiveControlGroupScopedDefaultSiteControl(scg_id, ad) - for ad in deleted_defaults - for scg_id in site_control_group_ids + scoped_deleted = [ + ArchiveSiteScopedSiteControlGroupDefault(agg_id, site_id, dd.site_control_group_id, dd) + for dd in deleted_defaults + for agg_id, site_id in aggregator_site_ids ] return AggregatorBatchedEntities( - timestamp, SubscriptionResource.DEFAULT_SITE_CONTROL, site_control_scoped_actives, site_control_scoped_deleted + timestamp, SubscriptionResource.DEFAULT_SITE_CONTROL, scoped_actives, scoped_deleted ) # type: ignore diff --git a/src/envoy/notification/crud/common.py b/src/envoy/notification/crud/common.py index 32324309..c6d89c7c 100644 --- a/src/envoy/notification/crud/common.py +++ b/src/envoy/notification/crud/common.py @@ -1,9 +1,12 @@ from dataclasses import dataclass from typing import Optional, TypeVar -from envoy.server.model.archive.doe import ArchiveDynamicOperatingEnvelope, ArchiveSiteControlGroup +from envoy.server.model.archive.doe import ( + ArchiveDynamicOperatingEnvelope, + ArchiveSiteControlGroup, + ArchiveSiteControlGroupDefault, +) from envoy.server.model.archive.site import ( - ArchiveDefaultSiteControl, ArchiveSite, ArchiveSiteDER, ArchiveSiteDERAvailability, @@ -14,17 +17,9 @@ from envoy.server.model.archive.site_reading import ArchiveSiteReading, ArchiveSiteReadingType from envoy.server.model.archive.subscription import ArchiveSubscription from envoy.server.model.archive.tariff import ArchiveTariffGeneratedRate -from envoy.server.model.doe import DynamicOperatingEnvelope, SiteControlGroup +from envoy.server.model.doe import DynamicOperatingEnvelope, SiteControlGroup, SiteControlGroupDefault from envoy.server.model.server import RuntimeServerConfig -from envoy.server.model.site import ( - DefaultSiteControl, - Site, - SiteDER, - SiteDERAvailability, - SiteDERRating, - SiteDERSetting, - SiteDERStatus, -) +from envoy.server.model.site import Site, SiteDER, SiteDERAvailability, SiteDERRating, SiteDERSetting, SiteDERStatus from envoy.server.model.site_reading import SiteReading, SiteReadingType from envoy.server.model.subscription import Subscription from envoy.server.model.tariff import TariffGeneratedRate @@ -70,19 +65,23 @@ class ArchiveSiteScopedSiteControlGroup: @dataclass -class ControlGroupScopedDefaultSiteControl: - """DefaultSiteControl isn't scoped to a specific SiteControlGroup - for csip-aus it will need to be""" +class SiteScopedSiteControlGroupDefault: + """SiteControlGroupDefault isn't scoped to a specific site - for csip-aus it will need to be""" + aggregator_id: int + site_id: int site_control_group_id: int - original: DefaultSiteControl + original: SiteControlGroupDefault @dataclass -class ArchiveControlGroupScopedDefaultSiteControl: - """DefaultSiteControl isn't scoped to a specific SiteControlGroup - for csip-aus it will need to be""" +class ArchiveSiteScopedSiteControlGroupDefault: + """SiteControlGroupDefault isn't scoped to a specific site - for csip-aus it will need to be""" + aggregator_id: int + site_id: int site_control_group_id: int - original: ArchiveDefaultSiteControl + original: ArchiveSiteControlGroupDefault TResourceModel = TypeVar( @@ -98,7 +97,7 @@ class ArchiveControlGroupScopedDefaultSiteControl: SiteDERSetting, SiteDERStatus, Subscription, - DefaultSiteControl, + SiteControlGroupDefault, SiteControlGroup, RuntimeServerConfig, ) @@ -116,6 +115,6 @@ class ArchiveControlGroupScopedDefaultSiteControl: ArchiveSiteDERSetting, ArchiveSiteDERStatus, ArchiveSubscription, - ArchiveDefaultSiteControl, + ArchiveSiteControlGroupDefault, ArchiveSiteControlGroup, ) diff --git a/src/envoy/notification/task/check.py b/src/envoy/notification/task/check.py index 8b6e6128..17ec1328 100644 --- a/src/envoy/notification/task/check.py +++ b/src/envoy/notification/task/check.py @@ -28,9 +28,9 @@ select_subscriptions_for_resource, ) from envoy.notification.crud.common import ( - ControlGroupScopedDefaultSiteControl, SiteScopedFunctionSetAssignment, SiteScopedSiteControlGroup, + SiteScopedSiteControlGroupDefault, TArchiveResourceModel, TResourceModel, ) @@ -322,7 +322,7 @@ def entities_to_notification( elif resource == SubscriptionResource.DEFAULT_SITE_CONTROL: # DEFAULT_SITE_CONTROL: (aggregator_id: int, site_id: int, site_control_group_id: int) (_, site_id, site_control_group_id) = batch_key - default_site_control = cast(ControlGroupScopedDefaultSiteControl, entities[0]) if len(entities) > 0 else None + default_site_control = cast(SiteScopedSiteControlGroupDefault, entities[0]) if len(entities) > 0 else None return NotificationMapper.map_default_site_control_response( None if default_site_control is None else default_site_control.original, diff --git a/src/envoy/server/alembic/versions/ed20565cc862_added_sitecontroldefault.py b/src/envoy/server/alembic/versions/ed20565cc862_added_sitecontroldefault.py new file mode 100644 index 00000000..43829630 --- /dev/null +++ b/src/envoy/server/alembic/versions/ed20565cc862_added_sitecontroldefault.py @@ -0,0 +1,153 @@ +"""added_sitecontroldefault + +Revision ID: ed20565cc862 +Revises: 29a83be2701e +Create Date: 2025-12-15 13:48:28.058413 + +""" + +import sqlalchemy as sa +from alembic import op +from sqlalchemy.dialects import postgresql + +# revision identifiers, used by Alembic. +revision = "ed20565cc862" +down_revision = "29a83be2701e" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.create_table( + "archive_site_control_group_default", + sa.Column("site_control_group_default_id", sa.INTEGER(), nullable=False), + sa.Column("site_control_group_id", sa.INTEGER(), nullable=False), + sa.Column("created_time", sa.DateTime(timezone=True), nullable=False), + sa.Column("changed_time", sa.DateTime(timezone=True), nullable=False), + sa.Column("version", sa.INTEGER(), nullable=False), + sa.Column("import_limit_active_watts", sa.DECIMAL(precision=16, scale=2), nullable=True), + sa.Column("export_limit_active_watts", sa.DECIMAL(precision=16, scale=2), nullable=True), + sa.Column("generation_limit_active_watts", sa.DECIMAL(precision=16, scale=2), nullable=True), + sa.Column("load_limit_active_watts", sa.DECIMAL(precision=16, scale=2), nullable=True), + sa.Column("ramp_rate_percent_per_second", sa.Integer(), nullable=True), + sa.Column("archive_id", sa.Integer(), autoincrement=True, nullable=False), + sa.Column("archive_time", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=True), + sa.Column("deleted_time", sa.DateTime(timezone=True), nullable=True), + sa.PrimaryKeyConstraint("archive_id"), + ) + op.create_index( + op.f("ix_archive_site_control_group_default_deleted_time"), + "archive_site_control_group_default", + ["deleted_time"], + unique=False, + ) + op.create_index( + op.f("ix_archive_site_control_group_default_site_control_group_default_id"), + "archive_site_control_group_default", + ["site_control_group_default_id"], + unique=False, + ) + op.create_table( + "site_control_group_default", + sa.Column("site_control_group_default_id", sa.Integer(), nullable=False), + sa.Column("site_control_group_id", sa.Integer(), nullable=False), + sa.Column("created_time", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), + sa.Column("changed_time", sa.DateTime(timezone=True), nullable=False), + sa.Column("version", sa.INTEGER(), server_default="0", nullable=False), + sa.Column("import_limit_active_watts", sa.DECIMAL(precision=16, scale=2), nullable=True), + sa.Column("export_limit_active_watts", sa.DECIMAL(precision=16, scale=2), nullable=True), + sa.Column("generation_limit_active_watts", sa.DECIMAL(precision=16, scale=2), nullable=True), + sa.Column("load_limit_active_watts", sa.DECIMAL(precision=16, scale=2), nullable=True), + sa.Column("ramp_rate_percent_per_second", sa.Integer(), nullable=True), + sa.ForeignKeyConstraint( + ["site_control_group_id"], ["site_control_group.site_control_group_id"], ondelete="CASCADE" + ), + sa.PrimaryKeyConstraint("site_control_group_default_id"), + ) + op.create_index( + op.f("ix_site_control_group_default_changed_time"), "site_control_group_default", ["changed_time"], unique=False + ) + op.create_index( + op.f("ix_site_control_group_default_site_control_group_id"), + "site_control_group_default", + ["site_control_group_id"], + unique=False, + ) + op.drop_index("ix_archive_default_site_control_deleted_time", table_name="archive_default_site_control") + op.drop_table("archive_default_site_control") + op.drop_index("ix_default_site_control_changed_time", table_name="default_site_control") + op.drop_index("ix_default_site_control_site_id", table_name="default_site_control") + op.drop_table("default_site_control") + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.create_table( + "default_site_control", + sa.Column("default_site_control_id", sa.INTEGER(), autoincrement=True, nullable=False), + sa.Column("site_id", sa.INTEGER(), autoincrement=False, nullable=False), + sa.Column( + "created_time", + postgresql.TIMESTAMP(timezone=True), + server_default=sa.text("now()"), + autoincrement=False, + nullable=False, + ), + sa.Column("changed_time", postgresql.TIMESTAMP(timezone=True), autoincrement=False, nullable=False), + sa.Column("import_limit_active_watts", sa.NUMERIC(precision=16, scale=2), autoincrement=False, nullable=True), + sa.Column("export_limit_active_watts", sa.NUMERIC(precision=16, scale=2), autoincrement=False, nullable=True), + sa.Column( + "generation_limit_active_watts", sa.NUMERIC(precision=16, scale=2), autoincrement=False, nullable=True + ), + sa.Column("load_limit_active_watts", sa.NUMERIC(precision=16, scale=2), autoincrement=False, nullable=True), + sa.Column("ramp_rate_percent_per_second", sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column("version", sa.INTEGER(), server_default=sa.text("0"), autoincrement=False, nullable=False), + sa.ForeignKeyConstraint( + ["site_id"], ["site.site_id"], name="default_site_control_site_id_fkey", ondelete="CASCADE" + ), + sa.PrimaryKeyConstraint("default_site_control_id", name="default_site_control_pkey"), + ) + op.create_index("ix_default_site_control_site_id", "default_site_control", ["site_id"], unique=False) + op.create_index("ix_default_site_control_changed_time", "default_site_control", ["changed_time"], unique=False) + op.create_table( + "archive_default_site_control", + sa.Column("default_site_control_id", sa.INTEGER(), autoincrement=False, nullable=False), + sa.Column("site_id", sa.INTEGER(), autoincrement=False, nullable=False), + sa.Column("created_time", postgresql.TIMESTAMP(timezone=True), autoincrement=False, nullable=False), + sa.Column("changed_time", postgresql.TIMESTAMP(timezone=True), autoincrement=False, nullable=False), + sa.Column("import_limit_active_watts", sa.NUMERIC(precision=16, scale=2), autoincrement=False, nullable=True), + sa.Column("export_limit_active_watts", sa.NUMERIC(precision=16, scale=2), autoincrement=False, nullable=True), + sa.Column( + "generation_limit_active_watts", sa.NUMERIC(precision=16, scale=2), autoincrement=False, nullable=True + ), + sa.Column("load_limit_active_watts", sa.NUMERIC(precision=16, scale=2), autoincrement=False, nullable=True), + sa.Column("ramp_rate_percent_per_second", sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column("archive_id", sa.INTEGER(), autoincrement=True, nullable=False), + sa.Column( + "archive_time", + postgresql.TIMESTAMP(timezone=True), + server_default=sa.text("now()"), + autoincrement=False, + nullable=True, + ), + sa.Column("deleted_time", postgresql.TIMESTAMP(timezone=True), autoincrement=False, nullable=True), + sa.Column("version", sa.INTEGER(), autoincrement=False, nullable=False), + sa.PrimaryKeyConstraint("default_site_control_id", "archive_id", name="archive_default_site_control_pkey"), + ) + op.create_index( + "ix_archive_default_site_control_deleted_time", "archive_default_site_control", ["deleted_time"], unique=False + ) + op.drop_index(op.f("ix_site_control_group_default_site_control_group_id"), table_name="site_control_group_default") + op.drop_index(op.f("ix_site_control_group_default_changed_time"), table_name="site_control_group_default") + op.drop_table("site_control_group_default") + op.drop_index( + op.f("ix_archive_site_control_group_default_site_control_group_default_id"), + table_name="archive_site_control_group_default", + ) + op.drop_index( + op.f("ix_archive_site_control_group_default_deleted_time"), table_name="archive_site_control_group_default" + ) + op.drop_table("archive_site_control_group_default") + # ### end Alembic commands ### diff --git a/src/envoy/server/api/depends/default_doe.py b/src/envoy/server/api/depends/default_doe.py deleted file mode 100644 index 52d47473..00000000 --- a/src/envoy/server/api/depends/default_doe.py +++ /dev/null @@ -1,39 +0,0 @@ -from decimal import Decimal -from typing import Optional - -from fastapi import Request - -from envoy.server.model.config.default_doe import DefaultDoeConfiguration - - -class DefaultDoeDepends: - """Dependency class for populating the request state default_doe with an instance of DefaultDoeConfiguration""" - - import_limit_active_watts: Optional[Decimal] - export_limit_active_watts: Optional[Decimal] - generation_limit_active_watts: Optional[Decimal] - load_limit_active_watts: Optional[Decimal] - ramp_rate_percent_per_second: Optional[int] - - def __init__( - self, - import_limit_active_watts: Optional[Decimal] = None, - export_limit_active_watts: Optional[Decimal] = None, - generation_limit_active_watts: Optional[Decimal] = None, - load_limit_active_watts: Optional[Decimal] = None, - ramp_rate_percent_per_second: Optional[int] = None, - ): - self.import_limit_active_watts = import_limit_active_watts - self.export_limit_active_watts = export_limit_active_watts - self.generation_limit_active_watts = generation_limit_active_watts - self.load_limit_active_watts = load_limit_active_watts - self.ramp_rate_percent_per_second = ramp_rate_percent_per_second - - async def __call__(self, request: Request) -> None: - request.state.default_doe = DefaultDoeConfiguration( - import_limit_active_watts=self.import_limit_active_watts, - export_limit_active_watts=self.export_limit_active_watts, - generation_limit_active_watts=self.generation_limit_active_watts, - load_limit_active_watts=self.load_limit_active_watts, - ramp_rate_percent_per_second=self.ramp_rate_percent_per_second, - ) diff --git a/src/envoy/server/api/request.py b/src/envoy/server/api/request.py index 4d44fa5a..ce7c1bd7 100644 --- a/src/envoy/server/api/request.py +++ b/src/envoy/server/api/request.py @@ -4,7 +4,6 @@ from fastapi import HTTPException, Request -from envoy.server.model.config.default_doe import DefaultDoeConfiguration from envoy.server.request_scope import RawRequestClaims MAX_LIMIT = 500 @@ -51,15 +50,6 @@ def extract_request_claims(request: Request) -> RawRequestClaims: ) -def extract_default_doe(request: Request) -> Optional[DefaultDoeConfiguration]: - """If the DefaultDoeDepends is enabled a DefaultDoeConfiguration will be returned for this request or None - otherwise. This is a placeholder for static default DOE values""" - if request.state is not None: - return getattr(request.state, "default_doe", None) - - return None - - def extract_limit_from_paging_param(limit: Optional[list[int]] = None) -> int: """Given a sep2 paging parameter called limit (as an int) - return the value, defaulting to DEFAULT_LIMIT if not specified. Can raise HTTPException for invalid values""" diff --git a/src/envoy/server/api/sep2/der.py b/src/envoy/server/api/sep2/der.py index 91ea2547..9b1c7503 100644 --- a/src/envoy/server/api/sep2/der.py +++ b/src/envoy/server/api/sep2/der.py @@ -9,7 +9,6 @@ from envoy.server.api.error_handler import LoggedHttpException from envoy.server.api.request import ( extract_datetime_from_paging_param, - extract_default_doe, extract_limit_from_paging_param, extract_request_claims, extract_start_from_paging_param, @@ -348,7 +347,6 @@ async def get_derprogram_list( derp_list = await DERProgramManager.fetch_list_for_scope( db.session, scope=extract_request_claims(request).to_site_request_scope(site_id), - default_doe=extract_default_doe(request), start=extract_start_from_paging_param(start), limit=extract_limit_from_paging_param(limit), changed_after=extract_datetime_from_paging_param(after), diff --git a/src/envoy/server/api/sep2/derp.py b/src/envoy/server/api/sep2/derp.py index 80f7ad7d..4c9ea03d 100644 --- a/src/envoy/server/api/sep2/derp.py +++ b/src/envoy/server/api/sep2/derp.py @@ -8,7 +8,6 @@ from envoy.server.api.error_handler import LoggedHttpException from envoy.server.api.request import ( extract_datetime_from_paging_param, - extract_default_doe, extract_limit_from_paging_param, extract_request_claims, extract_start_from_paging_param, @@ -46,7 +45,6 @@ async def get_derprogram_list( derp_list = await DERProgramManager.fetch_list_for_scope( db.session, scope=extract_request_claims(request).to_site_request_scope(site_id), - default_doe=extract_default_doe(request), start=extract_start_from_paging_param(start), changed_after=extract_datetime_from_paging_param(after), limit=extract_limit_from_paging_param(limit), @@ -87,7 +85,6 @@ async def get_derprogram_list_fsa_scoped( derp_list = await DERProgramManager.fetch_list_for_scope( db.session, scope=extract_request_claims(request).to_site_request_scope(site_id), - default_doe=extract_default_doe(request), start=extract_start_from_paging_param(start), changed_after=extract_datetime_from_paging_param(after), limit=extract_limit_from_paging_param(limit), @@ -118,7 +115,6 @@ async def get_derprogram_doe(request: Request, site_id: int, der_program_id: int db.session, scope=extract_request_claims(request).to_site_request_scope(site_id), der_program_id=der_program_id, - default_doe=extract_default_doe(request), ) except BadRequestError as ex: raise LoggedHttpException(logger, ex, status_code=HTTPStatus.BAD_REQUEST, detail=ex.message) @@ -232,11 +228,10 @@ async def get_default_dercontrol( """ try: - derc_list = await DERControlManager.fetch_default_doe_controls_for_site( + derc_list = await DERControlManager.fetch_default_doe_controls_for_scope( db.session, scope=extract_request_claims(request).to_site_request_scope(site_id), der_program_id=der_program_id, - default_doe=extract_default_doe(request), ) except BadRequestError as ex: raise LoggedHttpException(logger, ex, status_code=HTTPStatus.BAD_REQUEST, detail=ex.message) diff --git a/src/envoy/server/crud/doe.py b/src/envoy/server/crud/doe.py index a9ca49c0..6554258c 100644 --- a/src/envoy/server/crud/doe.py +++ b/src/envoy/server/crud/doe.py @@ -3,6 +3,7 @@ from sqlalchemy import Select, func, literal_column, select from sqlalchemy.ext.asyncio import AsyncSession +from sqlalchemy.orm import selectinload from envoy.server.crud.common import localize_start_time, localize_start_time_for_entity from envoy.server.model.archive.doe import ArchiveDynamicOperatingEnvelope as ArchiveDOE @@ -363,6 +364,7 @@ async def _site_control_groups( changed_after: datetime, limit: Optional[int], fsa_id: Optional[int], + include_defaults: bool, ) -> Union[Sequence[SiteControlGroup], int]: """Internal utility for fetching/counting SiteControlGroup's @@ -383,6 +385,9 @@ async def _site_control_groups( if not is_counting: stmt = stmt.order_by(SiteControlGroup.primacy.asc(), SiteControlGroup.site_control_group_id.desc()) + if include_defaults: + stmt = stmt.options(selectinload(SiteControlGroup.site_control_group_default)) + resp = await session.execute(stmt) if is_counting: return resp.scalar_one() @@ -391,7 +396,12 @@ async def _site_control_groups( async def select_site_control_groups( - session: AsyncSession, start: Optional[int], changed_after: datetime, limit: Optional[int], fsa_id: Optional[int] + session: AsyncSession, + start: Optional[int], + changed_after: datetime, + limit: Optional[int], + fsa_id: Optional[int], + include_defaults: bool = False, ) -> Sequence[SiteControlGroup]: """Fetches SiteControlGroup with some basic pagination / filtering on change time. @@ -400,7 +410,7 @@ async def select_site_control_groups( Orders by 2030.5 requirements on DERProgram which is primacy ASC, primary key DESC""" # Test coverage will ensure that it's an entity list - return await _site_control_groups(False, session, start, changed_after, limit, fsa_id) # type: ignore [return-value] # noqa: E501 + return await _site_control_groups(False, session, start, changed_after, limit, fsa_id, include_defaults) # type: ignore [return-value] # noqa: E501 async def count_site_control_groups(session: AsyncSession, changed_after: datetime, fsa_id: Optional[int]) -> int: @@ -410,7 +420,15 @@ async def count_site_control_groups(session: AsyncSession, changed_after: dateti """ # Test coverage will ensure that it's an int - return await _site_control_groups(True, session, 0, changed_after, None, fsa_id) # type: ignore [return-value] + return await _site_control_groups( + True, + session, + 0, + changed_after, + None, + fsa_id, + False, + ) # type: ignore [return-value] async def count_site_control_groups_by_fsa_id(session: AsyncSession) -> dict[int, int]: @@ -425,11 +443,15 @@ async def count_site_control_groups_by_fsa_id(session: AsyncSession) -> dict[int async def select_site_control_group_by_id( - session: AsyncSession, site_control_group_id: int + session: AsyncSession, site_control_group_id: int, include_default: bool = False ) -> Optional[SiteControlGroup]: """Fetches a single SiteControlGroup with the specified site_control_group_id. Returns None if it can't be found.""" stmt = select(SiteControlGroup).where(SiteControlGroup.site_control_group_id == site_control_group_id).limit(1) + + if include_default: + stmt = stmt.options(selectinload(SiteControlGroup.site_control_group_default)) + resp = await session.execute(stmt) return resp.scalar_one_or_none() diff --git a/src/envoy/server/crud/site.py b/src/envoy/server/crud/site.py index 66d4e563..4515b778 100644 --- a/src/envoy/server/crud/site.py +++ b/src/envoy/server/crud/site.py @@ -6,7 +6,6 @@ from sqlalchemy import delete, func, select from sqlalchemy.dialects.postgresql import insert as psql_insert from sqlalchemy.ext.asyncio import AsyncSession -from sqlalchemy.orm import selectinload from envoy.server.crud import common from envoy.server.crud.aggregator import select_aggregator @@ -15,7 +14,6 @@ from envoy.server.model.aggregator import Aggregator from envoy.server.model.archive.doe import ArchiveDynamicOperatingEnvelope from envoy.server.model.archive.site import ( - ArchiveDefaultSiteControl, ArchiveSite, ArchiveSiteDER, ArchiveSiteDERAvailability, @@ -28,7 +26,6 @@ from envoy.server.model.archive.tariff import ArchiveTariffGeneratedRate from envoy.server.model.doe import DynamicOperatingEnvelope from envoy.server.model.site import ( - DefaultSiteControl, Site, SiteDER, SiteDERAvailability, @@ -341,21 +338,6 @@ async def delete_site_for_aggregator( # Site Groups assignments aren't archived - we can delete them directly await session.execute(delete(SiteGroupAssignment).where(SiteGroupAssignment.site_id == site_id)) - # Archive the DefaultSiteControl - default_site_control_id = ( - await session.execute( - (select(DefaultSiteControl.default_site_control_id).where(DefaultSiteControl.site_id == site_id)) - ) - ).scalar_one_or_none() - if default_site_control_id is not None: - await delete_rows_into_archive( - session, - DefaultSiteControl, - ArchiveDefaultSiteControl, - deleted_time, - lambda q: q.where(DefaultSiteControl.default_site_control_id == default_site_control_id), - ) - # Finally delete the site await delete_rows_into_archive( session, @@ -365,19 +347,3 @@ async def delete_site_for_aggregator( lambda q: q.where(Site.site_id == site_id), ) return True - - -async def select_site_with_default_site_control( - session: AsyncSession, site_id: int, aggregator_id: int -) -> Optional[Site]: - """Selects the unique Site with the specified site_id and aggregator_id, including the Site's DefaultSiteControls. - Returns None if a match isn't found""" - stmt = ( - select(Site) - .where((Site.aggregator_id == aggregator_id) & (Site.site_id == site_id)) - .options( - selectinload(Site.default_site_control), - ) - ) - resp = await session.execute(stmt) - return resp.scalar_one_or_none() diff --git a/src/envoy/server/main.py b/src/envoy/server/main.py index a2155d93..24a63d65 100644 --- a/src/envoy/server/main.py +++ b/src/envoy/server/main.py @@ -10,7 +10,6 @@ from envoy.notification.handler import enable_notification_client from envoy.server.api.depends.allow_nmi_updates import ALLOW_NMI_UPDATES_ATTR from envoy.server.api.depends.azure_ad_auth import AzureADAuthDepends -from envoy.server.api.depends.default_doe import DefaultDoeDepends from envoy.server.api.depends.lfdi_auth import LFDIAuthDepends from envoy.server.api.depends.nmi_validator import NMI_VALIDATOR_ATTR from envoy.server.api.depends.request_state_settings import RequestStateSettingsDepends @@ -22,8 +21,8 @@ ) from envoy.server.api.router import routers, unsecured_routers from envoy.server.database import enable_dynamic_azure_ad_database_credentials -from envoy.server.lifespan import generate_combined_lifespan_manager from envoy.server.endpoint_exclusion import generate_routers_with_excluded_endpoints +from envoy.server.lifespan import generate_combined_lifespan_manager from envoy.server.settings import AppSettings, settings # Setup logs @@ -42,10 +41,6 @@ def generate_app(new_settings: AppSettings) -> FastAPI: global_dependencies.append(Depends(RequestStateSettingsDepends(new_settings.href_prefix, new_settings.iana_pen))) - # if default DOE is specified - include the DefaultDoeDepends - if new_settings.use_global_default_doe_fallback: - global_dependencies.append(Depends(DefaultDoeDepends(**new_settings.default_doe_configuration))) - # Setup notification broker connection for sep2 pub/sub support if new_settings.enable_notifications: lifespan_managers.append(enable_notification_client(new_settings.rabbit_mq_broker_url)) diff --git a/src/envoy/server/manager/derp.py b/src/envoy/server/manager/derp.py index e41c015c..53370600 100644 --- a/src/envoy/server/manager/derp.py +++ b/src/envoy/server/manager/derp.py @@ -1,5 +1,5 @@ from datetime import datetime -from typing import Any, Optional +from typing import Optional from envoy_schema.server.schema.sep2.der import ( DefaultDERControl, @@ -19,7 +19,7 @@ select_site_control_group_by_id, select_site_control_groups, ) -from envoy.server.crud.site import select_single_site_with_site_id, select_site_with_default_site_control +from envoy.server.crud.site import select_single_site_with_site_id from envoy.server.exception import NotFoundError from envoy.server.manager.server import RuntimeServerConfigManager from envoy.server.manager.time import utc_now @@ -30,9 +30,7 @@ DERProgramMapper, ) from envoy.server.model.archive.doe import ArchiveDynamicOperatingEnvelope -from envoy.server.model.config.default_doe import DefaultDoeConfiguration -from envoy.server.model.doe import DynamicOperatingEnvelope, SiteControlGroup -from envoy.server.model.site import DefaultSiteControl +from envoy.server.model.doe import DynamicOperatingEnvelope, SiteControlGroup, SiteControlGroupDefault from envoy.server.request_scope import SiteRequestScope @@ -41,7 +39,6 @@ class DERProgramManager: async def fetch_list_for_scope( session: AsyncSession, scope: SiteRequestScope, - default_doe: Optional[DefaultDoeConfiguration], start: int, changed_after: datetime, limit: int, @@ -54,15 +51,14 @@ async def fetch_list_for_scope( now = utc_now() - site = await select_site_with_default_site_control(session, scope.site_id, scope.aggregator_id) + site = await select_single_site_with_site_id(session, scope.site_id, scope.aggregator_id) if not site: raise NotFoundError(f"site_id {scope.site_id} is not accessible / does not exist") - default_site_control = DERControlManager._resolve_default_site_control(default_doe, site.default_site_control) config = await RuntimeServerConfigManager.fetch_current_config(session) site_control_groups = await select_site_control_groups( - session, start=start, limit=limit, changed_after=changed_after, fsa_id=fsa_id + session, start=start, limit=limit, changed_after=changed_after, fsa_id=fsa_id, include_defaults=True ) site_control_group_count = await count_site_control_groups(session, changed_after, fsa_id=fsa_id) control_counts_by_group: list[tuple[SiteControlGroup, int]] = [] @@ -84,7 +80,6 @@ async def fetch_list_for_scope( scope, control_counts_by_group, site_control_group_count, - default_site_control, config.derpl_pollrate_seconds, fsa_id, ) @@ -94,25 +89,24 @@ async def fetch_doe_program_for_scope( session: AsyncSession, scope: SiteRequestScope, der_program_id: int, - default_doe: Optional[DefaultDoeConfiguration], ) -> DERProgramResponse: """Returns the DERProgram with the specified ID if site_id DNE is inaccessible to aggregator_id a NotFoundError will be raised""" - site = await select_site_with_default_site_control(session, scope.site_id, scope.aggregator_id) + site = await select_single_site_with_site_id(session, scope.site_id, scope.aggregator_id) if not site: raise NotFoundError(f"site_id {scope.site_id} is not accessible / does not exist") - default_site_control = DERControlManager._resolve_default_site_control(default_doe, site.default_site_control) - - site_control_group = await select_site_control_group_by_id(session, der_program_id) + site_control_group = await select_site_control_group_by_id(session, der_program_id, include_default=True) if not site_control_group: raise NotFoundError(f"der_program_id {der_program_id} is not accessible / does not exist") now = utc_now() total_does = await count_active_does_include_deleted(session, der_program_id, site, now, datetime.min) - return DERProgramMapper.doe_program_response(scope, total_does, site_control_group, default_site_control) + return DERProgramMapper.doe_program_response( + scope, total_does, site_control_group, site_control_group.site_control_group_default + ) class DERControlManager: @@ -207,80 +201,32 @@ async def fetch_active_doe_controls_for_scope( ) @staticmethod - async def fetch_default_doe_controls_for_site( + async def fetch_default_doe_controls_for_scope( session: AsyncSession, scope: SiteRequestScope, der_program_id: int, - default_doe: Optional[DefaultDoeConfiguration], ) -> DefaultDERControl: - """Returns a default DOE control for a site or raises a NotFoundError if the site / defaults are inaccessible - or not configured""" - site = await select_site_with_default_site_control(session, scope.site_id, scope.aggregator_id) - if not site: - raise NotFoundError(f"site_id {scope.site_id} is not accessible / does not exist") - - default_site_control = DERControlManager._resolve_default_site_control(default_doe, site.default_site_control) - if default_site_control is None: - raise NotFoundError(f"There is no default DefaultDERControl configured for site {scope.site_id}") + """Returns a default DOE control for DERProgram - raises an error if the referenced DERProgram DNE""" + + scg = await select_site_control_group_by_id(session, der_program_id, include_default=True) + if not scg: + raise NotFoundError(f"DERProgram {der_program_id} for site {scope.site_id} is not accessible / missing.") + + scg_default = scg.site_control_group_default + if scg_default is None: + scg_default = SiteControlGroupDefault( + site_control_group_default_id=0, + site_control_group_id=scg.site_control_group_id, + created_time=scg.created_time, + changed_time=scg.created_time, # This is deliberately set to created_time instead of changed_time + version=0, + ) - # fetch runtime server config config = await RuntimeServerConfigManager.fetch_current_config(session) - return DERControlMapper.map_to_default_response( - scope, default_site_control, scope.display_site_id, der_program_id, config.site_control_pow10_encoding - ) - - @staticmethod - def _resolve_default_site_control( - default_doe: Optional[DefaultDoeConfiguration], - default_site_control: Optional[DefaultSiteControl], - ) -> Optional[DefaultSiteControl]: - """ - Coalesce site control entity with fallback configuration values. For each field, the - entity's non-None value takes precedence, otherwise the fallback configuration value is used. - - Args: - default_doe (Optional[DefaultDoeConfiguration]): Fallback configuration - providing default values. - default_site_control (Optional[DefaultSiteControl]): Site control entity - loaded from the database whose non-None fields take precedence. - - Returns: - Optional[DefaultSiteControl]: DefaultSiteControl instance with values - coalesced from entity and fallback configuration. Returns None if both inputs are None. - """ - - def _prefer_left(left: Any, right: Any) -> Any: - return left if left is not None else right - - if default_doe is None: - return default_site_control - - if default_site_control is not None: - return DefaultSiteControl( - version=default_site_control.version, - default_site_control_id=default_site_control.default_site_control_id, - import_limit_active_watts=_prefer_left( - default_site_control.import_limit_active_watts, default_doe.import_limit_active_watts - ), - export_limit_active_watts=_prefer_left( - default_site_control.export_limit_active_watts, default_doe.export_limit_active_watts - ), - generation_limit_active_watts=_prefer_left( - default_site_control.generation_limit_active_watts, default_doe.generation_limit_active_watts - ), - load_limit_active_watts=_prefer_left( - default_site_control.load_limit_active_watts, default_doe.load_limit_active_watts - ), - ramp_rate_percent_per_second=_prefer_left( - default_site_control.ramp_rate_percent_per_second, default_doe.ramp_rate_percent_per_second - ), - ) - return DefaultSiteControl( - version=0, - import_limit_active_watts=default_doe.import_limit_active_watts, - export_limit_active_watts=default_doe.export_limit_active_watts, - generation_limit_active_watts=default_doe.generation_limit_active_watts, - load_limit_active_watts=default_doe.load_limit_active_watts, - ramp_rate_percent_per_second=default_doe.ramp_rate_percent_per_second, + scope, + scg_default, + scope.display_site_id, + der_program_id, + config.site_control_pow10_encoding, ) diff --git a/src/envoy/server/mapper/csip_aus/doe.py b/src/envoy/server/mapper/csip_aus/doe.py index 9d6594f1..9c7103bd 100644 --- a/src/envoy/server/mapper/csip_aus/doe.py +++ b/src/envoy/server/mapper/csip_aus/doe.py @@ -22,8 +22,7 @@ from envoy.server.mapper.sep2.mrid import MridMapper, ResponseSetType from envoy.server.mapper.sep2.response import SPECIFIC_RESPONSE_REQUIRED, ResponseListMapper from envoy.server.model.archive.doe import ArchiveDynamicOperatingEnvelope, ArchiveSiteControlGroup -from envoy.server.model.doe import DynamicOperatingEnvelope, SiteControlGroup -from envoy.server.model.site import DefaultSiteControl +from envoy.server.model.doe import DynamicOperatingEnvelope, SiteControlGroup, SiteControlGroupDefault from envoy.server.request_scope import AggregatorRequestScope, BaseRequestScope, DeviceOrAggregatorRequestScope @@ -152,7 +151,7 @@ def map_to_response( @staticmethod def map_to_default_response( scope: BaseRequestScope, - default_doe: DefaultSiteControl, + default_doe: SiteControlGroupDefault, display_site_id: int, der_program_id: int, pow10_multipier: int, @@ -288,13 +287,13 @@ def doe_program_response( rq_scope: Union[AggregatorRequestScope, DeviceOrAggregatorRequestScope], total_controls: Optional[int], site_control_group: Union[SiteControlGroup, ArchiveSiteControlGroup], - default_doe: Optional[DefaultSiteControl], + site_control_group_default: Optional[SiteControlGroupDefault], ) -> DERProgramResponse: """Returns a DERProgram response for a SiteControlGroup""" # The default control link will only be included if we have a default DOE configured for this site default_der_link: Optional[Link] = None - if default_doe is not None: + if site_control_group_default is not None: default_der_link = Link.model_validate( { "href": DERControlMapper.default_control_href( @@ -340,7 +339,6 @@ def doe_program_list_response( rq_scope: DeviceOrAggregatorRequestScope, site_control_groups_with_control_count: list[tuple[SiteControlGroup, int]], total_site_control_groups: int, - default_doe: Optional[DefaultSiteControl], pollrate_seconds: int, fsa_id: Optional[int], ) -> DERProgramListResponse: @@ -353,7 +351,9 @@ def doe_program_list_response( "pollRate": pollrate_seconds, "subscribable": SubscribableType.resource_supports_non_conditional_subscriptions, "DERProgram": [ - DERProgramMapper.doe_program_response(rq_scope, control_count, group, default_doe) + DERProgramMapper.doe_program_response( + rq_scope, control_count, group, group.site_control_group_default + ) for group, control_count in site_control_groups_with_control_count ], "all_": total_site_control_groups, diff --git a/src/envoy/server/mapper/sep2/mrid.py b/src/envoy/server/mapper/sep2/mrid.py index a2a4a11f..2e45c30f 100644 --- a/src/envoy/server/mapper/sep2/mrid.py +++ b/src/envoy/server/mapper/sep2/mrid.py @@ -1,7 +1,7 @@ from datetime import datetime, timezone from envoy.server.mapper.constants import MridType, PricingReadingType, ResponseSetType -from envoy.server.model.site import DefaultSiteControl +from envoy.server.model.doe import SiteControlGroupDefault from envoy.server.request_scope import BaseRequestScope # constant maximum values for the various mrid components (max values for an unsigned int representation) @@ -88,10 +88,9 @@ def decode_iana_pen(mrid: str) -> int: class MridMapper: @staticmethod - def encode_default_doe_mrid(scope: BaseRequestScope, default_site_control: DefaultSiteControl) -> str: + def encode_default_doe_mrid(scope: BaseRequestScope, scg_default: SiteControlGroupDefault) -> str: """Encodes a valid MRID for representing the default DOE""" - id_value: int = default_site_control.default_site_control_id or DEFAULT_DOE_ID - return encode_mrid(MridType.DEFAULT_DOE, id_value, scope.iana_pen) + return encode_mrid(MridType.DEFAULT_DOE, scg_default.site_control_group_default_id, scope.iana_pen) @staticmethod def encode_doe_program_mrid(scope: BaseRequestScope, site_control_group_id: int, site_id: int) -> str: diff --git a/src/envoy/server/mapper/sep2/pub_sub.py b/src/envoy/server/mapper/sep2/pub_sub.py index 8d07b938..347fafdd 100644 --- a/src/envoy/server/mapper/sep2/pub_sub.py +++ b/src/envoy/server/mapper/sep2/pub_sub.py @@ -52,15 +52,8 @@ from envoy.server.mapper.sep2.metering import READING_SET_ALL_ID, MirrorMeterReadingMapper from envoy.server.mapper.sep2.pricing import TimeTariffIntervalMapper from envoy.server.model.archive.doe import ArchiveDynamicOperatingEnvelope, ArchiveSiteControlGroup -from envoy.server.model.doe import DynamicOperatingEnvelope, SiteControlGroup -from envoy.server.model.site import ( - DefaultSiteControl, - Site, - SiteDERAvailability, - SiteDERRating, - SiteDERSetting, - SiteDERStatus, -) +from envoy.server.model.doe import DynamicOperatingEnvelope, SiteControlGroup, SiteControlGroupDefault +from envoy.server.model.site import Site, SiteDERAvailability, SiteDERRating, SiteDERSetting, SiteDERStatus from envoy.server.model.site_reading import SiteReading from envoy.server.model.subscription import Subscription, SubscriptionCondition, SubscriptionResource from envoy.server.model.tariff import TariffGeneratedRate @@ -763,7 +756,7 @@ def map_function_set_assignments_list_to_response( @staticmethod def map_default_site_control_response( - default_control: Optional[DefaultSiteControl], + scg_default: Optional[SiteControlGroupDefault], der_program_id: int, pow10_multipier: int, sub: Subscription, @@ -777,9 +770,9 @@ def map_default_site_control_response( ) resource_model: Optional[DefaultDERControl] = None - if default_control is not None: + if scg_default is not None: resource_model = DERControlMapper.map_to_default_response( - scope, default_control, scope.display_site_id, der_program_id, pow10_multipier + scope, scg_default, scope.display_site_id, der_program_id, pow10_multipier ) resource_model.type = XSI_TYPE_DEFAULT_DER_CONTROL diff --git a/src/envoy/server/model/archive/doe.py b/src/envoy/server/model/archive/doe.py index 2287f64c..87204d2f 100644 --- a/src/envoy/server/model/archive/doe.py +++ b/src/envoy/server/model/archive/doe.py @@ -7,6 +7,7 @@ import envoy.server.model as original_models from envoy.server.model.archive.base import ARCHIVE_TABLE_PREFIX, ArchiveBase +from envoy.server.model.constants import DOE_DECIMAL_PLACES class ArchiveSiteControlGroup(ArchiveBase): @@ -25,6 +26,32 @@ class ArchiveSiteControlGroup(ArchiveBase): changed_time: Mapped[datetime] = mapped_column(DateTime(timezone=True)) +class ArchiveSiteControlGroupDefault(ArchiveBase): + """Represents fields that map to a subset of the attributes defined in CSIP-AUS' DefaultDERControl resource. These + default values fall underneath a specific SiteControlGroup.""" + + __tablename__ = ARCHIVE_TABLE_PREFIX + original_models.doe.SiteControlGroupDefault.__tablename__ # type: ignore + site_control_group_default_id: Mapped[int] = mapped_column(INTEGER, index=True) + site_control_group_id: Mapped[int] = mapped_column(INTEGER, nullable=False) + + created_time: Mapped[datetime] = mapped_column(DateTime(timezone=True)) # When this record was created + changed_time: Mapped[datetime] = mapped_column(DateTime(timezone=True)) + + version: Mapped[int] = mapped_column(INTEGER) # Incremented whenever this record is changed + + import_limit_active_watts: Mapped[Optional[Decimal]] = mapped_column( + DECIMAL(16, DOE_DECIMAL_PLACES), nullable=True + ) # Constraint on imported active power + export_limit_active_watts: Mapped[Optional[Decimal]] = mapped_column( + DECIMAL(16, DOE_DECIMAL_PLACES), nullable=True + ) # Constraint on exported active power + generation_limit_active_watts: Mapped[Optional[Decimal]] = mapped_column( + DECIMAL(16, DOE_DECIMAL_PLACES), nullable=True + ) + load_limit_active_watts: Mapped[Optional[Decimal]] = mapped_column(DECIMAL(16, DOE_DECIMAL_PLACES), nullable=True) + ramp_rate_percent_per_second: Mapped[Optional[int]] = mapped_column(nullable=True) # hundredths of percent per sec + + class ArchiveDynamicOperatingEnvelope(ArchiveBase): """Represents a dynamic operating envelope for a site at a particular time interval""" diff --git a/src/envoy/server/model/archive/site.py b/src/envoy/server/model/archive/site.py index c880601d..5c10e036 100644 --- a/src/envoy/server/model/archive/site.py +++ b/src/envoy/server/model/archive/site.py @@ -212,33 +212,3 @@ class ArchiveSiteDERStatus(ArchiveBase): storage_mode_status_time: Mapped[Optional[datetime]] = mapped_column(DateTime(timezone=True), nullable=True) storage_connect_status: Mapped[Optional[ConnectStatusType]] = mapped_column(INTEGER, nullable=True) storage_connect_status_time: Mapped[Optional[datetime]] = mapped_column(DateTime(timezone=True), nullable=True) - - -class ArchiveDefaultSiteControl(ArchiveBase): - """Represents fields that map to a subset of the attributes defined in CSIP-AUS' DefaultDERControl resource. - This entity is linked to a Site.""" - - __tablename__ = ARCHIVE_TABLE_PREFIX + original_models.site.DefaultSiteControl.__tablename__ # type: ignore - default_site_control_id: Mapped[int] = mapped_column(INTEGER, primary_key=True) - site_id: Mapped[int] = mapped_column(INTEGER) - - created_time: Mapped[datetime] = mapped_column(DateTime(timezone=True)) - changed_time: Mapped[datetime] = mapped_column(DateTime(timezone=True)) - - version: Mapped[int] = mapped_column(INTEGER) # Incremented whenever this record is changed - - import_limit_active_watts: Mapped[Optional[Decimal]] = mapped_column( - DECIMAL(16, original_models.site.DOE_DECIMAL_PLACES), nullable=True - ) # Constraint on imported active power - export_limit_active_watts: Mapped[Optional[Decimal]] = mapped_column( - DECIMAL(16, original_models.site.DOE_DECIMAL_PLACES), nullable=True - ) # Constraint on exported active/reactive power - generation_limit_active_watts: Mapped[Optional[Optional[Decimal]]] = mapped_column( - DECIMAL(16, original_models.site.DOE_DECIMAL_PLACES), nullable=True - ) - load_limit_active_watts: Mapped[Optional[Optional[Decimal]]] = mapped_column( - DECIMAL(16, original_models.site.DOE_DECIMAL_PLACES), nullable=True - ) - ramp_rate_percent_per_second: Mapped[Optional[int]] = mapped_column( - nullable=True - ) # Constraint on exported active/reactive power diff --git a/src/envoy/server/model/config/default_doe.py b/src/envoy/server/model/config/default_doe.py deleted file mode 100644 index a44386d0..00000000 --- a/src/envoy/server/model/config/default_doe.py +++ /dev/null @@ -1,18 +0,0 @@ -from dataclasses import dataclass -from decimal import Decimal -from typing import Optional - - -# TODO: rename to DefaultSiteControlConfiguration as part of DOE->SiteControl refactor. -@dataclass -class DefaultDoeConfiguration: - """The globally configured Default dynamic operating envelope (DOE) values to be used as a fallback if - one is/are not defined for a particular site. - - """ - - import_limit_active_watts: Optional[Decimal] = None - export_limit_active_watts: Optional[Decimal] = None - generation_limit_active_watts: Optional[Decimal] = None - load_limit_active_watts: Optional[Decimal] = None - ramp_rate_percent_per_second: Optional[int] = None diff --git a/src/envoy/server/model/doe.py b/src/envoy/server/model/doe.py index f1273f8f..ed35512a 100644 --- a/src/envoy/server/model/doe.py +++ b/src/envoy/server/model/doe.py @@ -2,7 +2,7 @@ from decimal import Decimal from typing import Optional -from sqlalchemy import BOOLEAN, DECIMAL, VARCHAR, BigInteger, DateTime, ForeignKey, Index, func +from sqlalchemy import BOOLEAN, DECIMAL, INTEGER, VARCHAR, BigInteger, DateTime, ForeignKey, Index, func from sqlalchemy.orm import Mapped, mapped_column, relationship from envoy.server.model.base import Base @@ -36,6 +36,10 @@ class SiteControlGroup(Base): lazy="raise", back_populates="site_control_group" ) + site_control_group_default: Mapped[Optional["SiteControlGroupDefault"]] = relationship( + back_populates="site_control_group", lazy="raise", passive_deletes=True, uselist=False + ) # The default DOE + Index( "ix_site_control_group_primacy_site_control_group_id", "primacy", @@ -43,6 +47,40 @@ class SiteControlGroup(Base): ), +class SiteControlGroupDefault(Base): + """Represents fields that map to a subset of the attributes defined in CSIP-AUS' DefaultDERControl resource. These + default values fall underneath a specific SiteControlGroup.""" + + __tablename__ = "site_control_group_default" + site_control_group_default_id: Mapped[int] = mapped_column(primary_key=True) + site_control_group_id: Mapped[int] = mapped_column( + ForeignKey("site_control_group.site_control_group_id", ondelete="CASCADE"), nullable=False, index=True + ) + + created_time: Mapped[datetime] = mapped_column( + DateTime(timezone=True), server_default=func.now() + ) # When this record was created + changed_time: Mapped[datetime] = mapped_column(DateTime(timezone=True), index=True) + + version: Mapped[int] = mapped_column(INTEGER, server_default="0") # Incremented whenever this record is changed + + import_limit_active_watts: Mapped[Optional[Decimal]] = mapped_column( + DECIMAL(16, DOE_DECIMAL_PLACES), nullable=True + ) # Constraint on imported active power + export_limit_active_watts: Mapped[Optional[Decimal]] = mapped_column( + DECIMAL(16, DOE_DECIMAL_PLACES), nullable=True + ) # Constraint on exported active power + generation_limit_active_watts: Mapped[Optional[Decimal]] = mapped_column( + DECIMAL(16, DOE_DECIMAL_PLACES), nullable=True + ) + load_limit_active_watts: Mapped[Optional[Decimal]] = mapped_column(DECIMAL(16, DOE_DECIMAL_PLACES), nullable=True) + ramp_rate_percent_per_second: Mapped[Optional[int]] = mapped_column(nullable=True) # hundredths of percent per sec + + site_control_group: Mapped["SiteControlGroup"] = relationship( + back_populates="site_control_group_default", lazy="raise" + ) + + # TODO: Rename this and related archive to SiteControl. These entities will eventually hold more than # just DOE related information, e.g. set-point control, etc. class DynamicOperatingEnvelope(Base): diff --git a/src/envoy/server/model/site.py b/src/envoy/server/model/site.py index e9bbcb45..e6db6b77 100644 --- a/src/envoy/server/model/site.py +++ b/src/envoy/server/model/site.py @@ -32,7 +32,7 @@ from sqlalchemy.orm import Mapped, mapped_column, relationship from envoy.server.model import Base -from envoy.server.model.constants import DOE_DECIMAL_PLACES, PERCENT_DECIMAL_PLACES +from envoy.server.model.constants import PERCENT_DECIMAL_PLACES class Site(Base): @@ -73,9 +73,6 @@ class Site(Base): cascade="all, delete", passive_deletes=True, ) # What DER live underneath/behind this site - default_site_control: Mapped[Optional["DefaultSiteControl"]] = relationship( - back_populates="site", lazy="raise", passive_deletes=True, uselist=False - ) # The default DOE + other controls that apply to this site # NOTE: We're defining Default are set on a per Site basis @@ -383,36 +380,3 @@ class SiteLogEvent(Base): __table_args__ = ( Index("site_log_event_site_id_created_time_log_event_id_idx", "site_id", "created_time", "log_event_id"), ) - - -# TODO: deally this would be in the model.doe module. This causes a circular import issue due to the relationship -# mapping between Site and this model. The recommended solution is to use a type.TYPE_CHECKING if statement before the -# imports. However, this causes an issue with the `assertical` testing package that needs to be looked into. -class DefaultSiteControl(Base): - """Represents fields that map to a subset of the attributes defined in CSIP-AUS' DefaultDERControl resource. - This entity is linked to a Site.""" - - __tablename__ = "default_site_control" - default_site_control_id: Mapped[int] = mapped_column(INTEGER, primary_key=True) - site_id: Mapped[int] = mapped_column(ForeignKey("site.site_id", ondelete="CASCADE"), nullable=False, index=True) - - created_time: Mapped[datetime] = mapped_column( - DateTime(timezone=True), server_default=func.now() - ) # When this record was created - changed_time: Mapped[datetime] = mapped_column(DateTime(timezone=True), index=True) - - version: Mapped[int] = mapped_column(INTEGER, server_default="0") # Incremented whenever this record is changed - - import_limit_active_watts: Mapped[Optional[Decimal]] = mapped_column( - DECIMAL(16, DOE_DECIMAL_PLACES), nullable=True - ) # Constraint on imported active power - export_limit_active_watts: Mapped[Optional[Decimal]] = mapped_column( - DECIMAL(16, DOE_DECIMAL_PLACES), nullable=True - ) # Constraint on exported active power - generation_limit_active_watts: Mapped[Optional[Decimal]] = mapped_column( - DECIMAL(16, DOE_DECIMAL_PLACES), nullable=True - ) - load_limit_active_watts: Mapped[Optional[Decimal]] = mapped_column(DECIMAL(16, DOE_DECIMAL_PLACES), nullable=True) - ramp_rate_percent_per_second: Mapped[Optional[int]] = mapped_column(nullable=True) # hundredths of percent per sec - - site: Mapped["Site"] = relationship(back_populates="default_site_control", lazy="raise") diff --git a/src/envoy/server/settings.py b/src/envoy/server/settings.py index 7922a9b0..db81e240 100644 --- a/src/envoy/server/settings.py +++ b/src/envoy/server/settings.py @@ -1,6 +1,5 @@ -from functools import cached_property import importlib.metadata -from decimal import Decimal +from functools import cached_property from typing import Any, Dict, Optional from pydantic import Field, model_validator @@ -8,7 +7,7 @@ from envoy.server.api.depends.allow_nmi_updates import DEFAULT_ALLOW_NMI_UPDATES from envoy.server.endpoint_exclusion import EndpointExclusionSet -from envoy.server.manager.nmi_validator import NmiValidator, DNSPParticipantId +from envoy.server.manager.nmi_validator import DNSPParticipantId, NmiValidator from envoy.settings import CommonSettings @@ -49,16 +48,6 @@ class AppSettings(CommonSettings): cert_header: str = "x-forwarded-client-cert" # either client certificate in PEM format or the sha256 fingerprint - # Global fallback default doe for sites that do not have these configured. - use_global_default_doe_fallback: bool = True - default_doe_import_active_watts: Optional[str] = None # Constant default DERControl import as a decimal float - default_doe_export_active_watts: Optional[str] = None # Constant default DERControl export as a decimal float - default_doe_load_active_watts: Optional[str] = None # Constant default DERControl load limit as a decimal float - default_doe_generation_active_watts: Optional[str] = ( - None # Constant default DERControl generation limit as a decimal float - ) - default_doe_ramp_rate_percent_per_second: Optional[int] = None # Constant default DERControl ramp rate setpoint. - allow_device_registration: bool = False # True: LFDI auth will allow unknown certs to register single EndDevices nmi_validation: NmiValidationSettings = Field(default_factory=NmiValidationSettings) @@ -78,24 +67,6 @@ def fastapi_kwargs(self) -> Dict[str, Any]: "version": self.version, } - @property - def default_doe_configuration(self) -> Dict[str, Any]: - return { - "import_limit_active_watts": ( - Decimal(self.default_doe_import_active_watts) if self.default_doe_import_active_watts else None - ), - "export_limit_active_watts": ( - Decimal(self.default_doe_export_active_watts) if self.default_doe_export_active_watts else None - ), - "load_limit_active_watts": ( - Decimal(self.default_doe_load_active_watts) if self.default_doe_load_active_watts else None - ), - "generation_limit_active_watts": ( - Decimal(self.default_doe_generation_active_watts) if self.default_doe_generation_active_watts else None - ), - "ramp_rate_percent_per_second": self.default_doe_ramp_rate_percent_per_second, - } - def generate_settings() -> AppSettings: """Generates and configures a new instance of the AppSettings""" diff --git a/tests/conftest.py b/tests/conftest.py index c5097ae0..8f7f8797 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -65,18 +65,6 @@ def pg_empty_config( if href_prefix_marker is not None: os.environ["HREF_PREFIX"] = str(href_prefix_marker.args[0]) - no_default_doe_marker = request.node.get_closest_marker("no_default_doe") - if no_default_doe_marker is None: - os.environ["USE_GLOBAL_DEFAULT_DOE_FALLBACK"] = "true" - os.environ["DEFAULT_DOE_IMPORT_ACTIVE_WATTS"] = str(DEFAULT_DOE_IMPORT_ACTIVE_WATTS) - os.environ["DEFAULT_DOE_EXPORT_ACTIVE_WATTS"] = str(DEFAULT_DOE_EXPORT_ACTIVE_WATTS) - os.environ["DEFAULT_DOE_LOAD_ACTIVE_WATTS"] = str(DEFAULT_DOE_LOAD_ACTIVE_WATTS) - os.environ["DEFAULT_DOE_GENERATION_ACTIVE_WATTS"] = str(DEFAULT_DOE_GENERATION_ACTIVE_WATTS) - os.environ["DEFAULT_DOE_RAMP_RATE_PERCENT_PER_SECOND"] = str(DEFAULT_DOE_RAMP_RATE_PERCENT_PER_SECOND) - - else: - os.environ["USE_GLOBAL_DEFAULT_DOE_FALLBACK"] = "false" - if request.node.get_closest_marker("admin_ro_user"): os.environ["READ_ONLY_USER"] = READONLY_USER_NAME os.environ["READ_ONLY_KEYS"] = f'["{READONLY_USER_KEY_1}", "{READONLY_USER_KEY_2}"]' diff --git a/tests/data/sql/base_config.sql b/tests/data/sql/base_config.sql index 6261ff5a..08f1bb9d 100644 --- a/tests/data/sql/base_config.sql +++ b/tests/data/sql/base_config.sql @@ -47,9 +47,22 @@ INSERT INTO public.site("site_id", "nmi", "aggregator_id", "timezone_id", "creat SELECT pg_catalog.setval('public.site_site_id_seq', 7, true); INSERT INTO public.site_control_group("site_control_group_id", "description", "primacy", "fsa_id", "created_time", "changed_time") -VALUES (1, 'Dynamic Operating Envelopes', 0, 1, '2000-01-01 00:00:00Z', '2021-04-05 10:01:00.500'); +VALUES (1, 'Dynamic Operating Envelopes #1', 0, 1, '2000-01-01 00:00:00Z', '2021-04-05 10:01:00.500'); -SELECT pg_catalog.setval('public.site_control_group_site_control_group_id_seq', 2, true); +INSERT INTO public.site_control_group("site_control_group_id", "description", "primacy", "fsa_id", "created_time", "changed_time") +VALUES (2, 'Dynamic Operating Envelopes #2', 1, 1, '2000-01-01 00:00:00Z', '2021-04-05 10:02:00.500'); + +INSERT INTO public.site_control_group("site_control_group_id", "description", "primacy", "fsa_id", "created_time", "changed_time") +VALUES (3, 'Dynamic Operating Envelopes #3', 2, 1, '2000-01-01 00:00:00Z', '2021-04-05 10:03:00.500'); + +SELECT pg_catalog.setval('public.site_control_group_site_control_group_id_seq', 4, true); + +-- DERProgram #1 and DERProgram #3 have defaults. +INSERT INTO public.site_control_group_default(site_control_group_default_id, site_control_group_id, import_limit_active_watts, export_limit_active_watts, generation_limit_active_watts, load_limit_active_watts, ramp_rate_percent_per_second, created_time, changed_time) +VALUES (1, 1, 10.10, 9.99, 8.88, 7.77, 6, '2000-01-01 00:00:00', '2023-05-01 01:02:02.500'); +INSERT INTO public.site_control_group_default(site_control_group_default_id, site_control_group_id, import_limit_active_watts, export_limit_active_watts, generation_limit_active_watts, load_limit_active_watts, ramp_rate_percent_per_second, created_time, changed_time) +VALUES (2, 3, 20.20, 19.19, 18.18, 17.17, 16, '2000-01-01 00:00:00', '2023-05-01 02:02:02.500'); +SELECT pg_catalog.setval('public.site_control_group_default_site_control_group_default_id_seq', 3, true); -- Calculation log 1/2 have the same interval but calculation log 2 has a more recent created time @@ -439,12 +452,5 @@ INSERT INTO public.site_log_event (site_log_event_id, site_id, created_time, det VALUES (5, 1, '2023-05-01 05:05:05.500', NULL, NULL, 5, 52, 53, 54, 4); SELECT pg_catalog.setval('public.site_log_event_site_log_event_id_seq', 6, true); - -INSERT INTO public.default_site_control(default_site_control_id, site_id, import_limit_active_watts, export_limit_active_watts, generation_limit_active_watts, load_limit_active_watts, ramp_rate_percent_per_second, created_time, changed_time) -VALUES (1, 1, 10.10, 9.99, 8.88, 7.77, 6, '2023-05-01 02:02:02.500', '2023-05-01 02:02:02.500'); -INSERT INTO public.default_site_control(default_site_control_id, site_id, import_limit_active_watts, export_limit_active_watts, generation_limit_active_watts, load_limit_active_watts, ramp_rate_percent_per_second, created_time, changed_time) -VALUES (2, 3, 20.20, 19.19, 18.18, 17.17, 16, '2023-05-01 02:02:02.500', '2023-05-01 02:02:02.500'); -SELECT pg_catalog.setval('public.default_site_control_default_site_control_id_seq', 3, true); - INSERT INTO public.runtime_server_config(runtime_server_config_id, changed_time, created_time, dcap_pollrate_seconds, edevl_pollrate_seconds, fsal_pollrate_seconds, derpl_pollrate_seconds, derl_pollrate_seconds, mup_postrate_seconds, site_control_pow10_encoding) VALUES (1, '2023-05-01 01:01:01.500', '2023-05-01 01:01:01.500', 300, 300, 300, 60, 60, 60, -2); diff --git a/tests/integration/admin/test_config.py b/tests/integration/admin/test_config.py index 3f58d0a1..1c378556 100644 --- a/tests/integration/admin/test_config.py +++ b/tests/integration/admin/test_config.py @@ -1,26 +1,17 @@ import json -from decimal import Decimal from http import HTTPStatus -from typing import Optional import pytest from assertical.asserts.generator import assert_class_instance_equality from assertical.asserts.time import assert_nowish from assertical.fake.generator import generate_class_instance from assertical.fixtures.postgres import generate_async_session -from envoy_schema.admin.schema.config import ( - ControlDefaultRequest, - ControlDefaultResponse, - RuntimeServerConfigRequest, - RuntimeServerConfigResponse, - UpdateDefaultValue, -) -from envoy_schema.admin.schema.uri import ServerConfigRuntimeUri, SiteControlDefaultConfigUri +from envoy_schema.admin.schema.config import RuntimeServerConfigRequest, RuntimeServerConfigResponse +from envoy_schema.admin.schema.uri import ServerConfigRuntimeUri from httpx import AsyncClient -from sqlalchemy import delete, select +from sqlalchemy import delete from envoy.server.model.server import RuntimeServerConfig -from envoy.server.model.site import DefaultSiteControl from tests.integration.response import read_response_body_string @@ -98,85 +89,3 @@ async def test_get_update_server_config(admin_client_auth: AsyncClient, pg_base_ assert ( third_update_response.site_control_pow10_encoding == third_config_request.site_control_pow10_encoding ), "This was the updated field" - - -@pytest.mark.parametrize( - "site_id, expected", - [ - (1, (Decimal("10.10"), Decimal("9.99"), Decimal("8.88"), Decimal("7.77"))), - (2, (None, None, None, None)), - (3, (Decimal("20.20"), Decimal("19.19"), Decimal("18.18"), Decimal("17.17"))), - (5, (None, None, None, None)), - (6, (None, None, None, None)), - (99, None), - ], -) -@pytest.mark.anyio -async def test_get_and_update_site_control_default( - pg_base_config, - admin_client_auth: AsyncClient, - site_id: int, - expected: Optional[tuple], -): - version_before = 0 - async with generate_async_session(pg_base_config) as session: - db_record = ( - await session.execute(select(DefaultSiteControl).where(DefaultSiteControl.site_id == site_id)) - ).scalar_one_or_none() - if db_record: - version_before = db_record.version - - resp = await admin_client_auth.get(SiteControlDefaultConfigUri.format(site_id=site_id)) - if expected is None: - assert resp.status_code == HTTPStatus.NOT_FOUND - else: - assert resp.status_code == HTTPStatus.OK - body = read_response_body_string(resp) - config: ControlDefaultResponse = ControlDefaultResponse(**json.loads(body)) - assert expected == ( - config.server_default_import_limit_watts, - config.server_default_export_limit_watts, - config.server_default_generation_limit_watts, - config.server_default_load_limit_watts, - ) - - # now do an update for certain fields - config_request = ControlDefaultRequest( - import_limit_watts=UpdateDefaultValue(value=None), - export_limit_watts=UpdateDefaultValue(value=Decimal("1.11")), - generation_limit_watts=None, - load_limit_watts=None, - ramp_rate_percent_per_second=None, - ) - resp = await admin_client_auth.post( - SiteControlDefaultConfigUri.format(site_id=site_id), content=config_request.model_dump_json() - ) - if not expected: - assert resp.status_code == HTTPStatus.NOT_FOUND - else: - assert resp.status_code == HTTPStatus.NO_CONTENT - - # and refetch - resp = await admin_client_auth.get(SiteControlDefaultConfigUri.format(site_id=site_id)) - - # Make sure only the fields we updated did an update - if expected is None: - assert resp.status_code == HTTPStatus.NOT_FOUND - else: - assert resp.status_code == HTTPStatus.OK - body = read_response_body_string(resp) - config: ControlDefaultResponse = ControlDefaultResponse(**json.loads(body)) - - assert (None, Decimal("1.11"), expected[2], expected[3]) == ( - config.server_default_import_limit_watts, - config.server_default_export_limit_watts, - config.server_default_generation_limit_watts, - config.server_default_load_limit_watts, - ) - - # Version number in the DB should be getting updated - async with generate_async_session(pg_base_config) as session: - db_record = ( - await session.execute(select(DefaultSiteControl).where(DefaultSiteControl.site_id == site_id)) - ).scalar_one() - assert db_record.version == version_before + 1, "The version field should be updated per update" diff --git a/tests/integration/admin/test_doe.py b/tests/integration/admin/test_doe.py index 0b8d78ef..b9e3caa9 100644 --- a/tests/integration/admin/test_doe.py +++ b/tests/integration/admin/test_doe.py @@ -15,15 +15,20 @@ DynamicOperatingEnvelopeRequest, DynamicOperatingEnvelopeResponse, ) -from envoy_schema.admin.schema.uri import DoeUri +from envoy_schema.admin.schema.site_control import ( + SiteControlGroupDefaultRequest, + SiteControlGroupDefaultResponse, + UpdateDefaultValue, +) +from envoy_schema.admin.schema.uri import DoeUri, SiteControlGroupDefaultUri from httpx import AsyncClient from sqlalchemy import func, select from envoy.admin.crud.doe import count_all_does from envoy.admin.mapper.doe import DEFAULT_DOE_SITE_CONTROL_GROUP_ID from envoy.server.api.request import MAX_LIMIT -from envoy.server.model.archive.doe import ArchiveDynamicOperatingEnvelope -from envoy.server.model.doe import DynamicOperatingEnvelope +from envoy.server.model.archive.doe import ArchiveDynamicOperatingEnvelope, ArchiveSiteControlGroupDefault +from envoy.server.model.doe import DynamicOperatingEnvelope, SiteControlGroup, SiteControlGroupDefault from tests.integration.admin.test_site import _build_query_string from tests.integration.response import read_response_body_string @@ -192,3 +197,116 @@ async def test_get_all_does( assert_list_type(DynamicOperatingEnvelopeResponse, site_page.does, len(expected_doe_ids)) assert expected_doe_ids == [d.dynamic_operating_envelope_id for d in site_page.does] + + +@pytest.mark.parametrize( + "site_control_group_id, expected", + [ + (1, (Decimal("10.10"), Decimal("9.99"), Decimal("8.88"), Decimal("7.77"))), + (2, None), + (3, (Decimal("20.20"), Decimal("19.19"), Decimal("18.18"), Decimal("17.17"))), + (99, None), + ], +) +@pytest.mark.anyio +async def test_get_and_update_site_control_default( + pg_base_config, + admin_client_auth: AsyncClient, + site_control_group_id: int, + expected: Optional[tuple], +): + version_before = 0 + async with generate_async_session(pg_base_config) as session: + default_db_record = ( + await session.execute( + select(SiteControlGroupDefault).where( + SiteControlGroupDefault.site_control_group_id == site_control_group_id + ) + ) + ).scalar_one_or_none() + default_exists = default_db_record is not None + if default_db_record: + version_before = default_db_record.version + + scg_db_record = ( + await session.execute( + select(SiteControlGroup).where(SiteControlGroup.site_control_group_id == site_control_group_id) + ) + ).scalar_one_or_none() + scg_exists = scg_db_record is not None + + resp = await admin_client_auth.get(SiteControlGroupDefaultUri.format(group_id=site_control_group_id)) + if expected is None: + assert resp.status_code == HTTPStatus.NOT_FOUND + else: + assert resp.status_code == HTTPStatus.OK + body = read_response_body_string(resp) + config: SiteControlGroupDefaultResponse = SiteControlGroupDefaultResponse(**json.loads(body)) + assert expected == ( + config.server_default_import_limit_watts, + config.server_default_export_limit_watts, + config.server_default_generation_limit_watts, + config.server_default_load_limit_watts, + ) + + # now do an update for certain fields + config_request = SiteControlGroupDefaultRequest( + import_limit_watts=UpdateDefaultValue(value=None), + export_limit_watts=UpdateDefaultValue(value=Decimal("1.11")), + generation_limit_watts=None, + load_limit_watts=None, + ramp_rate_percent_per_second=None, + ) + resp = await admin_client_auth.post( + SiteControlGroupDefaultUri.format(group_id=site_control_group_id), content=config_request.model_dump_json() + ) + if scg_exists: + assert resp.status_code == HTTPStatus.NO_CONTENT + else: + assert resp.status_code == HTTPStatus.NOT_FOUND + + # and refetch + resp = await admin_client_auth.get(SiteControlGroupDefaultUri.format(group_id=site_control_group_id)) + + # Make sure only the fields we updated did an update + if not scg_exists: + assert resp.status_code == HTTPStatus.NOT_FOUND + else: + assert resp.status_code == HTTPStatus.OK + body = read_response_body_string(resp) + config: SiteControlGroupDefaultResponse = SiteControlGroupDefaultResponse(**json.loads(body)) + + assert ( + None, + Decimal("1.11"), + expected[2] if expected is not None else None, + expected[3] if expected is not None else None, + ) == ( + config.server_default_import_limit_watts, + config.server_default_export_limit_watts, + config.server_default_generation_limit_watts, + config.server_default_load_limit_watts, + ) + + # Version number in the DB should be getting updated + # + # Archive records should be generated + async with generate_async_session(pg_base_config) as session: + default_db_record = ( + await session.execute( + select(SiteControlGroupDefault).where( + SiteControlGroupDefault.site_control_group_id == site_control_group_id + ) + ) + ).scalar_one() + assert default_db_record.version == version_before + 1, "The version field should be updated per update" + + archive_records = (await session.execute(select(ArchiveSiteControlGroupDefault))).scalars().all() + if default_exists: + assert len(archive_records) == 1 + assert archive_records[0].import_limit_active_watts == expected[0] + assert archive_records[0].export_limit_active_watts == expected[1] + assert archive_records[0].generation_limit_active_watts == expected[2] + assert archive_records[0].load_limit_active_watts == expected[3] + else: + assert len(archive_records) == 0 diff --git a/tests/integration/admin/test_notification.py b/tests/integration/admin/test_notification.py index 03f6632a..2bbb6b52 100644 --- a/tests/integration/admin/test_notification.py +++ b/tests/integration/admin/test_notification.py @@ -9,15 +9,20 @@ from assertical.fake.generator import generate_class_instance from assertical.fake.http import HTTPMethod, MockedAsyncClient from assertical.fixtures.postgres import generate_async_session -from envoy_schema.admin.schema.config import ControlDefaultRequest, RuntimeServerConfigRequest, UpdateDefaultValue +from envoy_schema.admin.schema.config import RuntimeServerConfigRequest from envoy_schema.admin.schema.doe import DynamicOperatingEnvelopeRequest from envoy_schema.admin.schema.pricing import TariffGeneratedRateRequest from envoy_schema.admin.schema.site import SiteUpdateRequest -from envoy_schema.admin.schema.site_control import SiteControlGroupRequest, SiteControlRequest +from envoy_schema.admin.schema.site_control import ( + SiteControlGroupDefaultRequest, + SiteControlGroupRequest, + SiteControlRequest, + UpdateDefaultValue, +) from envoy_schema.admin.schema.uri import ( DoeUri, ServerConfigRuntimeUri, - SiteControlDefaultConfigUri, + SiteControlGroupDefaultUri, SiteControlGroupListUri, SiteControlUri, SiteUri, @@ -794,10 +799,10 @@ async def test_update_server_config_fsa_notification_no_change( @pytest.mark.anyio -async def test_update_site_default_config_notification( +async def test_update_site_control_group_default_notification( admin_client_auth: AsyncClient, notifications_enabled: MockedAsyncClient, pg_base_config ): - """Tests that updating site default config generates subscription notifications for DefaultDERControl""" + """Tests that updating site control group default generates subscription notifications for DefaultDERControl""" subscription1_uri = "http://my.example:542/uri" @@ -805,13 +810,13 @@ async def test_update_site_default_config_notification( # Clear any other subs first await session.execute(delete(Subscription)) - # Will pickup site default updates for site 2 + # Will pickup default updates for site 2, derp 3 await session.execute( insert(Subscription).values( aggregator_id=1, changed_time=datetime.now(), resource_type=SubscriptionResource.DEFAULT_SITE_CONTROL, - resource_id=None, + resource_id=3, scoped_site_id=2, notification_uri=subscription1_uri, entity_limit=10, @@ -821,7 +826,7 @@ async def test_update_site_default_config_notification( await session.commit() # now do an update for certain fields - config_request = ControlDefaultRequest( + config_request = SiteControlGroupDefaultRequest( import_limit_watts=UpdateDefaultValue(value=None), export_limit_watts=UpdateDefaultValue(value=Decimal("2.34")), generation_limit_watts=None, @@ -829,14 +834,14 @@ async def test_update_site_default_config_notification( ramp_rate_percent_per_second=None, ) - # Update default controls for site 1 and site 2 + # Update default controls for DERP2 and DERP3 resp = await admin_client_auth.post( - SiteControlDefaultConfigUri.format(site_id=1), content=config_request.model_dump_json() + SiteControlGroupDefaultUri.format(group_id=2), content=config_request.model_dump_json() ) assert resp.status_code == HTTPStatus.NO_CONTENT resp = await admin_client_auth.post( - SiteControlDefaultConfigUri.format(site_id=2), content=config_request.model_dump_json() + SiteControlGroupDefaultUri.format(group_id=3), content=config_request.model_dump_json() ) assert resp.status_code == HTTPStatus.NO_CONTENT @@ -911,9 +916,9 @@ async def test_create_site_control_groups_with_active_subscription( for r in notifications_enabled.logged_requests if r.uri == subscription2_uri and str(primacy) in r.content - and "/edev/1/derp/3" not in r.content - and "/edev/2/derp/3" in r.content - and "/edev/4/derp/3" not in r.content + and "/edev/1/" not in r.content + and "/edev/2/derp/5" in r.content + and "/edev/4/" not in r.content ] ) == 1 diff --git a/tests/integration/admin/test_site_control.py b/tests/integration/admin/test_site_control.py index 233e0292..53e319fe 100644 --- a/tests/integration/admin/test_site_control.py +++ b/tests/integration/admin/test_site_control.py @@ -81,12 +81,12 @@ async def test_get_site_control_group_by_id( @pytest.mark.parametrize( "start, limit, after, expected_group_ids", [ - (None, None, None, [1]), - (1, None, None, []), + (None, None, None, [1, 2, 3]), + (1, None, None, [2, 3]), (0, 1, None, [1]), (0, 0, None, []), - (None, None, datetime(2021, 4, 5, 10, 1, 0, 500000, tzinfo=timezone.utc), [1]), - (None, None, datetime(2021, 4, 5, 10, 2, 0, 500000, tzinfo=timezone.utc), []), + (None, None, datetime(2021, 4, 5, 10, 1, 0, 500000, tzinfo=timezone.utc), [1, 2, 3]), + (None, None, datetime(2021, 4, 5, 10, 2, 0, 500000, tzinfo=timezone.utc), [2, 3]), ], ) @pytest.mark.anyio diff --git a/tests/integration/func_sets/test_der.py b/tests/integration/func_sets/test_der.py index 362fde33..4d0b882c 100644 --- a/tests/integration/func_sets/test_der.py +++ b/tests/integration/func_sets/test_der.py @@ -534,10 +534,7 @@ async def test_get_associated_derprogram_list( expected_doe_count: Optional[int], valid_headers: dict, ): - """Tests getting DERPrograms for various sites/der and validates access constraints - - Being a virtual entity - we don't go too hard on validating the paging (it'll always - be a single element or a 404)""" + """Tests getting DERPrograms for various sites/der and validates access constraints""" # Test a known site href = uri.AssociatedDERProgramListUri.format(site_id=site_id, der_id=der_id) @@ -552,6 +549,6 @@ async def test_get_associated_derprogram_list( body = read_response_body_string(response) assert len(body) > 0 parsed_response: DERProgramListResponse = DERProgramListResponse.from_xml(body) - assert parsed_response.all_ == 1 - assert parsed_response.results == 1 + assert parsed_response.all_ == 3 + assert parsed_response.results == 3 assert parsed_response.DERProgram[0].DERControlListLink.all_ == expected_doe_count diff --git a/tests/integration/func_sets/test_derp.py b/tests/integration/func_sets/test_derp.py index 137be36d..56bfa2ef 100644 --- a/tests/integration/func_sets/test_derp.py +++ b/tests/integration/func_sets/test_derp.py @@ -120,26 +120,6 @@ async def test_get_derprogram_list( ): """Tests getting DERPrograms for various sites and validates access constraints""" - # preload some extra derps - async with generate_async_session(pg_base_config) as session: - session.add( - generate_class_instance( - SiteControlGroup, - seed=101, - site_control_group_id=2, - changed_time=datetime(2021, 4, 5, 10, 2, 0, tzinfo=timezone.utc), - ) - ) - session.add( - generate_class_instance( - SiteControlGroup, - seed=202, - site_control_group_id=3, - changed_time=datetime(2021, 4, 5, 10, 3, 0, tzinfo=timezone.utc), - ) - ) - await session.commit() - path = uri_derp_list_format.format(site_id=site_id) + build_paging_params(start, limit, after) response = await client.get(path, headers=agg_1_headers) @@ -164,14 +144,14 @@ async def test_get_derprogram_list( @pytest.mark.parametrize( "start, limit, after, site_id, fsa_id, expected_derp_ids_with_count, expected_status", [ - (None, 99, None, 1, 1, [(1, 3), (2, 0)], HTTPStatus.OK), - (None, 99, None, 1, 2, [(3, 0)], HTTPStatus.OK), + (None, 99, None, 1, 1, [(1, 3), (2, 0), (3, 0)], HTTPStatus.OK), + (None, 99, None, 1, 2, [], HTTPStatus.OK), (None, 99, None, 1, 3, [(4, 0)], HTTPStatus.OK), - (None, 99, datetime(2021, 4, 5, 10, 2, 0, tzinfo=timezone.utc), 1, 1, [(2, 0)], HTTPStatus.OK), + (None, 99, datetime(2021, 4, 5, 10, 2, 0, tzinfo=timezone.utc), 1, 1, [(2, 0), (3, 0)], HTTPStatus.OK), (None, 1, None, 1, 1, [(1, 3)], HTTPStatus.OK), - (1, 99, None, 1, 1, [(2, 0)], HTTPStatus.OK), + (1, 99, None, 1, 1, [(2, 0), (3, 0)], HTTPStatus.OK), (None, 99, None, 3, 1, None, HTTPStatus.NOT_FOUND), # Belongs to agg 2 - (None, 99, None, 4, 1, [(1, 0), (2, 0)], HTTPStatus.OK), + (None, 99, None, 4, 1, [(1, 0), (2, 0), (3, 0)], HTTPStatus.OK), (None, 99, None, 5, 1, None, HTTPStatus.NOT_FOUND), # Belongs to device cert (None, 99, None, 99, 1, None, HTTPStatus.NOT_FOUND), # DNE (None, 99, None, 0, 1, None, HTTPStatus.FORBIDDEN), # Virtual aggregator device cant access DERPs @@ -195,24 +175,6 @@ async def test_get_derprogram_list_fsa_scoped( # preload some extra derps async with generate_async_session(pg_base_config) as session: - session.add( - generate_class_instance( - SiteControlGroup, - seed=101, - site_control_group_id=2, - fsa_id=1, - changed_time=datetime(2021, 4, 5, 10, 2, 0, tzinfo=timezone.utc), - ) - ) - session.add( - generate_class_instance( - SiteControlGroup, - seed=202, - site_control_group_id=3, - fsa_id=2, - changed_time=datetime(2021, 4, 5, 10, 3, 0, tzinfo=timezone.utc), - ) - ) session.add( generate_class_instance( SiteControlGroup, @@ -236,11 +198,11 @@ async def test_get_derprogram_list_fsa_scoped( parsed_response: DERProgramListResponse = DERProgramListResponse.from_xml(body) assert parsed_response.href == uri_derp_list_fsa_format.format(site_id=site_id, fsa_id=fsa_id) assert parsed_response.results == len(expected_derp_ids_with_count) - assert len(parsed_response.DERProgram) == len(expected_derp_ids_with_count) - actual_derp_ids_with_count = [ - (int(derp.href.split("/")[-1]), derp.DERControlListLink.all_) for derp in parsed_response.DERProgram - ] + derps = [] if parsed_response.DERProgram is None else parsed_response.DERProgram + assert len(derps) == len(expected_derp_ids_with_count) + + actual_derp_ids_with_count = [(int(derp.href.split("/")[-1]), derp.DERControlListLink.all_) for derp in derps] assert expected_derp_ids_with_count == actual_derp_ids_with_count @@ -547,58 +509,41 @@ async def test_get_dercontrol_list_all_expired( @pytest.mark.anyio -@pytest.mark.no_default_doe async def test_get_default_doe_not_configured(client: AsyncClient, uri_derc_default_control_format, agg_1_headers): - """Tests getting the default DOE with no default configured returns 404""" + """Tests getting the default DOE with no default configured returns an "empty" default""" - # test a known site in base_config that does not have anything set either - path = uri_derc_default_control_format.format(site_id=2, der_program_id=1) + # test a known DERProgram in base_config that does not have a default set + path = uri_derc_default_control_format.format(site_id=2, der_program_id=2) response = await client.get(path, headers=agg_1_headers) - assert_response_header(response, HTTPStatus.NOT_FOUND) - assert_error_response(response) + assert_response_header(response, HTTPStatus.OK) + body = read_response_body_string(response) + assert len(body) > 0 + + parsed_response: DefaultDERControl = DefaultDERControl.from_xml(body) + assert parsed_response.href == path + assert parsed_response.DERControlBase_ is not None + assert parsed_response.DERControlBase_.opModImpLimW is None + assert parsed_response.DERControlBase_.opModExpLimW is None + assert parsed_response.DERControlBase_.opModGenLimW is None + assert parsed_response.DERControlBase_.opModLoadLimW is None + assert parsed_response.setGradW is None + assert parsed_response.version == 0 + assert parsed_response.mRID @pytest.mark.anyio -async def test_get_default_invalid_site_id(client: AsyncClient, uri_derc_default_control_format, agg_1_headers): - """Tests getting the default DOE with no default configured returns 404""" +async def test_get_default_invalid_derp_id(client: AsyncClient, uri_derc_default_control_format, agg_1_headers): + """Tests getting the default DOE for a non existent DERProgram returns 404""" # test trying to fetch a site unavailable to this aggregator - path = uri_derc_default_control_format.format(site_id=3, der_program_id=1) + path = uri_derc_default_control_format.format(site_id=1, der_program_id=99) response = await client.get(path, headers=agg_1_headers) assert_response_header(response, HTTPStatus.NOT_FOUND) assert_error_response(response) -@pytest.mark.anyio -async def test_get_fallback_default_doe(client: AsyncClient, uri_derc_default_control_format, agg_1_headers): - """Tests getting the default DOE""" - - # test a known site - path = uri_derc_default_control_format.format(site_id=2, der_program_id=1) - response = await client.get(path, headers=agg_1_headers) - - assert_response_header(response, HTTPStatus.OK) - body = read_response_body_string(response) - assert len(body) > 0 - - parsed_response: DefaultDERControl = DefaultDERControl.from_xml(body) - - assert ( - parsed_response.DERControlBase_.opModImpLimW.value - == DERControlMapper.map_to_active_power( - DEFAULT_DOE_IMPORT_ACTIVE_WATTS, DEFAULT_SITE_CONTROL_POW10_ENCODING - ).value - ) - assert ( - parsed_response.DERControlBase_.opModExpLimW.value - == DERControlMapper.map_to_active_power( - DEFAULT_DOE_EXPORT_ACTIVE_WATTS, DEFAULT_SITE_CONTROL_POW10_ENCODING - ).value - ) - - @pytest.mark.anyio async def test_get_site_specific_default_doe(client: AsyncClient, uri_derc_default_control_format, agg_1_headers): """Tests getting the default DOE""" diff --git a/tests/integration/func_sets/test_pub_sub.py b/tests/integration/func_sets/test_pub_sub.py index 268c9cea..68dca4bc 100644 --- a/tests/integration/func_sets/test_pub_sub.py +++ b/tests/integration/func_sets/test_pub_sub.py @@ -25,7 +25,7 @@ from envoy.server.crud.site import VIRTUAL_END_DEVICE_SITE_ID from envoy.server.crud.subscription import select_subscription_by_id from envoy.server.manager.der_constants import PUBLIC_SITE_DER_ID -from envoy.server.model.subscription import Subscription +from envoy.server.model.subscription import Subscription, SubscriptionResource from tests.data.certificates.certificate1 import TEST_CERTIFICATE_FINGERPRINT as AGG_1_VALID_CERT from tests.data.certificates.certificate4 import TEST_CERTIFICATE_FINGERPRINT as AGG_2_VALID_CERT from tests.data.certificates.certificate5 import TEST_CERTIFICATE_FINGERPRINT as AGG_3_VALID_CERT @@ -281,6 +281,63 @@ async def test_delete_subscription( assert (initial_count - 1) == after_count +@pytest.mark.parametrize( + "use_aggregator_edev, derp_id", + product([True, False], [1, 2, 3]), +) +@pytest.mark.anyio +async def test_create_derp_default_subscription( + pg_base_config, client: AsyncClient, sub_list_uri_format: str, use_aggregator_edev: bool, derp_id: int +): + """When creating a sub check to see if it persists and is correctly assigned to the aggregator""" + + notification_uri = "https://example.com/456?foo=bar" + edev_id: int = 0 if use_aggregator_edev else 1 + + insert_request: Sep2Subscription = generate_class_instance(Sep2Subscription) + insert_request.encoding = SubscriptionEncoding.XML + insert_request.notificationURI = notification_uri + insert_request.subscribedResource = f"/edev/{edev_id}/derp/{derp_id}/dderc" + response = await client.post( + sub_list_uri_format.format(site_id=0), # All subscriptions should be made to /edev/0/sub + headers={cert_header: urllib.parse.quote(AGG_1_VALID_CERT)}, + content=Sep2Subscription.to_xml(insert_request), + ) + assert_response_header(response, HTTPStatus.CREATED, expected_content_type=None) + assert len(read_response_body_string(response)) == 0 + inserted_href = read_location_header(response) + + # now lets grab the sub we just created + response = await client.get(inserted_href, headers={cert_header: urllib.parse.quote(AGG_1_VALID_CERT)}) + assert_response_header(response, HTTPStatus.OK) + response_body = read_response_body_string(response) + assert len(response_body) > 0 + parsed_response: Sep2Subscription = Sep2Subscription.from_xml(response_body) + assert parsed_response.href in inserted_href + assert parsed_response.notificationURI == insert_request.notificationURI + assert parsed_response.subscribedResource == insert_request.subscribedResource + assert parsed_response.limit == insert_request.limit + + # check that other aggregators can't fetch it + response = await client.get(inserted_href, headers={cert_header: urllib.parse.quote(AGG_2_VALID_CERT)}) + assert_response_header(response, HTTPStatus.NOT_FOUND) + assert_error_response(response) + + # Validate the DB record is properly scoped + async with generate_async_session(pg_base_config) as session: + resp = await session.execute(select(Subscription).order_by(Subscription.subscription_id.desc()).limit(1)) + created_sub = resp.scalars().first() + assert_nowish(created_sub.changed_time) + assert_nowish(created_sub.created_time) + if use_aggregator_edev: + assert created_sub.scoped_site_id is None, "Aggregator scoped requests are site unscoped" + else: + assert created_sub.scoped_site_id == edev_id, "Regular requests are site scoped" + assert created_sub.notification_uri == notification_uri + assert created_sub.resource_id == derp_id + assert created_sub.resource_type == SubscriptionResource.DEFAULT_SITE_CONTROL + + @pytest.mark.parametrize( "use_aggregator_edev, notification_uri", product([True, False], ["https://example.com/456?foo=bar", "http://example.com:123", "https://example.com"]), @@ -292,11 +349,12 @@ async def test_create_doe_subscription( """When creating a sub check to see if it persists and is correctly assigned to the aggregator""" edev_id: int = 0 if use_aggregator_edev else 1 + derp_id = 2 insert_request: Sep2Subscription = generate_class_instance(Sep2Subscription) insert_request.encoding = SubscriptionEncoding.XML insert_request.notificationURI = notification_uri - insert_request.subscribedResource = f"/edev/{edev_id}/derp/1/derc" + insert_request.subscribedResource = f"/edev/{edev_id}/derp/{derp_id}/derc" response = await client.post( sub_list_uri_format.format(site_id=0), # All subscriptions should be made to /edev/0/sub headers={cert_header: urllib.parse.quote(AGG_1_VALID_CERT)}, @@ -333,6 +391,8 @@ async def test_create_doe_subscription( else: assert created_sub.scoped_site_id == edev_id, "Regular requests are site scoped" assert created_sub.notification_uri == notification_uri + assert created_sub.resource_id == derp_id + assert created_sub.resource_type == SubscriptionResource.DYNAMIC_OPERATING_ENVELOPE async def do_test_renewal( diff --git a/tests/unit/admin/crud/test_doe.py b/tests/unit/admin/crud/test_doe.py index 4cb840f2..d5007038 100644 --- a/tests/unit/admin/crud/test_doe.py +++ b/tests/unit/admin/crud/test_doe.py @@ -11,7 +11,7 @@ from assertical.asserts.type import assert_list_type from assertical.fake.generator import clone_class_instance, generate_class_instance from assertical.fixtures.postgres import generate_async_session -from sqlalchemy import func, select +from sqlalchemy import func, select, update from envoy.admin.crud.doe import ( cancel_then_insert_does, @@ -221,6 +221,16 @@ async def test_supersede_matching_does_for_site( pg_base_config, doe_list: list[DynamicOperatingEnvelope], expected_doe_update_ids: list[int] ): async with generate_async_session(pg_base_config) as session: + await session.execute( + update(SiteControlGroup).where(SiteControlGroup.site_control_group_id == 1).values(primacy=11) + ) + await session.execute( + update(SiteControlGroup).where(SiteControlGroup.site_control_group_id == 2).values(primacy=22) + ) + await session.execute( + update(SiteControlGroup).where(SiteControlGroup.site_control_group_id == 3).values(primacy=1) + ) + original_superseded_values = dict( ( await session.execute( @@ -230,8 +240,6 @@ async def test_supersede_matching_does_for_site( .tuples() .all() ) - session.add(generate_class_instance(SiteControlGroup, seed=1, site_control_group_id=2, primacy=22)) - session.add(generate_class_instance(SiteControlGroup, seed=2, site_control_group_id=3, primacy=33)) await session.commit() site_id = 99 @@ -275,11 +283,9 @@ async def test_supersede_then_insert_does_many_sites( original_doe_count = ( await session.execute(select(func.count()).select_from(DynamicOperatingEnvelope)) ).scalar_one() - session.add(generate_class_instance(SiteControlGroup, seed=1, site_control_group_id=2, primacy=22)) - session.add(generate_class_instance(SiteControlGroup, seed=2, site_control_group_id=3, primacy=33)) await session.commit() - expected_primacy_by_group_id = {1: 0, 2: 22, 3: 33} + expected_primacy_by_group_id = {1: 0, 2: 1, 3: 2} changed_time = datetime(2021, 11, 4, 2, 3, 4, tzinfo=timezone.utc) does = [ generate_class_instance( @@ -415,24 +421,6 @@ async def extra_site_control_groups(pg_base_config): # Current database entry has changed time '2021-04-05 10:01:00.500' async with generate_async_session(pg_base_config) as session: - session.add( - generate_class_instance( - SiteControlGroup, - seed=101, - primacy=2, - site_control_group_id=2, - changed_time=datetime(2021, 4, 5, 10, 2, 0, 500000, tzinfo=timezone.utc), - ) - ) - session.add( - generate_class_instance( - SiteControlGroup, - seed=202, - primacy=1, - site_control_group_id=3, - changed_time=datetime(2021, 4, 5, 10, 3, 0, 500000, tzinfo=timezone.utc), - ) - ) session.add( generate_class_instance( SiteControlGroup, diff --git a/tests/unit/admin/crud/test_site.py b/tests/unit/admin/crud/test_site.py index b36f310d..388335b6 100644 --- a/tests/unit/admin/crud/test_site.py +++ b/tests/unit/admin/crud/test_site.py @@ -307,8 +307,8 @@ async def test_select_all_site_groups( @pytest.mark.anyio async def test_select_single_site_no_scoping_missing_site_ids(pg_base_config, missing_site_id: int): async with generate_async_session(pg_base_config) as session: - for groups, der, site_default in product([True, False], [True, False], [True, False]): - assert (await select_single_site_no_scoping(session, missing_site_id, groups, der, site_default)) is None + for groups, der in product([True, False], [True, False]): + assert (await select_single_site_no_scoping(session, missing_site_id, groups, der)) is None @pytest.mark.parametrize( @@ -350,14 +350,13 @@ def der_to_expected_tuple( ders[0].site_der_status.site_der_status_id if ders[0].site_der_status else None, ) - for include_groups, include_der, include_site_default in product([True, False], [True, False], [True, False]): + for include_groups, include_der in product([True, False], [True, False]): async with generate_async_session(pg_base_config) as session: site = await select_single_site_no_scoping( session, site_id, include_groups=include_groups, include_der=include_der, - include_site_default=include_site_default, ) if include_groups: @@ -371,13 +370,3 @@ def der_to_expected_tuple( else: with pytest.raises(InvalidRequestError): assert len(site.site_ders) == 0 - - if include_site_default: - assert expected_site_import_watts == ( - site.default_site_control.import_limit_active_watts - if site.default_site_control is not None - else None - ) - else: - with pytest.raises(InvalidRequestError): - assert site.default_site_control.default_site_control_id == 0 diff --git a/tests/unit/admin/manager/test_config.py b/tests/unit/admin/manager/test_config.py deleted file mode 100644 index 92437f6d..00000000 --- a/tests/unit/admin/manager/test_config.py +++ /dev/null @@ -1,66 +0,0 @@ -import unittest.mock as mock -from decimal import Decimal - -import pytest -from assertical.fixtures.postgres import generate_async_session -from envoy_schema.admin.schema.config import ControlDefaultRequest, UpdateDefaultValue -from sqlalchemy import select - -from envoy.admin.manager.config import ConfigManager -from envoy.server.model.site import DefaultSiteControl - - -@pytest.mark.parametrize( - "site_id, control_request", - [ - ( - 1, - ControlDefaultRequest( - import_limit_watts=UpdateDefaultValue(value=None), - export_limit_watts=UpdateDefaultValue(value=None), - load_limit_watts=UpdateDefaultValue(value=None), - generation_limit_watts=UpdateDefaultValue(value=None), - ramp_rate_percent_per_second=UpdateDefaultValue(value=None), - ), - ), - ( - 3, - ControlDefaultRequest( - import_limit_watts=UpdateDefaultValue(value=Decimal(11)), - export_limit_watts=UpdateDefaultValue(value=Decimal(12)), - load_limit_watts=UpdateDefaultValue(value=Decimal(13)), - generation_limit_watts=UpdateDefaultValue(value=Decimal(14)), - ramp_rate_percent_per_second=UpdateDefaultValue(value=Decimal(15)), - ), - ), - ], -) -@mock.patch("envoy.admin.manager.config.NotificationManager.notify_changed_deleted_entities") -@pytest.mark.anyio -async def test_update_site_control_default_all_vals_update( - mock_notify_changed_deleted_entities: mock.MagicMock, - pg_base_config, - site_id: int, - control_request: ControlDefaultRequest, -): - """Tests that the values for existing/new control defaults can be correctly updated""" - async with generate_async_session(pg_base_config) as session: - version_before = ( - await session.execute(select(DefaultSiteControl.version).where(DefaultSiteControl.site_id == site_id)) - ).scalar_one() - - async with generate_async_session(pg_base_config) as session: - await ConfigManager.update_site_control_default(session, site_id, control_request) - - # Check the DB - async with generate_async_session(pg_base_config) as session: - result = await session.execute(select(DefaultSiteControl).where(DefaultSiteControl.site_id == site_id)) - saved_result = result.scalar_one() - assert saved_result.import_limit_active_watts == control_request.import_limit_watts.value - assert saved_result.export_limit_active_watts == control_request.export_limit_watts.value - assert saved_result.generation_limit_active_watts == control_request.generation_limit_watts.value - assert saved_result.load_limit_active_watts == control_request.load_limit_watts.value - assert saved_result.ramp_rate_percent_per_second == control_request.ramp_rate_percent_per_second.value - assert saved_result.version == version_before + 1, "This should be incremented as part of the update" - - mock_notify_changed_deleted_entities.assert_called_once() diff --git a/tests/unit/admin/manager/test_site_control.py b/tests/unit/admin/manager/test_site_control.py new file mode 100644 index 00000000..10ad3e92 --- /dev/null +++ b/tests/unit/admin/manager/test_site_control.py @@ -0,0 +1,101 @@ +import unittest.mock as mock +from decimal import Decimal + +import pytest +from assertical.fixtures.postgres import generate_async_session +from envoy_schema.admin.schema.site_control import SiteControlGroupDefaultRequest, UpdateDefaultValue +from sqlalchemy import func, select + +from envoy.admin.manager.site_control import SiteControlGroupManager +from envoy.server.model.archive.doe import ArchiveSiteControlGroupDefault +from envoy.server.model.doe import SiteControlGroupDefault + + +@pytest.mark.parametrize( + "group_id, control_request", + [ + ( + 1, + SiteControlGroupDefaultRequest( + import_limit_watts=UpdateDefaultValue(value=None), + export_limit_watts=UpdateDefaultValue(value=None), + load_limit_watts=UpdateDefaultValue(value=None), + generation_limit_watts=UpdateDefaultValue(value=None), + ramp_rate_percent_per_second=UpdateDefaultValue(value=None), + ), + ), + ( + 1, + SiteControlGroupDefaultRequest( + import_limit_watts=UpdateDefaultValue(value=Decimal(11)), + export_limit_watts=UpdateDefaultValue(value=Decimal(12)), + load_limit_watts=UpdateDefaultValue(value=Decimal(13)), + generation_limit_watts=UpdateDefaultValue(value=Decimal(14)), + ramp_rate_percent_per_second=UpdateDefaultValue(value=Decimal(15)), + ), + ), + ( + 2, + SiteControlGroupDefaultRequest( + import_limit_watts=UpdateDefaultValue(value=Decimal(11)), + export_limit_watts=UpdateDefaultValue(value=Decimal(12)), + load_limit_watts=UpdateDefaultValue(value=Decimal(13)), + generation_limit_watts=UpdateDefaultValue(value=Decimal(14)), + ramp_rate_percent_per_second=UpdateDefaultValue(value=Decimal(15)), + ), + ), + ( + 3, + SiteControlGroupDefaultRequest( + import_limit_watts=UpdateDefaultValue(value=Decimal(11)), + export_limit_watts=UpdateDefaultValue(value=Decimal(12)), + load_limit_watts=UpdateDefaultValue(value=Decimal(13)), + generation_limit_watts=UpdateDefaultValue(value=Decimal(14)), + ramp_rate_percent_per_second=UpdateDefaultValue(value=Decimal(15)), + ), + ), + ], +) +@mock.patch("envoy.admin.manager.config.NotificationManager.notify_changed_deleted_entities") +@pytest.mark.anyio +async def test_update_site_control_default_all_vals_update( + mock_notify_changed_deleted_entities: mock.MagicMock, + pg_base_config, + group_id: int, + control_request: SiteControlGroupDefaultRequest, +): + """Tests that the values for existing/new control defaults can be correctly updated""" + async with generate_async_session(pg_base_config) as session: + version_before = ( + await session.execute( + select(SiteControlGroupDefault.version).where(SiteControlGroupDefault.site_control_group_id == group_id) + ) + ).scalar_one_or_none() + + async with generate_async_session(pg_base_config) as session: + await SiteControlGroupManager.update_site_control_default(session, group_id, control_request) + + # Check the DB + async with generate_async_session(pg_base_config) as session: + result = await session.execute( + select(SiteControlGroupDefault).where(SiteControlGroupDefault.site_control_group_id == group_id) + ) + saved_result = result.scalar_one() + assert saved_result.import_limit_active_watts == control_request.import_limit_watts.value + assert saved_result.export_limit_active_watts == control_request.export_limit_watts.value + assert saved_result.generation_limit_active_watts == control_request.generation_limit_watts.value + assert saved_result.load_limit_active_watts == control_request.load_limit_watts.value + assert saved_result.ramp_rate_percent_per_second == control_request.ramp_rate_percent_per_second.value + + if version_before is None: + assert saved_result.version == 1 + assert ( + await session.execute(select(func.count()).select_from(ArchiveSiteControlGroupDefault)) + ).scalar_one() == 0, "No archive rows if this is a new default" + else: + assert saved_result.version == version_before + 1, "This should be incremented as part of the update" + assert ( + await session.execute(select(func.count()).select_from(ArchiveSiteControlGroupDefault)) + ).scalar_one() == 1, "Old values should've been archived" + + mock_notify_changed_deleted_entities.assert_called_once() diff --git a/tests/unit/notification/crud/test_batch.py b/tests/unit/notification/crud/test_batch.py index d86ae94e..c57ff3bf 100644 --- a/tests/unit/notification/crud/test_batch.py +++ b/tests/unit/notification/crud/test_batch.py @@ -32,12 +32,12 @@ select_subscriptions_for_resource, ) from envoy.notification.crud.common import ( - ArchiveControlGroupScopedDefaultSiteControl, ArchiveSiteScopedFunctionSetAssignment, ArchiveSiteScopedSiteControlGroup, - ControlGroupScopedDefaultSiteControl, + ArchiveSiteScopedSiteControlGroupDefault, SiteScopedFunctionSetAssignment, SiteScopedSiteControlGroup, + SiteScopedSiteControlGroupDefault, TResourceModel, ) from envoy.notification.exception import NotificationError @@ -45,9 +45,12 @@ from envoy.server.manager.der_constants import PUBLIC_SITE_DER_ID from envoy.server.model.aggregator import NULL_AGGREGATOR_ID from envoy.server.model.archive.base import ArchiveBase -from envoy.server.model.archive.doe import ArchiveDynamicOperatingEnvelope, ArchiveSiteControlGroup +from envoy.server.model.archive.doe import ( + ArchiveDynamicOperatingEnvelope, + ArchiveSiteControlGroup, + ArchiveSiteControlGroupDefault, +) from envoy.server.model.archive.site import ( - ArchiveDefaultSiteControl, ArchiveSite, ArchiveSiteDER, ArchiveSiteDERAvailability, @@ -58,15 +61,8 @@ from envoy.server.model.archive.site_reading import ArchiveSiteReading, ArchiveSiteReadingType from envoy.server.model.archive.tariff import ArchiveTariffGeneratedRate from envoy.server.model.base import Base -from envoy.server.model.doe import DynamicOperatingEnvelope, SiteControlGroup -from envoy.server.model.site import ( - DefaultSiteControl, - SiteDER, - SiteDERAvailability, - SiteDERRating, - SiteDERSetting, - SiteDERStatus, -) +from envoy.server.model.doe import DynamicOperatingEnvelope, SiteControlGroupDefault +from envoy.server.model.site import SiteDER, SiteDERAvailability, SiteDERRating, SiteDERSetting, SiteDERStatus from envoy.server.model.site_reading import SiteReading, SiteReadingType from envoy.server.model.subscription import Subscription, SubscriptionCondition, SubscriptionResource from envoy.server.model.tariff import TariffGeneratedRate @@ -300,11 +296,13 @@ def test_get_batch_key_invalid(): ), ( SubscriptionResource.DEFAULT_SITE_CONTROL, - ControlGroupScopedDefaultSiteControl( + SiteScopedSiteControlGroupDefault( + aggregator_id=11, + site_id=22, site_control_group_id=33, - original=DefaultSiteControl(site_id=11, site=Site(site_id=11, aggregator_id=22)), + original=generate_class_instance(SiteControlGroupDefault), ), - (22, 11, 33), + (11, 22, 33), ), ], ) @@ -352,10 +350,13 @@ def test_get_subscription_filter_id_invalid(): ), ( SubscriptionResource.DEFAULT_SITE_CONTROL, - ControlGroupScopedDefaultSiteControl( - site_control_group_id=4, original=generate_class_instance(DefaultSiteControl, seed=101) + SiteScopedSiteControlGroupDefault( + aggregator_id=11, + site_id=22, + site_control_group_id=33, + original=generate_class_instance(SiteControlGroupDefault), ), - 4, + 33, ), ], ) @@ -402,10 +403,13 @@ def test_get_site_id_invalid(): ), ( SubscriptionResource.DEFAULT_SITE_CONTROL, - ControlGroupScopedDefaultSiteControl( - site_control_group_id=4, original=generate_class_instance(DefaultSiteControl, seed=101, site_id=5) + SiteScopedSiteControlGroupDefault( + aggregator_id=11, + site_id=22, + site_control_group_id=33, + original=generate_class_instance(SiteControlGroupDefault), ), - 5, + 22, ), ( SubscriptionResource.FUNCTION_SET_ASSIGNMENTS, @@ -2240,179 +2244,106 @@ async def test_fetch_der_status_by_timestamp_with_archive(pg_base_config): @pytest.mark.parametrize( - "timestamp,expected_ids", + "timestamp,expected_agg_site_scg_ids", [ + ( + datetime(2023, 5, 1, 1, 2, 2, 500000, tzinfo=timezone.utc), + [(0, 5, 1), (0, 6, 1), (1, 1, 1), (1, 2, 1), (1, 4, 1), (2, 3, 1)], # One for every site and default #1 + ), ( datetime(2023, 5, 1, 2, 2, 2, 500000, tzinfo=timezone.utc), - [1, 2], + [(0, 5, 3), (0, 6, 3), (1, 1, 3), (1, 2, 3), (1, 4, 3), (2, 3, 3)], # One for every site and default #2 ), ( - datetime(2022, 2, 3, 4, 5, 8), # timestamp mismatch + datetime(2022, 2, 3, 4, 5, 8, tzinfo=timezone.utc), # timestamp mismatch [], ), ], ) @pytest.mark.anyio -async def test_fetch_default_site_controls_by_changed_at(pg_base_config, timestamp: datetime, expected_ids: list[int]): - """Tests that entities are filtered/returned correctly""" +async def test_fetch_default_site_controls_by_changed_at( + pg_base_config, timestamp: datetime, expected_agg_site_scg_ids: list[tuple[int, int, int]] +): + """Tests that entities are filtered/returned correctly + + expected_agg_site_scg_ids should be a tuple of [agg_id, site_id, site_control_group_id]""" async with generate_async_session(pg_base_config) as session: # Need to unroll the batching into a single list (batching is tested elsewhere) batch = await fetch_default_site_controls_by_changed_at(session, timestamp) assert_batched_entities( batch, - ControlGroupScopedDefaultSiteControl, - ArchiveControlGroupScopedDefaultSiteControl, - len(expected_ids), + SiteScopedSiteControlGroupDefault, + ArchiveSiteScopedSiteControlGroupDefault, + len(expected_agg_site_scg_ids), 0, ) list_entities = [e for _, entities in batch.models_by_batch_key.items() for e in entities] - list_entities.sort(key=lambda default: default.original.default_site_control_id) + list_entities.sort(key=lambda default: (default.aggregator_id, default.site_id, default.site_control_group_id)) - assert all([isinstance(e, ControlGroupScopedDefaultSiteControl) for e in list_entities]) - assert all([e.site_control_group_id == 1 for e in list_entities]), "The only SiteControlGroup ID in the DB" - for i in range(len(expected_ids)): - assert list_entities[i].original.default_site_control_id == expected_ids[i] + assert all([isinstance(e, SiteScopedSiteControlGroupDefault) for e in list_entities]) + for i in range(len(expected_agg_site_scg_ids)): + agg_id, site_id, scg_id = expected_agg_site_scg_ids[i] + assert list_entities[i].aggregator_id == agg_id, f"Mismatch at idx[{i}]" + assert list_entities[i].site_id == site_id, f"Mismatch at idx[{i}]" + assert list_entities[i].site_control_group_id == scg_id, f"Mismatch at idx[{i}]" - assert all([isinstance(e.original.site, Site) for e in list_entities]), "Site relationship populated" - - -@pytest.mark.anyio -async def test_fetch_default_site_controls_by_changed_at_multiple_groups(pg_base_config): - """Tests that multiple SiteControlGroup's generate copies of the defaults per SiteControlGroup ID""" - timestamp = datetime(2023, 5, 1, 2, 2, 2, 500000, tzinfo=timezone.utc) - expected_default_group_ids = [(1, 1), (1, 99), (2, 1), (2, 99)] - - async with generate_async_session(pg_base_config) as session: - session.add(generate_class_instance(SiteControlGroup, site_control_group_id=99)) - await session.commit() - - async with generate_async_session(pg_base_config) as session: - # Need to unroll the batching into a single list (batching is tested elsewhere) - batch = await fetch_default_site_controls_by_changed_at(session, timestamp) - assert_batched_entities( - batch, - ControlGroupScopedDefaultSiteControl, - ArchiveControlGroupScopedDefaultSiteControl, - len(expected_default_group_ids), - 0, - ) - assert expected_default_group_ids == [ - (e.original.default_site_control_id, e.site_control_group_id) - for _, entities in batch.models_by_batch_key.items() - for e in entities - ] + assert all([isinstance(e.original, SiteControlGroupDefault) for e in list_entities]) @pytest.mark.anyio async def test_fetch_default_site_controls_by_timestamp_with_archive(pg_base_config): """Tests that entities are filtered/returned correctly and include archive data""" - # This matches the changed_time on default 1 and 2 + # This matches the changed_time on derp 3 default timestamp = datetime(2023, 5, 1, 2, 2, 2, 500000, tzinfo=timezone.utc) - expected_active_default_ids = [1, 2] - expected_deleted_default_ids = [21, 24, 25] + + # Combination of agg_id, site_id, site_control_group_id + expected_active_default_ids = [(0, 5, 3), (0, 6, 3), (1, 1, 3), (1, 2, 3), (1, 4, 3), (2, 3, 3)] + expected_deleted_default_ids = [(0, 5, 21), (0, 6, 21), (1, 1, 21), (1, 2, 21), (1, 4, 21), (2, 3, 21)] # inject a bunch of archival data async with generate_async_session(pg_base_config) as session: - # Inject a parent "archive" site that was deleted - the "newest" deleted value will be used - session.add(generate_class_instance(ArchiveSite, seed=11, aggregator_id=1, site_id=70)) - session.add( - generate_class_instance( - ArchiveSite, - seed=22, - aggregator_id=1, - site_id=70, - deleted_time=timestamp - timedelta(seconds=10), - ) - ) - session.add( - generate_class_instance( - ArchiveSite, - seed=33, - aggregator_id=1, - site_id=70, - deleted_time=timestamp - timedelta(seconds=5), # Doesn't need to match the timestamp - nmi="deleted70", - timezone_id="Australia/Brisbane", - ) - ) - - # This deleted site will be ignored in favour of the version in the active table - session.add( - generate_class_instance( - ArchiveSite, - seed=44, - aggregator_id=1, - site_id=1, - deleted_time=timestamp, - ) - ) - # Inject archive defaults (only most recent is used) session.add( generate_class_instance( - ArchiveDefaultSiteControl, + ArchiveSiteControlGroupDefault, seed=55, - site_id=1, - default_site_control_id=21, + site_control_group_default_id=2, + site_control_group_id=21, ) ) session.add( generate_class_instance( - ArchiveDefaultSiteControl, + ArchiveSiteControlGroupDefault, seed=66, - site_id=1, - default_site_control_id=21, + site_control_group_default_id=2, + site_control_group_id=21, deleted_time=timestamp - timedelta(seconds=5), ) ) session.add( generate_class_instance( - ArchiveDefaultSiteControl, + ArchiveSiteControlGroupDefault, seed=77, - site_id=70, - default_site_control_id=21, + site_control_group_default_id=2, + site_control_group_id=21, deleted_time=timestamp, ramp_rate_percent_per_second=21, # for identifying this record later ) ) # No deleted time so ignored - session.add(generate_class_instance(ArchiveDefaultSiteControl, seed=88, site_id=1, default_site_control_id=22)) - - # Wrong deleted time so ignored session.add( generate_class_instance( - ArchiveDefaultSiteControl, - seed=99, - site_id=1, - default_site_control_id=23, - deleted_time=timestamp - timedelta(seconds=5), + ArchiveSiteControlGroupDefault, + seed=88, + site_control_group_id=22, + site_control_group_default_id=21, + deleted_time=None, ) ) - # These will be picked up - session.add( - generate_class_instance( - ArchiveDefaultSiteControl, - seed=1010, - site_id=2, - default_site_control_id=24, - deleted_time=timestamp, - ramp_rate_percent_per_second=24, # for identifying this record later - ) - ) - session.add( - generate_class_instance( - ArchiveDefaultSiteControl, - seed=1111, - site_id=3, - default_site_control_id=25, - deleted_time=timestamp, - ramp_rate_percent_per_second=25, # for identifying this record later - ) - ) await session.commit() # Now see if the fetch grabs everything @@ -2421,41 +2352,34 @@ async def test_fetch_default_site_controls_by_timestamp_with_archive(pg_base_con batch = await fetch_default_site_controls_by_changed_at(session, timestamp) assert_batched_entities( batch, - ControlGroupScopedDefaultSiteControl, - ArchiveControlGroupScopedDefaultSiteControl, + SiteScopedSiteControlGroupDefault, + ArchiveSiteScopedSiteControlGroupDefault, len(expected_active_default_ids), len(expected_deleted_default_ids), ) active_list_entities = [e for _, entities in batch.models_by_batch_key.items() for e in entities] - active_list_entities.sort(key=lambda e: e.original.default_site_control_id) + active_list_entities.sort( + key=lambda default: (default.aggregator_id, default.site_id, default.site_control_group_id) + ) deleted_list_entities = [e for _, entities in batch.deleted_by_batch_key.items() for e in entities] - deleted_list_entities.sort(key=lambda e: e.original.default_site_control_id) + deleted_list_entities.sort( + key=lambda default: (default.aggregator_id, default.site_id, default.site_control_group_id) + ) assert set(expected_active_default_ids) == set( - [e.original.default_site_control_id for e in active_list_entities] + [(e.aggregator_id, e.site_id, e.site_control_group_id) for e in active_list_entities] ) assert set(expected_deleted_default_ids) == set( - [e.original.default_site_control_id for e in deleted_list_entities] - ) - - # Ensure the parent ORM relationship is populated for deleted/active instances - assert all([isinstance(e.original.site, Site) for v_list in batch.models_by_batch_key.values() for e in v_list]) - assert all( - [ - hasattr(e.original, "site") - and (isinstance(e.original.site, Site) or isinstance(e.original.site, ArchiveSite)) - for v_list in batch.deleted_by_batch_key.values() - for e in v_list - ] + [(e.aggregator_id, e.site_id, e.site_control_group_id) for e in deleted_list_entities] ) # Validate the deleted entities are the ones we expect (lean on the fact we setup a property on the # archive type in a particular way for the expected matches) assert all( [ - e.original.ramp_rate_percent_per_second == e.original.default_site_control_id + e.original.ramp_rate_percent_per_second == e.site_control_group_id for v_list in batch.deleted_by_batch_key.values() for e in v_list ] @@ -2464,7 +2388,7 @@ async def test_fetch_default_site_controls_by_timestamp_with_archive(pg_base_con # Sanity check that a different timestamp yields nothing empty_batch = await fetch_sites_by_changed_at(session, timestamp - timedelta(milliseconds=50)) assert_batched_entities( - empty_batch, ControlGroupScopedDefaultSiteControl, ArchiveControlGroupScopedDefaultSiteControl, 0, 0 + empty_batch, SiteScopedSiteControlGroupDefault, ArchiveSiteScopedSiteControlGroupDefault, 0, 0 ) assert len(empty_batch.models_by_batch_key) == 0 assert len(empty_batch.deleted_by_batch_key) == 0 @@ -2662,8 +2586,6 @@ async def test_fetch_site_control_groups_by_timestamp_with_archive(pg_base_confi # Sanity check that a different timestamp yields nothing empty_batch = await fetch_sites_by_changed_at(session, timestamp - timedelta(milliseconds=50)) - assert_batched_entities( - empty_batch, ControlGroupScopedDefaultSiteControl, ArchiveControlGroupScopedDefaultSiteControl, 0, 0 - ) + assert_batched_entities(empty_batch, SiteScopedSiteControlGroup, ArchiveSiteScopedSiteControlGroup, 0, 0) assert len(empty_batch.models_by_batch_key) == 0 assert len(empty_batch.deleted_by_batch_key) == 0 diff --git a/tests/unit/notification/task/test_check.py b/tests/unit/notification/task/test_check.py index 7917bf46..fa848510 100644 --- a/tests/unit/notification/task/test_check.py +++ b/tests/unit/notification/task/test_check.py @@ -15,9 +15,9 @@ from envoy.notification.crud.batch import AggregatorBatchedEntities, get_batch_key from envoy.notification.crud.common import ( - ControlGroupScopedDefaultSiteControl, SiteScopedFunctionSetAssignment, SiteScopedSiteControlGroup, + SiteScopedSiteControlGroupDefault, ) from envoy.notification.exception import NotificationError from envoy.notification.task.check import ( @@ -589,8 +589,8 @@ def test_all_entity_batches(input_changed: dict[tuple, list], input_deleted: dic (SubscriptionResource.SITE_DER_STATUS, SiteDERStatus, 987941), (SubscriptionResource.FUNCTION_SET_ASSIGNMENTS, SiteScopedFunctionSetAssignment, None), (SubscriptionResource.FUNCTION_SET_ASSIGNMENTS, SiteScopedFunctionSetAssignment, 241214), - (SubscriptionResource.DEFAULT_SITE_CONTROL, ControlGroupScopedDefaultSiteControl, None), - (SubscriptionResource.DEFAULT_SITE_CONTROL, ControlGroupScopedDefaultSiteControl, 331241), + (SubscriptionResource.DEFAULT_SITE_CONTROL, SiteScopedSiteControlGroupDefault, None), + (SubscriptionResource.DEFAULT_SITE_CONTROL, SiteScopedSiteControlGroupDefault, 331241), (SubscriptionResource.SITE_CONTROL_GROUP, SiteScopedSiteControlGroup, None), (SubscriptionResource.SITE_CONTROL_GROUP, SiteScopedSiteControlGroup, 442119), ], diff --git a/tests/unit/server/crud/test_doe.py b/tests/unit/server/crud/test_doe.py index d2dbcac2..eb0bbcad 100644 --- a/tests/unit/server/crud/test_doe.py +++ b/tests/unit/server/crud/test_doe.py @@ -10,6 +10,7 @@ from assertical.fake.generator import clone_class_instance, generate_class_instance from assertical.fixtures.postgres import generate_async_session from sqlalchemy import select, update +from sqlalchemy.exc import InvalidRequestError from envoy.admin.crud.doe import cancel_then_insert_does from envoy.server.crud.doe import ( @@ -267,9 +268,6 @@ async def test_select_and_count_active_does_include_deleted_multiple_groups( # Migrate ever async with generate_async_session(pg_additional_does) as session: - session.add(generate_class_instance(SiteControlGroup, site_control_group_id=2)) - await session.flush() - await session.execute( update(DOE) .values(site_control_group_id=2) @@ -618,34 +616,16 @@ async def test_select_doe_at_timestamp_pagination( async def extra_site_control_groups(pg_base_config): # Current database entry has changed time '2021-04-05 10:01:00.500' + # + # We will slot this in between 1 and 2 async with generate_async_session(pg_base_config) as session: session.add( generate_class_instance( SiteControlGroup, seed=101, - primacy=2, - site_control_group_id=2, - fsa_id=1, - changed_time=datetime(2021, 4, 5, 10, 2, 0, 500000, tzinfo=timezone.utc), - ) - ) - session.add( - generate_class_instance( - SiteControlGroup, - seed=202, - primacy=1, - site_control_group_id=3, - fsa_id=3, - changed_time=datetime(2021, 4, 5, 10, 3, 0, 500000, tzinfo=timezone.utc), - ) - ) - session.add( - generate_class_instance( - SiteControlGroup, - seed=303, primacy=1, site_control_group_id=4, - fsa_id=1, + fsa_id=3, changed_time=datetime(2021, 4, 5, 10, 4, 0, 500000, tzinfo=timezone.utc), ) ) @@ -654,17 +634,21 @@ async def extra_site_control_groups(pg_base_config): @pytest.mark.parametrize( - "site_control_group_id, expected_primacy", - [(1, 0), (3, 1), (99, None), (None, None)], + "site_control_group_id, expected_primacy, expected_default_import_limit_active_watts", + [(1, 0, Decimal("10.10")), (2, 1, None), (3, 2, Decimal("20.20")), (99, None, None), (None, None, None)], ) @pytest.mark.anyio async def test_select_site_control_group_by_id( - extra_site_control_groups, site_control_group_id: int, expected_primacy: Optional[int] + extra_site_control_groups, + site_control_group_id: int, + expected_primacy: Optional[int], + expected_default_import_limit_active_watts: Optional[Decimal], ): """Tests that select_site_control_group_by_code works with a variety of success/failure cases""" + # With include async with generate_async_session(extra_site_control_groups) as session: - result = await select_site_control_group_by_id(session, site_control_group_id) + result = await select_site_control_group_by_id(session, site_control_group_id, include_default=True) if expected_primacy is None: assert result is None @@ -672,25 +656,45 @@ async def test_select_site_control_group_by_id( assert isinstance(result, SiteControlGroup) assert result.primacy == expected_primacy assert result.site_control_group_id == site_control_group_id + if expected_default_import_limit_active_watts is None: + assert ( + result.site_control_group_default is None + or result.site_control_group_default.import_limit_active_watts is None + ) + else: + assert ( + result.site_control_group_default is not None + and result.site_control_group_default.import_limit_active_watts + == expected_default_import_limit_active_watts + ) + + # Without include + async with generate_async_session(extra_site_control_groups) as session: + result = await select_site_control_group_by_id(session, site_control_group_id, include_default=False) + if expected_primacy is None: + assert result is None + else: + with pytest.raises(InvalidRequestError): + result.site_control_group_default.created_time @pytest.mark.parametrize( "start, limit, changed_after, fsa_id, expected_ids, expected_count", [ - (0, 99, datetime.min, None, [1, 4, 3, 2], 4), - (0, 99, datetime.min, 1, [1, 4, 2], 3), + (0, 99, datetime.min, None, [1, 4, 2, 3], 4), + (0, 99, datetime.min, 1, [1, 2, 3], 3), (0, 99, datetime.min, 2, [], 0), - (0, 99, datetime.min, 3, [3], 1), - (1, 2, datetime.min, None, [4, 3], 4), - (1, 1, datetime.min, 1, [4], 3), + (0, 99, datetime.min, 3, [4], 1), + (1, 2, datetime.min, None, [4, 2], 4), + (1, 1, datetime.min, 1, [2], 3), (99, 99, datetime.min, None, [], 4), - (0, 99, datetime(2021, 4, 5, 10, 1, 0, tzinfo=timezone.utc), None, [1, 4, 3, 2], 4), - (3, 99, datetime(2021, 4, 5, 10, 1, 0, tzinfo=timezone.utc), None, [2], 4), - (0, 99, datetime(2021, 4, 5, 10, 2, 0, tzinfo=timezone.utc), None, [4, 3, 2], 3), + (0, 99, datetime(2021, 4, 5, 10, 1, 0, tzinfo=timezone.utc), None, [1, 4, 2, 3], 4), + (3, 99, datetime(2021, 4, 5, 10, 1, 0, tzinfo=timezone.utc), None, [3], 4), + (0, 99, datetime(2021, 4, 5, 10, 2, 0, tzinfo=timezone.utc), None, [4, 2, 3], 3), (0, 99, datetime(2021, 4, 5, 10, 3, 0, tzinfo=timezone.utc), None, [4, 3], 2), (0, 99, datetime(2021, 4, 5, 10, 4, 0, tzinfo=timezone.utc), None, [4], 1), (0, 99, datetime(2021, 4, 5, 10, 5, 0, tzinfo=timezone.utc), None, [], 0), - (0, 99, datetime(2021, 4, 5, 10, 2, 0, tzinfo=timezone.utc), 1, [4, 2], 2), + (0, 99, datetime(2021, 4, 5, 10, 2, 0, tzinfo=timezone.utc), 1, [2, 3], 2), ], ) @pytest.mark.anyio @@ -703,14 +707,45 @@ async def test_select_and_count_site_control_groups( expected_ids: list[int], expected_count: int, ): - async with generate_async_session(extra_site_control_groups) as session: - actual_groups = await select_site_control_groups(session, start, changed_after, limit, fsa_id) - assert expected_ids == [e.site_control_group_id for e in actual_groups] - assert_list_type(SiteControlGroup, actual_groups, len(expected_ids)) - - actual_count = await count_site_control_groups(session, changed_after, fsa_id) - assert isinstance(actual_count, int) - assert actual_count == expected_count + # This is a cache of known site_control_group_default values, keyed by the SiteControlGroup id + # + # If an ID isn't in here - it's because it doesn't have a default + # + # These values correspond to the SiteControlGroupDefault values in base_config.sql + scg_default_import_limit: dict[int, Decimal] = { + 1: Decimal("10.10"), + 3: Decimal("20.20"), + } + + for include_defaults in [True, False]: + async with generate_async_session(extra_site_control_groups) as session: + actual_groups = await select_site_control_groups( + session, start, changed_after, limit, fsa_id, include_defaults=include_defaults + ) + assert expected_ids == [e.site_control_group_id for e in actual_groups] + assert_list_type(SiteControlGroup, actual_groups, len(expected_ids)) + + actual_count = await count_site_control_groups(session, changed_after, fsa_id) + assert isinstance(actual_count, int) + assert actual_count == expected_count + + if include_defaults: + for g in actual_groups: + expected_import_default = scg_default_import_limit.get(g.site_control_group_id, None) + if expected_import_default is None: + assert g.site_control_group_default is None + else: + assert ( + g.site_control_group_default is not None + ), f"SiteControlGroup ID: {g.site_control_group_id}" + assert ( + g.site_control_group_default.import_limit_active_watts == expected_import_default + ), f"SiteControlGroup ID: {g.site_control_group_id}" + + else: + for g in actual_groups: + with pytest.raises(InvalidRequestError): + g.site_control_group_default.created_time @pytest.mark.anyio @@ -735,7 +770,7 @@ async def test_count_site_control_groups_by_fsa_id_empty_db(pg_empty_config): (datetime(2021, 4, 5, 10, 1, 0, tzinfo=timezone.utc), [1, 3]), (datetime(2021, 4, 5, 10, 2, 0, tzinfo=timezone.utc), [1, 3]), (datetime(2021, 4, 5, 10, 2, 0, tzinfo=timezone.utc), [1, 3]), - (datetime(2021, 4, 5, 10, 4, 0, tzinfo=timezone.utc), [1]), + (datetime(2021, 4, 5, 10, 4, 0, tzinfo=timezone.utc), [3]), (datetime(2021, 4, 5, 10, 6, 0, tzinfo=timezone.utc), []), ], ) diff --git a/tests/unit/server/crud/test_site.py b/tests/unit/server/crud/test_site.py index b6b51e99..1111d476 100644 --- a/tests/unit/server/crud/test_site.py +++ b/tests/unit/server/crud/test_site.py @@ -1,6 +1,5 @@ from dataclasses import dataclass from datetime import datetime, timezone -from decimal import Decimal from itertools import product from typing import Callable, Optional, Union @@ -25,14 +24,12 @@ select_single_site_with_lfdi, select_single_site_with_sfdi, select_single_site_with_site_id, - select_site_with_default_site_control, upsert_site_for_aggregator, ) from envoy.server.manager.time import utc_now from envoy.server.model.archive.base import ArchiveBase from envoy.server.model.archive.doe import ArchiveDynamicOperatingEnvelope from envoy.server.model.archive.site import ( - ArchiveDefaultSiteControl, ArchiveSite, ArchiveSiteDER, ArchiveSiteDERAvailability, @@ -45,15 +42,7 @@ from envoy.server.model.archive.tariff import ArchiveTariffGeneratedRate from envoy.server.model.base import Base from envoy.server.model.doe import DynamicOperatingEnvelope -from envoy.server.model.site import ( - DefaultSiteControl, - Site, - SiteDER, - SiteDERAvailability, - SiteDERRating, - SiteDERSetting, - SiteDERStatus, -) +from envoy.server.model.site import Site, SiteDER, SiteDERAvailability, SiteDERRating, SiteDERSetting, SiteDERStatus from envoy.server.model.site_reading import SiteReading, SiteReadingType from envoy.server.model.subscription import Subscription, SubscriptionCondition from envoy.server.model.tariff import TariffGeneratedRate @@ -753,16 +742,6 @@ async def snapshot_all_site_tables(session: AsyncSession, agg_id: int, site_id: ) ) - snapshot.append( - await count_table_rows( - session, - DefaultSiteControl, - None, - ArchiveDefaultSiteControl, - lambda q: q.where(DefaultSiteControl.site_id == site_id), - ) - ) - return snapshot @@ -859,53 +838,6 @@ async def test_delete_site_for_aggregator( assert site is None, "If the delete was NOT committed but the site DNE - it should continue to not exist" -@pytest.mark.parametrize( - "site_id, agg_id, expected_vals", - [ - (1, 1, (1, 1, Decimal("10.10"), Decimal("9.99"), Decimal("8.88"), Decimal("7.77"), 6)), - (2, 1, None), - (3, 2, (2, 3, Decimal("20.20"), Decimal("19.19"), Decimal("18.18"), Decimal("17.17"), 16)), - (3, 1, None), - (99, 99, None), - ], -) -@pytest.mark.anyio -async def test_select_site_with_default_site_control( - pg_base_config, - site_id: int, - agg_id: int, - expected_vals: Optional[ - tuple[int, int, Optional[Decimal], Optional[Decimal], Optional[Decimal], Optional[Decimal], Optional[Decimal]] - ], -): - """Tests the site to default site control relationship""" - async with generate_async_session(pg_base_config) as session: - site = await select_site_with_default_site_control(session, site_id=site_id, aggregator_id=agg_id) - - if expected_vals is None: - if site is not None: - assert site.default_site_control is None - else: - default_site_control = site.default_site_control - ( - default_site_control_id, - site_id, - import_limit_active_watts, - export_limit_active_watts, - generation_limit_active_watts, - load_limit_active_watts, - ramp_rate_percent_per_second, - ) = expected_vals - assert isinstance(default_site_control, DefaultSiteControl) - assert default_site_control.site_id == site_id - assert default_site_control.default_site_control_id == default_site_control_id - assert default_site_control.import_limit_active_watts == import_limit_active_watts - assert default_site_control.export_limit_active_watts == export_limit_active_watts - assert default_site_control.generation_limit_active_watts == generation_limit_active_watts - assert default_site_control.load_limit_active_watts == load_limit_active_watts - assert default_site_control.ramp_rate_percent_per_second == ramp_rate_percent_per_second - - @pytest.mark.anyio async def test_insert_site_for_aggregator(pg_base_config): """Tests that the insert can do inserts""" diff --git a/tests/unit/server/manager/test_derp.py b/tests/unit/server/manager/test_derp.py index 65b14831..a15455aa 100644 --- a/tests/unit/server/manager/test_derp.py +++ b/tests/unit/server/manager/test_derp.py @@ -17,17 +17,16 @@ from envoy.server.exception import NotFoundError from envoy.server.manager.derp import DERControlManager, DERProgramManager from envoy.server.mapper.csip_aus.doe import DERControlListSource -from envoy.server.model.config.default_doe import DefaultDoeConfiguration from envoy.server.model.config.server import RuntimeServerConfig -from envoy.server.model.doe import DynamicOperatingEnvelope, SiteControlGroup -from envoy.server.model.site import DefaultSiteControl, Site +from envoy.server.model.doe import DynamicOperatingEnvelope, SiteControlGroup, SiteControlGroupDefault +from envoy.server.model.site import Site from envoy.server.request_scope import DeviceOrAggregatorRequestScope, SiteRequestScope @pytest.mark.anyio @mock.patch("envoy.server.manager.derp.select_site_control_groups") @mock.patch("envoy.server.manager.derp.count_site_control_groups") -@mock.patch("envoy.server.manager.derp.select_site_with_default_site_control") +@mock.patch("envoy.server.manager.derp.select_single_site_with_site_id") @mock.patch("envoy.server.manager.derp.count_active_does_include_deleted") @mock.patch("envoy.server.manager.derp.DERProgramMapper") @mock.patch("envoy.server.manager.derp.utc_now") @@ -37,14 +36,13 @@ async def test_program_fetch_list_for_scope( mock_utc_now: mock.MagicMock, mock_DERProgramMapper: mock.MagicMock, mock_count_active_does_include_deleted: mock.MagicMock, - mock_select_site_with_default_site_control: mock.MagicMock, + mock_select_single_site_with_site_id: mock.MagicMock, mock_count_site_control_groups: mock.MagicMock, mock_select_site_control_groups: mock.MagicMock, ): """Tests that the underlying dependencies pipe their outputs correctly into the downstream inputs""" # Arrange existing_site = generate_class_instance(Site) - default_doe = generate_class_instance(DefaultDoeConfiguration) mapped_list = generate_class_instance(DERProgramListResponse) scope = generate_class_instance(SiteRequestScope) now = datetime(2020, 1, 2, tzinfo=timezone.utc) @@ -60,7 +58,7 @@ async def test_program_fetch_list_for_scope( mock_utc_now.return_value = now mock_session = create_mock_session() - mock_select_site_with_default_site_control.return_value = existing_site + mock_select_single_site_with_site_id.return_value = existing_site mock_count_site_control_groups.return_value = site_control_group_count mock_select_site_control_groups.return_value = site_control_groups mock_DERProgramMapper.doe_program_list_response = mock.Mock(return_value=mapped_list) @@ -72,16 +70,14 @@ async def test_program_fetch_list_for_scope( mock_fetch_current_config.return_value = config # Act - result = await DERProgramManager.fetch_list_for_scope( - mock_session, scope, default_doe, start, changed_after, limit, fsa_id - ) + result = await DERProgramManager.fetch_list_for_scope(mock_session, scope, start, changed_after, limit, fsa_id) # Assert assert result is mapped_list - mock_select_site_with_default_site_control.assert_called_once_with(mock_session, scope.site_id, scope.aggregator_id) + mock_select_single_site_with_site_id.assert_called_once_with(mock_session, scope.site_id, scope.aggregator_id) mock_select_site_control_groups.assert_called_once_with( - mock_session, start=start, limit=limit, changed_after=changed_after, fsa_id=fsa_id + mock_session, start=start, limit=limit, changed_after=changed_after, fsa_id=fsa_id, include_defaults=True ) # One call to control count for each site control group @@ -93,29 +89,28 @@ async def test_program_fetch_list_for_scope( @pytest.mark.anyio -@mock.patch("envoy.server.manager.derp.select_site_with_default_site_control") +@mock.patch("envoy.server.manager.derp.select_single_site_with_site_id") @mock.patch("envoy.server.manager.derp.count_active_does_include_deleted") @mock.patch("envoy.server.manager.derp.DERProgramMapper") async def test_program_fetch_list_scope_dne( mock_DERProgramMapper: mock.MagicMock, mock_count_active_does_include_deleted: mock.MagicMock, - mock_select_site_with_default_site_control: mock.MagicMock, + mock_select_single_site_with_site_id: mock.MagicMock, ): """Checks that if the crud layer indicates site doesn't exist then the manager will raise an exception""" # Arrange - default_doe = generate_class_instance(DefaultDoeConfiguration) fsa_id = 11 mock_session = create_mock_session() - mock_select_site_with_default_site_control.return_value = None + mock_select_single_site_with_site_id.return_value = None scope = generate_class_instance(SiteRequestScope) # Act with pytest.raises(NotFoundError): - await DERProgramManager.fetch_list_for_scope(mock_session, scope, default_doe, 1, datetime.min, 2, fsa_id) + await DERProgramManager.fetch_list_for_scope(mock_session, scope, 1, datetime.min, 2, fsa_id) # Assert - mock_select_site_with_default_site_control.assert_called_once_with(mock_session, scope.site_id, scope.aggregator_id) + mock_select_single_site_with_site_id.assert_called_once_with(mock_session, scope.site_id, scope.aggregator_id) mock_count_active_does_include_deleted.assert_not_called() mock_DERProgramMapper.doe_program_list_response.assert_not_called() assert_mock_session(mock_session) @@ -123,7 +118,7 @@ async def test_program_fetch_list_scope_dne( @pytest.mark.anyio @mock.patch("envoy.server.manager.derp.select_site_control_group_by_id") -@mock.patch("envoy.server.manager.derp.select_site_with_default_site_control") +@mock.patch("envoy.server.manager.derp.select_single_site_with_site_id") @mock.patch("envoy.server.manager.derp.count_active_does_include_deleted") @mock.patch("envoy.server.manager.derp.DERProgramMapper") @mock.patch("envoy.server.manager.derp.utc_now") @@ -131,7 +126,7 @@ async def test_program_fetch_for_scope( mock_utc_now: mock.MagicMock, mock_DERProgramMapper: mock.MagicMock, mock_count_active_does_include_deleted: mock.MagicMock, - mock_select_site_with_default_site_control: mock.MagicMock, + mock_select_single_site_with_site_id: mock.MagicMock, mock_select_site_control_group_by_id: mock.MagicMock, ): """Tests that the underlying dependencies pipe their outputs correctly into the downstream inputs""" @@ -140,26 +135,25 @@ async def test_program_fetch_for_scope( derp_id = 142124 existing_site = generate_class_instance(Site) mapped_program = generate_class_instance(DERProgramResponse) - default_doe = generate_class_instance(DefaultDoeConfiguration) scope = generate_class_instance(SiteRequestScope) now = datetime(2011, 2, 3, tzinfo=timezone.utc) group = generate_class_instance(SiteControlGroup) mock_session = create_mock_session() - mock_select_site_with_default_site_control.return_value = existing_site + mock_select_single_site_with_site_id.return_value = existing_site mock_count_active_does_include_deleted.return_value = doe_count mock_DERProgramMapper.doe_program_response = mock.Mock(return_value=mapped_program) mock_utc_now.return_value = now mock_select_site_control_group_by_id.return_value = group # Act - result = await DERProgramManager.fetch_doe_program_for_scope(mock_session, scope, derp_id, default_doe) + result = await DERProgramManager.fetch_doe_program_for_scope(mock_session, scope, derp_id) # Assert assert result is mapped_program - mock_select_site_control_group_by_id.assert_called_once_with(mock_session, derp_id) - mock_select_site_with_default_site_control.assert_called_once_with(mock_session, scope.site_id, scope.aggregator_id) + mock_select_site_control_group_by_id.assert_called_once_with(mock_session, derp_id, include_default=True) + mock_select_single_site_with_site_id.assert_called_once_with(mock_session, scope.site_id, scope.aggregator_id) mock_count_active_does_include_deleted.assert_called_once_with( mock_session, derp_id, existing_site, now, datetime.min ) @@ -169,31 +163,30 @@ async def test_program_fetch_for_scope( @pytest.mark.anyio @mock.patch("envoy.server.manager.derp.select_site_control_group_by_id") -@mock.patch("envoy.server.manager.derp.select_site_with_default_site_control") +@mock.patch("envoy.server.manager.derp.select_single_site_with_site_id") @mock.patch("envoy.server.manager.derp.count_active_does_include_deleted") @mock.patch("envoy.server.manager.derp.DERProgramMapper") async def test_program_fetch_site_dne( mock_DERProgramMapper: mock.MagicMock, mock_count_active_does_include_deleted: mock.MagicMock, - mock_select_site_with_default_site_control: mock.MagicMock, + mock_select_single_site_with_site_id: mock.MagicMock, mock_select_site_control_group_by_id: mock.MagicMock, ): """Checks that if the crud layer indicates site doesn't exist then the manager will raise an exception""" # Arrange - default_doe = generate_class_instance(DefaultDoeConfiguration) derp_id = 76662 mock_session = create_mock_session() - mock_select_site_with_default_site_control.return_value = None + mock_select_single_site_with_site_id.return_value = None scope = generate_class_instance(SiteRequestScope) # Act with pytest.raises(NotFoundError): - await DERProgramManager.fetch_doe_program_for_scope(mock_session, scope, derp_id, default_doe) + await DERProgramManager.fetch_doe_program_for_scope(mock_session, scope, derp_id) # Assert mock_select_site_control_group_by_id.assert_not_called() - mock_select_site_with_default_site_control.assert_called_once_with(mock_session, scope.site_id, scope.aggregator_id) + mock_select_single_site_with_site_id.assert_called_once_with(mock_session, scope.site_id, scope.aggregator_id) mock_count_active_does_include_deleted.assert_not_called() mock_DERProgramMapper.doe_program_response.assert_not_called() assert_mock_session(mock_session) @@ -201,32 +194,31 @@ async def test_program_fetch_site_dne( @pytest.mark.anyio @mock.patch("envoy.server.manager.derp.select_site_control_group_by_id") -@mock.patch("envoy.server.manager.derp.select_site_with_default_site_control") +@mock.patch("envoy.server.manager.derp.select_single_site_with_site_id") @mock.patch("envoy.server.manager.derp.count_active_does_include_deleted") @mock.patch("envoy.server.manager.derp.DERProgramMapper") async def test_program_fetch_site_control_group_dne( mock_DERProgramMapper: mock.MagicMock, mock_count_active_does_include_deleted: mock.MagicMock, - mock_select_site_with_default_site_control: mock.MagicMock, + mock_select_single_site_with_site_id: mock.MagicMock, mock_select_site_control_group_by_id: mock.MagicMock, ): """Checks that if the crud layer indicates site doesn't exist then the manager will raise an exception""" # Arrange - default_doe = generate_class_instance(DefaultDoeConfiguration) derp_id = 76662 mock_session = create_mock_session() - mock_select_site_with_default_site_control.return_value = generate_class_instance(Site) + mock_select_single_site_with_site_id.return_value = generate_class_instance(Site) mock_select_site_control_group_by_id.return_value = None scope = generate_class_instance(SiteRequestScope) # Act with pytest.raises(NotFoundError): - await DERProgramManager.fetch_doe_program_for_scope(mock_session, scope, derp_id, default_doe) + await DERProgramManager.fetch_doe_program_for_scope(mock_session, scope, derp_id) # Assert - mock_select_site_control_group_by_id.assert_called_once_with(mock_session, derp_id) - mock_select_site_with_default_site_control.assert_called_once_with(mock_session, scope.site_id, scope.aggregator_id) + mock_select_site_control_group_by_id.assert_called_once_with(mock_session, derp_id, include_default=True) + mock_select_single_site_with_site_id.assert_called_once_with(mock_session, scope.site_id, scope.aggregator_id) mock_count_active_does_include_deleted.assert_not_called() mock_DERProgramMapper.doe_program_response.assert_not_called() assert_mock_session(mock_session) @@ -481,24 +473,20 @@ async def test_fetch_active_doe_controls_for_site( @pytest.mark.anyio -@mock.patch("envoy.server.manager.derp.select_site_with_default_site_control") @mock.patch("envoy.server.manager.derp.DERControlMapper") -@mock.patch("envoy.server.manager.derp.DERControlManager._resolve_default_site_control") @mock.patch("envoy.server.manager.derp.RuntimeServerConfigManager.fetch_current_config") -async def test_fetch_default_doe_controls_for_site( +@mock.patch("envoy.server.manager.derp.select_site_control_group_by_id") +async def test_fetch_default_doe_controls_for_scope( + mock_select_site_control_group_by_id: mock.MagicMock, mock_fetch_current_config: mock.MagicMock, - mock_resolve_default_site_control: mock.MagicMock, mock_DERControlMapper: mock.MagicMock, - mock_select_site_with_default_site_control: mock.MagicMock, ): """Tests that the underlying dependencies pipe their outputs correctly into the downstream inputs""" # Arrange - default_doe = generate_class_instance(DefaultDoeConfiguration) derp_id = 771263 - returned_site = generate_class_instance(Site, generate_relationships=True) - mock_resolve_default_site_control.return_value = returned_site.default_site_control - mock_select_site_with_default_site_control.return_value = returned_site + returned_scg = generate_class_instance(SiteControlGroup, generate_relationships=True) + mock_select_site_control_group_by_id.return_value = returned_scg mapped_control = generate_class_instance(DefaultDERControl) mock_DERControlMapper.map_to_default_response = mock.Mock(return_value=mapped_control) @@ -510,28 +498,31 @@ async def test_fetch_default_doe_controls_for_site( mock_fetch_current_config.return_value = config # Act - result = await DERControlManager.fetch_default_doe_controls_for_site(mock_session, scope, derp_id, default_doe) + result = await DERControlManager.fetch_default_doe_controls_for_scope(mock_session, scope, derp_id) # Assert assert result is mapped_control - mock_select_site_with_default_site_control.assert_called_once_with(mock_session, scope.site_id, scope.aggregator_id) + mock_select_site_control_group_by_id.assert_called_once_with(mock_session, derp_id, include_default=True) mock_DERControlMapper.map_to_default_response.assert_called_once_with( - scope, returned_site.default_site_control, scope.display_site_id, derp_id, config.site_control_pow10_encoding + scope, + returned_scg.site_control_group_default, + scope.display_site_id, + derp_id, + config.site_control_pow10_encoding, ) assert_mock_session(mock_session) @pytest.mark.anyio -@mock.patch("envoy.server.manager.derp.select_site_with_default_site_control") +@mock.patch("envoy.server.manager.derp.select_site_control_group_by_id") @mock.patch("envoy.server.manager.derp.DERControlMapper") -async def test_fetch_default_doe_controls_for_site_bad_site( +async def test_fetch_default_doe_controls_for_scope_bad_derp( mock_DERControlMapper: mock.MagicMock, - mock_select_site_with_default_site_control: mock.MagicMock, + mock_select_site_control_group_by_id: mock.MagicMock, ): - """Tests that the underlying dependencies pipe their outputs correctly into the downstream inputs""" + """Tests that a missing SiteControlGroup results in an error""" # Arrange - default_doe = generate_class_instance(DefaultDoeConfiguration) derp_id = 771263 mapped_control = generate_class_instance(DefaultDERControl) @@ -539,169 +530,62 @@ async def test_fetch_default_doe_controls_for_site_bad_site( mock_session = create_mock_session() scope: SiteRequestScope = generate_class_instance(SiteRequestScope) - mock_select_site_with_default_site_control.return_value = None + mock_select_site_control_group_by_id.return_value = None mock_DERControlMapper.map_to_default_response = mock.Mock(return_value=mapped_control) # Act with pytest.raises(NotFoundError): - await DERControlManager.fetch_default_doe_controls_for_site(mock_session, scope, derp_id, default_doe) + await DERControlManager.fetch_default_doe_controls_for_scope(mock_session, scope, derp_id) # Assert - mock_select_site_with_default_site_control.assert_called_once_with(mock_session, scope.site_id, scope.aggregator_id) + mock_select_site_control_group_by_id.assert_called_once_with(mock_session, derp_id, include_default=True) mock_DERControlMapper.map_to_default_response.assert_not_called() assert_mock_session(mock_session) @pytest.mark.anyio -@mock.patch("envoy.server.manager.derp.select_site_with_default_site_control") -@mock.patch("envoy.server.manager.derp.DERControlMapper") +@mock.patch("envoy.server.manager.derp.select_site_control_group_by_id") +@mock.patch("envoy.server.manager.derp.DERControlMapper.map_to_default_response") +@mock.patch("envoy.server.manager.derp.RuntimeServerConfigManager.fetch_current_config") async def test_fetch_default_doe_controls_for_site_no_default( - mock_DERControlMapper: mock.MagicMock, - mock_select_site_with_default_site_control: mock.MagicMock, + mock_fetch_current_config: mock.MagicMock, + mock_map_to_default_response: mock.MagicMock, + mock_select_site_control_group_by_id: mock.MagicMock, ): - """Tests that the underlying dependencies pipe their outputs correctly into the downstream inputs""" + """Tests that a SiteControlGroup with no default generates a empty "default" """ # Arrange - default_doe = None derp_id = 88123 - returned_site = generate_class_instance(Site, generate_relationships=False) - + returned_scg = generate_class_instance(SiteControlGroup, site_control_group_default=None) mapped_control = generate_class_instance(DefaultDERControl) + config = generate_class_instance(RuntimeServerConfig) mock_session = create_mock_session() scope: SiteRequestScope = generate_class_instance(SiteRequestScope) - mock_select_site_with_default_site_control.return_value = returned_site - mock_DERControlMapper.map_to_default_response = mock.Mock(return_value=mapped_control) - - # Act - with pytest.raises(NotFoundError): - await DERControlManager.fetch_default_doe_controls_for_site(mock_session, scope, derp_id, default_doe) - - # Assert - mock_select_site_with_default_site_control.assert_called_once_with(mock_session, scope.site_id, scope.aggregator_id) - mock_DERControlMapper.map_to_default_response.assert_not_called() - - assert_mock_session(mock_session) - - -@pytest.mark.anyio -@mock.patch("envoy.server.manager.derp.select_site_with_default_site_control") -@mock.patch("envoy.server.manager.derp.DERControlMapper") -@mock.patch("envoy.server.manager.derp.RuntimeServerConfigManager.fetch_current_config") -async def test_fetch_default_doe_controls_for_site_no_global_default( - mock_fetch_current_config: mock.MagicMock, - mock_DERControlMapper: mock.MagicMock, - mock_select_site_with_default_site_control: mock.MagicMock, -): - """Tests that the underlying dependencies pipe their outputs correctly into the downstream inputs""" - # Arrange - default_doe = None - derp_id = 88144 - - returned_site = generate_class_instance(Site, generate_relationships=True) - - mapped_control = None - - mock_session = create_mock_session() - scope: SiteRequestScope = generate_class_instance(SiteRequestScope) - - mock_select_site_with_default_site_control.return_value = returned_site - mock_DERControlMapper.map_to_default_response = mock.Mock(return_value=mapped_control) - - mock_fetch_current_config.return_value = RuntimeServerConfig() + mock_select_site_control_group_by_id.return_value = returned_scg + mock_map_to_default_response.return_value = mapped_control + mock_fetch_current_config.return_value = config # Act - await DERControlManager.fetch_default_doe_controls_for_site(mock_session, scope, derp_id, default_doe) + result = await DERControlManager.fetch_default_doe_controls_for_scope(mock_session, scope, derp_id) # Assert - mock_select_site_with_default_site_control.assert_called_once_with(mock_session, scope.site_id, scope.aggregator_id) - mock_DERControlMapper.map_to_default_response.assert_called_once() + assert result is mapped_control + mock_select_site_control_group_by_id.assert_called_once_with(mock_session, derp_id, include_default=True) + mock_map_to_default_response.assert_called_once() + + empty_default: SiteControlGroupDefault = mock_map_to_default_response.call_args_list[0].args[1] + assert isinstance(empty_default, SiteControlGroupDefault) + assert empty_default.created_time == returned_scg.created_time + assert ( + empty_default.changed_time == returned_scg.created_time + ), "Yes - changed_time should be set to parent creation time" + assert empty_default.version == 0 + assert empty_default.export_limit_active_watts is None + assert empty_default.import_limit_active_watts is None + assert empty_default.generation_limit_active_watts is None + assert empty_default.load_limit_active_watts is None + assert empty_default.ramp_rate_percent_per_second is None assert_mock_session(mock_session) - - -@pytest.mark.parametrize( - "default_doe_config, default_site_control, expected", - [ - # misc - (None, None, None), - ( - DefaultDoeConfiguration(100, 200, 300, 400, 50), - DefaultSiteControl(import_limit_active_watts=0, load_limit_active_watts=0, version=999), - (999, 0, 200, 300, 0, 50), - ), - ( - DefaultDoeConfiguration(None, 200, 300, None, 50), - DefaultSiteControl(import_limit_active_watts=0, load_limit_active_watts=0, version=888), - (888, 0, 200, 300, 0, 50), - ), - ( - DefaultDoeConfiguration(None, None, None, None, None), - DefaultSiteControl(import_limit_active_watts=0, load_limit_active_watts=0, version=777), - (777, 0, None, None, 0, None), - ), - # No site control - ( - DefaultDoeConfiguration(100, 200, 300, 400, 50), - None, - (0, 100, 200, 300, 400, 50), - ), - # Partial site control - ( - DefaultDoeConfiguration( - import_limit_active_watts=100, - export_limit_active_watts=200, - generation_limit_active_watts=300, - load_limit_active_watts=400, - ramp_rate_percent_per_second=50, - ), - DefaultSiteControl(import_limit_active_watts=111, load_limit_active_watts=444, version=555), - (555, 111, 200, 300, 444, 50), - ), - # Full site control - ( - DefaultDoeConfiguration( - import_limit_active_watts=100, - export_limit_active_watts=200, - generation_limit_active_watts=300, - load_limit_active_watts=400, - ramp_rate_percent_per_second=50, - ), - DefaultSiteControl( - import_limit_active_watts=1, - export_limit_active_watts=2, - generation_limit_active_watts=3, - load_limit_active_watts=4, - ramp_rate_percent_per_second=5, - version=6, - ), - (6, 1, 2, 3, 4, 5), - ), - ( - None, - DefaultSiteControl( - import_limit_active_watts=1, - export_limit_active_watts=2, - generation_limit_active_watts=3, - load_limit_active_watts=4, - ramp_rate_percent_per_second=5, - version=7, - ), - (7, 1, 2, 3, 4, 5), - ), - ], -) -def test_resolve_default_site_control(default_doe_config, default_site_control, expected): - """Tests all combos of resolution""" - result = DERControlManager._resolve_default_site_control(default_doe_config, default_site_control) - - if expected is None: - assert result is None - else: - assert result.version == expected[0] - assert result.import_limit_active_watts == expected[1] - assert result.export_limit_active_watts == expected[2] - assert result.generation_limit_active_watts == expected[3] - assert result.load_limit_active_watts == expected[4] - assert result.ramp_rate_percent_per_second == expected[5] diff --git a/tests/unit/server/mapper/csip_aus/test_doe.py b/tests/unit/server/mapper/csip_aus/test_doe.py index 8656da4b..ca4fc927 100644 --- a/tests/unit/server/mapper/csip_aus/test_doe.py +++ b/tests/unit/server/mapper/csip_aus/test_doe.py @@ -20,9 +20,7 @@ from envoy.server.mapper.csip_aus.doe import DERControlListSource, DERControlMapper, DERProgramMapper from envoy.server.model.archive.doe import ArchiveDynamicOperatingEnvelope, ArchiveSiteControlGroup -from envoy.server.model.config.default_doe import DefaultDoeConfiguration -from envoy.server.model.doe import DynamicOperatingEnvelope, SiteControlGroup -from envoy.server.model.site import DefaultSiteControl +from envoy.server.model.doe import DynamicOperatingEnvelope, SiteControlGroup, SiteControlGroupDefault from envoy.server.request_scope import BaseRequestScope, DeviceOrAggregatorRequestScope @@ -162,7 +160,7 @@ def test_map_derc_to_response( @pytest.mark.parametrize("optional_is_none", [True, False]) def test_map_default_to_response(optional_is_none: bool): """Simple sanity check on the mapper to ensure things don't break with a variety of values.""" - doe_default = generate_class_instance(DefaultSiteControl, seed=101, optional_is_none=optional_is_none) + doe_default = generate_class_instance(SiteControlGroupDefault, seed=101, optional_is_none=optional_is_none) scope = generate_class_instance(BaseRequestScope, href_prefix="/my/prefix/") pow_10 = 1 derp_id = 2 @@ -270,7 +268,7 @@ def test_map_derp_doe_program_response_with_default_doe(total_does: Optional[int scope: DeviceOrAggregatorRequestScope = generate_class_instance( DeviceOrAggregatorRequestScope, display_site_id=54122 ) - default_doe = generate_class_instance(DefaultDoeConfiguration) + default_doe = generate_class_instance(SiteControlGroupDefault) result = DERProgramMapper.doe_program_response(scope, total_does, site_control_group, default_doe) assert result is not None @@ -334,35 +332,54 @@ def test_map_derp_doe_program_response_no_default_doe(scg_type: type[Union[SiteC @pytest.mark.parametrize( - "control_groups_with_counts, total_control_groups, default_doe, fsa_id", + "control_groups_with_counts, total_control_groups, fsa_id", [ - ([], 0, None, None), - ([], 0, None, 123), - ([], 0, generate_class_instance(DefaultSiteControl), None), + ([], 0, None), + ([], 0, 123), + ([], 0, None), ( [ - (generate_class_instance(SiteControlGroup, seed=101), 99), + ( + generate_class_instance( + SiteControlGroup, + seed=101, + site_control_group_default=generate_class_instance(SiteControlGroupDefault), + ), + 99, + ), (generate_class_instance(SiteControlGroup, seed=202, optional_is_none=True), 77), ], 456, None, - None, ), ( [ (generate_class_instance(SiteControlGroup, seed=101), 99), - (generate_class_instance(SiteControlGroup, seed=202, optional_is_none=True), 77), + ( + generate_class_instance( + SiteControlGroup, + seed=202, + optional_is_none=True, + site_control_group_default=generate_class_instance(SiteControlGroupDefault), + ), + 77, + ), ], 456, - generate_class_instance(DefaultSiteControl), None, ), ( [ - (generate_class_instance(SiteControlGroup, seed=101), 11), + ( + generate_class_instance( + SiteControlGroup, + seed=101, + site_control_group_default=generate_class_instance(SiteControlGroupDefault), + ), + 11, + ), ], 789, - generate_class_instance(DefaultSiteControl, optional_is_none=True), None, ), ( @@ -370,7 +387,6 @@ def test_map_derp_doe_program_response_no_default_doe(scg_type: type[Union[SiteC (generate_class_instance(SiteControlGroup, seed=101), 11), ], 789, - generate_class_instance(DefaultSiteControl, optional_is_none=True), 123, ), ], @@ -378,14 +394,13 @@ def test_map_derp_doe_program_response_no_default_doe(scg_type: type[Union[SiteC def test_map_derp_doe_program_list_response( control_groups_with_counts: list[tuple[SiteControlGroup, int]], total_control_groups: int, - default_doe: Optional[DefaultSiteControl], fsa_id: Optional[int], ): """Shows that encoding a list of site_control_groups works with various counts""" scope: DeviceOrAggregatorRequestScope = generate_class_instance(DeviceOrAggregatorRequestScope, href_prefix="/foo") poll_rate = 3 result = DERProgramMapper.doe_program_list_response( - scope, control_groups_with_counts, total_control_groups, default_doe, poll_rate, fsa_id + scope, control_groups_with_counts, total_control_groups, poll_rate, fsa_id ) assert result is not None assert isinstance(result, DERProgramListResponse) @@ -407,7 +422,7 @@ def test_map_derp_doe_program_list_response( assert_list_type(DERProgramResponse, result.DERProgram, len(control_groups_with_counts)) for derp, (group, group_count) in zip(result.DERProgram, control_groups_with_counts): derp.DERControlListLink.all_ == group_count - if default_doe is not None: + if group.site_control_group_default is not None: assert derp.DefaultDERControlLink is not None assert f"/{group.site_control_group_id}" in derp.DefaultDERControlLink.href else: diff --git a/tests/unit/server/mapper/sep2/test_mrid.py b/tests/unit/server/mapper/sep2/test_mrid.py index dbdc6173..1200ae2e 100644 --- a/tests/unit/server/mapper/sep2/test_mrid.py +++ b/tests/unit/server/mapper/sep2/test_mrid.py @@ -19,7 +19,7 @@ decode_mrid_type, encode_mrid, ) -from envoy.server.model.site import DefaultSiteControl +from envoy.server.model.doe import SiteControlGroupDefault from envoy.server.request_scope import BaseRequestScope @@ -149,9 +149,9 @@ def test_all_default_encodings_unique(): scope1 = generate_class_instance(BaseRequestScope, seed=1, iana_pen=0) all_generated_mrids = [] - default_control = generate_class_instance(DefaultSiteControl) + scg_default = generate_class_instance(SiteControlGroupDefault) - assert_and_append_mrid(MridMapper.encode_default_doe_mrid(scope1, default_control), all_generated_mrids) + assert_and_append_mrid(MridMapper.encode_default_doe_mrid(scope1, scg_default), all_generated_mrids) assert_and_append_mrid(MridMapper.encode_doe_program_mrid(scope1, 0, 0), all_generated_mrids) assert_and_append_mrid(MridMapper.encode_doe_mrid(scope1, 0), all_generated_mrids) assert_and_append_mrid(MridMapper.encode_function_set_assignment_mrid(scope1, 0, 0), all_generated_mrids) @@ -171,11 +171,11 @@ def test_encode_default_doe_mrid(): scope1 = generate_class_instance(BaseRequestScope, seed=1, iana_pen=123) scope2 = generate_class_instance(BaseRequestScope, seed=1, iana_pen=456) - default_control = generate_class_instance(DefaultSiteControl) - mrid1 = MridMapper.encode_default_doe_mrid(scope1, default_control) + scg_default = generate_class_instance(SiteControlGroupDefault) + mrid1 = MridMapper.encode_default_doe_mrid(scope1, scg_default) assert_mrid(mrid1) - assert mrid1 == MridMapper.encode_default_doe_mrid(scope1, default_control), "Mrid should be stable" - assert mrid1 != MridMapper.encode_default_doe_mrid(scope2, default_control), "PEN number should affect things" + assert mrid1 == MridMapper.encode_default_doe_mrid(scope1, scg_default), "Mrid should be stable" + assert mrid1 != MridMapper.encode_default_doe_mrid(scope2, scg_default), "PEN number should affect things" assert decode_mrid_type(mrid1) == MridType.DEFAULT_DOE @@ -370,7 +370,7 @@ def do_test(encode: Callable[[BaseRequestScope], str]): # This is using a different scope (pen) and is therefore an error MridMapper.decode_and_validate_mrid_type(scope2, mrid1) - default_control = generate_class_instance(DefaultSiteControl) + default_control = generate_class_instance(SiteControlGroupDefault) do_test(lambda s: MridMapper.encode_default_doe_mrid(s, default_control)) do_test(lambda s: MridMapper.encode_doe_program_mrid(s, 1, 2)) do_test(lambda s: MridMapper.encode_doe_mrid(s, 1)) diff --git a/tests/unit/server/mapper/sep2/test_pub_sub.py b/tests/unit/server/mapper/sep2/test_pub_sub.py index 65b13e0f..28ff39e0 100644 --- a/tests/unit/server/mapper/sep2/test_pub_sub.py +++ b/tests/unit/server/mapper/sep2/test_pub_sub.py @@ -51,15 +51,8 @@ SubscriptionMapper, _map_to_notification_status, ) -from envoy.server.model.doe import DynamicOperatingEnvelope, SiteControlGroup -from envoy.server.model.site import ( - DefaultSiteControl, - Site, - SiteDERAvailability, - SiteDERRating, - SiteDERSetting, - SiteDERStatus, -) +from envoy.server.model.doe import DynamicOperatingEnvelope, SiteControlGroup, SiteControlGroupDefault +from envoy.server.model.site import Site, SiteDERAvailability, SiteDERRating, SiteDERSetting, SiteDERStatus from envoy.server.model.site_reading import SiteReading from envoy.server.model.subscription import Subscription, SubscriptionCondition, SubscriptionResource from envoy.server.model.tariff import TariffGeneratedRate @@ -941,7 +934,7 @@ def test_NotificationMapper_map_function_set_assignments_list_to_response( @pytest.mark.parametrize("notification_type", list(NotificationType)) def test_NotificationMapper_map_default_site_control_response(notification_type: NotificationType): - all_set = generate_class_instance(DefaultSiteControl, seed=1, optional_is_none=False) + all_set = generate_class_instance(SiteControlGroupDefault, seed=1, optional_is_none=False) sub = generate_class_instance(Subscription, seed=303) scope = generate_class_instance(SiteRequestScope, seed=1001, href_prefix="/custom/prefix")