Allow admins to see policy server-flagged events

Author: turt2live

changelog.d/18585.feature

@@ -0,0 +1 @@
+When admins enable themselves to see soft-failed events, they will also see if the cause is due to the policy server flagging them as spam via `unsigned`.

docs/admin_api/client_server_api_extensions.md

@@ -22,4 +22,30 @@ To receive soft failed events in APIs like `/sync` and `/messages`, set `return_
 to `true` in the admin client config. When `false`, the normal behaviour of these endpoints is to
 exclude soft failed events.
 
+**Note**: If the policy server flagged the event as spam and that caused soft failure, that will be indicated
+in the event's `unsigned` content like so:
+
+```json
+{
+  "type": "m.room.message",
+  "other": "event_fields_go_here",
+  "unsigned": {
+    "io.element.synapse.soft_failed": true,
+    "io.element.synapse.policy_server_spammy": true
+  }
+}
+```
+
+Default: `false`
+
+## See events marked spammy by policy servers
+
+Learn more about policy servers from [MSC4284](https://github.com/matrix-org/matrix-spec-proposals/pull/4284).
+
+Similar to `return_soft_failed_events`, clients logged in with admin accounts can see events which were
+flagged by the policy server as spammy (and thus soft failed) by setting `return_policy_server_spammy_events`
+to `true`. If `return_soft_failed_events` is `true`, then `return_policy_server_spammy_events` is implied
+`true`. When `false`, the normal behaviour of Client-Server API endpoints is retained (unless `return_soft_failed_events`
+is `true`, per above).
+
 Default: `false`

rust/src/events/internal_metadata.rs

@@ -54,6 +54,7 @@ enum EventInternalMetadataData {
     RecheckRedaction(bool),
     SoftFailed(bool),
     ProactivelySend(bool),
+    PolicyServerSpammy(bool),
     Redacted(bool),
     TxnId(Box<str>),
     TokenId(i64),
@@ -96,6 +97,13 @@ impl EventInternalMetadataData {
                     .to_owned()
                     .into_any(),
             ),
+            EventInternalMetadataData::PolicyServerSpammy(o) => (
+                pyo3::intern!(py, "policy_server_spammy"),
+                o.into_pyobject(py)
+                    .unwrap_infallible()
+                    .to_owned()
+                    .into_any(),
+            ),
             EventInternalMetadataData::Redacted(o) => (
                 pyo3::intern!(py, "redacted"),
                 o.into_pyobject(py)
@@ -155,6 +163,11 @@ impl EventInternalMetadataData {
                     .extract()
                     .with_context(|| format!("'{key_str}' has invalid type"))?,
             ),
+            "policy_server_spammy" => EventInternalMetadataData::PolicyServerSpammy(
+                value
+                    .extract()
+                    .with_context(|| format!("'{key_str}' has invalid type"))?,
+            ),
             "redacted" => EventInternalMetadataData::Redacted(
                 value
                     .extract()
@@ -427,6 +440,17 @@ impl EventInternalMetadata {
         set_property!(self, ProactivelySend, obj);
     }
 
+    #[getter]
+    fn get_policy_server_spammy(&self) -> PyResult<bool> {
+        Ok(get_property_opt!(self, PolicyServerSpammy)
+            .copied()
+            .unwrap_or(false))
+    }
+    #[setter]
+    fn set_policy_server_spammy(&mut self, obj: bool) {
+        set_property!(self, PolicyServerSpammy, obj);
+    }
+
     #[getter]
     fn get_redacted(&self) -> PyResult<bool> {
         let bool = get_property!(self, Redacted)?;

synapse/events/utils.py

@@ -538,8 +538,11 @@ def serialize_event(
             d["content"] = dict(d["content"])
             d["content"]["redacts"] = e.redacts
 
-    if config.include_admin_metadata and e.internal_metadata.is_soft_failed():
-        d["unsigned"]["io.element.synapse.soft_failed"] = True
+    if config.include_admin_metadata:
+        if e.internal_metadata.is_soft_failed():
+            d["unsigned"]["io.element.synapse.soft_failed"] = True
+        if e.internal_metadata.policy_server_spammy:
+            d["unsigned"]["io.element.synapse.policy_server_spammy"] = True
 
     only_event_fields = config.only_event_fields
     if only_event_fields:

synapse/federation/federation_base.py

@@ -174,6 +174,7 @@ async def _check_sigs_and_hash(
                 "Event not allowed by policy server, soft-failing %s", pdu.event_id
             )
             pdu.internal_metadata.soft_failed = True
+            pdu.internal_metadata.policy_server_spammy = True
             # Note: we don't redact the event so admins can inspect the event after the
             # fact. Other processes may redact the event, but that won't be applied to
             # the database copy of the event until the server's config requires it.

synapse/handlers/message.py

@@ -1111,6 +1111,9 @@ async def _create_and_send_nonmember_event_locked(
 
                 policy_allowed = await self._policy_handler.is_event_allowed(event)
                 if not policy_allowed:
+                    # We shouldn't need to set the metadata because the raise should
+                    # cause the request to be denied, but just in case:
+                    event.internal_metadata.policy_server_spammy = True
                     logger.warning(
                         "Event not allowed by policy server, rejecting %s",
                         event.event_id,

synapse/storage/admin_client_config.py

@@ -15,8 +15,12 @@ def __init__(self, account_data: Optional[JsonMapping]):
         # `unsigned` portion of the event to inform clients that the event
         # is soft-failed.
         self.return_soft_failed_events: bool = False
+        self.return_policy_server_spammy_events: bool = False
 
         if account_data:
             self.return_soft_failed_events = account_data.get(
                 "return_soft_failed_events", False
             )
+            self.return_policy_server_spammy_events = account_data.get(
+                "return_policy_server_spammy_events", False
+            )

synapse/synapse_rust/events.pyi

@@ -33,6 +33,9 @@ class EventInternalMetadata:
     proactively_send: bool
     redacted: bool
 
+    policy_server_spammy: bool
+    """whether the policy server indicated that this event is spammy"""
+
     txn_id: str
     """The transaction ID, if it was set when the event was created."""
     token_id: int

synapse/visibility.py

@@ -120,10 +120,26 @@ async def filter_events_for_client(
     client_config = await storage.main.get_admin_client_config_for_user(user_id)
     if not (
         filter_send_to_client
-        and client_config.return_soft_failed_events
+        and (
+            client_config.return_soft_failed_events
+            or client_config.return_policy_server_spammy_events
+        )
         and await storage.main.is_server_admin(UserID.from_string(user_id))
     ):
-        events = [e for e in events if not e.internal_metadata.is_soft_failed()]
+        # `return_soft_failed_events` implies `return_policy_server_spammy_events`, so
+        # we want to check when they've asked for *just* `return_policy_server_spammy_events`
+        if not client_config.return_soft_failed_events:
+            events = [
+                e
+                for e in events
+                # Return non-soft-failed events as well as those explicitly marked
+                # as spam by a policy server. This excludes events that were soft
+                # failed by other means.
+                if not e.internal_metadata.is_soft_failed()
+                or e.internal_metadata.policy_server_spammy
+            ]
+        else:
+            events = [e for e in events if not e.internal_metadata.is_soft_failed()]
     if len(events_before_filtering) != len(events):
         if filtered_event_logger.isEnabledFor(logging.DEBUG):
             filtered_event_logger.debug(