Merge e3430769b29a0767431b8daf488aa0cae3d6a1fb into 6ddbb0361222da276cf0f9b76dd2dff92862327e

Request to raise the defensive version cap for poetry-core from 1.9.1 to
2.1.3.

My understanding is that the major version bump of poetry signals the
transition to standardized pyproject.toml metadata, but does not affect
backwards compatibility.

This is a subset of the changes in #18432

Fixes #18200

### Pull Request Checklist

<!-- Please read
https://element-hq.github.io/synapse/latest/development/contributing_guide.html
before submitting your pull request -->

* [x] Pull request is based on the develop branch
* [x] Pull request includes a [changelog
file](https://element-hq.github.io/synapse/latest/development/contributing_guide.html#changelog).
The entry should:
- Be a short description of your change which makes sense to users.
"Fixed a bug that prevented receiving messages from other servers."
instead of "Moved X method from `EventStore` to `EventWorkerStore`.".
  - Use markdown where necessary, mostly for `code blocks`.
  - End with either a period (.) or an exclamation mark (!).
  - Start with a capital letter.
- Feel free to credit yourself, by adding a sentence "Contributed by
@github_username." or "Contributed by [Your Name]." to the end of the
entry.
* [x] [Code
style](https://element-hq.github.io/synapse/latest/code_style.html) is
correct (run the
[linters](https://element-hq.github.io/synapse/latest/development/contributing_guide.html#run-the-linters))

.github/workflows/release-artifacts.yml

@@ -111,7 +111,7 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        os: [ubuntu-22.04, macos-13]
+        os: [ubuntu-24.04, macos-13]
         arch: [x86_64, aarch64]
         # is_pr is a flag used to exclude certain jobs from the matrix on PRs.
         # It is not read by the rest of the workflow.
@@ -139,7 +139,7 @@ jobs:
           python-version: "3.x"
 
       - name: Install cibuildwheel
-        run: python -m pip install cibuildwheel==2.23.0
+        run: python -m pip install cibuildwheel==3.0.0
 
       - name: Set up QEMU to emulate aarch64
         if: matrix.arch == 'aarch64'

CHANGES.md

@@ -1,3 +1,21 @@
+# Synapse 1.133.0 (2025-07-01)
+
+Pre-built wheels are now built using the [manylinux_2_28](https://github.com/pypa/manylinux#manylinux_2_28-almalinux-8-based) base, which is expected to be compatible with distros using glibc 2.28 or later, including:
+
+ - Debian 10+
+ - Ubuntu 18.10+
+ - Fedora 29+
+ - CentOS/RHEL 8+
+
+Previously, wheels were built using the [manylinux2014](https://github.com/pypa/manylinux#manylinux2014-centos-7-based-glibc-217) base, which was expected to be compatible with distros using glibc 2.17 or later.
+
+### Bugfixes
+
+- Bump `cibuildwheel` to 3.0.0 to fix the `manylinux` wheel builds. ([\#18615](https://github.com/element-hq/synapse/issues/18615))
+
+
+
+
 # Synapse 1.133.0rc1 (2025-06-24)
 
 ### Features

changelog.d/18070.feature

@@ -0,0 +1 @@
+Support for [MSC4235](https://github.com/matrix-org/matrix-spec-proposals/pull/4235): via query param for hierarchy endpoint. Contributed by Krishan (@kfiven).

changelog.d/18238.feature

@@ -0,0 +1 @@
+If enabled by the user, server admins will see [soft failed](https://spec.matrix.org/v1.13/server-server-api/#soft-failure) events over the Client-Server API.

changelog.d/18418.removal

@@ -0,0 +1 @@
+Stop adding the "origin" field to newly-created events (PDUs).

changelog.d/18519.doc

@@ -0,0 +1 @@
+Fix documentation of the Delete Room Admin API's status field.

changelog.d/18575.misc

@@ -0,0 +1 @@
+Raise poetry-core version cap to 2.1.3.

changelog.d/18585.feature

@@ -0,0 +1 @@
+When admins enable themselves to see soft-failed events, they will also see if the cause is due to the policy server flagging them as spam via `unsigned`.

changelog.d/18625.misc

@@ -0,0 +1 @@
+Log the room ID we're purging state for.

contrib/graph/graph.py

@@ -45,6 +45,10 @@ def make_graph(pdus: List[dict], filename_prefix: str) -> None:
     colors = {"red", "green", "blue", "yellow", "purple"}
 
     for pdu in pdus:
+        # TODO: The "origin" field has since been removed from events generated
+        # by Synapse. We should consider removing it here as well but since this
+        # is part of `contrib/`, it is left for the community to revise and ensure things
+        # still work correctly.
         origins.add(pdu.get("origin"))
 
     color_map = {color: color for color in colors if color in origins}

debian/changelog

@@ -1,3 +1,9 @@
+matrix-synapse-py3 (1.133.0) stable; urgency=medium
+
+  * New synapse release 1.133.0.
+
+ -- Synapse Packaging team <packages@matrix.org>  Tue, 01 Jul 2025 13:13:24 +0000
+
 matrix-synapse-py3 (1.133.0~rc1) stable; urgency=medium
 
   * New Synapse release 1.133.0rc1.

docs/SUMMARY.md

@@ -74,6 +74,7 @@
       - [Users](admin_api/user_admin_api.md)
       - [Server Version](admin_api/version_api.md)
       - [Federation](usage/administration/admin_api/federation.md)
+      - [Client-Server API Extensions](admin_api/client_server_api_extensions.md)
     - [Manhole](manhole.md)
     - [Monitoring](metrics-howto.md)
       - [Reporting Homeserver Usage Statistics](usage/administration/monitoring/reporting_homeserver_usage_statistics.md)

docs/admin_api/client_server_api_extensions.md

@@ -0,0 +1,51 @@
+# Client-Server API Extensions
+
+Server administrators can set special account data to change how the Client-Server API behaves for
+their clients. Setting the account data, or having it already set, as a non-admin has no effect.
+
+All configuration options can be set through the `io.element.synapse.admin_client_config` global 
+account data on the admin's user account. 
+
+Example:
+```
+PUT /_matrix/client/v3/user/{adminUserId}/account_data/io.element.synapse.admin_client_config
+{
+    "return_soft_failed_events": true
+}
+```
+
+## See soft failed events
+
+Learn more about soft failure from [the spec](https://spec.matrix.org/v1.14/server-server-api/#soft-failure).
+
+To receive soft failed events in APIs like `/sync` and `/messages`, set `return_soft_failed_events`
+to `true` in the admin client config. When `false`, the normal behaviour of these endpoints is to
+exclude soft failed events.
+
+**Note**: If the policy server flagged the event as spam and that caused soft failure, that will be indicated
+in the event's `unsigned` content like so:
+
+```json
+{
+  "type": "m.room.message",
+  "other": "event_fields_go_here",
+  "unsigned": {
+    "io.element.synapse.soft_failed": true,
+    "io.element.synapse.policy_server_spammy": true
+  }
+}
+```
+
+Default: `false`
+
+## See events marked spammy by policy servers
+
+Learn more about policy servers from [MSC4284](https://github.com/matrix-org/matrix-spec-proposals/pull/4284).
+
+Similar to `return_soft_failed_events`, clients logged in with admin accounts can see events which were
+flagged by the policy server as spammy (and thus soft failed) by setting `return_policy_server_spammy_events`
+to `true`. If `return_soft_failed_events` is `true`, then `return_policy_server_spammy_events` is implied
+`true`. When `false`, the normal behaviour of Client-Server API endpoints is retained (unless `return_soft_failed_events`
+is `true`, per above).
+
+Default: `false`

docs/admin_api/event_reports.md

@@ -117,7 +117,6 @@ It returns a JSON body like the following:
         "hashes": {
             "sha256": "xK1//xnmvHJIOvbgXlkI8eEqdvoMmihVDJ9J4SNlsAw"
         },
-        "origin": "matrix.org",
         "origin_server_ts": 1592291711430,
         "prev_events": [
             "$YK4arsKKcc0LRoe700pS8DSjOvUT4NDv0HfInlMFw2M"

docs/admin_api/rooms.md

@@ -806,7 +806,7 @@ A response body like the following is returned:
         }, {
             "delete_id": "delete_id2",
             "room_id": "!roomid:example.com",
-            "status": "purging",
+            "status": "active",
             "shutdown_room": {
                 "kicked_users": [
                     "@foobar:example.com"
@@ -843,7 +843,7 @@ A response body like the following is returned:
 
 ```json
 {
-    "status": "purging",
+    "status": "active",
     "delete_id": "bHkCNQpHqOaFhPtK",
     "room_id": "!roomid:example.com",
     "shutdown_room": {
@@ -876,8 +876,8 @@ The following fields are returned in the JSON response body:
   - `delete_id` - The ID for this purge
   - `room_id` - The ID of the room being deleted
   - `status` - The status will be one of:
-    - `shutting_down` - The process is removing users from the room.
+    - `scheduled` - The deletion is waiting to be started
-    - `purging` - The process is purging the room and event data from database.
+    - `active` - The process is purging the room and event data from database.
     - `complete` - The process has completed successfully.
     - `failed` - The process is aborted, an error has occurred.
   - `error` - A string that shows an error message if `status` is `failed`.

pyproject.toml

@@ -101,7 +101,7 @@ module-name = "synapse.synapse_rust"
 
 [tool.poetry]
 name = "matrix-synapse"
-version = "1.133.0rc1"
+version = "1.133.0"
 description = "Homeserver for the Matrix decentralised comms protocol"
 authors = ["Matrix.org Team and Contributors <packages@matrix.org>"]
 license = "AGPL-3.0-or-later"
@@ -374,7 +374,7 @@ tomli = ">=1.2.3"
 # runtime errors caused by build system changes.
 # We are happy to raise these upper bounds upon request,
 # provided we check that it's safe to do so (i.e. that CI passes).
-requires = ["poetry-core>=1.1.0,<=1.9.1", "setuptools_rust>=1.3,<=1.10.2"]
+requires = ["poetry-core>=1.1.0,<=2.1.3", "setuptools_rust>=1.3,<=1.10.2"]
 build-backend = "poetry.core.masonry.api"
 
 

rust/src/events/internal_metadata.rs

@@ -54,6 +54,7 @@ enum EventInternalMetadataData {
     RecheckRedaction(bool),
     SoftFailed(bool),
     ProactivelySend(bool),
+    PolicyServerSpammy(bool),
     Redacted(bool),
     TxnId(Box<str>),
     TokenId(i64),
@@ -96,6 +97,13 @@ impl EventInternalMetadataData {
                     .to_owned()
                     .into_any(),
             ),
+            EventInternalMetadataData::PolicyServerSpammy(o) => (
+                pyo3::intern!(py, "policy_server_spammy"),
+                o.into_pyobject(py)
+                    .unwrap_infallible()
+                    .to_owned()
+                    .into_any(),
+            ),
             EventInternalMetadataData::Redacted(o) => (
                 pyo3::intern!(py, "redacted"),
                 o.into_pyobject(py)
@@ -155,6 +163,11 @@ impl EventInternalMetadataData {
                     .extract()
                     .with_context(|| format!("'{key_str}' has invalid type"))?,
             ),
+            "policy_server_spammy" => EventInternalMetadataData::PolicyServerSpammy(
+                value
+                    .extract()
+                    .with_context(|| format!("'{key_str}' has invalid type"))?,
+            ),
             "redacted" => EventInternalMetadataData::Redacted(
                 value
                     .extract()
@@ -427,6 +440,17 @@ impl EventInternalMetadata {
         set_property!(self, ProactivelySend, obj);
     }
 
+    #[getter]
+    fn get_policy_server_spammy(&self) -> PyResult<bool> {
+        Ok(get_property_opt!(self, PolicyServerSpammy)
+            .copied()
+            .unwrap_or(false))
+    }
+    #[setter]
+    fn set_policy_server_spammy(&mut self, obj: bool) {
+        set_property!(self, PolicyServerSpammy, obj);
+    }
+
     #[getter]
     fn get_redacted(&self) -> PyResult<bool> {
         let bool = get_property!(self, Redacted)?;

synapse/api/constants.py

@@ -290,6 +290,9 @@ class AccountDataTypes:
     MSC4155_INVITE_PERMISSION_CONFIG: Final = (
         "org.matrix.msc4155.invite_permission_config"
     )
+    # Synapse-specific behaviour. See "Client-Server API Extensions" documentation
+    # in Admin API for more information.
+    SYNAPSE_ADMIN_CLIENT_CONFIG: Final = "io.element.synapse.admin_client_config"
 
 
 class HistoryVisibility:

synapse/appservice/api.py

@@ -555,6 +555,9 @@ class ApplicationServiceApi(SimpleHttpClient):
                         )
                         and service.is_interested_in_user(e.state_key)
                     ),
+                    # Appservices are considered 'trusted' by the admin and should have
+                    # applicable metadata on their events.
+                    include_admin_metadata=True,
                 ),
             )
             for e in events

synapse/config/experimental.py

@@ -561,6 +561,9 @@ class ExperimentalConfig(Config):
         # MSC4076: Add `disable_badge_count`` to pusher configuration
         self.msc4076_enabled: bool = experimental.get("msc4076_enabled", False)
 
+        # MSC4235: Add `via` param to hierarchy endpoint
+        self.msc4235_enabled: bool = experimental.get("msc4235_enabled", False)
+
         # MSC4263: Preventing MXID enumeration via key queries
         self.msc4263_limit_key_queries_to_users_who_share_rooms = experimental.get(
             "msc4263_limit_key_queries_to_users_who_share_rooms",

synapse/events/__init__.py

@@ -208,7 +208,6 @@ class EventBase(metaclass=abc.ABCMeta):
     depth: DictProperty[int] = DictProperty("depth")
     content: DictProperty[JsonDict] = DictProperty("content")
     hashes: DictProperty[Dict[str, str]] = DictProperty("hashes")
-    origin: DictProperty[str] = DictProperty("origin")
     origin_server_ts: DictProperty[int] = DictProperty("origin_server_ts")
     room_id: DictProperty[str] = DictProperty("room_id")
     sender: DictProperty[str] = DictProperty("sender")

synapse/events/builder.py

@@ -302,8 +302,8 @@ def create_local_event_from_event_dict(
     event_dict: JsonDict,
     internal_metadata_dict: Optional[JsonDict] = None,
 ) -> EventBase:
-    """Takes a fully formed event dict, ensuring that fields like `origin`
+    """Takes a fully formed event dict, ensuring that fields like
-    and `origin_server_ts` have correct values for a locally produced event,
+    `origin_server_ts` have correct values for a locally produced event,
     then signs and hashes it.
     """
 
@@ -319,7 +319,6 @@ def create_local_event_from_event_dict(
     if format_version == EventFormatVersions.ROOM_V1_V2:
         event_dict["event_id"] = _create_event_id(clock, hostname)
 
-    event_dict["origin"] = hostname
     event_dict.setdefault("origin_server_ts", time_now)
 
     event_dict.setdefault("unsigned", {})

synapse/events/utils.py

@@ -421,11 +421,31 @@ class SerializeEventConfig:
     # False, that state will be removed from the event before it is returned.
     # Otherwise, it will be kept.
     include_stripped_room_state: bool = False
+    # When True, sets unsigned flags to help clients identify events which
+    # only server admins can see through other configuration. For example,
+    # whether an event was soft failed by the server.
+    include_admin_metadata: bool = False
+
+    # Developer note: when adding properties, update make_config_for_admin() below.
 
 
 _DEFAULT_SERIALIZE_EVENT_CONFIG = SerializeEventConfig()
 
 
+def make_config_for_admin(existing: SerializeEventConfig) -> SerializeEventConfig:
+    # Developer note: when adding properties, update test_make_serialize_config_for_admin_retains_other_fields
+    return SerializeEventConfig(
+        # Set the options which are only available to server admins
+        include_admin_metadata=True,
+        # And copy the rest
+        as_client_event=existing.as_client_event,
+        event_format=existing.event_format,
+        requester=existing.requester,
+        only_event_fields=existing.only_event_fields,
+        include_stripped_room_state=existing.include_stripped_room_state,
+    )
+
+
 def serialize_event(
     e: Union[JsonDict, EventBase],
     time_now_ms: int,
@@ -528,6 +548,12 @@ def serialize_event(
             d["content"] = dict(d["content"])
             d["content"]["redacts"] = e.redacts
 
+    if config.include_admin_metadata:
+        if e.internal_metadata.is_soft_failed():
+            d["unsigned"]["io.element.synapse.soft_failed"] = True
+        if e.internal_metadata.policy_server_spammy:
+            d["unsigned"]["io.element.synapse.policy_server_spammy"] = True
+
     only_event_fields = config.only_event_fields
     if only_event_fields:
         if not isinstance(only_event_fields, list) or not all(
@@ -576,6 +602,15 @@ class EventClientSerializer:
         if not isinstance(event, EventBase):
             return event
 
+        # Force-enable server admin metadata because the only time an event with
+        # relevant metadata will be when the admin requested it via their admin
+        # client config account data. Also, it's "just" some `unsigned` flags, so
+        # shouldn't cause much in terms of problems to downstream consumers.
+        if config.requester is not None and await self._store.is_server_admin(
+            config.requester.user
+        ):
+            config = make_config_for_admin(config)
+
         serialized_event = serialize_event(event, time_now, config=config)
 
         new_unsigned = {}

synapse/events/validator.py

@@ -67,7 +67,6 @@ class EventValidator:
             "auth_events",
             "content",
             "hashes",
-            "origin",
             "prev_events",
             "sender",
             "type",
@@ -77,13 +76,6 @@ class EventValidator:
             if k not in event:
                 raise SynapseError(400, "Event does not have key %s" % (k,))
 
-        # Check that the following keys have string values
-        event_strings = ["origin"]
-
-        for s in event_strings:
-            if not isinstance(getattr(event, s), str):
-                raise SynapseError(400, "'%s' not a string type" % (s,))
-
         # Depending on the room version, ensure the data is spec compliant JSON.
         if event.room_version.strict_canonicaljson:
             validate_canonicaljson(event.get_pdu_json())

synapse/federation/federation_base.py

@@ -174,6 +174,7 @@ class FederationBase:
                 "Event not allowed by policy server, soft-failing %s", pdu.event_id
             )
             pdu.internal_metadata.soft_failed = True
+            pdu.internal_metadata.policy_server_spammy = True
             # Note: we don't redact the event so admins can inspect the event after the
             # fact. Other processes may redact the event, but that won't be applied to
             # the database copy of the event until the server's config requires it.
@@ -322,8 +323,7 @@ def event_from_pdu_json(pdu_json: JsonDict, room_version: RoomVersion) -> EventB
         SynapseError: if the pdu is missing required fields or is otherwise
             not a valid matrix event
     """
-    # we could probably enforce a bunch of other fields here (room_id, sender,
+    # we could probably enforce a bunch of other fields here (room_id, sender, etc.)
-    # origin, etc etc)
     assert_params_in_dict(pdu_json, ("type", "depth"))
 
     # Strip any unauthorized values from "unsigned" if they exist

synapse/handlers/message.py

@@ -1111,6 +1111,9 @@ class EventCreationHandler:
 
                 policy_allowed = await self._policy_handler.is_event_allowed(event)
                 if not policy_allowed:
+                    # We shouldn't need to set the metadata because the raise should
+                    # cause the request to be denied, but just in case:
+                    event.internal_metadata.policy_server_spammy = True
                     logger.warning(
                         "Event not allowed by policy server, rejecting %s",
                         event.event_id,

synapse/handlers/room_summary.py

@@ -111,7 +111,15 @@ class RoomSummaryHandler:
         # If a user tries to fetch the same page multiple times in quick succession,
         # only process the first attempt and return its result to subsequent requests.
         self._pagination_response_cache: ResponseCache[
-            Tuple[str, str, bool, Optional[int], Optional[int], Optional[str]]
+            Tuple[
+                str,
+                str,
+                bool,
+                Optional[int],
+                Optional[int],
+                Optional[str],
+                Optional[Tuple[str, ...]],
+            ]
         ] = ResponseCache(
             hs.get_clock(),
             "get_room_hierarchy",
@@ -126,6 +134,7 @@ class RoomSummaryHandler:
         max_depth: Optional[int] = None,
         limit: Optional[int] = None,
         from_token: Optional[str] = None,
+        remote_room_hosts: Optional[Tuple[str, ...]] = None,
     ) -> JsonDict:
         """
         Implementation of the room hierarchy C-S API.
@@ -143,6 +152,9 @@ class RoomSummaryHandler:
             limit: An optional limit on the number of rooms to return per
                 page. Must be a positive integer.
             from_token: An optional pagination token.
+            remote_room_hosts: An optional list of remote homeserver server names. If defined,
+                each host will be used to try and fetch the room hierarchy. Must be a tuple so
+                that it can be hashed by the `RoomSummaryHandler._pagination_response_cache`.
 
         Returns:
             The JSON hierarchy dictionary.
@@ -162,6 +174,7 @@ class RoomSummaryHandler:
                 max_depth,
                 limit,
                 from_token,
+                remote_room_hosts,
             ),
             self._get_room_hierarchy,
             requester.user.to_string(),
@@ -170,6 +183,7 @@ class RoomSummaryHandler:
             max_depth,
             limit,
             from_token,
+            remote_room_hosts,
         )
 
     async def _get_room_hierarchy(
@@ -180,6 +194,7 @@ class RoomSummaryHandler:
         max_depth: Optional[int] = None,
         limit: Optional[int] = None,
         from_token: Optional[str] = None,
+        remote_room_hosts: Optional[Tuple[str, ...]] = None,
     ) -> JsonDict:
         """See docstring for SpaceSummaryHandler.get_room_hierarchy."""
 
@@ -199,7 +214,7 @@ class RoomSummaryHandler:
 
         if not local_room:
             room_hierarchy = await self._summarize_remote_room_hierarchy(
-                _RoomQueueEntry(requested_room_id, ()),
+                _RoomQueueEntry(requested_room_id, remote_room_hosts or ()),
                 False,
             )
             root_room_entry = room_hierarchy[0]
@@ -240,7 +255,7 @@ class RoomSummaryHandler:
             processed_rooms = set(pagination_session["processed_rooms"])
         else:
             # The queue of rooms to process, the next room is last on the stack.
-            room_queue = [_RoomQueueEntry(requested_room_id, ())]
+            room_queue = [_RoomQueueEntry(requested_room_id, remote_room_hosts or ())]
 
             # Rooms we have already processed.
             processed_rooms = set()

synapse/rest/client/room.py

@@ -1538,6 +1538,7 @@ class RoomHierarchyRestServlet(RestServlet):
         super().__init__()
         self._auth = hs.get_auth()
         self._room_summary_handler = hs.get_room_summary_handler()
+        self.msc4235_enabled = hs.config.experimental.msc4235_enabled
 
     async def on_GET(
         self, request: SynapseRequest, room_id: str
@@ -1547,6 +1548,15 @@ class RoomHierarchyRestServlet(RestServlet):
         max_depth = parse_integer(request, "max_depth")
         limit = parse_integer(request, "limit")
 
+        # twisted.web.server.Request.args is incorrectly defined as Optional[Any]
+        remote_room_hosts = None
+        if self.msc4235_enabled:
+            args: Dict[bytes, List[bytes]] = request.args  # type: ignore
+            via_param = parse_strings_from_args(
+                args, "org.matrix.msc4235.via", required=False
+            )
+            remote_room_hosts = tuple(via_param or [])
+
         return 200, await self._room_summary_handler.get_room_hierarchy(
             requester,
             room_id,
@@ -1554,6 +1564,7 @@ class RoomHierarchyRestServlet(RestServlet):
             max_depth=max_depth,
             limit=limit,
             from_token=parse_string(request, "from"),
+            remote_room_hosts=remote_room_hosts,
         )
 
 

synapse/storage/admin_client_config.py

@@ -0,0 +1,22 @@
+import logging
+from typing import Optional
+
+from synapse.types import JsonMapping
+
+logger = logging.getLogger(__name__)
+
+
+class AdminClientConfig:
+    """Class to track various Synapse-specific admin-only client-impacting config options."""
+
+    def __init__(self, account_data: Optional[JsonMapping]):
+        self.return_soft_failed_events: bool = False
+        self.return_policy_server_spammy_events: bool = False
+
+        if account_data:
+            self.return_soft_failed_events = account_data.get(
+                "return_soft_failed_events", False
+            )
+            self.return_policy_server_spammy_events = account_data.get(
+                "return_policy_server_spammy_events", False
+            )

synapse/storage/controllers/purge_events.py

@@ -34,6 +34,7 @@ from synapse.metrics.background_process_metrics import wrap_as_background_proces
 from synapse.storage.database import LoggingTransaction
 from synapse.storage.databases import Databases
 from synapse.types.storage import _BackgroundUpdates
+from synapse.util.stringutils import shortstr
 
 if TYPE_CHECKING:
     from synapse.server import HomeServer
@@ -167,6 +168,12 @@ class PurgeEventsStorageController:
                 break
 
             (room_id, groups_to_sequences) = next_to_delete
+
+            logger.info(
+                "[purge] deleting state groups for room %s: %s",
+                room_id,
+                shortstr(groups_to_sequences.keys(), maxitems=10),
+            )
             made_progress = await self._delete_state_groups(
                 room_id, groups_to_sequences
             )

synapse/storage/databases/main/account_data.py

@@ -37,6 +37,7 @@ from synapse.api.constants import AccountDataTypes
 from synapse.api.errors import Codes, SynapseError
 from synapse.replication.tcp.streams import AccountDataStream
 from synapse.storage._base import db_to_json
+from synapse.storage.admin_client_config import AdminClientConfig
 from synapse.storage.database import (
     DatabasePool,
     LoggingDatabaseConnection,
@@ -578,6 +579,21 @@ class AccountDataWorkerStore(PushRulesWorkerStore, CacheInvalidationWorkerStore)
         )
         return InviteRulesConfig(data)
 
+    async def get_admin_client_config_for_user(self, user_id: str) -> AdminClientConfig:
+        """
+        Get the admin client configuration for the specified user.
+
+        The admin client config contains Synapse-specific settings that clients running
+        server admin accounts can use. They have no effect on non-admin users.
+
+        Args:
+            user_id: The user ID to get config for.
+        """
+        data = await self.get_global_account_data_by_type_for_user(
+            user_id, AccountDataTypes.SYNAPSE_ADMIN_CLIENT_CONFIG
+        )
+        return AdminClientConfig(data)
+
     def process_replication_rows(
         self,
         stream_name: str,

synapse/synapse_rust/events.pyi

@@ -33,6 +33,9 @@ class EventInternalMetadata:
     proactively_send: bool
     redacted: bool
 
+    policy_server_spammy: bool
+    """whether the policy server indicated that this event is spammy"""
+
     txn_id: str
     """The transaction ID, if it was set when the event was created."""
     token_id: int

synapse/util/caches/__init__.py

@@ -225,7 +225,7 @@ KNOWN_KEYS = {
         "depth",
         "event_id",
         "hashes",
-        "origin",
+        "origin",  # old events were created with an origin field.
         "origin_server_ts",
         "prev_events",
         "room_id",

synapse/visibility.py

@@ -48,7 +48,13 @@ from synapse.logging.opentracing import trace
 from synapse.storage.controllers import StorageControllers
 from synapse.storage.databases.main import DataStore
 from synapse.synapse_rust.events import event_visible_to_server
-from synapse.types import RetentionPolicy, StateMap, StrCollection, get_domain_from_id
+from synapse.types import (
+    RetentionPolicy,
+    StateMap,
+    StrCollection,
+    UserID,
+    get_domain_from_id,
+)
 from synapse.types.state import StateFilter
 from synapse.util import Clock
 
@@ -106,9 +112,30 @@ async def filter_events_for_client(
         of `user_id` at each event.
     """
     # Filter out events that have been soft failed so that we don't relay them
-    # to clients.
+    # to clients, unless they're a server admin and want that to happen.
     events_before_filtering = events
-    events = [e for e in events if not e.internal_metadata.is_soft_failed()]
+    client_config = await storage.main.get_admin_client_config_for_user(user_id)
+    if (
+        filter_send_to_client
+        and (
+            client_config.return_soft_failed_events
+            or client_config.return_policy_server_spammy_events
+        )
+        and await storage.main.is_server_admin(UserID.from_string(user_id))
+    ):
+        # `return_soft_failed_events` implies `return_policy_server_spammy_events`, so
+        # we want to check when they've asked for *just* `return_policy_server_spammy_events`
+        if not client_config.return_soft_failed_events:
+            events = [
+                e
+                for e in events
+                if not e.internal_metadata.is_soft_failed()
+                or e.internal_metadata.policy_server_spammy
+            ]
+        else:
+            events = events_before_filtering
+    else:
+        events = [e for e in events if not e.internal_metadata.is_soft_failed()]
     if len(events_before_filtering) != len(events):
         if filtered_event_logger.isEnabledFor(logging.DEBUG):
             filtered_event_logger.debug(

tests/crypto/test_event_signing.py

@@ -48,7 +48,6 @@ class EventSigningTestCase(unittest.TestCase):
     def test_sign_minimal(self) -> None:
         event_dict = {
             "event_id": "$0:domain",
-            "origin": "domain",
             "origin_server_ts": 1000000,
             "signatures": {},
             "type": "X",
@@ -64,7 +63,7 @@ class EventSigningTestCase(unittest.TestCase):
         self.assertTrue(hasattr(event, "hashes"))
         self.assertIn("sha256", event.hashes)
         self.assertEqual(
-            event.hashes["sha256"], "6tJjLpXtggfke8UxFhAKg82QVkJzvKOVOOSjUDK4ZSI"
+            event.hashes["sha256"], "A6Nco6sqoy18PPfPDVdYvoowfc0PVBk9g9OiyT3ncRM"
         )
 
         self.assertTrue(hasattr(event, "signatures"))
@@ -72,15 +71,14 @@ class EventSigningTestCase(unittest.TestCase):
         self.assertIn(KEY_NAME, event.signatures["domain"])
         self.assertEqual(
             event.signatures[HOSTNAME][KEY_NAME],
-            "2Wptgo4CwmLo/Y8B8qinxApKaCkBG2fjTWB7AbP5Uy+"
+            "PBc48yDVszWB9TRaB/+CZC1B+pDAC10F8zll006j+NN"
-            "aIbygsSdLOFzvdDjww8zUVKCmI02eP9xtyJxc/cLiBA",
+            "fe4PEMWcVuLaG63LFTK9e4rwJE8iLZMPtCKhDTXhpAQ",
         )
 
     def test_sign_message(self) -> None:
         event_dict = {
             "content": {"body": "Here is the message content"},
             "event_id": "$0:domain",
-            "origin": "domain",
             "origin_server_ts": 1000000,
             "type": "m.room.message",
             "room_id": "!r:domain",
@@ -98,7 +96,7 @@ class EventSigningTestCase(unittest.TestCase):
         self.assertTrue(hasattr(event, "hashes"))
         self.assertIn("sha256", event.hashes)
         self.assertEqual(
-            event.hashes["sha256"], "onLKD1bGljeBWQhWZ1kaP9SorVmRQNdN5aM2JYU2n/g"
+            event.hashes["sha256"], "rDCeYBepPlI891h/RkI2/Lkf9bt7u0TxFku4tMs7WKk"
         )
 
         self.assertTrue(hasattr(event, "signatures"))
@@ -106,6 +104,6 @@ class EventSigningTestCase(unittest.TestCase):
         self.assertIn(KEY_NAME, event.signatures["domain"])
         self.assertEqual(
             event.signatures[HOSTNAME][KEY_NAME],
-            "Wm+VzmOUOz08Ds+0NTWb1d4CZrVsJSikkeRxh6aCcUw"
+            "Ay4aj2b5oJ1k8INYZ9n3KnszCflM0emwcmQQ7vxpbdc"
-            "u6pNC78FunoD7KNWzqFn241eYHYMGCA5McEiVPdhzBA",
+            "Sv9bkJxIZdWX1IJllcZLq89+D3sSabE+vqPtZs9akDw",
         )

tests/events/test_utils.py

@@ -34,11 +34,13 @@ from synapse.events.utils import (
     _split_field,
     clone_event,
     copy_and_fixup_power_levels_contents,
+    format_event_raw,
+    make_config_for_admin,
     maybe_upsert_event_field,
     prune_event,
     serialize_event,
 )
-from synapse.types import JsonDict
+from synapse.types import JsonDict, create_requester
 from synapse.util.frozenutils import freeze
 
 
@@ -49,7 +51,13 @@ def MockEvent(**kwargs: Any) -> EventBase:
         kwargs["type"] = "fake_type"
     if "content" not in kwargs:
         kwargs["content"] = {}
-    return make_event_from_dict(kwargs)
+
+    # Move internal metadata out so we can call make_event properly
+    internal_metadata = kwargs.get("internal_metadata")
+    if internal_metadata is not None:
+        kwargs.pop("internal_metadata")
+
+    return make_event_from_dict(kwargs, internal_metadata_dict=internal_metadata)
 
 
 class TestMaybeUpsertEventField(stdlib_unittest.TestCase):
@@ -122,7 +130,7 @@ class PruneEventTestCase(stdlib_unittest.TestCase):
                 "prev_events": "prev_events",
                 "prev_state": "prev_state",
                 "auth_events": "auth_events",
-                "origin": "domain",
+                "origin": "domain",  # historical top-level field that still exists on old events
                 "origin_server_ts": 1234,
                 "membership": "join",
                 # Also include a key that should be removed.
@@ -139,7 +147,7 @@ class PruneEventTestCase(stdlib_unittest.TestCase):
                 "prev_events": "prev_events",
                 "prev_state": "prev_state",
                 "auth_events": "auth_events",
-                "origin": "domain",
+                "origin": "domain",  # historical top-level field that still exists on old events
                 "origin_server_ts": 1234,
                 "membership": "join",
                 "content": {},
@@ -148,13 +156,12 @@ class PruneEventTestCase(stdlib_unittest.TestCase):
             },
         )
 
-        # As of room versions we now redact the membership, prev_states, and origin keys.
+        # As of room versions we now redact the membership and prev_states keys.
         self.run_test(
             {
                 "type": "A",
                 "prev_state": "prev_state",
                 "membership": "join",
-                "origin": "example.com",
             },
             {"type": "A", "content": {}, "signatures": {}, "unsigned": {}},
             room_version=RoomVersions.V11,
@@ -238,7 +245,6 @@ class PruneEventTestCase(stdlib_unittest.TestCase):
             {
                 "type": "m.room.create",
                 "content": {"not_a_real_key": True},
-                "origin": "some_homeserver",
                 "nonsense_field": "some_random_garbage",
             },
             {
@@ -639,9 +645,18 @@ class CloneEventTestCase(stdlib_unittest.TestCase):
 
 
 class SerializeEventTestCase(stdlib_unittest.TestCase):
-    def serialize(self, ev: EventBase, fields: Optional[List[str]]) -> JsonDict:
+    def serialize(
+        self,
+        ev: EventBase,
+        fields: Optional[List[str]],
+        include_admin_metadata: bool = False,
+    ) -> JsonDict:
         return serialize_event(
-            ev, 1479807801915, config=SerializeEventConfig(only_event_fields=fields)
+            ev,
+            1479807801915,
+            config=SerializeEventConfig(
+                only_event_fields=fields, include_admin_metadata=include_admin_metadata
+            ),
         )
 
     def test_event_fields_works_with_keys(self) -> None:
@@ -760,6 +775,78 @@ class SerializeEventTestCase(stdlib_unittest.TestCase):
                 ["room_id", 4],  # type: ignore[list-item]
             )
 
+    def test_default_serialize_config_excludes_admin_metadata(self) -> None:
+        # We just really don't want this to be set to True accidentally
+        self.assertFalse(SerializeEventConfig().include_admin_metadata)
+
+    def test_event_flagged_for_admins(self) -> None:
+        # Default behaviour should be *not* to include it
+        self.assertEqual(
+            self.serialize(
+                MockEvent(
+                    type="foo",
+                    event_id="test",
+                    room_id="!foo:bar",
+                    content={"foo": "bar"},
+                    internal_metadata={"soft_failed": True},
+                ),
+                [],
+            ),
+            {
+                "type": "foo",
+                "event_id": "test",
+                "room_id": "!foo:bar",
+                "content": {"foo": "bar"},
+                "unsigned": {},
+            },
+        )
+
+        # When asked though, we should set it
+        self.assertEqual(
+            self.serialize(
+                MockEvent(
+                    type="foo",
+                    event_id="test",
+                    room_id="!foo:bar",
+                    content={"foo": "bar"},
+                    internal_metadata={"soft_failed": True},
+                ),
+                [],
+                True,
+            ),
+            {
+                "type": "foo",
+                "event_id": "test",
+                "room_id": "!foo:bar",
+                "content": {"foo": "bar"},
+                "unsigned": {"io.element.synapse.soft_failed": True},
+            },
+        )
+
+    def test_make_serialize_config_for_admin_retains_other_fields(self) -> None:
+        non_default_config = SerializeEventConfig(
+            include_admin_metadata=False,  # should be True in a moment
+            as_client_event=False,  # default True
+            event_format=format_event_raw,  # default format_event_for_client_v1
+            requester=create_requester("@example:example.org"),  # default None
+            only_event_fields=["foo"],  # default None
+            include_stripped_room_state=True,  # default False
+        )
+        admin_config = make_config_for_admin(non_default_config)
+        self.assertEqual(
+            admin_config.as_client_event, non_default_config.as_client_event
+        )
+        self.assertEqual(admin_config.event_format, non_default_config.event_format)
+        self.assertEqual(admin_config.requester, non_default_config.requester)
+        self.assertEqual(
+            admin_config.only_event_fields, non_default_config.only_event_fields
+        )
+        self.assertEqual(
+            admin_config.include_stripped_room_state,
+            admin_config.include_stripped_room_state,
+        )
+        self.assertTrue(admin_config.include_admin_metadata)
+
 
 class CopyPowerLevelsContentTestCase(stdlib_unittest.TestCase):
     def setUp(self) -> None:

tests/federation/test_federation_server.py

@@ -535,7 +535,6 @@ class StripUnsignedFromEventsTestCase(unittest.TestCase):
             "depth": 1000,
             "origin_server_ts": 1,
             "type": "m.room.member",
-            "origin": "test.servx",
             "content": {"membership": "join"},
             "auth_events": [],
             "unsigned": {"malicious garbage": "hackz", "more warez": "more hackz"},
@@ -552,7 +551,6 @@ class StripUnsignedFromEventsTestCase(unittest.TestCase):
             "depth": 1000,
             "origin_server_ts": 1,
             "type": "m.room.member",
-            "origin": "test.servx",
             "auth_events": [],
             "content": {"membership": "join"},
             "unsigned": {
@@ -579,7 +577,6 @@ class StripUnsignedFromEventsTestCase(unittest.TestCase):
             "depth": 1000,
             "origin_server_ts": 1,
             "type": "m.room.power_levels",
-            "origin": "test.servx",
             "content": {},
             "auth_events": [],
             "unsigned": {

tests/handlers/test_room_summary.py

@@ -1080,6 +1080,62 @@ class SpaceSummaryTestCase(unittest.HomeserverTestCase):
             self.assertEqual(federation_requests, 2)
             self._assert_hierarchy(result, expected)
 
+    def test_fed_remote_room_hosts(self) -> None:
+        """
+        Test if requested room is available over federation using via's.
+        """
+        fed_hostname = self.hs.hostname + "2"
+        fed_space = "#fed_space:" + fed_hostname
+        fed_subroom = "#fed_sub_room:" + fed_hostname
+
+        remote_room_hosts = tuple(fed_hostname)
+
+        requested_room_entry = _RoomEntry(
+            fed_space,
+            {
+                "room_id": fed_space,
+                "world_readable": True,
+                "join_rule": "public",
+                "room_type": RoomTypes.SPACE,
+            },
+            [
+                {
+                    "type": EventTypes.SpaceChild,
+                    "room_id": fed_space,
+                    "state_key": fed_subroom,
+                    "content": {"via": [fed_hostname]},
+                }
+            ],
+        )
+        child_room = {
+            "room_id": fed_subroom,
+            "world_readable": True,
+            "join_rule": "public",
+        }
+
+        async def summarize_remote_room_hierarchy(
+            _self: Any, room: Any, suggested_only: bool
+        ) -> Tuple[Optional[_RoomEntry], Dict[str, JsonDict], Set[str]]:
+            return requested_room_entry, {fed_subroom: child_room}, set()
+
+        expected = [
+            (fed_space, [fed_subroom]),
+            (fed_subroom, ()),
+        ]
+
+        with mock.patch(
+            "synapse.handlers.room_summary.RoomSummaryHandler._summarize_remote_room_hierarchy",
+            new=summarize_remote_room_hierarchy,
+        ):
+            result = self.get_success(
+                self.handler.get_room_hierarchy(
+                    create_requester(self.user),
+                    fed_space,
+                    remote_room_hosts=remote_room_hosts,
+                )
+            )
+        self._assert_hierarchy(result, expected)
+
 
 class RoomSummaryTestCase(unittest.HomeserverTestCase):
     servlets = [

tests/rest/client/test_relations.py

@@ -1181,7 +1181,7 @@ class BundledAggregationsTestCase(BaseRelationsTestCase):
                 bundled_aggregations,
             )
 
-        self._test_bundled_aggregations(RelationTypes.REFERENCE, assert_annotations, 6)
+        self._test_bundled_aggregations(RelationTypes.REFERENCE, assert_annotations, 8)
 
     def test_thread(self) -> None:
         """
@@ -1226,21 +1226,21 @@ class BundledAggregationsTestCase(BaseRelationsTestCase):
 
         # The "user" sent the root event and is making queries for the bundled
         # aggregations: they have participated.
-        self._test_bundled_aggregations(RelationTypes.THREAD, _gen_assert(True), 6)
+        self._test_bundled_aggregations(RelationTypes.THREAD, _gen_assert(True), 9)
         # The "user2" sent replies in the thread and is making queries for the
         # bundled aggregations: they have participated.
         #
         # Note that this re-uses some cached values, so the total number of
         # queries is much smaller.
         self._test_bundled_aggregations(
-            RelationTypes.THREAD, _gen_assert(True), 3, access_token=self.user2_token
+            RelationTypes.THREAD, _gen_assert(True), 6, access_token=self.user2_token
         )
 
         # A user with no interactions with the thread: they have not participated.
         user3_id, user3_token = self._create_user("charlie")
         self.helper.join(self.room, user=user3_id, tok=user3_token)
         self._test_bundled_aggregations(
-            RelationTypes.THREAD, _gen_assert(False), 3, access_token=user3_token
+            RelationTypes.THREAD, _gen_assert(False), 6, access_token=user3_token
         )
 
     def test_thread_with_bundled_aggregations_for_latest(self) -> None:
@@ -1287,7 +1287,7 @@ class BundledAggregationsTestCase(BaseRelationsTestCase):
                 bundled_aggregations["latest_event"].get("unsigned"),
             )
 
-        self._test_bundled_aggregations(RelationTypes.THREAD, assert_thread, 6)
+        self._test_bundled_aggregations(RelationTypes.THREAD, assert_thread, 9)
 
     def test_nested_thread(self) -> None:
         """