Merge branch 'master' into daniel/multipleexamples

Conflicts:
	specification/modules/third_party_invites.rst

api/client-server/v1/definitions/push_rule.json

@@ -1,4 +1,5 @@
 {
+  "title": "PushRule",
   "type": "object", 
   "properties": {
     "default": {
@@ -17,4 +18,4 @@
       "type": "array"
     }
   }
-}
+}

api/client-server/v1/definitions/push_ruleset.json

@@ -1,60 +1,65 @@
 {
-  "type": "object", 
+  "type": "object",
   "properties": {
     "content": {
       "items": {
-        "type": "object", 
+        "type": "object",
+        "title": "PushRule",
         "allOf": [
           {
             "$ref": "push_rule.json"
           }
         ]
-      }, 
+      },
       "type": "array"
-    }, 
+    },
     "override": {
       "items": {
-        "type": "object", 
+        "type": "object",
+        "title": "PushRule",
         "allOf": [
           {
             "$ref": "push_rule.json"
           }
         ]
-      }, 
+      },
       "type": "array"
-    }, 
+    },
     "sender": {
       "items": {
-        "type": "object", 
+        "type": "object",
+        "title": "PushRule",
         "allOf": [
           {
             "$ref": "push_rule.json"
           }
         ]
-      }, 
+      },
       "type": "array"
-    }, 
+    },
     "underride": {
       "items": {
-        "type": "object", 
+        "type": "object",
+        "title": "PushRule",
         "allOf": [
           {
             "$ref": "push_rule.json"
           }
         ]
-      }, 
+      },
       "type": "array"
-    }, 
+    },
     "room": {
       "items": {
-        "type": "object", 
+        "type": "object",
+        "title": "PushRule",
         "allOf": [
           {
             "$ref": "push_rule.json"
           }
         ]
-      }, 
+      },
       "type": "array"
     }
   }
-}
+}

api/client-server/v1/membership.yaml

@@ -73,9 +73,12 @@ paths:
     post:
       summary: Invite a user to participate in a particular room.
       description: |-
+        .. _invite-by-user-id-endpoint:
+
         *Note that there are two forms of this API, which are documented separately.
         This version of the API requires that the inviter knows the Matrix
-        identifier of the invitee.*
+        identifier of the invitee. The other is documented in the*
+        `third party invites section`_.
 
         This API invites a user to participate in a particular room.
         They do not start participating in the room until they actually join the
@@ -86,6 +89,8 @@ paths:
 
         If the user was invited to the room, the home server will append a
         ``m.room.member`` event to the room.
+
+        .. _third party invites section: `invite-by-third-party-id-endpoint`_
       security:
         - accessToken: []
       parameters:
@@ -132,106 +137,3 @@ paths:
           description: This request was rate-limited.
           schema:
             "$ref": "definitions/error.yaml"
-
-  "/rooms/{roomId}/invite":
-    post:
-      summary: Invite a user to participate in a particular room.
-      description: |-
-        *Note that there are two forms of this API, which are documented separately.
-        This version of the API does not require that the inviter know the Matrix
-        identifier of the invitee, and instead relies on third party identifiers.
-        The homeserver uses an identity server to perform the mapping from
-        third party identifier to a Matrix identifier.*
-
-        This API invites a user to participate in a particular room.
-        They do not start participating in the room until they actually join the
-        room.
-
-        Only users currently in a particular room can invite other users to
-        join that room.
-
-        If the identity server did know the Matrix user identifier for the
-        third party identifier, the home server will append a ``m.room.member``
-        event to the room.
-
-        If the identity server does not know a Matrix user identifier for the
-        passed third party identifier, the homeserver will issue an invitation
-        which can be accepted upon providing proof of ownership of the third
-        party identifier. This is achieved by the identity server generating a
-        token, which it gives to the inviting homeserver. The homeserver will
-        add an ``m.room.third_party_invite`` event into the graph for the room,
-        containing that token.
-
-        When the invitee binds the invited third party identifier to a Matrix
-        user ID, the identity server will give the user a list of pending
-        invitations, each containing:
-
-        - The room ID to which they were invited
-
-        - The token given to the homeserver
-
-        - A signature of the token, signed with the identity server's private key
-
-        - The matrix user ID who invited them to the room
-
-        If a token is requested from the identity server, the home server will
-        append a ``m.room.third_party_invite`` event to the room.
-      security:
-        - accessToken: []
-      parameters:
-        - in: path
-          type: string
-          name: roomId
-          description: The room identifier (not alias) to which to invite the user.
-          required: true
-          x-example: "!d41d8cd:matrix.org"
-        - in: body
-          name: body
-          required: true
-          schema:
-            type: object
-            example: |-
-              {
-                "id_server": "matrix.org",
-                "medium": "email",
-                "address": "cheeky@monkey.com",
-                "display_name": "A very cheeky monkey"
-              }
-            properties:
-              id_server:
-                type: string
-                description: The hostname+port of the identity server which should be used for third party identifier lookups.
-              medium:
-                type: string
-                # TODO: Link to identity service spec when it eixsts
-                description: The kind of address being passed in the address field, for example ``email``.
-              address:
-                type: string
-                description: The invitee's third party identifier.
-              display_name:
-                type: string
-                description: A user-friendly string describing who has been invited. It should not contain the address of the invitee, to avoid leaking mappings between third party identities and matrix user IDs.
-            required: ["id_server", "medium", "address", "display_name"]
-      responses:
-        200:
-          description: The user has been invited to join the room.
-          examples:
-            application/json: |-
-              {}
-          schema:
-            type: object
-        403:
-          description: |-
-            You do not have permission to invite the user to the room. A meaningful ``errcode`` and description error text will be returned. Example reasons for rejections are:
-
-            - The invitee has been banned from the room.
-            - The invitee is already a member of the room.
-            - The inviter is not currently in the room.
-            - The inviter's power level is insufficient to invite users to the room.
-          examples:
-            application/json: |-
-              {"errcode": "M_FORBIDDEN", "error": "@cheeky_monkey:matrix.org is banned from the room"}
-        429:
-          description: This request was rate-limited.
-          schema:
-            "$ref": "definitions/error.yaml"

api/client-server/v1/third_party_membership.yaml

@@ -0,0 +1,127 @@
+swagger: '2.0'
+info:
+  title: "Matrix Client-Server v1 Room Membership API for third party identifiers"
+  version: "1.0.0"
+host: localhost:8008
+schemes:
+  - https
+  - http
+basePath: /_matrix/client/api/v1
+consumes:
+  - application/json
+produces:
+  - application/json
+securityDefinitions:
+  accessToken:
+    type: apiKey
+    description: The user_id or application service access_token
+    name: access_token
+    in: query
+paths:
+  "/rooms/{roomId}/invite":
+    post:
+      summary: Invite a user to participate in a particular room.
+      description: |-
+        .. _invite-by-third-party-id-endpoint:
+
+        *Note that there are two forms of this API, which are documented separately.
+        This version of the API does not require that the inviter know the Matrix
+        identifier of the invitee, and instead relies on third party identifiers.
+        The homeserver uses an identity server to perform the mapping from
+        third party identifier to a Matrix identifier. The other is documented in the*
+        `joining rooms section`_.
+
+        This API invites a user to participate in a particular room.
+        They do not start participating in the room until they actually join the
+        room.
+
+        Only users currently in a particular room can invite other users to
+        join that room.
+
+        If the identity server did know the Matrix user identifier for the
+        third party identifier, the home server will append a ``m.room.member``
+        event to the room.
+
+        If the identity server does not know a Matrix user identifier for the
+        passed third party identifier, the homeserver will issue an invitation
+        which can be accepted upon providing proof of ownership of the third
+        party identifier. This is achieved by the identity server generating a
+        token, which it gives to the inviting homeserver. The homeserver will
+        add an ``m.room.third_party_invite`` event into the graph for the room,
+        containing that token.
+
+        When the invitee binds the invited third party identifier to a Matrix
+        user ID, the identity server will give the user a list of pending
+        invitations, each containing:
+
+        - The room ID to which they were invited
+
+        - The token given to the homeserver
+
+        - A signature of the token, signed with the identity server's private key
+
+        - The matrix user ID who invited them to the room
+
+        If a token is requested from the identity server, the home server will
+        append a ``m.room.third_party_invite`` event to the room.
+
+        .. _joining rooms section: `invite-by-user-id-endpoint`_
+      security:
+        - accessToken: []
+      parameters:
+        - in: path
+          type: string
+          name: roomId
+          description: The room identifier (not alias) to which to invite the user.
+          required: true
+          x-example: "!d41d8cd:matrix.org"
+        - in: body
+          name: body
+          required: true
+          schema:
+            type: object
+            example: |-
+              {
+                "id_server": "matrix.org",
+                "medium": "email",
+                "address": "cheeky@monkey.com",
+                "display_name": "A very cheeky monkey"
+              }
+            properties:
+              id_server:
+                type: string
+                description: The hostname+port of the identity server which should be used for third party identifier lookups.
+              medium:
+                type: string
+                # TODO: Link to identity service spec when it eixsts
+                description: The kind of address being passed in the address field, for example ``email``.
+              address:
+                type: string
+                description: The invitee's third party identifier.
+              display_name:
+                type: string
+                description: A user-friendly string describing who has been invited. It should not contain the address of the invitee, to avoid leaking mappings between third party identities and matrix user IDs.
+            required: ["id_server", "medium", "address", "display_name"]
+      responses:
+        200:
+          description: The user has been invited to join the room.
+          examples:
+            application/json: |-
+              {}
+          schema:
+            type: object
+        403:
+          description: |-
+            You do not have permission to invite the user to the room. A meaningful ``errcode`` and description error text will be returned. Example reasons for rejections are:
+
+            - The invitee has been banned from the room.
+            - The invitee is already a member of the room.
+            - The inviter is not currently in the room.
+            - The inviter's power level is insufficient to invite users to the room.
+          examples:
+            application/json: |-
+              {"errcode": "M_FORBIDDEN", "error": "@cheeky_monkey:matrix.org is banned from the room"}
+        429:
+          description: This request was rate-limited.
+          schema:
+            "$ref": "definitions/error.yaml"

api/client-server/v2_alpha/definitions/event_batch.json

@@ -5,6 +5,7 @@
       "type": "array",
       "description": "List of events",
       "items": {
+        "title": "Event",
         "type": "object"
       }
     }

api/client-server/v2_alpha/filter.yaml

@@ -56,7 +56,7 @@ paths:
                     "not_rooms": ["!726s6s6q:example.com"],
                     "not_senders": ["@spam:example.com"]
                   },
-                  "emphemeral": {
+                  "ephemeral": {
                     "types": ["m.receipt", "m.typing"],
                     "not_rooms": ["!726s6s6q:example.com"],
                     "not_senders": ["@spam:example.com"]
@@ -120,7 +120,7 @@ paths:
                     "not_rooms": ["!726s6s6q:example.com"],
                     "not_senders": ["@spam:example.com"]
                   },
-                  "emphemeral": {
+                  "ephemeral": {
                     "types": ["m.receipt", "m.typing"],
                     "not_rooms": ["!726s6s6q:example.com"],
                     "not_senders": ["@spam:example.com"]

api/client-server/v2_alpha/registration.yaml

@@ -17,7 +17,24 @@ paths:
       summary: Register for an account on this homeserver.
       description: |-
         Register for an account on this homeserver.
+
+        There are two kinds of user account:
+
+        - `user` accounts. These accounts may use the full API described in this specification.
+
+        - `guest` accounts. These accounts may have limited permissions and may not be supported by all servers.
+
       parameters:
+        - in: query
+          name: kind
+          type: string
+          x-example: guest
+          required: false
+          default: user
+          enum:
+            - guest
+            - user
+          description: The kind of account to register. Defaults to `user`.
         - in: body
           name: body
           schema:

api/client-server/v2_alpha/sync.yaml

@@ -241,6 +241,9 @@ paths:
                           "type": "m.room.member",
                           "state_key": "@bob:example.com",
                           "content": {"membership": "join"},
+                          "unsigned": {
+                            "prev_content": {"membership": "invite"}
+                          },
                           "origin_server_ts": 1417731086795
                         },
                         "$74686972643033:example.com": {

event-schemas/schema/v1/core-event-schema/event.json

@@ -5,6 +5,7 @@
   "properties": {
     "content": {
       "type": "object",
+      "title": "EventContent",
       "description": "The fields in this object will vary depending on the type of event. When interacting with the REST API, this is the HTTP body."
     },
     "type": {

event-schemas/schema/v1/core-event-schema/state_event.json

@@ -11,6 +11,7 @@
       "description": "A unique key which defines the overwriting semantics for this piece of room state. This value is often a zero-length string. The presence of this key makes this event a State Event. The key MUST NOT start with '_'."
     },
     "prev_content": {
+      "title": "EventContent",
       "type": "object",
       "description": "Optional. The previous ``content`` for this event. If there is no previous content, this key will be missing."
     }

event-schemas/schema/v1/m.room.member

@@ -8,6 +8,7 @@
     "properties": {
         "content": {
             "type": "object",
+            "title": "EventContent",
             "properties": {
                 "membership": {
                     "type": "string",

scripts/css/codehighlight.css

@@ -1,6 +1,16 @@
 pre.code .comment, code .comment { color: green }
 pre.code .keyword, code .keyword { color: darkred; font-weight: bold }
 pre.code .name.builtin, code .name.builtin { color: darkred; font-weight: bold }
-pre.code .literal.number, code .literal.number { color: blue }
 pre.code .name.tag, code .name.tag { color: darkgreen }
-pre.code .literal.string, code .literal.string { color: darkblue }
+pre.code .literal, code .literal { color: darkblue }
+pre.code .literal.number, code .literal.number { color: blue }
+
+
+/* HTTP Methods have class "name function" */
+pre.code.http .name.function,  code.http .name.function { color: black; font-weight: bold }
+/* HTTP Paths have class "name namespace" */
+pre.code.http .name.namespace, code.http .name.namespace { color: darkgreen }
+/* HTTP "HTTP" strings have class "keyword reserved" */
+pre.code.http .keyword.reserved, code.http .keyword.reserved { color: black; font-weight: bold }
+/* HTTP Header names have class "name attribute" */
+pre.code.http .name.attribute, code.http .name.attribute { color: black; font-weight: bold }

specification/events.rst

@@ -12,6 +12,42 @@ server-server and application-service APIs, and are described below.
 {{common_state_event_fields}}
 
 
+Differences between /v1 and /v2 events
+--------------------------------------
+
+There are a few differences between how events are formatted for sending
+between servers over federation and how they are formatted for sending between
+a server and its clients.
+
+Additionally there are a few differences between the format of events in the
+responses to client APIs with a /v1 prefix and responses APIs with a /v2
+prefix.
+
+Events in responses for APIs with the /v2 prefix are generated from an event
+formatted for federation by:
+
+* Removing the following keys:
+  ``auth_events``, ``prev_events``, ``hashes``, ``signatures``, ``depth``,
+  ``origin``, ``prev_state``.
+* Adding an ``age`` to the ``unsigned`` object which gives the time in
+  milliseconds that has ellapsed since the event was sent.
+* Adding a ``prev_content`` to the ``unsigned`` object if the event is
+  a ``state event`` which gives previous content of that state key.
+* Adding a ``redacted_because`` to the ``unsigned`` object if the event was
+  redacted which gives the event that redacted it.
+* Adding a ``transaction_id`` if the event was sent by the client requesting it.
+
+Events in responses for APIs with the /v1 prefix are generated from an event
+formatted for the /v2 prefix by:
+
+* Moving the folling keys from the ``unsigned`` object to the top level event
+  object: ``age``, ``redacted_because``, ``replaces_state``, ``prev_content``.
+* Removing the ``unsigned`` object.
+* Rename the ``sender`` key to ``user_id``.
+* If the event was an ``m.room.member`` with ``membership`` set to ``invite``
+  then adding a ``invite_room_state`` key to the top level event object.
+
+
 Size limits
 -----------
 

specification/modules/anonymous_access.rst

@@ -0,0 +1,50 @@
+Guest access
+================
+
+.. _module:guest-access:
+
+It may be desirable to allow users without a fully registered user account to
+ephemerally access Matrix rooms. This module specifies limited ways of doing so.
+
+Note that this is not currently a complete anonymous access solution; in
+particular, it only allows servers to provided anonymous access to rooms in
+which they are already participating, and relies on individual homeservers to
+adhere to the conventions which this module sets, rather than allowing all
+participating homeservers to enforce them.
+
+Events
+------
+
+{{m_room_guest_accessibility}}
+
+Client behaviour
+----------------
+A client can register for guest access using the FOO endpoint. From that point
+on, they can interact with a limited subset of the existing client-server API,
+as if they were a fully registered user, using the access token granted to them
+by the server.
+
+These users are only allowed to make calls in relation to rooms which have the
+``m.room.history_visibility`` event set to ``world_readable``.
+
+The APIs they are allowed to hit are:
+
+/rooms/{roomId}/messages
+/rooms/{roomId}/state
+/rooms/{roomId}/state/{eventType}/{stateKey}
+/events
+
+Server behaviour
+----------------
+Does the server need to handle any of the new events in a special way (e.g.
+typing timeouts, presence). Advice on how to persist events and/or requests are
+recommended to aid implementation. Federation-specific logic should be included
+here.
+
+Security considerations
+-----------------------
+This includes privacy leaks: for example leaking presence info. How do
+misbehaving clients or servers impact this module? This section should always be
+included, if only to say "we've thought about it but there isn't anything to do
+here".
+

specification/modules/instant_messaging.rst

@@ -116,12 +116,63 @@ of the client-server API will resolve this by attaching the transaction ID of th
 sending request to the event itself.
 
 
+Calculating the display name for a user
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Clients may wish to show the human-readable display name of a room member as
+part of a membership list, or when they send a message. However, different
+members may have conflicting display names. Display names MUST be disambiguated
+before showing them to the user, in order to prevent spoofing of other users.
+
+To ensure this is done consistently across clients, clients SHOULD use the
+following algorithm to calculate a disambiguated display name for a given user:
+
+1. Inspect the ``m.room.member`` state event for the relevant user id.
+2. If the ``m.room.member`` state event has no ``displayname`` field, or if
+   that field has a ``null`` value, use the raw user id as the display
+   name. Otherwise:
+3. If the ``m.room.member`` event has a ``displayname`` which is unique among
+   members of the room with ``membership: join`` or ``membership: invite``, use
+   the given ``displayname`` as the user-visible display name. Otherwise:
+4. The ``m.room.member`` event has a non-unique ``displayname``. This should be
+   disambiguated using the user id, for example "display name
+   (@id:homeserver.org)".
+
+   .. TODO-spec
+     what does it mean for a ``displayname`` to be 'unique'? Are we
+     case-sensitive?  Do we care about homograph attacks? See
+     https://matrix.org/jira/browse/SPEC-221.
+
+Developers should take note of the following when implementing the above
+algorithm:
+
+* The user-visible display name of one member can be affected by changes in the
+  state of another member. For example, if ``@user1:matrix.org`` is present in
+  a room, with ``displayname: Alice``, then when ``@user2:example.com`` joins
+  the room, also with ``displayname: Alice``, *both* users must be given
+  disambiguated display names. Similarly, when one of the users then changes
+  their display name, there is no longer a clash, and *both* users can be given
+  their chosen display name. Clients should be alert to this possibility and
+  ensure that all affected users are correctly renamed.
+
+* The display name of a room may also be affected by changes in the membership
+  list. This is due to the room name sometimes being based on user display
+  names (see `Calculating the display name for a room`_).
+
+* If the entire membership list is searched for clashing display names, this
+  leads to an O(N^2) implementation for building the list of room members. This
+  will be very inefficient for rooms with large numbers of members. It is
+  recommended that client implementations maintain a hash table mapping from
+  ``displayname`` to a list of room members using that name. Such a table can
+  then be used for efficient calculation of whether disambiguation is needed.
+
+
 Displaying membership information with messages
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 Clients may wish to show the display name and avatar URL of the room member who
 sent a message. This can be achieved by inspecting the ``m.room.member`` state
-event for that user ID.
+event for that user ID (see `Calculating the display name for a user`_).
 
 When a user paginates the message history, clients may wish to show the
 **historical** display name and avatar URL for a room member. This is possible
@@ -133,6 +184,85 @@ events update the old state. When paginated events are processed sequentially,
 the old state represents the state of the room *at the time the event was sent*.
 This can then be used to set the historical display name and avatar URL.
 
+
+Calculating the display name for a room
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Clients may wish to show a human-readable name for a room. There are a number
+of possibilities for choosing a useful name. To ensure that rooms are named
+consistently across clients, clients SHOULD use the following algorithm to
+choose a name:
+
+1. If the room has an `m.room.name`_ state event, use the name given by that
+   event.
+
+#. If the room has an `m.room.canonical_alias`_ state event, use the alias
+   given by that event.
+
+#. If neither of the above events are present, a name should be composed based
+   on the members of the room. Clients should consider `m.room.member`_ events
+   for users other than the logged-in user, with ``membership: join`` or
+   ``membership: invite``.
+
+   .. _active_members:
+
+   i. If there is only one such event, the display name for the room should be
+      the `disambiguated display name`_ of the corresponding user.
+
+   #. If there are two such events, they should be lexicographically sorted by
+      their ``state_key`` (i.e. the corresponding user IDs), and the display
+      name for the room should be the  `disambiguated display name`_ of both
+      users: "<user1> and <user2>", or a localised variant thereof.
+
+   #. If there are three or more such events, the display name for the room
+      should be based on the disambiguated display name of the user
+      corresponding to the first such event, under a lexicographical sorting
+      according to their ``state_key``. The display name should be in the
+      format "<user1> and <N> others" (or a localised variant thereof), where N
+      is the number of `m.room.member`_ events with ``membership: join`` or
+      ``membership: invite``, excluding the logged-in user and "user1".
+
+      For example, if Alice joins a room, where Bob (whose user id is
+      ``@superuser:example.com``), Carol (user id ``@carol:example.com``) and
+      Dan (user id ``@dan:matrix.org``) are in conversation, Alice's
+      client should show the room name as "Carol and 2 others".
+
+   .. TODO-spec
+     Sorting by user_id certainly isn't ideal, as IDs at the start of the
+     alphabet will end up dominating room names: they will all be called
+     "Arathorn and 15 others". Furthermore - user_ids are not necessarily
+     ASCII, which means we need to either specify a collation order, or specify
+     how to choose one.
+
+     Ideally we might sort by the time when the user was first invited to, or
+     first joined the room. But we don't have this information.
+
+     See https://matrix.org/jira/browse/SPEC-267 for further discussion.
+
+#. If the room has no ``m.room.name`` or ``m.room.canonical_alias`` events, and
+   no active members other than the current user, clients should consider
+   ``m.room.member`` events with ``membership: leave``. If such events exist, a
+   display name such as "Empty room (was <user1> and <N> others)" (or a
+   localised variant thereof) should be used, following similar rules as for
+   active members (see `above <active_members_>`_).
+
+#. A complete absence of ``m.room.name``, ``m.room.canonical_alias``, and
+   ``m.room.member`` events is likely to indicate a problem with creating the
+   room or synchronising the state table; however clients should still handle
+   this situation. A display name such as "Empty room" (or a localised variant
+   thereof) should be used in this situation.
+
+.. _`disambiguated display name`: `Calculating the display name for a user`_
+
+Clients SHOULD NOT use `m.room.aliases`_ events as a source for room names, as
+it is difficult for clients to agree on the best alias to use, and aliases can
+change unexpectedly.
+
+.. TODO-spec
+  How can we make this less painful for clients to implement, without forcing
+  an English-language implementation on them all?
+
+
 Server behaviour
 ----------------
 

specification/modules/third_party_invites.rst

@@ -1,23 +1,24 @@
 Third party invites
 ===================
 
-.. _module:third_party_invites:
+.. _module:third-party-invites:
 
 This module adds in support for inviting new members to a room where their
 Matrix user ID is not known, instead addressing them by a third party identifier
 such as an email address.
-
 There are two flows here; one if a Matrix user ID is known for the third party
 identifier, and one if not. Either way, the client calls ``/invite`` with the
 details of the third party identifier.
 
 The homeserver asks the identity server whether a Matrix user ID is known for
-that identifier. If it is, an invite is simply issued for that user.
+that identifier:
 
-If it is not, the homeserver asks the identity server to record the details of
+- If it is, an invite is simply issued for that user.
-the invitation, and to notify the invitee's homeserver of this pending invitation if it gets
+
-a binding for this identifier in the future. The identity server returns a token
+- If it is not, the homeserver asks the identity server to record the details of
-and public key to the inviting homeserver.
+  the invitation, and to notify the invitee's homeserver of this pending invitation if it gets
+  a binding for this identifier in the future. The identity server returns a token
+  and public key to the inviting homeserver.
 
 When the invitee's homeserver receives the notification of the binding, it
 should insert an ``m.room.member`` event into the room's graph for that user,
@@ -35,6 +36,8 @@ Client behaviour
 
 A client asks a server to invite a user by their third party identifier.
 
+{{third_party_membership_http_api}}
+
 Server behaviour
 ----------------
 
@@ -87,24 +90,24 @@ membership is questionable.
 
 For example:
 
-    If room R has two participating homeservers, H1, H2
+#. Room R has two participating homeservers, H1, H2
 
-    And user A on H1 invites a third party identifier to room R
+#. User A on H1 invites a third party identifier to room R
 
-    H1 asks the identity server for a binding to a Matrix user ID, and has none,
+#. H1 asks the identity server for a binding to a Matrix user ID, and has none,
-    so issues an ``m.room.third_party_invite`` event to the room.
+   so issues an ``m.room.third_party_invite`` event to the room.
 
-    When the third party user validates their identity, their homeserver H3
+#. When the third party user validates their identity, their homeserver H3
-    is notified and attempts to issue an ``m.room.member`` event to participate
+   is notified and attempts to issue an ``m.room.member`` event to participate
-    in the room.
+   in the room.
 
-    H3 validates the signature given to it by the identity server.
+#. H3 validates the signature given to it by the identity server.
 
-    H3 then asks H1 to join it to the room. H1 *must* validate the ``signed``
+#. H3 then asks H1 to join it to the room. H1 *must* validate the ``signed``
-    property *and* check ``key_validity_url``.
+   property *and* check ``key_validity_url``.
 
-    Having validated these things, H1 writes the invite event to the room, and H3
+#. Having validated these things, H1 writes the invite event to the room, and H3
-    begins participating in the room. H2 *must* accept this event.
+   begins participating in the room. H2 *must* accept this event.
 
 The reason that no other homeserver may reject the event based on checking
 ``key_validity_url`` is that we must ensure event acceptance is deterministic.
@@ -115,3 +118,32 @@ This relies on participating servers trusting each other, but that trust is
 already implied by the server-server protocol. Also, the public key signature
 verification must still be performed, so the attack surface here is minimized.
 
+Security considerations
+-----------------------
+
+There are a number of privary and trust implications to this module.
+
+It is important for user privacy that leaking the mapping between a matrix user
+ID and a third party identifier is hard. In particular, being able to look up
+all third party identifiers from a matrix user ID (and accordingly, being able
+to link each third party identifier) should be avoided wherever possible.
+To this end, when implementing this API care should be taken to avoid
+adding links between these two identifiers as room events. This mapping can be
+unintentionally created by specifying the third party identifier in the
+``display_name`` field of the ``m.room.third_party_invite`` event, and then
+observing which matrix user ID joins the room using that invite. Clients SHOULD
+set ``display_name`` to a value other than the third party identifier, e.g. the
+invitee's common name.
+
+Homeservers are not required to trust any particular identity server(s). It is
+generally a client's responsibility to decide which identity servers it trusts,
+not a homeserver's. Accordingly, this API takes identity servers as input from
+end users, and doesn't have any specific trusted set. It is possible some
+homeservers may want to supply defaults, or reject some identity servers for
+*its* users, but no homeserver is allowed to dictate which identity servers
+*other* homeservers' users trust.
+
+There is some risk of denial of service attacks by flooding homeservers or
+identity servers with many requests, or much state to store. Defending against
+these is left to the implementer's discretion.
+

specification/server_server_api.rst

@@ -15,9 +15,9 @@ There are three main kinds of communication that occur between home servers:
 
 Persisted Data Units (PDUs):
     These events are broadcast from one home server to any others that have
-    joined the same "context" (namely, a Room ID). They are persisted in
+    joined the same room (identified by Room ID). They are persisted in
     long-term storage and record the history of messages and state for a
-    context.
+    room.
 
     Like email, it is the responsibility of the originating server of a PDU
     to deliver that event to its recipient servers. However PDUs are signed
@@ -26,7 +26,7 @@ Persisted Data Units (PDUs):
 
 Ephemeral Data Units (EDUs):
     These events are pushed between pairs of home servers. They are not
-    persisted and are not part of the history of a "context", nor does the
+    persisted and are not part of the history of a room, nor does the
     receiving home server have to reply to them.
 
 Queries:
@@ -338,11 +338,11 @@ PDUs
 
 All PDUs have:
 
-- An ID
+- An ID to identify the PDU itself
-- A context
+- A room ID that it relates to
 - A declaration of their type
-- A list of other PDU IDs that have been seen recently on that context
+- A list of other PDU IDs that have been seen recently in that room (regardless
-  (regardless of which origin sent them)
+  of which origin sent them)
 
 
 Required PDU Fields
@@ -351,7 +351,7 @@ Required PDU Fields
 ==================== ================== =======================================
  Key                  Type               Description
 ==================== ================== =======================================
-``context``          String             Event context identifier
+``context``          String             Room identifier
 ``user_id``          String             The ID of the user sending the PDU
 ``origin``           String             DNS name of homeserver that created
                                         this PDU
@@ -363,7 +363,7 @@ Required PDU Fields
 ``content``          Object             The content of the PDU.
 ``prev_pdus``        List of (String,   The originating homeserver, PDU ids and
                      String, Object)    hashes of the most recent PDUs the
-                     Triplets           homeserver was aware of for the context
+                     Triplets           homeserver was aware of for the room
                                         when it made this PDU
 ``depth``            Integer            The maximum depth of the previous PDUs
                                         plus one
@@ -440,7 +440,7 @@ keys exist to support this:
 EDUs
 ----
 
-EDUs, by comparison to PDUs, do not have an ID, a context, or a list of
+EDUs, by comparison to PDUs, do not have an ID, a room ID, or a list of
 "previous" IDs. The only mandatory fields for these are the type, origin and
 destination home server names, and the actual nested content.
 
@@ -491,23 +491,23 @@ Retrieves a given PDU from the server. The response will contain a single new
 Transaction, inside which will be the requested PDU.
 
 
-To fetch all the state of a given context::
+To fetch all the state of a given room::
 
-  GET .../state/<context>/
+  GET .../state/<room_id>/
     Response: JSON encoding of a single Transaction containing multiple PDUs
 
-Retrieves a snapshot of the entire current state of the given context. The
+Retrieves a snapshot of the entire current state of the given room. The
 response will contain a single Transaction, inside which will be a list of PDUs
 that encode the state.
 
-To backfill events on a given context::
+To backfill events on a given room::
 
-  GET .../backfill/<context>/
+  GET .../backfill/<room_id>/
     Query args: v, limit
     Response: JSON encoding of a single Transaction containing multiple PDUs
 
 Retrieves a sliding-window history of previous PDUs that occurred on the given
-context. Starting from the PDU ID(s) given in the "v" argument, the PDUs that
+room. Starting from the PDU ID(s) given in the "v" argument, the PDUs that
 preceded it are retrieved, up to a total number given by the "limit" argument.
 These are then returned in a new Transaction containing all of the PDUs.
 

templating/build.py

@@ -84,6 +84,13 @@ def main(input_module, file_stream=None, out_dir=None, verbose=False):
         input_lines = input.split('\n\n')
         wrapper = TextWrapper(initial_indent=initial_indent, width=wrap)
         output_lines = [wrapper.fill(line) for line in input_lines]
+
+        for i in range(len(output_lines)):
+            line = output_lines[i]
+            in_bullet = line.startswith("- ")
+            if in_bullet:
+                output_lines[i] = line.replace("\n", "\n  " + initial_indent)
+
         return '\n\n'.join(output_lines)
 
     # make Jinja aware of the templates and filters

templating/matrix_templates/templates/http-api.tmpl

@@ -47,7 +47,9 @@ Response format:
 {% endfor %}
 {% endif -%}
 
-Example request::
+Example request:
+
+.. code:: http
 
   {{endpoint.example.req | indent_block(2)}}
 

templating/matrix_templates/templates/msgtypes.tmpl

@@ -18,6 +18,8 @@
 ================== ================= ===========================================
 
 {% endfor %}
-Example::
+Example:
+
+.. code:: json
 
     {{example | jsonify(4, 4)}}

templating/matrix_templates/units.py

@@ -28,7 +28,25 @@ ROOM_EVENT = "core-event-schema/room_event.json"
 STATE_EVENT = "core-event-schema/state_event.json"
 
 
-def get_json_schema_object_fields(obj, enforce_title=False):
+def resolve_references(path, schema):
+    if isinstance(schema, dict):
+        result = {}
+        for key, value in schema.items():
+            if key == "$ref":
+                path = os.path.join(os.path.dirname(path), value)
+                with open(path) as f:
+                    schema = json.load(f)
+                return resolve_references(path, schema)
+            else:
+                result[key] = resolve_references(path, value)
+        return result
+    elif isinstance(schema, list):
+        return [resolve_references(path, value) for value in schema]
+    else:
+        return schema
+
+
+def get_json_schema_object_fields(obj, enforce_title=False, include_parents=False):
     # Algorithm:
     # f.e. property => add field info (if field is object then recurse)
     if obj.get("type") != "object":
@@ -36,9 +54,9 @@ def get_json_schema_object_fields(obj, enforce_title=False):
             "get_json_schema_object_fields: Object %s isn't an object." % obj
         )
     if enforce_title and not obj.get("title"):
-        raise Exception(
+        # Force a default titile of "NO_TITLE" to make it obvious in the
-            "get_json_schema_object_fields: Nested object %s doesn't have a title." % obj
+        # specification output which parts of the schema are missing a title
-        )
+        obj["title"] = 'NO_TITLE'
 
     required_keys = obj.get("required")
     if not required_keys:
@@ -73,9 +91,15 @@ def get_json_schema_object_fields(obj, enforce_title=False):
             "Object %s has no properties or parents." % obj
         )
     if not props:  # parents only
+        if include_parents:
+            if obj["title"] == "NO_TITLE" and parents[0].get("title"):
+                obj["title"] = parents[0].get("title")
+            props = parents[0].get("properties")
+
+    if not props:
         return [{
             "title": obj["title"],
-            "parent": parents[0]["$ref"],
+            "parent": parents[0].get("$ref"),
             "no-table": True
         }]
 
@@ -91,7 +115,8 @@ def get_json_schema_object_fields(obj, enforce_title=False):
                 if prop_val == "object":
                     nested_object = get_json_schema_object_fields(
                         props[key_name]["additionalProperties"],
-                        enforce_title=True
+                        enforce_title=True,
+                        include_parents=include_parents,
                     )
                     key = props[key_name]["additionalProperties"].get(
                         "x-pattern", "string"
@@ -103,8 +128,9 @@ def get_json_schema_object_fields(obj, enforce_title=False):
                     value_type = "{string: %s}" % prop_val
             else:
                 nested_object = get_json_schema_object_fields(
-                    props[key_name], 
+                    props[key_name],
-                    enforce_title=True
+                    enforce_title=True,
+                    include_parents=include_parents,
                 )
                 value_type = "{%s}" % nested_object[0]["title"]
 
@@ -114,13 +140,17 @@ def get_json_schema_object_fields(obj, enforce_title=False):
             # if the items of the array are objects then recurse
             if props[key_name]["items"]["type"] == "object":
                 nested_object = get_json_schema_object_fields(
-                    props[key_name]["items"], 
+                    props[key_name]["items"],
-                    enforce_title=True
+                    enforce_title=True,
+                    include_parents=include_parents,
                 )
                 value_type = "[%s]" % nested_object[0]["title"]
                 tables += nested_object
             else:
-                value_type = "[%s]" % props[key_name]["items"]["type"]
+                value_type = props[key_name]["items"]["type"]
+                if isinstance(value_type, list):
+                    value_type = " or ".join(value_type)
+                value_type = "[%s]" % value_type
                 array_enums = props[key_name]["items"].get("enum")
                 if array_enums:
                     if len(array_enums) > 1:
@@ -137,12 +167,16 @@ def get_json_schema_object_fields(obj, enforce_title=False):
             if props[key_name].get("enum"):
                 if len(props[key_name].get("enum")) > 1:
                     value_type = "enum"
+                    if desc:
+                        desc += " "
                     desc += (
-                        " One of: %s" % json.dumps(props[key_name]["enum"])
+                        "One of: %s" % json.dumps(props[key_name]["enum"])
                     )
                 else:
+                    if desc:
+                        desc += " "
                     desc += (
-                        " Must be '%s'." % props[key_name]["enum"][0]
+                        "Must be '%s'." % props[key_name]["enum"][0]
                     )
             if isinstance(value_type, list):
                 value_type = " or ".join(value_type)
@@ -154,12 +188,22 @@ def get_json_schema_object_fields(obj, enforce_title=False):
             "desc": desc,
             "req_str": "**Required.** " if required else ""
         })
-    return tables
+
+    titles = set()
+    filtered = []
+    for table in tables:
+        if table.get("title") in titles:
+            continue
+
+        titles.add(table.get("title"))
+        filtered.append(table)
+
+    return filtered
 
 
 class MatrixUnits(Units):
 
-    def _load_swagger_meta(self, api, group_name):
+    def _load_swagger_meta(self, filepath, api, group_name):
         endpoints = []
         for path in api["paths"]:
             for method in api["paths"][path]:
@@ -262,7 +306,10 @@ class MatrixUnits(Units):
                             if is_array_of_objects:
                                 req_obj = req_obj["items"]
 
-                            req_tables = get_json_schema_object_fields(req_obj)
+                            req_tables = get_json_schema_object_fields(
+                                resolve_references(filepath, req_obj),
+                                include_parents=True,
+                            )
 
                             if req_tables > 1:
                                 for table in req_tables[1:]:
@@ -336,9 +383,15 @@ class MatrixUnits(Units):
                         elif param["in"] == "query":
                             qps[param["name"]] = param["x-example"]
                     query_string = "" if len(qps) == 0 else "?"+urllib.urlencode(qps)
-                    endpoint["example"]["req"] = "%s %s%s\n%s" % (
+                    if body:
-                        method.upper(), path_template, query_string, body
+                        endpoint["example"]["req"] = "%s %s%s HTTP/1.1\nContent-Type: application/json\n\n%s" % (
-                    )
+                            method.upper(), path_template, query_string, body
+                        )
+                    else:
+                        endpoint["example"]["req"] = "%s %s%s HTTP/1.1\n\n" % (
+                            method.upper(), path_template, query_string
+                        )
+
                 else:
                     self.log(
                         "The following parameters are missing examples :( \n %s" %
@@ -373,7 +426,10 @@ class MatrixUnits(Units):
                     elif res_type and Units.prop(good_response, "schema/properties"):
                         # response is an object:
                         schema = good_response["schema"]
-                        res_tables = get_json_schema_object_fields(schema)
+                        res_tables = get_json_schema_object_fields(
+                            resolve_references(filepath, schema),
+                            include_parents=True,
+                        )
                         for table in res_tables:
                             if "no-table" not in table:
                                 endpoint["res_tables"].append(table)
@@ -439,13 +495,16 @@ class MatrixUnits(Units):
                 if not filename.endswith(".yaml"):
                     continue
                 self.log("Reading swagger API: %s" % filename)
-                with open(os.path.join(path, filename), "r") as f:
+                filepath = os.path.join(path, filename)
+                with open(filepath, "r") as f:
                     # strip .yaml
                     group_name = filename[:-5].replace("-", "_")
                     if is_v2:
                         group_name = "v2_" + group_name
                     api = yaml.load(f.read())
-                    api["__meta"] = self._load_swagger_meta(api, group_name)
+                    api["__meta"] = self._load_swagger_meta(
+                        filepath, api, group_name
+                    )
                     apis[group_name] = api
         return apis