Merge pull request #1463 from turt2live/travis/s2s/pdus-and-edus

Improve documentation around EDUs and PDUs

api/server-server/definitions/edu.yaml

@@ -12,11 +12,9 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# TODO: Address any concerns about this being inaccurate (flagged as such in the EDUs section)
-
 type: object
 title: Ephemeral Data Unit
-description: An ephemeral data unit
+description: An ephemeral data unit.
 example:
   $ref: "../examples/edu.json"
 properties:
@@ -24,17 +22,7 @@ properties:
     type: string
     description: The type of ephemeral message.
     example: "m.presence"
-    required: true
-  origin:
-    type: string
-    description: The server name sending the ephemeral message.
-    example: "matrix.org"
-    required: true
-  destination:
-    type: string
-    description: The server name receiving the ephemeral message.
-    example: "elsewhere.com"
-    required: true
   content:
     type: object
     description: The content of the ephemeral message.
+required: ['edu_type', 'content']

api/server-server/definitions/unsigned_pdu.yaml

@@ -82,7 +82,9 @@ properties:
     example: 12
   auth_events:
     type: array
-    description: Event IDs and hashes for the "auth events" of this event.
+    description: |-
+      An event reference list containing the authorization events that would
+      allow this event to be in the room.
     items:
       type: array
       maxItems: 2
@@ -108,8 +110,35 @@ properties:
     example: "$def456:matrix.org"
   unsigned:
     type: object
-    description: Additional data added by the origin server but not covered by the ``signatures``.
+    title: Example Unsigned Data
+    description: |-
+      Additional data added by the origin server but not covered by the ``signatures``. More
+      keys than those defined here may be used.
     example: {"key": "value"}
+    properties:
+      age:
+        type: integer
+        description: The number of milliseconds that have passed since this message was sent.
+        example: 4612
+      replaces_state:
+        type: string
+        description: The event ID of the state event this event replaces.
+        example: "$state_event:domain.com"
+      prev_sender:
+        type: string
+        description: The sender of the replaced state event.
+        example: "@someone:domain.com"
+      prev_content:
+        type: object
+        description: The content of the replaced state event.
+        example: {
+          "membership": "join",
+          "displayname": "Bob"
+        }
+      redacted_because:
+        type: string
+        description: A reason for why the event was redacted.
+        example: "Inappropriate content"
 required:
   - event_id
   - room_id

api/server-server/examples/edu.json

@@ -1,7 +1,5 @@
 {
     "edu_type": "m.presence",
-    "origin": "blue",
-    "destination": "orange",
     "content": {
         "key": "value"
     }

api/server-server/examples/unsigned_pdu.json

@@ -20,5 +20,8 @@
     ],
     "content": {
         "key": "value"
+    },
+    "unsigned": {
+        "age": 4612
     }
 }

scripts/templating/matrix_templates/sections.py

@@ -189,3 +189,17 @@ class MatrixSections(Sections):
         template = self.env.get_template("apis.tmpl")
         apis = self.units.get("apis")
         return template.render(apis=apis)
+
+    def render_swagger_definition(self):
+        rendered = {}
+        template = self.env.get_template("schema-definition.tmpl")
+        subtitle_title_char = self.units.get("spec_targets")[
+            "relative_title_styles"
+        ]["subtitle"]
+        definitions = self.units.get("swagger_definitions")
+        for group, swagger_def in definitions.items():
+            rendered["definition_" + group] = template.render(
+                definition=swagger_def['definition'],
+                examples=swagger_def['examples'],
+                title_kind=subtitle_title_char)
+        return rendered

scripts/templating/matrix_templates/templates/schema-definition.tmpl

@@ -0,0 +1,21 @@
+{% import 'tables.tmpl' as tables -%}
+
+``{{definition.title}}`` Schema
+{{(11 + definition.title | length) * title_kind}}
+
+{% if 'description' in definition %}
+{{definition.description}}
+{% endif %}
+
+{% for table in definition.tables -%}
+{{"``"+table.title+"``" if table.title else "" }}
+{{ tables.paramtable(table.rows) }}
+{% endfor %}
+
+Example{% if examples | length > 1 %}s{% endif %}:
+
+{% for example in examples %}
+.. code:: json
+
+    {{example | jsonify(4, 4)}}
+{% endfor %}

scripts/templating/matrix_templates/units.py

@@ -43,6 +43,13 @@ HTTP_APIS = {
     os.path.join(matrix_doc_dir, "api/push-gateway"): "push",
     os.path.join(matrix_doc_dir, "api/server-server"): "ss",
 }
+SWAGGER_DEFINITIONS = {
+    os.path.join(matrix_doc_dir, "api/application-service/definitions"): "as",
+    os.path.join(matrix_doc_dir, "api/client-server/definitions"): "cs",
+    os.path.join(matrix_doc_dir, "api/identity/definitions"): "is",
+    os.path.join(matrix_doc_dir, "api/push-gateway/definitions"): "push",
+    os.path.join(matrix_doc_dir, "api/server-server/definitions"): "ss",
+}
 EVENT_EXAMPLES = os.path.join(matrix_doc_dir, "event-schemas/examples")
 EVENT_SCHEMA = os.path.join(matrix_doc_dir, "event-schemas/schema")
 CORE_EVENT_SCHEMA = os.path.join(matrix_doc_dir, "event-schemas/schema/core-event-schema")
@@ -654,6 +661,50 @@ class MatrixUnits(Units):
                     apis[group_name] = api
         return apis
 
+
+    def load_swagger_definitions(self):
+        defs = {}
+        for path, prefix in SWAGGER_DEFINITIONS.items():
+            self._load_swagger_definitions_in_dir(defs, path, prefix)
+        return defs
+
+    def _load_swagger_definitions_in_dir(self, defs, path, prefix, recurse=True):
+        if not os.path.exists(path):
+            return defs
+        for filename in os.listdir(path):
+            filepath = os.path.join(path, filename)
+            if os.path.isdir(filepath) and recurse:
+                safe_name = re.sub(r"[^a-zA-Z0-9_]", "_", filename)
+                dir_prefix = "_".join([prefix, safe_name])
+                # We don't recurse because we have to stop at some point
+                self._load_swagger_definitions_in_dir(
+                    defs, filepath, dir_prefix, recurse=False)
+            if not filename.endswith(".yaml"):
+                continue
+            filepath = os.path.join(path, filename)
+            logger.info("Reading swagger definition: %s" % filepath)
+            with open(filepath, "r") as f:
+                # strip .yaml
+                group_name = re.sub(r"[^a-zA-Z0-9_]", "_", filename[:-5])
+                group_name = "%s_%s" % (prefix, group_name)
+                definition = yaml.load(f.read(), OrderedLoader)
+                definition = resolve_references(filepath, definition)
+                if 'type' not in definition:
+                    continue
+                try:
+                    example = get_example_for_schema(definition)
+                except:
+                    example = None
+                    pass  # do nothing - we don't care
+                if 'title' not in definition:
+                    definition['title'] = "NO_TITLE"
+                definition['tables'] = get_tables_for_schema(definition)
+                defs[group_name] = {
+                    "definition": definition,
+                    "examples": [example] if example is not None else [],
+                }
+        return defs
+
     def load_common_event_fields(self):
         """Parse the core event schema files
 

specification/server_server_api.rst

@@ -180,78 +180,6 @@ PDUs
 Each PDU contains a single Room Event which the origin server wants to send to
 the destination.
 
-
-PDU Fields
-~~~~~~~~~~
-
-.. TODO-spec
-
-  Figure out how to embed swagger definitions in here (or improve the section)
-
-==================== ================== =======================================
- Key                  Type               Description
-==================== ================== =======================================
-``room_id``          String             **Required**. Room identifier.
-``sender``           String             **Required**. The ID of the user sending
-                                        the event.
-``origin``           String             **Required**. ``server_name`` of the
-                                        homeserver that created this event.
-``event_id``         String             **Required**. Unique identifier for the
-                                        event being sent.
-``origin_server_ts`` Integer            **Required**. Timestamp in milliseconds
-                                        on origin homeserver when this event
-                                        was created.
-``type``             String             **Required**. Event type.
-``state_key``        String             If this key is present, the event is a
-                                        state event, and it will replace
-                                        previous events with the same ``type``
-                                        and ``state_key`` in the room state.
-``content``          Object             **Required**. The content of the event.
-``prev_events``      List of (String,   **Required**. Event IDs and hashes of
-                     {String: String})  the most recent events in the room that
-                     pairs              the homeserver was aware of when it
-                                        made this event.
-``depth``            Integer            **Required**. The maximum depth of the
-                                        ``prev_events``, plus one.
-``auth_events``      List of (String,   **Required**. Event IDs and hashes for
-                     {String: String})  the "auth events" of this event.
-                     pairs
-``hashes``           {String: String}   **Required**. Hashes of the PDU,
-                                        following the algorithm specified in
-                                        `Signing Events`_.
-``signatures``       {String:           **Required**. Signatures of the redacted
-                     {String: String}}  PDU, following the algorithm specified
-                                        in `Signing Events`_.
-``redacts``          String             For redaction events, the ID of the
-                                        event being redacted.
-``unsigned``         Object             Additional data added by the origin
-                                        server but not covered by the
-                                        ``signatures``.
-==================== ================== =======================================
-
-Example:
-
-.. code:: json
-
- {
-  "room_id": "!UcYsUzyxTGDxLBEvLy:example.org",
-  "sender": "@alice:example.com",
-  "origin": "example.com",
-  "event_id": "$a4ecee13e2accdadf56c1025:example.com",
-  "origin_server_ts": 1404838188000,
-  "type": "m.room.message",
-  "prev_events": [
-    ["$af232176:example.org", {"sha256": "abase64encodedsha256hashshouldbe43byteslong"}]
-  ],
-  "hashes": {"sha256": "thishashcoversallfieldsincasethisisredacted"},
-  "signatures": {
-    "example.com": {
-      "ed25519:key_version:": "these86bytesofbase64signaturecoveressentialfieldsincludinghashessocancheckredactedpdus"
-    }
-  },
-  "content": {...}
- }
-
 The ``prev_events`` field of a PDU identifies the "parents" of the event, and
 thus establishes a partial ordering on events within the room by linking them
 into a Directed Acyclic Graph (DAG). The sending server should populate this
@@ -282,6 +210,8 @@ following subset of the room state:
 - The current ``m.room.join_rules`` event, if any.
 - The sender's current ``m.room.member`` event, if any.
 
+{{definition_ss_pdu}}
+
 Authorization of PDUs
 ~~~~~~~~~~~~~~~~~~~~~
 
@@ -460,32 +390,11 @@ give the homeserver an avenue for getting the information it needs.
 EDUs
 ----
 
-.. WARNING::
-  This section may be misleading or inaccurate.
-
 EDUs, by comparison to PDUs, do not have an ID, a room ID, or a list of
-"previous" IDs. The only mandatory fields for these are the type, origin and
+"previous" IDs. They are intended to be non-persistent data such as user
-destination homeserver names, and the actual nested content.
+presence, typing notifications, etc.
 
-======================== ============ =========================================
+{{definition_ss_edu}}
- Key                      Type          Description
-======================== ============ =========================================
-``edu_type``             String       The type of the ephemeral message.
-``origin``               String       The server name sending the ephemeral
-                                      message.
-``destination``          String       The server name receiving the ephemeral
-                                      message.
-``content``              Object       Content of the ephemeral message.
-======================== ============ =========================================
-
-.. code:: json
-
- {
-  "edu_type": "m.presence",
-  "origin": "blue",
-  "destination": "orange",
-  "content": {...}
- }
 
 Room State Resolution
 ---------------------