Use standard logic to parse core event schemas

Use process_data_type rather than reinventing our own wheel; doing so means
that the 'Required' fields are correctly annotated as such.

templating/matrix_templates/templates/common-event-fields.tmpl

@@ -1,8 +1,13 @@
 {% import 'tables.tmpl' as tables -%}
 
-{{common_event.title}} Fields
+{{common_event.type}} Fields
-{{(7 + common_event.title | length) * title_kind}}
+{{(7 + common_event.type | length) * title_kind}}
 
-{{common_event.desc | wrap(80)}}
+{{common_event.desc}}
 
-{{ tables.paramtable(common_event.rows, ["Key", "Type", "Description"]) }}
+{% for table in common_event.tables %}
+{{"``"+table.title+"``" if table.title else "" }}
+
+{{ tables.paramtable(table.rows, ["Key", "Type", "Description"]) }}
+
+{% endfor %}

templating/matrix_templates/units.py

@@ -155,6 +155,21 @@ def inherit_parents(obj):
 
 
 def get_json_schema_object_fields(obj, enforce_title=False):
+    """Parse a JSON schema object definition
+
+    Args:
+        obj(dict): definition from the JSON schema file. $refs should already
+            have been resolved.
+        enforce_title (bool): if True, and the definition has no "title",
+            the 'type' result will be set to 'NO_TITLE' (otherwise it will be
+            set to None)
+
+    Returns:
+        dict: with the following fields:
+          - type (str): title (normally the type name) for the object
+          - tables (list[TypeTable]): list of the tables for the type
+                definition
+    """
     # Algorithm:
     # f.e. property => add field info (if field is object then recurse)
     if obj.get("type") != "object":
@@ -600,41 +615,34 @@ class MatrixUnits(Units):
         return apis
 
     def load_common_event_fields(self):
+        """Parse the core event schema files
+
+        Returns:
+            dict: with the following properties:
+                "type": prop_type,
+                "desc": desc,
+                "tables": list[TypeTable]
+        """
         path = CORE_EVENT_SCHEMA
         event_types = {}
 
-        for (root, dirs, files) in os.walk(path):
+        for filename in os.listdir(path):
-            for filename in files:
+            if not filename.endswith(".yaml"):
-                if not filename.endswith(".yaml"):
+                continue
-                    continue
 
-                event_type = filename[:-5]  # strip the ".yaml"
+            filepath = os.path.join(path, filename)
-                filepath = os.path.join(root, filename)
-                with open(filepath) as f:
-                    try:
-                        event_info = yaml.load(f, OrderedLoader)
-                    except Exception as e:
-                        raise ValueError(
-                            "Error reading file %r" % (filepath,), e
-                        )
 
-                if "event" not in event_type:
+            event_type = filename[:-5]  # strip the ".yaml"
-                    continue  # filter ImageInfo and co
+            logger.info("Reading event schema: %s" % filepath)
 
-                table = TypeTable(
+            with open(filepath) as f:
-                    title=event_info["title"],
+                event_schema = yaml.load(f, OrderedLoader)
-                    desc=event_info["description"],
-                )
 
-                for prop in sorted(event_info["properties"]):
+            schema_info = process_data_type(
-                    prop_info = event_info["properties"][prop]
+                event_schema,
-                    table.add_row(TypeTableRow(
+                enforce_title=True,
-                        key=prop,
+            )
-                        typ=prop_info["type"],
+            event_types[event_type] = schema_info
-                        desc=prop_info.get("description", ""),
-                    ))
-
-                event_types[event_type] = table
         return event_types
 
     def load_apis(self, substitutions):
@@ -709,8 +717,12 @@ class MatrixUnits(Units):
             json_schema = yaml.load(f, OrderedLoader)
 
         schema = {
+            # one of "Message Event" or "State Event"
             "typeof": "",
             "typeof_info": "",
+
+            # event type, eg "m.room.member". Note *not* the type of the
+            # event object (which should always be 'object').
             "type": None,
             "title": None,
             "desc": None,
@@ -720,7 +732,8 @@ class MatrixUnits(Units):
             ]
         }
 
-        # add typeof
+        # before we resolve the references, see if the first reference is to
+        # the message event or state event schemas, and add typeof info if so.
         base_defs = {
             ROOM_EVENT: "Message Event",
             STATE_EVENT: "State Event"