Add more CI checks for OpenAPI definitions and JSON Schemas (#1656)

Signed-off-by: Kévin Commaille <zecakeh@tedomum.fr>
Co-authored-by: Richard van der Hoff <1389908+richvdh@users.noreply.github.com>

.github/workflows/main.yml

@@ -27,7 +27,7 @@ jobs:
         run: |
           npx @redocly/cli@latest lint data/api/*/*.yaml
 
-  check-examples:
+  check-event-examples:
     name: "🔎 Check Event schema examples"
     runs-on: ubuntu-latest
     steps:
@@ -46,6 +46,44 @@ jobs:
         run: |
           python scripts/check-event-schema-examples.py
   
+  check-openapi-examples:
+    name: "🔎 Check OpenAPI definitions examples"
+    runs-on: ubuntu-latest
+    steps:
+      - name: "📥 Source checkout"
+        uses: actions/checkout@v2
+      - name: "➕ Setup Python"
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.9'
+          cache: 'pip'
+          cache-dependency-path: scripts/requirements.txt
+      - name: "➕ Install dependencies"
+        run: |
+          pip install -r scripts/requirements.txt
+      - name: "🔎 Run validator"
+        run: |
+          python scripts/check-openapi-sources.py
+
+  check-schemas-examples:
+    name: "🔎 Check JSON Schemas inline examples"
+    runs-on: ubuntu-latest
+    steps:
+      - name: "📥 Source checkout"
+        uses: actions/checkout@v2
+      - name: "➕ Setup Python"
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.9'
+          cache: 'pip'
+          cache-dependency-path: scripts/requirements.txt
+      - name: "➕ Install dependencies"
+        run: |
+          pip install -r scripts/requirements.txt
+      - name: "🔎 Run validator"
+        run: |
+          python scripts/check-json-schemas.py
+
   calculate-baseurl:
     name: "⚙️ Calculate baseURL for later jobs"
     runs-on: ubuntu-latest

changelogs/internal/newsfragments/1656.feature

@@ -0,0 +1 @@
+Add more CI checks for OpenAPI definitions and JSON Schemas.

data/event-schemas/schema/core-event-schema/msgtype_infos/image_info.yaml

@@ -1,4 +1,3 @@
-$schema: http://json-schema.org/draft-04/schema#
 description: Metadata about an image.
 properties:
   h:

data/event-schemas/schema/core-event-schema/msgtype_infos/thumbnail_info.yaml

@@ -1,4 +1,3 @@
-$schema: http://json-schema.org/draft-04/schema#
 description: Metadata about a thumbnail image.
 properties:
   h:

data/event-schemas/schema/m.room.third_party_invite.yaml

@@ -1,5 +1,4 @@
 ---
-$schema: http://json-schema.org/draft-04/schema#
 allOf:
   - $ref: core-event-schema/state_event.yaml
 description: "Acts as an `m.room.member` invite event, where there isn't a target user_id to invite. This event contains a token and a public key whose private key must be used to sign the token. Any user who can present that signature may use this invitation to join the target room."

scripts/check-event-schema-examples.py

@@ -1,5 +1,9 @@
 #!/usr/bin/env python
-#
+
+# Validates the examples under `../data/event_schemas` against their JSON
+# schemas. In the process, the JSON schemas are validated against the JSON
+# Schema 2020-12 specification.
+
 # Copyright 2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -92,7 +96,8 @@ def check_example_file(examplepath, schemapath):
 
     print ("Checking schema for: %r %r" % (examplepath, schemapath))
     try:
-        jsonschema.validate(example, schema, resolver=resolver)
+        validator = jsonschema.Draft202012Validator(schema, resolver)
+        validator.validate(example)
     except Exception as e:
         raise ValueError("Error validating JSON schema for %r %r" % (
             examplepath, schemapath

scripts/check-json-schemas.py

@@ -0,0 +1,196 @@
+#!/usr/bin/env python3
+
+# Validates the JSON schemas under `../data`. The schemas are validated against
+# the JSON Schema 2020-12 specification, and their inline examples and default
+# values are validated against the schema.
+
+# Copyright 2023 Kévin Commaille
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import sys
+import json
+import os
+import traceback
+
+
+def import_error(module, package, debian, error):
+    sys.stderr.write((
+        "Error importing %(module)s: %(error)r\n"
+        "To install %(module)s run:\n"
+        "  pip install %(package)s\n"
+        "or on Debian run:\n"
+        "  sudo apt-get install python-%(debian)s\n"
+    ) % locals())
+    if __name__ == '__main__':
+        sys.exit(1)
+
+try:
+    import jsonschema
+except ImportError as e:
+    import_error("jsonschema", "jsonschema", "jsonschema", e)
+    raise
+
+try:
+    import yaml
+except ImportError as e:
+    import_error("yaml", "PyYAML", "yaml", e)
+    raise
+
+try:
+    import jsonpath
+except ImportError as e:
+    import_error("jsonpath", "python-jsonpath", "jsonpath", e)
+    raise
+
+try:
+    import attrs
+except ImportError as e:
+    import_error("attrs", "attrs", "attrs", e)
+    raise
+
+@attrs.define
+class SchemaDirReport:
+    files: int = 0
+    errors: int = 0
+
+    def add(self, other_report):
+        self.files += other_report.files
+        self.errors += other_report.errors
+
+def load_file(path):
+    if not path.startswith("file://"):
+        raise Exception(f"Bad ref: {path}")
+    path = path[len("file://"):]
+    with open(path, "r") as f:
+        if path.endswith(".json"):
+            return json.load(f)
+        else:
+            # We have to assume it's YAML because some of the YAML examples
+            # do not have file extensions.
+            return yaml.safe_load(f)
+        
+def check_example(path, schema, example):
+    # URI with scheme is necessary to make RefResolver work.
+    fileurl = "file://" + os.path.abspath(path)
+    resolver = jsonschema.RefResolver(fileurl, schema, handlers={"file": load_file})
+    validator = jsonschema.Draft202012Validator(schema, resolver)
+
+    validator.validate(example)
+
+def check_schema_examples(path, full_schema):
+    """Search objects with inline examples in the schema and check they validate
+    against the object's definition.
+    """
+    errors = []
+    matches = jsonpath.finditer(
+        # Recurse through all objects and filter out those that don't have an
+        # `example`, `examples` or `default` field.
+        "$..[?(@.example != undefined || @.examples != undefined || @.default != undefined)]",
+        full_schema
+    )
+
+    for match in matches:
+        schema = match.obj
+        if "example" in schema:
+            try:
+                check_example(path, schema, schema["example"])
+            except Exception as e:
+                example_path = f"{match.path}['example']"
+                print(f"Failed to validate example at {example_path}: {e}")
+                errors.append(e)
+
+        if "examples" in schema:
+            for index, example in enumerate(schema["examples"]):
+                try:
+                    check_example(path, schema, example)
+                except Exception as e:
+                    example_path = f"{match.path}['examples'][{index}]"
+                    print(f"Failed to validate example at {example_path}: {e}")
+                    errors.append(e)
+
+        if "default" in schema:
+            try:
+                check_example(path, schema, schema["default"])
+            except Exception as e:
+                example_path = f"{match.path}['default']"
+                print(f"Failed to validate example at {example_path}: {e}")
+                errors.append(e)
+    
+    if len(errors) > 0:
+        raise Exception(errors)
+
+
+def check_schema_file(schema_path):
+    with open(schema_path) as f:
+        schema = yaml.safe_load(f)
+
+    print(f"Checking schema: {schema_path}")
+
+    # Check schema is valid.
+    try:
+        validator = jsonschema.Draft202012Validator
+        validator.check_schema(schema)
+    except Exception as e:
+        print(f"Failed to validate JSON schema: {e}")
+        raise
+    
+    # Check schema examples are valid.
+    check_schema_examples(schema_path, schema)
+
+def check_schema_dir(schemadir: str) -> SchemaDirReport:
+    report = SchemaDirReport()
+    for root, dirs, files in os.walk(schemadir):
+        for schemadir in dirs:
+            dir_report = check_schema_dir(os.path.join(root, schemadir))
+            report.add(dir_report)
+        for filename in files:
+            if filename.startswith("."):
+                # Skip over any vim .swp files.
+                continue
+            if filename.endswith(".json"):
+                # Skip over any explicit examples (partial event definitions)
+                continue
+            try:
+                report.files += 1
+                check_schema_file(os.path.join(root, filename))
+            except Exception as e:
+                report.errors += 1
+    return report
+
+# The directory that this script is residing in.
+script_dir = os.path.dirname(os.path.realpath(__file__))
+# The directory of the project.
+project_dir = os.path.abspath(os.path.join(script_dir, "../"))
+print(f"Project dir: {project_dir}")
+
+# Directories to check, relative to the data folder.
+schema_dirs = [
+    "api/application-service/definitions",
+    "api/client-server/definitions",
+    "api/identity/definitions",
+    "api/server-server/definitions",
+    "event-schemas/schema",
+    "schemas",
+]
+
+report = SchemaDirReport()
+for schema_dir in schema_dirs:
+    dir_report = check_schema_dir(os.path.join(project_dir, "data", schema_dir))
+    report.add(dir_report)
+
+print(f"Found {report.errors} errors in {report.files} files")
+
+if report.errors:
+    sys.exit(1)
+    

scripts/check-openapi-sources.py

@@ -1,5 +1,10 @@
 #! /usr/bin/env python
-#
+
+# Validates the OpenAPI definitions under `../data/api`. Checks the request
+# parameters and body, and response body. The schemas are validated against the
+# JSON Schema 2020-12 specification and the examples are validated against those
+# schemas.
+
 # Copyright 2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -47,17 +52,39 @@ def check_schema(filepath, example, schema):
     example = resolve_references(filepath, example)
     schema = resolve_references(filepath, schema)
     resolver = jsonschema.RefResolver(filepath, schema, handlers={"file": load_file})
-    jsonschema.validate(example, schema, resolver=resolver)
+    validator = jsonschema.Draft202012Validator(schema, resolver)
+    validator.validate(example)
 
 
 def check_parameter(filepath, request, parameter):
-    schema = parameter.get("schema")
+    schema = parameter.get('schema')
-    example = schema.get('example')
+    example = parameter.get('example')
+
+    if not example:
+        example = schema.get('example')
 
     if example and schema:
         try:
-            print("Checking request schema for: %r %r" % (
+            print("Checking schema for request parameter: %r %r %r" % (
-                filepath, request
+                filepath, request, parameter.get("name")
+            ))
+            check_schema(filepath, example, schema)
+        except Exception as e:
+            raise ValueError("Error validating JSON schema for %r" % (
+                request
+            ), e)
+
+def check_request_body(filepath, request, body):
+    schema = body.get('schema')
+    example = body.get('example')
+
+    if not example:
+        example = schema.get('example')
+
+    if example and schema:
+        try:
+            print("Checking schema for request body: %r %r" % (
+                filepath, request,
             ))
             check_schema(filepath, example, schema)
         except Exception as e:
@@ -67,44 +94,59 @@ def check_parameter(filepath, request, parameter):
 
 
 def check_response(filepath, request, code, response):
-    example = response.get('examples', {}).get('application/json')
     schema = response.get('schema')
-    if example and schema:
+    if schema:
-        try:
+        for name, example in response.get('examples', {}).items():
-            print ("Checking response schema for: %r %r %r" % (
+            value = example.get('value')
-                filepath, request, code
+            if value:
-            ))
+                try:
-            check_schema(filepath, example, schema)
+                    print ("Checking response schema for: %r %r %r %r" % (
-        except jsonschema.SchemaError as error:
+                        filepath, request, code, name
-            for suberror in sorted(error.context, key=lambda e: e.schema_path):
+                    ))
-                print(list(suberror.schema_path), suberror.message, sep=", ")
+                    check_schema(filepath, value, schema)
-            raise ValueError("Error validating JSON schema for %r %r" % (
+                except jsonschema.SchemaError as error:
-                request, code
+                    for suberror in sorted(error.context, key=lambda e: e.schema_path):
-            ), e)
+                        print(list(suberror.schema_path), suberror.message, sep=", ")
-        except Exception as e:
+                    raise ValueError("Error validating JSON schema for %r %r" % (
-            raise ValueError("Error validating JSON schema for %r %r" % (
+                        request, code
-                request, code
+                    ), e)
-            ), e)
+                except Exception as e:
+                    raise ValueError("Error validating JSON schema for %r %r" % (
+                        request, code
+                    ), e)
 
 
 def check_openapi_file(filepath):
     with open(filepath) as f:
         openapi = yaml.safe_load(f)
 
+    openapi_version = openapi.get('openapi')
+    if not openapi_version:
+        # This is not an OpenAPI file, skip.
+        return
+    elif openapi_version != '3.1.0':
+        raise ValueError("File %r is not using the proper OpenAPI version: expected '3.1.0', got %r" % (filepath, openapi_version))
+
     for path, path_api in openapi.get('paths', {}).items():
 
         for method, request_api in path_api.items():
             request = "%s %s" % (method.upper(), path)
             for parameter in request_api.get('parameters', ()):
-                if parameter['in'] == 'body':
+                check_parameter(filepath, request, parameter)
-                    check_parameter(filepath, request, parameter)
+            
+            json_body = request_api.get('requestBody', {}).get('content', {}).get('application/json')
+            if json_body:
+                check_request_body(filepath, request, json_body)
 
             try:
                 responses = request_api['responses']
             except KeyError:
                 raise ValueError("No responses for %r" % (request,))
             for code, response in responses.items():
-                check_response(filepath, request, code, response)
+                json_response = response.get('content', {}).get('application/json')
+
+                if json_response:
+                    check_response(filepath, request, code, json_response)
 
 
 def resolve_references(path, schema):
@@ -171,7 +213,7 @@ if __name__ == '__main__':
 
     # Resolve the directory containing the OpenAPI sources,
     # relative to the script path
-    source_files_directory = os.path.realpath(os.path.join(script_directory, "../data"))
+    source_files_directory = os.path.realpath(os.path.join(script_directory, "../data/api"))
 
     # Walk the source path directory, looking for YAML files to check
     for (root, dirs, files) in os.walk(source_files_directory):

scripts/requirements.txt

@@ -4,6 +4,8 @@
 # we need at least version 4.0.0 for support of JSON Schema Draft 2020-12.
 jsonschema == 4.17.3
 
+python-jsonpath == 0.9.0
+attrs >= 23.1.0
 PyYAML >= 3.12
 requests >= 2.18.4
 towncrier == 23.6.0